@inproceedings{Sun26EvolutionaryBlackboxFramework,
        abstract = {Large language models (LLMs) remain susceptible to adversarial prompts that can bypass alignment mechanisms. Existing approaches to adversarial prompt generation typically rely on manual prompt engineering, helper LLMs, or white-box adversarial machine learning methods, which either lack scalability or require access to model internals. In this paper, we propose a novel black-box framework for automated adversarial prompt generation based on evolutionary algorithms. The framework is instantiated using a genetic algorithm and an evolution strategy and operates without access to internal model parameters, making it applicable to both open-source and proprietary LLMs. To improve search effectiveness under realistic query constraints, we introduce a novel population initialisation strategy based on templates, pre-prompts, and post-prompts. Evolutionary search is guided by heuristic, model-agnostic fitness signals derived from prompt goal semantic similarity, refusal based response assessment, and a small heuristic lexical bonus based on lightweight instruction-following indicators. We evaluate our framework across multiple LLMs using a refusal based attack success rate metric, demonstrating consistent improvements over direct dataset prompting and competitive performance against a state-of-the-art white-box baseline under comparable query budgets. Additional analyses examine fitness stabilisation and cross-model transferability for unseen models.},
        address = {New York, NY, USA},
        author = {Sun, Qiyang and Karafili, Erisa},
        booktitle = {Proceedings of the Sixteenth ACM Conference on Data and Application Security and Privacy},
        doi = {10.1145/3800506.3803495},
        isbn = {9798400725623},
        keywords = {jailbreak attacks, black-box attack, adversarial prompt generation, evolutionary algorithms, large language models},
        location = {Germany},
        numpages = {11},
        pages = {168--178},
        publisher = {Association for Computing Machinery},
        series = {CODASPY '26},
        title = {An Evolutionary Black-Box Framework for Adversarial Prompt Generation in Large Language Models},
        year = {2026}
}