Skip to content

FEAT: Best-of-N jailbreak attack + CharNoiseConverter #2277

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Richard Lundeen (rlundeen2) merged 4 commits into from
Aug 28, 2026
Merged

FEAT: Best-of-N jailbreak attack + CharNoiseConverter #2277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3351fff
FEAT: Best-of-N jailbreak attack + CharNoiseConverter
francose Jul 27, 2026
145fbaf
FIX: n_samples int validation, doc scorer rubric, converter table
francose Jul 27, 2026
532b212
REFAC: Express Best-of-N as a technique
rlundeen2 Aug 27, 2026
5aa6e63
Merge origin/main into feat/best-of-n-technique
rlundeen2 Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 58 additions & 57 deletions doc/code/converters/0_converters.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,63 +89,64 @@
"31 text text BinaryConverter\n",
"32 text text BrailleConverter\n",
"33 text text CaesarConverter\n",
"34 text text CharSwapConverter\n",
"35 text text CharacterSpaceConverter\n",
"36 text text CodeAttackConverter\n",
"37 text text CodeChameleonConverter\n",
"38 text text ColloquialWordswapConverter\n",
"39 text text DecompositionConverter\n",
"40 text text DenylistConverter\n",
"41 text text DiacriticConverter\n",
"42 text text EcojiConverter\n",
"43 text text EmojiConverter\n",
"44 text text FirstLetterConverter\n",
"45 text text FlipConverter\n",
"46 text text IPAConverter\n",
"47 text text ImagePromptStyleConverter\n",
"48 text text InsertPunctuationConverter\n",
"49 text text JsonStringConverter\n",
"50 text text LLMGenericTextConverter\n",
"51 text text LeetspeakConverter\n",
"52 text text MaliciousQuestionGeneratorConverter\n",
"53 text text MathObfuscationConverter\n",
"54 text text MathPromptConverter\n",
"55 text text MorseConverter\n",
"56 text text NatoConverter\n",
"57 text text NegationTrapConverter\n",
"58 text text NoiseConverter\n",
"59 text text PersuasionConverter\n",
"60 text text PolicyPuppetryConverter\n",
"61 text text PuzzledConverter\n",
"62 text text ROT13Converter\n",
"63 text text RandomCapitalLettersConverter\n",
"64 text text RandomTranslationConverter\n",
"65 text text RepeatTokenConverter\n",
"66 text text SATAMaskingConverter\n",
"67 text text ScientificTranslationConverter\n",
"68 text text SearchReplaceConverter\n",
"69 text text SelectiveTextConverter\n",
"70 text text SneakyBitsSmugglerConverter\n",
"71 text text StringJoinConverter\n",
"72 text text SuffixAppendConverter\n",
"73 text text SuperscriptConverter\n",
"74 text text TaskFramingConverter\n",
"75 text text TatweelConverter\n",
"76 text text TemplateSegmentConverter\n",
"77 text text TenseConverter\n",
"78 text text TextJailbreakConverter\n",
"79 text text ToneConverter\n",
"80 text text ToxicSentenceGeneratorConverter\n",
"81 text text TranslationConverter\n",
"82 text text UnicodeConfusableConverter\n",
"83 text text UnicodeReplacementConverter\n",
"84 text text UnicodeSubstitutionConverter\n",
"85 text text UrlConverter\n",
"86 text text VariationConverter\n",
"87 text text VariationSelectorSmugglerConverter\n",
"88 text text VigenereConverter\n",
"89 text text ZalgoConverter\n",
"90 text text ZeroWidthConverter\n"
"34 text text CharNoiseConverter\n",
"35 text text CharSwapConverter\n",
"36 text text CharacterSpaceConverter\n",
"37 text text CodeAttackConverter\n",
"38 text text CodeChameleonConverter\n",
"39 text text ColloquialWordswapConverter\n",
"40 text text DecompositionConverter\n",
"41 text text DenylistConverter\n",
"42 text text DiacriticConverter\n",
"43 text text EcojiConverter\n",
"44 text text EmojiConverter\n",
"45 text text FirstLetterConverter\n",
"46 text text FlipConverter\n",
"47 text text IPAConverter\n",
"48 text text ImagePromptStyleConverter\n",
"49 text text InsertPunctuationConverter\n",
"50 text text JsonStringConverter\n",
"51 text text LLMGenericTextConverter\n",
"52 text text LeetspeakConverter\n",
"53 text text MaliciousQuestionGeneratorConverter\n",
"54 text text MathObfuscationConverter\n",
"55 text text MathPromptConverter\n",
"56 text text MorseConverter\n",
"57 text text NatoConverter\n",
"58 text text NegationTrapConverter\n",
"59 text text NoiseConverter\n",
"60 text text PersuasionConverter\n",
"61 text text PolicyPuppetryConverter\n",
"62 text text PuzzledConverter\n",
"63 text text ROT13Converter\n",
"64 text text RandomCapitalLettersConverter\n",
"65 text text RandomTranslationConverter\n",
"66 text text RepeatTokenConverter\n",
"67 text text SATAMaskingConverter\n",
"68 text text ScientificTranslationConverter\n",
"69 text text SearchReplaceConverter\n",
"70 text text SelectiveTextConverter\n",
"71 text text SneakyBitsSmugglerConverter\n",
"72 text text StringJoinConverter\n",
"73 text text SuffixAppendConverter\n",
"74 text text SuperscriptConverter\n",
"75 text text TaskFramingConverter\n",
"76 text text TatweelConverter\n",
"77 text text TemplateSegmentConverter\n",
"78 text text TenseConverter\n",
"79 text text TextJailbreakConverter\n",
"80 text text ToneConverter\n",
"81 text text ToxicSentenceGeneratorConverter\n",
"82 text text TranslationConverter\n",
"83 text text UnicodeConfusableConverter\n",
"84 text text UnicodeReplacementConverter\n",
"85 text text UnicodeSubstitutionConverter\n",
"86 text text UrlConverter\n",
"87 text text VariationConverter\n",
"88 text text VariationSelectorSmugglerConverter\n",
"89 text text VigenereConverter\n",
"90 text text ZalgoConverter\n",
"91 text text ZeroWidthConverter\n"
]
}
],
Expand Down
4 changes: 4 additions & 0 deletions doc/code/converters/1_text_to_text_converters.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@
" ArabiziConverter,\n",
" BidiConverter,\n",
" CharacterSpaceConverter,\n",
" CharNoiseConverter,\n",
" CharSwapConverter,\n",
" CodeAttackConverter,\n",
" CodeChameleonConverter,\n",
Expand Down Expand Up @@ -440,6 +441,9 @@
")\n",
"print(\"CharSwap:\", await char_swap.convert_async(prompt=prompt)) # type: ignore\n",
"\n",
"# CharNoise nudges printable ASCII characters to an adjacent codepoint\n",
"print(\"CharNoise:\", await CharNoiseConverter(noise_probability=0.2).convert_async(prompt=prompt)) # type: ignore\n",
"\n",
"# Insert punctuation adds punctuation marks\n",
"insert_punct = InsertPunctuationConverter(word_swap_ratio=0.2)\n",
"print(\"Insert Punctuation:\", await insert_punct.convert_async(prompt=prompt)) # type: ignore\n",
Expand Down
4 changes: 4 additions & 0 deletions doc/code/converters/1_text_to_text_converters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@
ArabiziConverter,
BidiConverter,
CharacterSpaceConverter,
CharNoiseConverter,
CharSwapConverter,
CodeAttackConverter,
CodeChameleonConverter,
Expand Down Expand Up @@ -182,6 +183,9 @@
)
print("CharSwap:", await char_swap.convert_async(prompt=prompt)) # type: ignore

# CharNoise nudges printable ASCII characters to an adjacent codepoint
print("CharNoise:", await CharNoiseConverter(noise_probability=0.2).convert_async(prompt=prompt)) # type: ignore

# Insert punctuation adds punctuation marks
insert_punct = InsertPunctuationConverter(word_swap_ratio=0.2)
print("Insert Punctuation:", await insert_punct.convert_async(prompt=prompt)) # type: ignore
Expand Down
7 changes: 6 additions & 1 deletion doc/code/scenarios/0_attack_techniques.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@
" any scenario can use (the `role_play_*` variants, `many_shot`, `tap`, the `crescendo_*` variants, `red_teaming`,\n",
" `context_compliance`). Registered by default.\n",
"- [`extra.py`](../../../pyrit/setup/initializers/techniques/extra.py) — opt-in techniques that are\n",
" not part of the default set (`pair`, `violent_durian`, `skeleton_key`).\n",
" not part of the default set (`pair`, `violent_durian`, `skeleton_key`, `best_of_n`).\n",
"- [`airt.py`](../../../pyrit/setup/initializers/techniques/airt.py) — source-owned techniques that\n",
" belong to a specific AIRT scenario. Unlike `core`/`extra`, these are imported directly by their\n",
" owning scenario and are *not* part of the default aggregation.\n",
Expand Down Expand Up @@ -646,6 +646,11 @@
"consider whether it would be better expressed as a registered technique so scenarios can select it\n",
"by name and tag.\n",
"\n",
"`best_of_n` is another example. Its factory configures a\n",
"`PromptSendingAttack` with a 20-sample retry budget and stochastic character-swap, capitalization,\n",
"and ASCII-noise converters. `PromptSendingAttack` already re-runs the converters for each attempt\n",
"and stops when the objective scorer succeeds, so this technique does not need a separate attack class.\n",
"\n",
"## Defining your own\n",
"\n",
"To add a technique, register a factory. The simplest form names an attack class and tags it:\n",
Expand Down
7 changes: 6 additions & 1 deletion doc/code/scenarios/0_attack_techniques.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
# any scenario can use (the `role_play_*` variants, `many_shot`, `tap`, the `crescendo_*` variants, `red_teaming`,
# `context_compliance`). Registered by default.
# - [`extra.py`](../../../pyrit/setup/initializers/techniques/extra.py) — opt-in techniques that are
# not part of the default set (`pair`, `violent_durian`, `skeleton_key`).
# not part of the default set (`pair`, `violent_durian`, `skeleton_key`, `best_of_n`).
# - [`airt.py`](../../../pyrit/setup/initializers/techniques/airt.py) — source-owned techniques that
# belong to a specific AIRT scenario. Unlike `core`/`extra`, these are imported directly by their
# owning scenario and are *not* part of the default aggregation.
Expand Down Expand Up @@ -209,6 +209,11 @@
# consider whether it would be better expressed as a registered technique so scenarios can select it
# by name and tag.
#
# `best_of_n` is another example. Its factory configures a
# `PromptSendingAttack` with a 20-sample retry budget and stochastic character-swap, capitalization,
# and ASCII-noise converters. `PromptSendingAttack` already re-runs the converters for each attempt
# and stops when the objective scorer succeeds, so this technique does not need a separate attack class.
#
# ## Defining your own
#
# To add a technique, register a factory. The simplest form names an attack class and tags it:
Expand Down
8 changes: 8 additions & 0 deletions doc/references.bib
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -814,6 +814,14 @@ @misc{hiddenlayer2025policypuppetry
note = {HiddenLayer Innovation Hub. Introduces the Policy Puppetry prompt injection technique},
}

@article{hughes2024bestofn,
title = {Best-of-N Jailbreaking},
author = {John Hughes and Sara Price and Aengus Lynch and Rylan Schaeffer and Fazl Barez and Sanmi Koyejo and Henry Sleight and Erik Jones and Ethan Perez and Mrinank Sharma},
journal = {arXiv preprint arXiv:2412.03556},
year = {2024},
url = {https://arxiv.org/abs/2412.03556},
}

@inproceedings{dong2025sata,
title = {{SATA}: A Paradigm for {LLM} Jailbreak via Simple Assistive Task Linkage},
author = {Xiaoning Dong and Wenbo Hu and Wei Xu and Tianxing He},
Expand Down
2 changes: 2 additions & 0 deletions pyrit/converter/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
from pyrit.converter.binary_converter import BinaryConverter
from pyrit.converter.braille_converter import BrailleConverter
from pyrit.converter.caesar_converter import CaesarConverter
from pyrit.converter.char_noise_converter import CharNoiseConverter
from pyrit.converter.character_space_converter import CharacterSpaceConverter
from pyrit.converter.charswap_attack_converter import CharSwapConverter
from pyrit.converter.code_attack_converter import CodeAttackConverter
Expand Down Expand Up @@ -165,6 +166,7 @@
"BinaryConverter": "pyrit.converter.binary_converter",
"BrailleConverter": "pyrit.converter.braille_converter",
"CaesarConverter": "pyrit.converter.caesar_converter",
"CharNoiseConverter": "pyrit.converter.char_noise_converter",
"CharSwapConverter": "pyrit.converter.charswap_attack_converter",
"CharacterSpaceConverter": "pyrit.converter.character_space_converter",
"CodeAttackConverter": "pyrit.converter.code_attack_converter",
Expand Down
67 changes: 67 additions & 0 deletions pyrit/converter/char_noise_converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

import random

from pyrit.converter.converter import Converter, ConverterResult
from pyrit.models import ComponentIdentifier, PromptDataType


class CharNoiseConverter(Converter):
"""
Nudges printable ASCII characters to an adjacent codepoint.

Each character is shifted one step up or down with probability ``noise_probability``,
kept inside the printable ASCII range. Non-ASCII characters are left alone. Unlike
``NoiseConverter`` this uses no LLM, and each call draws fresh randomness.
"""

SUPPORTED_INPUT_TYPES = ("text",)
SUPPORTED_OUTPUT_TYPES = ("text",)

def __init__(self, *, noise_probability: float = 0.05) -> None:
"""
Args:
noise_probability (float): Per-character probability in [0.0, 1.0] of
nudging a character to an adjacent codepoint. Defaults to 0.05.

Raises:
ValueError: If ``noise_probability`` is outside [0.0, 1.0].
"""
if not 0.0 <= noise_probability <= 1.0:
raise ValueError("noise_probability must be between 0.0 and 1.0")
self.noise_probability = noise_probability

def _build_identifier(self) -> ComponentIdentifier:
return self._create_identifier(params={"noise_probability": self.noise_probability})

def _noise(self, text: str) -> str:
out = []
for ch in text:
if not " " <= ch <= "~" or random.random() >= self.noise_probability:
out.append(ch)
continue

offset = 1 if ch == " " else -1 if ch == "~" else random.choice((-1, 1))
ch = chr(ord(ch) + offset)
out.append(ch)
return "".join(out)

async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult:
"""
Apply one fresh pass of ASCII noise to the prompt.

Args:
prompt (str): The text prompt to perturb.
input_type (PromptDataType): The input data type. Only ``text`` is supported.

Returns:
ConverterResult: The perturbed prompt.

Raises:
ValueError: If the input type is not supported.
"""
if not self.input_supported(input_type):
raise ValueError("Input type not supported")

return ConverterResult(output_text=self._noise(prompt), output_type="text")
Loading
Loading