From d1c115f12e51e461a09ac42549ecb013bb42d4a4 Mon Sep 17 00:00:00 2001 From: Oleksandr Zakharchuk Date: Sun, 2 Aug 2026 07:18:52 +0000 Subject: [PATCH] Sampling: Support logit_bias on the exllamav3 backend Passes logit_bias through to the exllamav3 backend instead of dropping it with an unsupported-parameter warning. Adds a logit_bias() method to ExllamaV3SamplerBuilder that prepends SS_LogitBias (it must run before the logits are transformed), calls it from the exllamav3 model when a request sets a bias, and removes logit_bias from UNSUPPORTED_PARAMS. Depends on turboderp-org/exllamav3#274 (adds SS_LogitBias). --- backends/exllamav3/model.py | 4 ++++ backends/exllamav3/sampler.py | 5 +++++ common/sampling.py | 1 - 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/backends/exllamav3/model.py b/backends/exllamav3/model.py index a0902327..939adf29 100644 --- a/backends/exllamav3/model.py +++ b/backends/exllamav3/model.py @@ -1136,6 +1136,10 @@ async def generate_gen( sampler_builder = ExllamaV3SamplerBuilder() + # Apply logit bias first so it lands ahead of the other steps + if params.logit_bias: + sampler_builder.logit_bias(params.logit_bias) + # Penalties # Set penalty range diff --git a/backends/exllamav3/sampler.py b/backends/exllamav3/sampler.py index 403cdcde..c75f1d2e 100644 --- a/backends/exllamav3/sampler.py +++ b/backends/exllamav3/sampler.py @@ -12,6 +12,7 @@ SS_Sample, SS_Base, SS_AdaptiveP, + SS_LogitBias, ) @@ -41,6 +42,10 @@ def top_p(self, top_p): def min_p(self, min_p): self.stack.append(SS_MinP(min_p)) + def logit_bias(self, logit_bias): + # Must run before the logits are transformed, so prepend it to the stack + self.stack.insert(0, SS_LogitBias(logit_bias)) + def greedy(self): self.stack.append(SS_Argmax()) diff --git a/common/sampling.py b/common/sampling.py index 94533642..3b000ad2 100644 --- a/common/sampling.py +++ b/common/sampling.py @@ -34,7 +34,6 @@ "xtc_probability": 0.0, "dry_multiplier": 0.0, "mirostat_mode": 0, - "logit_bias": None, "temp_exponent": 1.0, }