diff --git a/backends/exllamav3/model.py b/backends/exllamav3/model.py index a0902327..939adf29 100644 --- a/backends/exllamav3/model.py +++ b/backends/exllamav3/model.py @@ -1136,6 +1136,10 @@ async def generate_gen( sampler_builder = ExllamaV3SamplerBuilder() + # Apply logit bias first so it lands ahead of the other steps + if params.logit_bias: + sampler_builder.logit_bias(params.logit_bias) + # Penalties # Set penalty range diff --git a/backends/exllamav3/sampler.py b/backends/exllamav3/sampler.py index 403cdcde..c75f1d2e 100644 --- a/backends/exllamav3/sampler.py +++ b/backends/exllamav3/sampler.py @@ -12,6 +12,7 @@ SS_Sample, SS_Base, SS_AdaptiveP, + SS_LogitBias, ) @@ -41,6 +42,10 @@ def top_p(self, top_p): def min_p(self, min_p): self.stack.append(SS_MinP(min_p)) + def logit_bias(self, logit_bias): + # Must run before the logits are transformed, so prepend it to the stack + self.stack.insert(0, SS_LogitBias(logit_bias)) + def greedy(self): self.stack.append(SS_Argmax()) diff --git a/common/sampling.py b/common/sampling.py index 94533642..3b000ad2 100644 --- a/common/sampling.py +++ b/common/sampling.py @@ -34,7 +34,6 @@ "xtc_probability": 0.0, "dry_multiplier": 0.0, "mirostat_mode": 0, - "logit_bias": None, "temp_exponent": 1.0, }