Sampling: Support logit_bias on the exllamav3 backend

backends/exllamav3/model.py

@@ -1136,6 +1136,10 @@ async def generate_gen(
 
         sampler_builder = ExllamaV3SamplerBuilder()
 
+        # Apply logit bias first so it lands ahead of the other steps
+        if params.logit_bias:
+            sampler_builder.logit_bias(params.logit_bias)
+
         # Penalties
 
         # Set penalty range

backends/exllamav3/sampler.py

@@ -12,6 +12,7 @@
     SS_Sample,
     SS_Base,
     SS_AdaptiveP,
+    SS_LogitBias,
 )
 
 
@@ -41,6 +42,10 @@ def top_p(self, top_p):
     def min_p(self, min_p):
         self.stack.append(SS_MinP(min_p))
 
+    def logit_bias(self, logit_bias):
+        # Must run before the logits are transformed, so prepend it to the stack
+        self.stack.insert(0, SS_LogitBias(logit_bias))
+
     def greedy(self):
         self.stack.append(SS_Argmax())
 

common/sampling.py

@@ -34,7 +34,6 @@
     "xtc_probability": 0.0,
     "dry_multiplier": 0.0,
     "mirostat_mode": 0,
-    "logit_bias": None,
     "temp_exponent": 1.0,
 }