Dirky
9df9689fdc
* feat: Added XTC logit processor * feat: Excluding newline/EOS chars from XTC * fix: Avoiding non-vectorized operation * feat: Moved XTC to samplers + correcting threshold allowed range * feat: Added XTC sampler to mlx_lm.chat and mlx_lm.generate * test: Adding test for apply_xtc * test: Updated test to match the new XTC behavior * feat: Simplified XTC implementation * feat: XTC special tokens exclusion simplification * fix: Corrected wrong compare for xtc_probability and threshold verif in server * test: Resetting probs and logprobs to avoid failures in CI * Fix tests * Change the special tokens name a bit * Fix it in mlx_lm.server --------- Co-authored-by: Dirky <dirky@dirky.fr> Co-authored-by: Angelos Katharopoulos <a_katharopoulos@apple.com>
122 lines
4.6 KiB
Python
122 lines
4.6 KiB
Python
import unittest
|
|
|
|
import mlx.core as mx
|
|
|
|
from mlx_lm.sample_utils import apply_min_p, apply_top_k, apply_top_p, apply_xtc
|
|
|
|
|
|
class TestSampleUtils(unittest.TestCase):
|
|
def test_apply_top_p(self):
|
|
probs = mx.array([0.9, 0.0, 0.0, 0.1])[None]
|
|
logits = mx.log(probs)
|
|
|
|
new_logits = apply_top_p(logits, 0.3)
|
|
actual_probs = mx.softmax(new_logits.squeeze())
|
|
self.assertEqual(actual_probs.tolist(), [1.0, 0.0, 0.0, 0.0])
|
|
|
|
new_logits = apply_top_p(logits, 0.95)
|
|
actual_probs = mx.softmax(new_logits.squeeze())
|
|
self.assertTrue(mx.allclose(probs.squeeze(), actual_probs))
|
|
|
|
probs = mx.array([0.0, 0.5, 0.4, 0.1])[None]
|
|
logits = mx.log(probs)
|
|
new_logits = apply_top_p(logits, 0.4)
|
|
actual_probs = mx.softmax(new_logits.squeeze())
|
|
self.assertEqual(actual_probs.tolist(), [0.0, 1.0, 0.0, 0.0])
|
|
|
|
new_logits = apply_top_p(logits, 0.6)
|
|
actual_probs = mx.softmax(new_logits.squeeze())
|
|
self.assertEqual(
|
|
[round(p, 4) for p in actual_probs.tolist()], [0.0, 0.5556, 0.4444, 0.0]
|
|
)
|
|
|
|
new_logits = apply_top_p(logits, 0.95)
|
|
actual_probs = mx.softmax(new_logits.squeeze())
|
|
actual_rounded = [round(p, 4) for p in actual_probs.tolist()]
|
|
expected_rounded = [0.0, 0.5, 0.4, 0.1]
|
|
self.assertEqual(actual_rounded, expected_rounded)
|
|
self.assertAlmostEqual(sum(actual_probs.tolist()), 1.0)
|
|
|
|
# Batch mode works
|
|
probs = mx.array([[0.9, 0.0, 0.0, 0.1], [0.0, 0.8, 0.1, 0.1]])
|
|
logits = mx.log(probs)
|
|
new_logits = apply_top_p(logits, 0.5)
|
|
actual_probs = mx.softmax(new_logits, axis=-1)
|
|
self.assertEqual(
|
|
actual_probs.tolist(), [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]]
|
|
)
|
|
|
|
def test_apply_min_p(self):
|
|
probs = mx.array([0.9, 0.0, 0.0, 0.1])[None]
|
|
logits = mx.log(probs)
|
|
new_logits = apply_min_p(logits, 0.8)
|
|
actual_probs = mx.softmax(new_logits.squeeze())
|
|
self.assertEqual(actual_probs.tolist(), [1.0, 0.0, 0.0, 0.0])
|
|
|
|
probs = mx.array([0.9, 0.0, 0.0, 0.1])[None]
|
|
logits = mx.log(probs)
|
|
new_logits = apply_min_p(logits, 0.05)
|
|
actual_probs = mx.softmax(new_logits.squeeze())
|
|
self.assertTrue(mx.allclose(actual_probs, mx.squeeze(probs)))
|
|
|
|
# Batch mode works
|
|
probs = mx.array([[0.9, 0.0, 0.0, 0.1], [0.0, 0.8, 0.0, 0.1]])
|
|
logits = mx.log(probs)
|
|
new_logits = apply_min_p(logits, 0.7)
|
|
actual_probs = mx.softmax(new_logits, axis=-1)
|
|
self.assertEqual(
|
|
actual_probs.tolist(), [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]]
|
|
)
|
|
|
|
def test_apply_top_k(self):
|
|
probs = mx.array([0.9, 0.0, 0.0, 0.1])[None]
|
|
logits = mx.log(probs)
|
|
|
|
new_logits = apply_top_k(logits, 1)
|
|
actual_probs = mx.softmax(new_logits.squeeze())
|
|
self.assertEqual(actual_probs.tolist(), [1.0, 0.0, 0.0, 0.0])
|
|
|
|
probs = mx.array([0.6, 0.0, 0.1, 0.3])[None]
|
|
logits = mx.log(probs)
|
|
new_logits = apply_top_k(logits, 2)
|
|
actual_probs = mx.softmax(new_logits.squeeze())
|
|
self.assertEqual(
|
|
[round(p, 4) for p in actual_probs.tolist()], [0.6667, 0.0, 0.0, 0.3333]
|
|
)
|
|
|
|
# Batch mode works
|
|
probs = mx.array([[0.9, 0.0, 0.0, 0.1], [0.0, 0.8, 0.0, 0.1]])
|
|
logits = mx.log(probs)
|
|
|
|
new_logits = apply_top_k(logits, 1)
|
|
actual_probs = mx.softmax(new_logits, axis=-1)
|
|
self.assertEqual(
|
|
actual_probs.tolist(), [[1.0, 0.0, 0.0, 0.0], [0.0, 1.0, 0.0, 0.0]]
|
|
)
|
|
|
|
def test_apply_xtc(self):
|
|
# Test the threshold
|
|
probs = mx.array([[0.4, 0.3, 0.15, 0.15]])
|
|
new_probs = mx.softmax(apply_xtc(mx.log(probs), 1, 0.2, []), -1)
|
|
expected = mx.array([[0, 0.5, 0.25, 0.25]])
|
|
self.assertTrue(mx.allclose(new_probs, expected))
|
|
probs = mx.array([[0.4, 0.3, 0.15, 0.15]])
|
|
new_probs = mx.softmax(apply_xtc(mx.log(probs), 1, 0.1, []), -1)
|
|
expected = mx.array([[0, 0.0, 0.5, 0.5]])
|
|
self.assertTrue(mx.allclose(new_probs, expected))
|
|
|
|
# Test the special tokens
|
|
probs = mx.array([[0.4, 0.3, 0.15, 0.15]])
|
|
new_probs = mx.softmax(apply_xtc(mx.log(probs), 1, 0.1, [0]), -1)
|
|
expected = mx.array([[4 / 7, 0.0, 1.5 / 7, 1.5 / 7]])
|
|
self.assertTrue(mx.allclose(new_probs, expected))
|
|
|
|
# Test that with probability 0 the probs don't change
|
|
probs = mx.array([[0.4, 0.3, 0.15, 0.15]])
|
|
new_probs = mx.softmax(apply_xtc(mx.log(probs), 0, 0.1, [0]), -1)
|
|
self.assertTrue(mx.allclose(new_probs, probs))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|