From 68b46cdb46d7500acf82894e10aa9434dc7212d5 Mon Sep 17 00:00:00 2001 From: Advait Jain Date: Wed, 6 May 2026 23:26:08 -0700 Subject: [PATCH] Default CPU benchmarks to :memory cache in litert-lm CLI. LiteRT-LM-PiperOrigin-RevId: 911763178 --- python/litert_lm_cli/model.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/litert_lm_cli/model.py b/python/litert_lm_cli/model.py index 099b3be5..1eccd197 100644 --- a/python/litert_lm_cli/model.py +++ b/python/litert_lm_cli/model.py @@ -503,6 +503,11 @@ def benchmark( try: backend_val = _parse_backend(backend, npu_library_dir) + cache_dir_val = ( + ":memory" + if isinstance(backend_val, litert_lm.Backend.CPU) + else ":nocache" + ) if is_android: if not _HAS_ADB: @@ -512,7 +517,6 @@ def benchmark( backend=backend_val, prefill_tokens=prefill_tokens, decode_tokens=decode_tokens, - cache_dir=":nocache", max_num_tokens=max_num_tokens, ) else: @@ -521,7 +525,7 @@ def benchmark( backend=backend_val, prefill_tokens=prefill_tokens, decode_tokens=decode_tokens, - cache_dir=":nocache", + cache_dir=cache_dir_val, enable_speculative_decoding=enable_speculative_decoding, max_num_tokens=max_num_tokens, )