bump high level API

manishshettym · manishshettym · commit 6bb3522f0833 · 2023-09-17T21:23:02.000-07:00
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -430,6 +430,7 @@ def tokenize(self, text: bytes, add_bos: bool = True) -> List[int]:
         n_tokens = llama_cpp.llama_tokenize_with_model(
             self.model,
             text,
+            len(text)
             tokens,
             n_ctx,
             add_bos,
@@ -440,6 +441,7 @@ def tokenize(self, text: bytes, add_bos: bool = True) -> List[int]:
             n_tokens = llama_cpp.llama_tokenize_with_model(
                 self.model,
                 text,
+                len(text),
                 tokens,
                 n_tokens,
                 add_bos,
