Apply suggestion from @Copilot

mikepapadim · Copilot · web-flow · commit 2df92cc763fa · 2025-08-04T13:02:39.000+03:00
Co-authored-by: Copilot &lt;175728472+Copilot@users.noreply.github.com&gt;
diff --git a/src/main/java/com/example/tokenizer/impl/Phi3Tokenizer.java b/src/main/java/com/example/tokenizer/impl/Phi3Tokenizer.java
@@ -33,12 +33,15 @@ public class Phi3Tokenizer implements Tokenizer {
     private final int[] tokenType;
     private final int byte0;
 
+    /** Number of base tokens in the vocabulary; tokens after this index are considered special. */
+    private static final int BASE_TOKENS = 32000;
+
     public Phi3Tokenizer(Map<String, Object> metadata, Vocabulary vocabulary) {
         int[] tokenTypes = (int[]) metadata.get("tokenizer.ggml.token_type");
         List<Pair<Integer, Integer>> merges = Collections.emptyList();
 
         int allTokens = vocabulary.size();
-        int baseTokens = 32000; // assume all tokens after the base ones are special.
+        int baseTokens = BASE_TOKENS; // assume all tokens after the base ones are special.
         //int reservedSpecialTokens = allTokens - baseTokens;
         List<String> specialTokensList = Arrays.stream(vocabulary.tokens(), baseTokens, allTokens).toList();
 
