Complete generateTokensPhi3 implementation and refine Phi3 tokenizer logic

mikepapadim · mikepapadim · commit 8adb2bbf9fab · 2025-08-01T16:50:40.000+03:00
diff --git a/src/main/java/com/example/inference/InferenceEngine.java b/src/main/java/com/example/inference/InferenceEngine.java
@@ -2,14 +2,14 @@
 
 import com.example.auxiliary.LastRunMetrics;
 import com.example.inference.sampler.Sampler;
-import com.example.inference.state.Phi3State;
 import com.example.inference.state.State;
 import com.example.model.Configuration;
 import com.example.model.Model;
 import com.example.tokenizer.impl.Tokenizer;
 import com.example.tornadovm.TornadoVMMasterPlan;
 import uk.ac.manchester.tornado.api.types.arrays.FloatArray;
 
+import java.io.ByteArrayOutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -217,7 +217,57 @@ public static List<Integer> generateTokensQwen3(Model model, State state, int st
 
     public static List<Integer> generateTokensPhi3(Model model, State state, int startPosition, List<Integer> promptTokens, Set<Integer> stopTokens, int maxTokens, Sampler sampler, boolean echo,
             IntConsumer onTokenGenerated) {
-        return null;
+
+        long startNanos = System.nanoTime();
+        if (maxTokens < 0 || model.configuration().contextLength() < maxTokens) {
+            maxTokens = model.configuration().contextLength();
+        }
+        List<Integer> generatedTokens = new ArrayList<>(maxTokens);
+        int token = state.latestToken; // BOS?
+        int nextToken;
+        int promptIndex = 0;
+        ByteArrayOutputStream baos = new ByteArrayOutputStream(5);
+        for (int position = startPosition; position < maxTokens; ++position) {
+
+            model.forward(state, token, position);
+            if (promptIndex < promptTokens.size()) {
+                // Force-pick token from prompt.
+                nextToken = promptTokens.get(promptIndex++);
+                if (echo) {
+                    // log prompt token (different color?)
+                    System.out.println("NextToken: " + nextToken);
+                    //System.err.print(Tokenizer.replaceControlCharacters(model.tokenizer().decode(List.of(nextToken))));
+                    String decoded = model.tokenizer().decode(List.of(nextToken));
+                    System.err.print(Tokenizer.replaceControlCharacters(model.tokenizer().decode(List.of(nextToken))));
+
+                    //                    System.err.print(de(decoded, baos));
+                }
+            } else {
+                nextToken = sampler.sampleToken(state.logits);
+                if (echo) {
+                    // log inferred token
+                    System.err.print(Tokenizer.replaceControlCharacters(model.tokenizer().decode(List.of(nextToken))));
+                }
+                generatedTokens.add(nextToken);
+                if (onTokenGenerated != null) {
+                    onTokenGenerated.accept(nextToken);
+                }
+                if (stopTokens.contains(nextToken)) {
+                    break;
+                }
+            }
+            state.latestToken = token = nextToken;
+            if (position == 2000) {
+                break;
+            }
+        }
+
+        long elapsedNanos = System.nanoTime() - startNanos;
+        int totalTokens = promptIndex + generatedTokens.size();
+        System.err.printf("%n%.2f tokens/s (%d)%n", totalTokens / (elapsedNanos / 1_000_000_000.0), totalTokens);
+
+        return generatedTokens;
+
     }
 
     public static List<Integer> generateTokensGPULlama(Model model, State state, int startPosition, List<Integer> promptTokens, Set<Integer> stopTokens, int maxTokens, Sampler sampler, boolean echo,
@@ -406,4 +456,5 @@ public static List<Integer> generateTokensGPUPhi3(Model model, State state, int
             IntConsumer onTokenGenerated, TornadoVMMasterPlan tornadoVMPlan) {
         return null;
     }
-}
+
+}
diff --git a/src/main/java/com/example/model/phi3/Phi3.java b/src/main/java/com/example/model/phi3/Phi3.java
@@ -66,7 +66,7 @@ public void forward(State state, int token, int position) {
     @Override
     public List<Integer> generateTokens(State state, int startPosition, List<Integer> promptTokens, Set<Integer> stopTokens, int maxTokens, Sampler sampler, boolean echo,
             IntConsumer onTokenGenerated) {
-        return InferenceEngine.generateTokensQwen3(this, state, startPosition, promptTokens, stopTokens, maxTokens, sampler, echo, onTokenGenerated);
+        return InferenceEngine.generateTokensPhi3(this, state, startPosition, promptTokens, stopTokens, maxTokens, sampler, echo, onTokenGenerated);
     }
 
     @Override
diff --git a/src/main/java/com/example/tokenizer/impl/Phi3Tokenizer.java b/src/main/java/com/example/tokenizer/impl/Phi3Tokenizer.java
@@ -73,12 +73,12 @@ public Map<String, Integer> getSpecialTokens() {
 
     @Override
     public boolean isSpecialToken(int tokenIndex) {
-        return false;
+        return specialTokens.containsValue(tokenIndex);
     }
 
     @Override
     public boolean shouldDisplayToken(int token) {
-        return false;
+        return !isSpecialToken(token);
     }
 
     @Override

Original file line number	Diff line number	Diff line change
`@@ -66,7 +66,7 @@ public void forward(State state, int token, int position) {`
`66`	`66`	`@Override`
`67`	`67`	`public List<Integer> generateTokens(State state, int startPosition, List<Integer> promptTokens, Set<Integer> stopTokens, int maxTokens, Sampler sampler, boolean echo,`
`68`	`68`	`IntConsumer onTokenGenerated) {`
`69`		`- return InferenceEngine.generateTokensQwen3(this, state, startPosition, promptTokens, stopTokens, maxTokens, sampler, echo, onTokenGenerated);`
	`69`	`+ return InferenceEngine.generateTokensPhi3(this, state, startPosition, promptTokens, stopTokens, maxTokens, sampler, echo, onTokenGenerated);`
`70`	`70`	`}`
`71`	`71`
`72`	`72`	`@Override`
Original file line number	Diff line number	Diff line change
`@@ -73,12 +73,12 @@ public Map<String, Integer> getSpecialTokens() {`
`73`	`73`
`74`	`74`	`@Override`
`75`	`75`	`public boolean isSpecialToken(int tokenIndex) {`
`76`		`- return false;`
	`76`	`+ return specialTokens.containsValue(tokenIndex);`
`77`	`77`	`}`
`78`	`78`
`79`	`79`	`@Override`
`80`	`80`	`public boolean shouldDisplayToken(int token) {`
`81`		`- return false;`
	`81`	`+ return !isSpecialToken(token);`
`82`	`82`	`}`
`83`	`83`
`84`	`84`	`@Override`