fix(parser) When ignoring a potential match highlighting can terminate early (highlightjs#2650)

joshgoebel · web-flow · commit 730f891cb841 · 2020-09-04T20:24:34.000-04:00
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,9 @@
 ## Version 10.2.0 (next up)
 
+Parser Engine:
+
+- (fix) When ignoring a potential match highlighting can terminate early (#2649) [Josh Goebel][]
+
 New themes:
 
 - *Gradient Light* by [Samia Ali]()
diff --git a/src/highlight.js b/src/highlight.js
@@ -261,6 +261,14 @@ const HLJS = function(hljs) {
       }
     }
 
+    /**
+     * Advance a single character
+     */
+    function advanceOne() {
+      mode_buffer += codeToHighlight[index];
+      index += 1;
+    }
+
     /**
      * Handle matching but then ignoring a sequence of text
      *
@@ -275,7 +283,7 @@ const HLJS = function(hljs) {
       } else {
         // no need to move the cursor, we still have additional regexes to try and
         // match at this very spot
-        continueScanAtSamePosition = true;
+        resumeScanAtSamePosition = true;
         return 0;
       }
     }
@@ -478,23 +486,32 @@ const HLJS = function(hljs) {
     var relevance = 0;
     var index = 0;
     var iterations = 0;
-    var continueScanAtSamePosition = false;
+    var resumeScanAtSamePosition = false;
 
     try {
       top.matcher.considerAll();
 
       for (;;) {
         iterations++;
-        if (continueScanAtSamePosition) {
+        if (resumeScanAtSamePosition) {
           // only regexes not matched previously will now be
           // considered for a potential match
-          continueScanAtSamePosition = false;
+          resumeScanAtSamePosition = false;
         } else {
           top.matcher.lastIndex = index;
           top.matcher.considerAll();
         }
+
         const match = top.matcher.exec(codeToHighlight);
         // console.log("match", match[0], match.rule && match.rule.begin)
+
+        // if our failure to match was the result of a "resumed scan" then we
+        // need to advance one position and revert to full scanning before we
+        // decide there are truly no more matches at all to be had
+        if (!match && top.matcher.resumingScanAtSamePosition()) {
+          advanceOne();
+          continue;
+        }
         if (!match) break;
 
         const beforeMatch = codeToHighlight.substring(index, match.index);
diff --git a/src/lib/mode_compiler.js b/src/lib/mode_compiler.js
@@ -142,6 +142,10 @@ export function compileLanguage(language) {
       return matcher;
     }
 
+    resumingScanAtSamePosition() {
+      return this.regexIndex != 0;
+    }
+
     considerAll() {
       this.regexIndex = 0;
     }
diff --git a/test/parser/index.js b/test/parser/index.js
@@ -1,7 +1,9 @@
 'use strict';
 
 describe('hljs', function() {
+  require('./look-ahead-end-matchers');
+  require('./resume-scan');
   require('./reuse-endsWithParent');
   require('./should-not-destroyData');
-  require('./look-ahead-end-matchers');
+
 });
diff --git a/test/parser/resume-scan.js b/test/parser/resume-scan.js
@@ -0,0 +1,13 @@
+const hljs = require('../../build');
+
+describe("bugs", function () {
+
+  describe("resume scan when a match is ignored", () => {
+    it("should continue to highlight later matches", () => {
+      hljs.highlight('java', 'ImmutablePair.of(Stuff.class, "bar")').value
+        .should.equal(
+          'ImmutablePair.of(Stuff.class, <span class="hljs-string">&quot;bar&quot;</span>)'
+      )
+    })
+  })
+})

Original file line number	Diff line number	Diff line change
`@@ -142,6 +142,10 @@ export function compileLanguage(language) {`
`142`	`142`	`return matcher;`
`143`	`143`	`}`
`144`	`144`
	`145`	`+ resumingScanAtSamePosition() {`
	`146`	`+ return this.regexIndex != 0;`
	`147`	`+ }`
	`148`	`+`
`145`	`149`	`considerAll() {`
`146`	`150`	`this.regexIndex = 0;`
`147`	`151`	`}`