Skip to content

Commit a3f6aad

Browse files
alexdimaalexdima
committed
Fixes microsoft#117264: Abandon tokenizing a line after 500ms if possible.
1 parent 4970694 commit a3f6aad

File tree

8 files changed

+32
-18
lines changed

8 files changed

+32
-18
lines changed

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@
8383
"vscode-proxy-agent": "^0.11.0",
8484
"vscode-regexpp": "^3.1.0",
8585
"vscode-ripgrep": "^1.12.1",
86-
"vscode-textmate": "5.4.1",
86+
"vscode-textmate": "5.5.0",
8787
"xterm": "4.16.0-beta.2",
8888
"xterm-addon-search": "0.9.0-beta.6",
8989
"xterm-addon-serialize": "0.7.0-beta.3",

‎remote/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
"vscode-proxy-agent": "^0.11.0",
2424
"vscode-regexpp": "^3.1.0",
2525
"vscode-ripgrep": "^1.12.1",
26-
"vscode-textmate": "5.4.1",
26+
"vscode-textmate": "5.5.0",
2727
"xterm": "4.16.0-beta.2",
2828
"xterm-addon-search": "0.9.0-beta.6",
2929
"xterm-addon-serialize": "0.7.0-beta.3",

‎remote/web/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
"jschardet": "3.0.0",
1010
"tas-client-umd": "0.1.4",
1111
"vscode-oniguruma": "1.6.1",
12-
"vscode-textmate": "5.4.1",
12+
"vscode-textmate": "5.5.0",
1313
"xterm": "4.16.0-beta.2",
1414
"xterm-addon-search": "0.9.0-beta.6",
1515
"xterm-addon-unicode11": "0.4.0-beta.1",

‎remote/web/yarn.lock

+4-4
Original file line numberDiff line numberDiff line change
@@ -108,10 +108,10 @@ vscode-oniguruma@1.6.1:
108108
resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.6.1.tgz#2bf4dfcfe3dd2e56eb549a3068c8ee39e6c30ce5"
109109
integrity sha512-vc4WhSIaVpgJ0jJIejjYxPvURJavX6QG41vu0mGhqywMkQqulezEqEQ3cO3gc8GvcOpX6ycmKGqRoROEMBNXTQ==
110110

111-
vscode-textmate@5.4.1:
112-
version "5.4.1"
113-
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.4.1.tgz#09d566724fc76b60b3ad9791eebf1f0b50f29e5a"
114-
integrity sha512-4CvPHmfuZQaXrcCpathdh6jo7myuR+MU8BvscgQADuponpbqfmu2rwTOtCXhGwwEgStvJF8V4s9FwMKRVLNmKQ==
111+
vscode-textmate@5.5.0:
112+
version "5.5.0"
113+
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.5.0.tgz#d83776562c07d1e3181c2c7f1b3d5f20afcab483"
114+
integrity sha512-jToQkPGMNKn0eyKyitYeINJF0NoD240aYyKPIWJv5W2jfPt++jIRg0OSergubtGhbw6SoefkvBYEpX7TsfoSUQ==
115115

116116
xterm-addon-search@0.9.0-beta.6:
117117
version "0.9.0-beta.6"

‎remote/yarn.lock

+4-4
Original file line numberDiff line numberDiff line change
@@ -512,10 +512,10 @@ vscode-ripgrep@^1.12.1:
512512
https-proxy-agent "^4.0.0"
513513
proxy-from-env "^1.1.0"
514514

515-
vscode-textmate@5.4.1:
516-
version "5.4.1"
517-
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.4.1.tgz#09d566724fc76b60b3ad9791eebf1f0b50f29e5a"
518-
integrity sha512-4CvPHmfuZQaXrcCpathdh6jo7myuR+MU8BvscgQADuponpbqfmu2rwTOtCXhGwwEgStvJF8V4s9FwMKRVLNmKQ==
515+
vscode-textmate@5.5.0:
516+
version "5.5.0"
517+
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.5.0.tgz#d83776562c07d1e3181c2c7f1b3d5f20afcab483"
518+
integrity sha512-jToQkPGMNKn0eyKyitYeINJF0NoD240aYyKPIWJv5W2jfPt++jIRg0OSergubtGhbw6SoefkvBYEpX7TsfoSUQ==
519519

520520
vscode-windows-ca-certs@^0.3.0:
521521
version "0.3.0"

‎src/vs/workbench/services/textMate/browser/abstractTextMateService.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -484,7 +484,13 @@ class TMTokenization extends Disposable {
484484
}
485485

486486
public tokenize2(line: string, state: StackElement): TokenizationResult2 {
487-
let textMateResult = this._grammar.tokenizeLine2(line, state);
487+
const textMateResult = this._grammar.tokenizeLine2(line, state, 500);
488+
489+
if (textMateResult.stoppedEarly) {
490+
console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`);
491+
// return the state at the beginning of the line
492+
return new TokenizationResult2(textMateResult.tokens, state);
493+
}
488494

489495
if (this._containsEmbeddedLanguages) {
490496
let seenLanguages = this._seenLanguages;

‎src/vs/workbench/services/textMate/common/textMateService.ts

+10-2
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ export interface IGrammar {
3333
/**
3434
* Tokenize `lineText` using previous line state `prevState`.
3535
*/
36-
tokenizeLine(lineText: string, prevState: StackElement | null): ITokenizeLineResult;
36+
tokenizeLine(lineText: string, prevState: StackElement | null, timeLimit?: number): ITokenizeLineResult;
3737
/**
3838
* Tokenize `lineText` using previous line state `prevState`.
3939
* The result contains the tokens in binary format, resolved with the following information:
@@ -44,14 +44,18 @@ export interface IGrammar {
4444
* - background color
4545
* e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET`
4646
*/
47-
tokenizeLine2(lineText: string, prevState: StackElement | null): ITokenizeLineResult2;
47+
tokenizeLine2(lineText: string, prevState: StackElement | null, timeLimit?: number): ITokenizeLineResult2;
4848
}
4949
export interface ITokenizeLineResult {
5050
readonly tokens: IToken[];
5151
/**
5252
* The `prevState` to be passed on to the next line tokenization.
5353
*/
5454
readonly ruleStack: StackElement;
55+
/**
56+
* Did tokenization stop early due to reaching the time limit.
57+
*/
58+
readonly stoppedEarly: boolean;
5559
}
5660
/**
5761
* Helpers to manage the "collapsed" metadata of an entire StackElement stack.
@@ -97,6 +101,10 @@ export interface ITokenizeLineResult2 {
97101
* The `prevState` to be passed on to the next line tokenization.
98102
*/
99103
readonly ruleStack: StackElement;
104+
/**
105+
* Did tokenization stop early due to reaching the time limit.
106+
*/
107+
readonly stoppedEarly: boolean;
100108
}
101109
export interface IToken {
102110
startIndex: number;

‎yarn.lock

+4-4
Original file line numberDiff line numberDiff line change
@@ -10476,10 +10476,10 @@ vscode-telemetry-extractor@^1.9.5:
1047610476
ts-morph "^12.2.0"
1047710477
vscode-ripgrep "^1.12.1"
1047810478

10479-
vscode-textmate@5.4.1:
10480-
version "5.4.1"
10481-
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.4.1.tgz#09d566724fc76b60b3ad9791eebf1f0b50f29e5a"
10482-
integrity sha512-4CvPHmfuZQaXrcCpathdh6jo7myuR+MU8BvscgQADuponpbqfmu2rwTOtCXhGwwEgStvJF8V4s9FwMKRVLNmKQ==
10479+
vscode-textmate@5.5.0:
10480+
version "5.5.0"
10481+
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.5.0.tgz#d83776562c07d1e3181c2c7f1b3d5f20afcab483"
10482+
integrity sha512-jToQkPGMNKn0eyKyitYeINJF0NoD240aYyKPIWJv5W2jfPt++jIRg0OSergubtGhbw6SoefkvBYEpX7TsfoSUQ==
1048310483

1048410484
vscode-windows-ca-certs@^0.3.0:
1048510485
version "0.3.0"

0 commit comments

Comments
 (0)