diff --git a/.gitignore b/.gitignore index 4377bfda..e17b261b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,4 @@ node_modules .DS_Store *.cache *.pyc - +package-lock.json diff --git a/.travis.yml b/.travis.yml index 8df07568..bf2912a0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,23 @@ language: node_js node_js: - - 0.10 + - "6" + +env: + - CXX=g++-4.8 + +dist: trusty + +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-4.8 install: - make script: - - make test + - make ci-test + - 'curl -s https://raw.githubusercontent.com/atom/ci/master/build-package.sh | sh' diff --git a/LICENSE b/LICENSE index 79a03ca5..e6b42c6f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License -Copyright (c) 2015 MagicStack Inc. http://magic.io +Copyright (c) 2015-present MagicStack Inc. http://magic.io Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/Makefile b/Makefile index 3ecb8621..21b1edbf 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ -.PHONY: all test release devenv publish +.PHONY: all ci-test test release devenv publish all: devenv release -test: release +ci-test: release # Run tests ./node_modules/.bin/syntaxdev test --tests test/**/*.py --syntax grammars/src/MagicPython.syntax.yaml ./node_modules/.bin/syntaxdev test --tests test/**/*.re --syntax grammars/src/MagicRegExp.syntax.yaml @@ -15,8 +15,16 @@ test: release ] ; \ then echo "Error: package.version != git.tag" && exit 1 ; fi +update-test: +# Run tests and overwrite the output + ./node_modules/.bin/syntaxdev test --tests test/**/*.py --syntax grammars/src/MagicPython.syntax.yaml --overwrite-tests + ./node_modules/.bin/syntaxdev test --tests test/**/*.re --syntax grammars/src/MagicRegExp.syntax.yaml --overwrite-tests + +test: ci-test + atom -t test/atom-spec + devenv: - npm install syntaxdev@0.0.12 + npm install --dev release: ./node_modules/.bin/syntaxdev build-plist --in grammars/src/MagicPython.syntax.yaml --out grammars/MagicPython.tmLanguage @@ -27,6 +35,9 @@ release: ./node_modules/.bin/syntaxdev scopes --syntax grammars/src/MagicPython.syntax.yaml > misc/scopes + ./node_modules/.bin/syntaxdev atom-spec --package-name MagicPython --tests test/**/*.py --syntax grammars/src/MagicPython.syntax.yaml --out test/atom-spec/python-spec.js + ./node_modules/.bin/syntaxdev atom-spec --package-name MagicPython --tests test/**/*.re --syntax grammars/src/MagicRegExp.syntax.yaml --out test/atom-spec/python-re-spec.js + publish: test apm publish patch rm -rf ./node_modules/syntaxdev diff --git a/README.md b/README.md index dd281085..64d9eb43 100644 --- a/README.md +++ b/README.md @@ -3,24 +3,24 @@ This is a package with preferences and syntax highlighter for cutting edge Python 3, although Python 2 is well supported, too. The syntax is compatible with [Sublime Text](http://www.sublimetext.com), [Atom](http://atom.io) and -[Visual Studio Code](http://code.visualstudio.com). -It is meant to be a drop-in replacement for the default Python package. +[Visual Studio Code](http://code.visualstudio.com). It is meant to be a drop-in +replacement for the default Python package. -MagicPython correctly highlights all Python 3.5 and 3.6 syntax features, -including type annotations, f-strings and regular expressions. It is built -from scratch for robustness with an extensive test suite. +**Attention VSCode users**: MagicPython is used as the _default_ +Python highlighter in Visual Studio Code. Don't install it unless you +want or need the cutting edge version of it. You will likely see no +difference because you're already using MagicPython. -We are proud to say that MagicPython has been included into the -[github/linguist](https://github.com/github/linguist), a library used -by GitHub.com to process languages. +MagicPython correctly highlights all Python 3 syntax features, +including type annotations, f-strings and regular expressions. It is +built from scratch for robustness with an extensive test suite. - -![](https://magicstack.github.io/MagicPython/example.png) - -Type hints in comments require support by the color scheme. The one +Type hints in comments require support by the color scheme. The one used in the screenshot is [Chromodynamics](https://github.com/MagicStack/Chromodynamics). +![](https://magicstack.github.io/MagicPython/example.png) + ## Installation Instructions diff --git a/grammars/MagicPython.cson b/grammars/MagicPython.cson index 3f9fa44f..d837d6a3 100644 --- a/grammars/MagicPython.cson +++ b/grammars/MagicPython.cson @@ -15,9 +15,12 @@ fileTypes: [ "gyp" "gypi" "wsgi" + "kv" + "Snakefile" + "smk" + "tac" ] -first_line_match: "^#!/.*\\bpython[\\d\\.]*\\b" -firstLineMatch: "^#!/.*\\bpython[\\d\\.]*\\b" +firstLineMatch: "^#![ \\t]*/.*\\bpython[\\d\\.]*\\b" uuid: "742deb57-6e38-4192-bed6-410746efd85d" patterns: [ { @@ -42,6 +45,9 @@ repository: { include: "#function-declaration" } + { + include: "#generator" + } { include: "#statement-keyword" } @@ -126,7 +132,8 @@ repository: ] "docstring-statement": begin: "^(?=\\s*[rR]?(\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\"))" - end: "(?<=\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\")" + comment: "the string either terminates correctly or by the beginning of a new line (this is for single line docstrings that aren't terminated) AND it's not followed by another docstring" + end: "((?<=\\1)|^)(?!\\s*[rR]?(\\'\\'\\'|\\\"\\\"\\\"|\\'|\\\"))" patterns: [ { include: "#docstring" @@ -183,7 +190,7 @@ repository: { name: "string.quoted.docstring.single.python" begin: "(\\'|\\\")" - end: "(\\1)|((?= | <= | < | >) (?# 5) + | (:=) (?# 6) + ''' captures: "1": @@ -561,6 +645,8 @@ repository: name: "keyword.operator.arithmetic.python" "5": name: "keyword.operator.comparison.python" + "6": + name: "keyword.operator.assignment.python" punctuation: patterns: [ { @@ -648,8 +734,6 @@ repository: name: "storage.type.imaginary.number.python" "2": name: "invalid.illegal.dec.python" - "3": - name: "invalid.illegal.dec.python" "number-hex": name: "constant.numeric.hex.python" match: ''' @@ -713,18 +797,6 @@ repository: { include: "#regexp-double-one-line" } - { - include: "#fregexp-single-three-line" - } - { - include: "#fregexp-double-three-line" - } - { - include: "#fregexp-single-one-line" - } - { - include: "#fregexp-double-one-line" - } ] string: patterns: [ @@ -867,7 +939,7 @@ repository: } ] "f-expression": - comment: "All valid Python expressions, except comments and line cont" + comment: "All valid Python expressions, except comments and line continuation" patterns: [ { include: "#expression-bare" @@ -910,27 +982,32 @@ repository: name: "constant.language.python" match: "\\\\$" "string-formatting": - name: "constant.character.format.placeholder.other.python" + name: "meta.format.percent.python" match: ''' (?x) - % (\\([\\w\\s]*\\))? - [-+#0 ]* - (\\d+|\\*)? (\\.(\\d+|\\*))? - ([hlL])? - [diouxXeEfFgGcrsa%] + ( + % (\\([\\w\\s]*\\))? + [-+#0 ]* + (\\d+|\\*)? (\\.(\\d+|\\*))? + ([hlL])? + [diouxXeEfFgGcrsab%] + ) ''' + captures: + "1": + name: "constant.character.format.placeholder.other.python" "string-brace-formatting": patterns: [ { - name: "constant.character.format.placeholder.other.python" + name: "meta.format.brace.python" match: ''' (?x) - (?: + ( {{ | }} | (?: { - \\w*? (\\.[[:alpha:]_]\\w*? | \\[[^\\]'"]+\\])*? + \\w* (\\.[[:alpha:]_]\\w* | \\[[^\\]'"]+\\])* (![rsa])? ( : \\w? [<>=^]? [-+ ]? \\#? \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )? @@ -939,36 +1016,36 @@ repository: ''' captures: - "2": - name: "storage.type.format.python" + "1": + name: "constant.character.format.placeholder.other.python" "3": - name: "support.other.format.python" + name: "storage.type.format.python" + "4": + name: "storage.type.format.python" } { - name: "constant.character.format.placeholder.other.python" - begin: ''' + name: "meta.format.brace.python" + match: ''' (?x) - \\{ - \\w*? (\\.[[:alpha:]_]\\w*? | \\[[^\\]'"]+\\])*? + ( + { + \\w* (\\.[[:alpha:]_]\\w* | \\[[^\\]'"]+\\])* (![rsa])? (:) - (?=[^'"}\\n]*\\}) + [^'"{}\\n]* (?: + \\{ [^'"}\\n]*? \\} [^'"{}\\n]* + )* + } + ) ''' - end: "\\}" - beginCaptures: - "2": - name: "storage.type.format.python" + captures: + "1": + name: "constant.character.format.placeholder.other.python" "3": - name: "support.other.format.python" - patterns: [ - { - match: ''' - (?x) \\{ [^'"}\\n]*? \\} (?=.*?\\}) - - ''' - } - ] + name: "storage.type.format.python" + "4": + name: "storage.type.format.python" } ] "fstring-formatting": @@ -980,48 +1057,46 @@ repository: include: "#fstring-formatting-singe-brace" } ] - "fstring-formatting-braces": - patterns: [ - { - comment: "empty braces are illegal" - match: "({)(\\s*?)(})" - captures: - "1": - name: "constant.character.format.placeholder.other.python" - "2": - name: "invalid.illegal.brace.python" - "3": - name: "constant.character.format.placeholder.other.python" - } - { - name: "constant.character.escape.python" - match: "({{|}})" - } - ] "fstring-formatting-singe-brace": name: "invalid.illegal.brace.python" match: "(}(?!}))" import: comment: ''' - Import statements + Import statements used to correctly mark `from`, `import`, and `as` ''' patterns: [ { - match: ''' - (?x) - \\s* \\b(from)\\b (\\s*\\.+\\s*) (import)? - - ''' - captures: + begin: "\\b(?) - - ''' - end: "(\\)|(?=\\'))|((?=(?) - - ''' - end: "(\\)|(?=\\'\\'\\'))" - beginCaptures: - "1": - name: "punctuation.parenthesis.named.begin.regexp support.other.parenthesis.regexp" - "2": - name: "entity.name.tag.named.group.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.named.end.regexp support.other.parenthesis.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#single-three-fregexp-expression" - } - { - include: "#comments-string-single-three" - } - ] - "single-three-fregexp-lookahead": - begin: "(\\()\\?=" - end: "(\\)|(?=\\'\\'\\'))" - beginCaptures: - "0": - name: "keyword.operator.lookahead.regexp" - "1": - name: "punctuation.parenthesis.lookahead.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#single-three-fregexp-expression" - } - { - include: "#comments-string-single-three" - } - ] - "single-three-fregexp-lookahead-negative": - begin: "(\\()\\?!" - end: "(\\)|(?=\\'\\'\\'))" - beginCaptures: - "0": - name: "keyword.operator.lookahead.negative.regexp" - "1": - name: "punctuation.parenthesis.lookahead.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.negative.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#single-three-fregexp-expression" - } - { - include: "#comments-string-single-three" - } - ] - "single-three-fregexp-lookbehind": - begin: "(\\()\\?<=" - end: "(\\)|(?=\\'\\'\\'))" - beginCaptures: - "0": - name: "keyword.operator.lookbehind.regexp" - "1": - name: "punctuation.parenthesis.lookbehind.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookbehind.end.regexp keyword.operator.lookbehind.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#single-three-fregexp-expression" - } - { - include: "#comments-string-single-three" - } - ] - "single-three-fregexp-lookbehind-negative": - begin: "(\\()\\?) - - ''' - end: "(\\)|(?=\"))|((?=(?) - - ''' - end: "(\\)|(?=\"\"\"))" - beginCaptures: - "1": - name: "punctuation.parenthesis.named.begin.regexp support.other.parenthesis.regexp" - "2": - name: "entity.name.tag.named.group.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.named.end.regexp support.other.parenthesis.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#double-three-fregexp-expression" - } - { - include: "#comments-string-double-three" - } - ] - "double-three-fregexp-lookahead": - begin: "(\\()\\?=" - end: "(\\)|(?=\"\"\"))" - beginCaptures: - "0": - name: "keyword.operator.lookahead.regexp" - "1": - name: "punctuation.parenthesis.lookahead.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#double-three-fregexp-expression" - } - { - include: "#comments-string-double-three" - } - ] - "double-three-fregexp-lookahead-negative": - begin: "(\\()\\?!" - end: "(\\)|(?=\"\"\"))" - beginCaptures: - "0": - name: "keyword.operator.lookahead.negative.regexp" - "1": - name: "punctuation.parenthesis.lookahead.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.negative.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#double-three-fregexp-expression" - } - { - include: "#comments-string-double-three" - } - ] - "double-three-fregexp-lookbehind": - begin: "(\\()\\?<=" - end: "(\\)|(?=\"\"\"))" - beginCaptures: - "0": - name: "keyword.operator.lookbehind.regexp" - "1": - name: "punctuation.parenthesis.lookbehind.begin.regexp" - endCaptures: - "1": - name: "punctuation.parenthesis.lookbehind.end.regexp keyword.operator.lookbehind.regexp" - "2": - name: "invalid.illegal.newline.python" - patterns: [ - { - include: "#double-three-fregexp-expression" - } - { - include: "#comments-string-double-three" - } - ] - "double-three-fregexp-lookbehind-negative": - begin: "(\\()\\?=^]? [-+ ]? \\#? \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=}) @@ -4612,20 +3861,20 @@ repository: "1": name: "storage.type.format.python" "2": - name: "support.other.format.python" + name: "storage.type.format.python" } { include: "#fstring-terminator-single-tail" } ] "fstring-terminator-single-tail": - begin: "(![rsa])?(:)(?=.*?{)" + begin: "((?:=?)(?:![rsa])?)(:)(?=.*?{)" end: "(?=})|(?=\\n)" beginCaptures: "1": name: "storage.type.format.python" "2": - name: "support.other.format.python" + name: "storage.type.format.python" patterns: [ { include: "#fstring-illegal-single-brace" @@ -4634,35 +3883,35 @@ repository: include: "#fstring-single-brace" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "([bcdeEfFgGnosxX%])(?=})" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(\\.\\d+)" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(,)" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(\\d+)" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(\\#)" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "([-+ ])" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "([<>=^])" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(\\w)" } ] @@ -4728,7 +3977,7 @@ repository: ] "fstring-raw-quoted-multi-line": name: "meta.fstring.python" - begin: "(\\b(?:[R][fF]|[fF][R]))('''|\"\"\")" + begin: "(\\b(?:[rR][fF]|[fF][rR]))('''|\"\"\")" end: "(\\2)" beginCaptures: "1": @@ -4822,12 +4071,16 @@ repository: patterns: [ { name: "storage.type.format.python" - match: "(![rsa])(?=})" + match: "(=(![rsa])?)(?=})" + } + { + name: "storage.type.format.python" + match: "(=?![rsa])(?=})" } { match: ''' (?x) - (![rsa])? + ( (?: =?) (?: ![rsa])? ) ( : \\w? [<>=^]? [-+ ]? \\#? \\d* ,? (\\.\\d+)? [bcdeEfFgGnosxX%]? )(?=}) @@ -4836,20 +4089,20 @@ repository: "1": name: "storage.type.format.python" "2": - name: "support.other.format.python" + name: "storage.type.format.python" } { include: "#fstring-terminator-multi-tail" } ] "fstring-terminator-multi-tail": - begin: "(![rsa])?(:)(?=.*?{)" + begin: "((?:=?)(?:![rsa])?)(:)(?=.*?{)" end: "(?=})" beginCaptures: "1": name: "storage.type.format.python" "2": - name: "support.other.format.python" + name: "storage.type.format.python" patterns: [ { include: "#fstring-illegal-multi-brace" @@ -4858,35 +4111,35 @@ repository: include: "#fstring-multi-brace" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "([bcdeEfFgGnosxX%])(?=})" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(\\.\\d+)" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(,)" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(\\d+)" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(\\#)" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "([-+ ])" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "([<>=^])" } { - name: "support.other.format.python" + name: "storage.type.format.python" match: "(\\w)" } ] diff --git a/grammars/MagicPython.tmLanguage b/grammars/MagicPython.tmLanguage index fe97f739..a1c3f8e7 100644 --- a/grammars/MagicPython.tmLanguage +++ b/grammars/MagicPython.tmLanguage @@ -1,7 +1,7 @@ - + name MagicPython @@ -22,11 +22,13 @@ gyp gypi wsgi + kv + Snakefile + smk + tac - first_line_match - ^#!/.*\bpython[\d\.]*\b firstLineMatch - ^#!/.*\bpython[\d\.]*\b + ^#![ \t]*/.*\bpython[\d\.]*\b uuid 742deb57-6e38-4192-bed6-410746efd85d patterns @@ -65,6 +67,10 @@ include #function-declaration + + include + #generator + include #statement-keyword @@ -181,8 +187,10 @@ begin ^(?=\s*[rR]?(\'\'\'|\"\"\"|\'|\")) + comment + the string either terminates correctly or by the beginning of a new line (this is for single line docstrings that aren't terminated) AND it's not followed by another docstring end - (?<=\'\'\'|\"\"\"|\'|\") + ((?<=\1)|^)(?!\s*[rR]?(\'\'\'|\"\"\"|\'|\")) patterns @@ -284,7 +292,7 @@ begin (\'|\") end - (\1)|((?<!\\)\n) + (\1)|(\n) beginCaptures 1 @@ -324,7 +332,7 @@ begin ([rR])(\'|\") end - (\2)|((?<!\\)\n) + (\2)|(\n) beginCaptures 1 @@ -401,19 +409,6 @@ - codetags - - match - (?:\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\b) - captures - - 1 - - name - keyword.codetag.notation.python - - - statement-keyword patterns @@ -424,13 +419,37 @@ match \b((async\s+)?\s*def)\b + + name + keyword.control.flow.python + comment + if `as` is eventually followed by `:` or line continuation +it's probably control flow like: + with foo as bar, \ + Foo as Bar: + try: + do_stuff() + except Exception as e: + pass + + match + \b(?<!\.)as\b(?=.*[:\\]) + + + name + keyword.control.import.python + comment + other legal use of `as` is in an import + match + \b(?<!\.)as\b + name keyword.control.flow.python match (?x) \b(?<!\.)( - as | async | continue | del | assert | break | finally | for + async | continue | del | assert | break | finally | for | from | elif | else | if | except | pass | raise | return | try | while | with )\b @@ -452,6 +471,22 @@ match \b(?<!\.)(class)\b + + match + (?x) + ^\s*( + case | match + )(?=\s*([-+\w\d(\[{'":#]|$))\b + + captures + + 1 + + name + keyword.control.flow.python + + + expression-bare @@ -484,6 +519,10 @@ include #lambda + + include + #generator + include #illegal-operator @@ -504,6 +543,10 @@ include #list + + include + #odd-function-call + include #round-braces @@ -598,8 +641,10 @@ member-access + name + meta.member.access.python begin - \.\s*(?!\.) + (\.)\s*(?!\.) end (?x) # stop when you've just read non-whitespace followed by non-word @@ -610,6 +655,14 @@ (^|(?<=\s))(?=[^\\\w\s]) | $ + beginCaptures + + 1 + + name + punctuation.separator.period.python + + patterns @@ -620,6 +673,10 @@ include #member-access-base + + include + #member-access-attribute + member-access-base @@ -652,6 +709,17 @@ + member-access-attribute + + comment + Highlight attribute access in otherwise non-specialized cases. + name + meta.attribute.python + match + (?x) + \b ([[:alpha:]_]\w*) \b + + special-names name @@ -659,8 +727,13 @@ match (?x) \b + # we want to see "enough", meaning 2 or more upper-case + # letters in the beginning of the constant + # + # for more details refer to: + # https://github.com/MagicStack/MagicPython/issues/42 ( - _* [[:upper:]]{2} + _* [[:upper:]] [_\d]* [[:upper:]] ) [[:upper:]\d]* (_\w*)? \b @@ -732,6 +805,36 @@ + odd-function-call + + comment + A bit obscured function call where there may have been an +arbitrary number of other operations to get the function. +E.g. "arr[idx](args)" + + begin + (?x) + (?<= \] | \) ) \s* + (?=\() + + end + (\)) + endCaptures + + 1 + + name + punctuation.definition.arguments.end.python + + + patterns + + + include + #function-arguments + + + round-braces begin @@ -846,6 +949,8 @@ | (\*\* | \* | \+ | - | % | // | / | @) (?# 4) | (!= | == | >= | <= | < | >) (?# 5) + + | (:=) (?# 6) captures @@ -874,6 +979,11 @@ name keyword.operator.comparison.python + 6 + + name + keyword.operator.assignment.python + punctuation @@ -1003,11 +1113,6 @@ name invalid.illegal.dec.python - 3 - - name - invalid.illegal.dec.python - number-hex @@ -1108,22 +1213,6 @@ include #regexp-double-one-line - - include - #fregexp-single-three-line - - - include - #fregexp-double-three-line - - - include - #fregexp-single-one-line - - - include - #fregexp-double-one-line - string @@ -1346,7 +1435,7 @@ f-expression comment - All valid Python expressions, except comments and line cont + All valid Python expressions, except comments and line continuation patterns @@ -1406,15 +1495,25 @@ string-formatting name - constant.character.format.placeholder.other.python + meta.format.percent.python match (?x) - % (\([\w\s]*\))? - [-+#0 ]* - (\d+|\*)? (\.(\d+|\*))? - ([hlL])? - [diouxXeEfFgGcrsa%] + ( + % (\([\w\s]*\))? + [-+#0 ]* + (\d+|\*)? (\.(\d+|\*))? + ([hlL])? + [diouxXeEfFgGcrsab%] + ) + captures + + 1 + + name + constant.character.format.placeholder.other.python + + string-brace-formatting @@ -1422,14 +1521,14 @@ name - constant.character.format.placeholder.other.python + meta.format.brace.python match (?x) - (?: + ( {{ | }} | (?: { - \w*? (\.[[:alpha:]_]\w*? | \[[^\]'"]+\])*? + \w* (\.[[:alpha:]_]\w* | \[[^\]'"]+\])* (![rsa])? ( : \w? [<>=^]? [-+ ]? \#? \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )? @@ -1438,52 +1537,57 @@ captures - 2 + 1 name - storage.type.format.python + constant.character.format.placeholder.other.python 3 name - support.other.format.python + storage.type.format.python + + 4 + + name + storage.type.format.python name - constant.character.format.placeholder.other.python - begin + meta.format.brace.python + match (?x) - \{ - \w*? (\.[[:alpha:]_]\w*? | \[[^\]'"]+\])*? + ( + { + \w* (\.[[:alpha:]_]\w* | \[[^\]'"]+\])* (![rsa])? (:) - (?=[^'"}\n]*\}) + [^'"{}\n]* (?: + \{ [^'"}\n]*? \} [^'"{}\n]* + )* + } + ) - end - \} - beginCaptures + captures - 2 + 1 name - storage.type.format.python + constant.character.format.placeholder.other.python 3 name - support.other.format.python + storage.type.format.python - - patterns - + 4 - match - (?x) \{ [^'"}\n]*? \} (?=.*?\}) - + name + storage.type.format.python - + @@ -1501,80 +1605,73 @@ - fstring-formatting-braces + fstring-formatting-singe-brace + name + invalid.illegal.brace.python + match + (}(?!})) + + import + + comment + Import statements used to correctly mark `from`, `import`, and `as` + patterns - comment - empty braces are illegal - match - ({)(\s*?)(}) - captures + begin + \b(?<!\.)(from)\b(?=.+import) + end + $|(?=import) + beginCaptures 1 name - constant.character.format.placeholder.other.python + keyword.control.import.python - 2 + + patterns + name - invalid.illegal.brace.python + punctuation.separator.period.python + match + \.+ - 3 - name - constant.character.format.placeholder.other.python + include + #expression - - - - name - constant.character.escape.python - match - ({{|}}) + - - - fstring-formatting-singe-brace - - name - invalid.illegal.brace.python - match - (}(?!})) - - import - - comment - Import statements - - patterns - - match - (?x) - \s* \b(from)\b (\s*\.+\s*) (import)? - - captures + begin + \b(?<!\.)(import)\b + end + $ + beginCaptures 1 name keyword.control.import.python - 3 + + patterns + name keyword.control.import.python + match + \b(?<!\.)as\b - - - - name - keyword.control.import.python - match - \b(?<!\.)import\b + + include + #expression + + @@ -1759,10 +1856,20 @@ member-access-class + name + meta.member.access.python begin - \.\s*(?!\.) + (\.)\s*(?!\.) end (?<=\S)(?=\W)|$ + beginCaptures + + 1 + + name + punctuation.separator.period.python + + patterns @@ -1836,6 +1943,12 @@ meta.function.lambda.parameters.python patterns + + name + keyword.operator.positional.parameter.python + match + / + name keyword.operator.unpacking.parameter.python @@ -1947,14 +2060,48 @@ - function-declaration + generator - name - meta.function.python + comment + Match "for ... in" construct used in generators and for loops to +correctly identify the "in" as a control flow keyword. + begin - (?x) - \s* - (?:\b(async) \s+)? \b(def)\s+ + \bfor\b + beginCaptures + + 0 + + name + keyword.control.flow.python + + + end + \bin\b + endCaptures + + 0 + + name + keyword.control.flow.python + + + patterns + + + include + #expression + + + + function-declaration + + name + meta.function.python + begin + (?x) + \s* + (?:\b(async) \s+)? \b(def)\s+ (?= [[:alpha:]_][[:word:]]* \s* \( ) @@ -2050,6 +2197,12 @@ patterns + + name + keyword.operator.positional.parameter.python + match + / + name keyword.operator.unpacking.parameter.python @@ -2290,6 +2443,8 @@ #special-names + name + meta.indexed-name.python match (?x) \b ([[:alpha:]_]\w*) \b @@ -2334,7 +2489,7 @@ begin (?x) ^\s* - (@) \s* (?=[[:alpha:]_]\w*) + ((@)) \s* (?=[[:alpha:]_]\w*) end (?x) @@ -2350,6 +2505,11 @@ name entity.name.function.decorator.python + 2 + + name + punctuation.definition.decorator.python + endCaptures @@ -2393,8 +2553,16 @@ entity.name.function.decorator.python match (?x) - ([[:alpha:]_]\w*) | \. + ([[:alpha:]_]\w*) | (\.) + captures + + 2 + + name + punctuation.separator.period.python + + include @@ -2474,6 +2642,8 @@ name meta.function-call.python + comment + Regular function call of the type "name(args)" begin (?x) \b(?= @@ -2529,12 +2699,7 @@ function-arguments begin - (?x) - (?: - (\() - (?:\s*(\*\*|\*))? - ) - + (\() end (?=\))(?!\)\s*\() beginCaptures @@ -2544,32 +2709,25 @@ name punctuation.definition.arguments.begin.python - 2 - - name - keyword.operator.unpacking.arguments.python - contentName meta.function-call.arguments.python patterns + + name + punctuation.separator.arguments.python + match + (,) + match (?x) - (?: - (,) - (?:\s*(\*\*|\*))? - ) + (?:(?<=[,(])|^) \s* (\*{1,2}) captures 1 - - name - punctuation.separator.arguments.python - - 2 name keyword.operator.unpacking.arguments.python @@ -2710,14 +2868,14 @@ match (?x) (?<!\.) \b( - __import__ | abs | all | any | ascii | bin | callable - | chr | compile | copyright | credits | delattr | dir | divmod - | enumerate | eval | exec | exit | filter | format | getattr - | globals | hasattr | hash | help | hex | id | input - | isinstance | issubclass | iter | len | license | locals | map - | max | memoryview | min | next | oct | open | ord | pow | print - | quit | range | reload | repr | reversed | round - | setattr | sorted | sum | vars | zip + __import__ | abs | aiter | all | any | anext | ascii | bin + | breakpoint | callable | chr | compile | copyright | credits + | delattr | dir | divmod | enumerate | eval | exec | exit + | filter | format | getattr | globals | hasattr | hash | help + | hex | id | input | isinstance | issubclass | iter | len + | license | locals | map | max | memoryview | min | next + | oct | open | ord | pow | print | quit | range | reload | repr + | reversed | round | setattr | sorted | sum | vars | zip )\b @@ -2762,24 +2920,29 @@ indirectly through syntactic constructs (?x) \b( __(?: - abs | add | aenter | aexit | aiter | and | anext | await - | bool | call | ceil | cmp | coerce | complex | contains - | copy | deepcopy | del | delattr | delete | delitem - | delslice | dir | div | divmod | enter | eq | exit | float - | floor | floordiv | format | ge | get | getattr - | getattribute | getinitargs | getitem | getnewargs - | getslice | getstate | gt | hash | hex | iadd | iand | idiv - | ifloordiv | ilshift | imod | imul | index | init - | instancecheck | int | invert | ior | ipow | irshift | isub - | iter | itruediv | ixor | le | len | long | lshift | lt - | missing | mod | mul | ne | neg | new | next | nonzero | oct | or - | pos | pow | radd | rand | rdiv | rdivmod | reduce - | reduce_ex | repr | reversed | rfloordiv | rlshift | rmod - | rmul | ror | round | rpow | rrshift | rshift | rsub - | rtruediv | rxor | set | setattr | setitem | setslice - | setstate | sizeof | str | sub | subclasscheck | truediv - | trunc | unicode | xor | matmul | rmatmul | imatmul - | init_subclass | set_name | fspath | bytes | prepare + abs | add | aenter | aexit | aiter | and | anext + | await | bool | call | ceil | class_getitem + | cmp | coerce | complex | contains | copy + | deepcopy | del | delattr | delete | delitem + | delslice | dir | div | divmod | enter | eq + | exit | float | floor | floordiv | format | ge + | get | getattr | getattribute | getinitargs + | getitem | getnewargs | getslice | getstate | gt + | hash | hex | iadd | iand | idiv | ifloordiv | + | ilshift | imod | imul | index | init + | instancecheck | int | invert | ior | ipow + | irshift | isub | iter | itruediv | ixor | le + | len | long | lshift | lt | missing | mod | mul + | ne | neg | new | next | nonzero | oct | or | pos + | pow | radd | rand | rdiv | rdivmod | reduce + | reduce_ex | repr | reversed | rfloordiv | + | rlshift | rmod | rmul | ror | round | rpow + | rrshift | rshift | rsub | rtruediv | rxor | set + | setattr | setitem | set_name | setslice + | setstate | sizeof | str | sub | subclasscheck + | truediv | trunc | unicode | xor | matmul + | rmatmul | imatmul | init_subclass | set_name + | fspath | bytes | prepare | length_hint )__ )\b @@ -2800,12 +2963,12 @@ indirectly through syntactic constructs (?x) \b( __(?: - all | bases | builtins | class | code | debug | defaults | dict - | doc | file | func | kwdefaults | members - | metaclass | methods | module | mro | name - | qualname | self | signature | slots | subclasses - | version | weakref | wrapped | annotations | classcell - | spec | path | package | future + all | annotations | bases | builtins | class + | closure | code | debug | defaults | dict | doc | file | func + | globals | kwdefaults | match_args | members | metaclass | methods + | module | mro | mro_entries | name | qualname | post_init | self + | signature | slots | subclasses | version | weakref | wrapped + | classcell | spec | path | package | future | traceback )__ )\b @@ -2834,18 +2997,33 @@ indirectly through syntactic constructs illegal-names - name - keyword.control.flow.python match (?x) - \b( - and | as | assert | async | await | break | class | continue | def - | del | elif | else | except | exec | finally | for | from | global - | if | import | in | is | (?<=\.)lambda | lambda(?=\s*[\.=]) - | nonlocal | not | or | pass | raise | return | try | while | with - | yield + \b(?: + ( + and | assert | async | await | break | class | continue | def + | del | elif | else | except | finally | for | from | global + | if | in | is | (?<=\.)lambda | lambda(?=\s*[\.=]) + | nonlocal | not | or | pass | raise | return | try | while | with + | yield + ) | ( + as | import + ) )\b + captures + + 1 + + name + keyword.control.flow.python + + 2 + + name + keyword.control.import.python + + special-variables @@ -2970,6 +3148,42 @@ indirectly through syntactic constructs + fstring-formatting-braces + + patterns + + + comment + empty braces are illegal + match + ({)(\s*?)(}) + captures + + 1 + + name + constant.character.format.placeholder.other.python + + 2 + + name + invalid.illegal.brace.python + + 3 + + name + constant.character.format.placeholder.other.python + + + + + name + constant.character.escape.python + match + ({{|}}) + + + regexp-base-common patterns @@ -3180,6 +3394,19 @@ indirectly through syntactic constructs + codetags + + match + (?:\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\b) + captures + + 1 + + name + keyword.codetag.notation.python + + + comments-base name @@ -4322,1947 +4549,41 @@ indirectly through syntactic constructs keyword.operator.negation.regexp 3 - - name - constant.character.set.regexp - - - endCaptures - - 1 - - name - punctuation.character.set.end.regexp constant.other.set.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #regexp-charecter-set-escapes - - - name - constant.character.set.regexp - match - [^\n] - - - - - - double-one-regexp-named-group - - name - meta.named.regexp - begin - (?x) - (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) - - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp - - 2 - - name - entity.name.tag.named.group.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-regexp-expression - - - - double-one-regexp-comments - - name - comment.regexp - begin - \(\?# - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - punctuation.comment.begin.regexp - - - endCaptures - - 1 - - name - punctuation.comment.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #codetags - - - - double-one-regexp-lookahead - - begin - (\()\?= - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookahead.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-regexp-expression - - - - double-one-regexp-lookahead-negative - - begin - (\()\?! - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookahead.negative.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-regexp-expression - - - - double-one-regexp-lookbehind - - begin - (\()\?<= - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-regexp-expression - - - - double-one-regexp-lookbehind-negative - - begin - (\()\?<! - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.negative.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.negative.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-regexp-expression - - - - double-one-regexp-conditional - - begin - (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.conditional.regexp - - 1 - - name - punctuation.parenthesis.conditional.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.conditional.negative.regexp punctuation.parenthesis.conditional.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-regexp-expression - - - - double-one-regexp-parentheses-non-capturing - - begin - \(\?: - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-regexp-expression - - - - double-one-regexp-parentheses - - begin - \( - end - (\)|(?="))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-regexp-expression - - - - double-three-regexp-expression - - patterns - - - include - #regexp-base-expression - - - include - #double-three-regexp-character-set - - - include - #double-three-regexp-comments - - - include - #regexp-flags - - - include - #double-three-regexp-named-group - - - include - #regexp-backreference - - - include - #double-three-regexp-lookahead - - - include - #double-three-regexp-lookahead-negative - - - include - #double-three-regexp-lookbehind - - - include - #double-three-regexp-lookbehind-negative - - - include - #double-three-regexp-conditional - - - include - #double-three-regexp-parentheses-non-capturing - - - include - #double-three-regexp-parentheses - - - include - #comments-string-double-three - - - - double-three-regexp-character-set - - patterns - - - match - (?x) - \[ \^? \] (?! .*?\]) - - - - name - meta.character.set.regexp - begin - (\[)(\^)?(\])? - end - (\]|(?=""")) - beginCaptures - - 1 - - name - punctuation.character.set.begin.regexp constant.other.set.regexp - - 2 - - name - keyword.operator.negation.regexp - - 3 - - name - constant.character.set.regexp - - - endCaptures - - 1 - - name - punctuation.character.set.end.regexp constant.other.set.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #regexp-charecter-set-escapes - - - name - constant.character.set.regexp - match - [^\n] - - - - - - double-three-regexp-named-group - - name - meta.named.regexp - begin - (?x) - (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) - - end - (\)|(?=""")) - beginCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp - - 2 - - name - entity.name.tag.named.group.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-regexp-expression - - - include - #comments-string-double-three - - - - double-three-regexp-comments - - name - comment.regexp - begin - \(\?# - end - (\)|(?=""")) - beginCaptures - - 0 - - name - punctuation.comment.begin.regexp - - - endCaptures - - 1 - - name - punctuation.comment.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #codetags - - - - double-three-regexp-lookahead - - begin - (\()\?= - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.lookahead.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-regexp-expression - - - include - #comments-string-double-three - - - - double-three-regexp-lookahead-negative - - begin - (\()\?! - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.lookahead.negative.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-regexp-expression - - - include - #comments-string-double-three - - - - double-three-regexp-lookbehind - - begin - (\()\?<= - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-regexp-expression - - - include - #comments-string-double-three - - - - double-three-regexp-lookbehind-negative - - begin - (\()\?<! - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.negative.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.negative.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-regexp-expression - - - include - #comments-string-double-three - - - - double-three-regexp-conditional - - begin - (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) - end - (\)|(?=""")) - beginCaptures - - 0 - - name - keyword.operator.conditional.regexp - - 1 - - name - punctuation.parenthesis.conditional.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.conditional.negative.regexp punctuation.parenthesis.conditional.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-regexp-expression - - - include - #comments-string-double-three - - - - double-three-regexp-parentheses-non-capturing - - begin - \(\?: - end - (\)|(?=""")) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-regexp-expression - - - include - #comments-string-double-three - - - - double-three-regexp-parentheses - - begin - \( - end - (\)|(?=""")) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-regexp-expression - - - include - #comments-string-double-three - - - - regexp-single-one-line - - name - string.regexp.quoted.single.python - begin - \b(([uU]r)|([bB]r)|(r[bB]?))(\') - end - (\')|(?<!\\)(\n) - beginCaptures - - 2 - - name - invalid.deprecated.prefix.python - - 3 - - name - storage.type.string.python - - 4 - - name - storage.type.string.python - - 5 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-regexp-expression - - - - regexp-single-three-line - - name - string.regexp.quoted.multi.python - begin - \b(([uU]r)|([bB]r)|(r[bB]?))(\'\'\') - end - (\'\'\') - beginCaptures - - 2 - - name - invalid.deprecated.prefix.python - - 3 - - name - storage.type.string.python - - 4 - - name - storage.type.string.python - - 5 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-regexp-expression - - - - regexp-double-one-line - - name - string.regexp.quoted.single.python - begin - \b(([uU]r)|([bB]r)|(r[bB]?))(") - end - (")|(?<!\\)(\n) - beginCaptures - - 2 - - name - invalid.deprecated.prefix.python - - 3 - - name - storage.type.string.python - - 4 - - name - storage.type.string.python - - 5 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-one-regexp-expression - - - - regexp-double-three-line - - name - string.regexp.quoted.multi.python - begin - \b(([uU]r)|([bB]r)|(r[bB]?))(""") - end - (""") - beginCaptures - - 2 - - name - invalid.deprecated.prefix.python - - 3 - - name - storage.type.string.python - - 4 - - name - storage.type.string.python - - 5 - - name - punctuation.definition.string.begin.python - - - endCaptures - - 1 - - name - punctuation.definition.string.end.python - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #double-three-regexp-expression - - - - single-one-fregexp-expression - - patterns - - - include - #fregexp-base-expression - - - include - #single-one-regexp-character-set - - - include - #single-one-regexp-comments - - - include - #regexp-flags - - - include - #single-one-regexp-named-group - - - include - #regexp-backreference - - - include - #single-one-fregexp-lookahead - - - include - #single-one-fregexp-lookahead-negative - - - include - #single-one-fregexp-lookbehind - - - include - #single-one-fregexp-lookbehind-negative - - - include - #single-one-fregexp-conditional - - - include - #single-one-fregexp-parentheses-non-capturing - - - include - #single-one-fregexp-parentheses - - - - single-one-fregexp-named-group - - name - meta.named.regexp - begin - (?x) - (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) - - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp - - 2 - - name - entity.name.tag.named.group.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-one-fregexp-lookahead - - begin - (\()\?= - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookahead.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-one-fregexp-lookahead-negative - - begin - (\()\?! - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookahead.negative.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-one-fregexp-lookbehind - - begin - (\()\?<= - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-one-fregexp-lookbehind-negative - - begin - (\()\?<! - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.negative.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.negative.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-one-fregexp-conditional - - begin - (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - keyword.operator.conditional.regexp - - 1 - - name - punctuation.parenthesis.conditional.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.conditional.negative.regexp punctuation.parenthesis.conditional.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-one-fregexp-parentheses-non-capturing - - begin - \(\?: - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-one-fregexp-parentheses - - begin - \( - end - (\)|(?=\'))|((?=(?<!\\)\n)) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-one-fregexp-expression - - - - single-three-fregexp-expression - - patterns - - - include - #fregexp-base-expression - - - include - #single-three-regexp-character-set - - - include - #single-three-regexp-comments - - - include - #regexp-flags - - - include - #single-three-regexp-named-group - - - include - #regexp-backreference - - - include - #single-three-fregexp-lookahead - - - include - #single-three-fregexp-lookahead-negative - - - include - #single-three-fregexp-lookbehind - - - include - #single-three-fregexp-lookbehind-negative - - - include - #single-three-fregexp-conditional - - - include - #single-three-fregexp-parentheses-non-capturing - - - include - #single-three-fregexp-parentheses - - - include - #comments-string-single-three - - - - single-three-fregexp-named-group - - name - meta.named.regexp - begin - (?x) - (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) - - end - (\)|(?=\'\'\')) - beginCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.begin.regexp - - 2 - - name - entity.name.tag.named.group.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.named.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-lookahead - - begin - (\()\?= - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.lookahead.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-lookahead-negative - - begin - (\()\?! - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.lookahead.negative.regexp - - 1 - - name - punctuation.parenthesis.lookahead.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookahead.negative.regexp punctuation.parenthesis.lookahead.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-lookbehind - - begin - (\()\?<= - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-lookbehind-negative - - begin - (\()\?<! - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.lookbehind.negative.regexp - - 1 - - name - punctuation.parenthesis.lookbehind.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.lookbehind.negative.regexp punctuation.parenthesis.lookbehind.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-conditional - - begin - (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - keyword.operator.conditional.regexp - - 1 - - name - punctuation.parenthesis.conditional.begin.regexp - - - endCaptures - - 1 - - name - keyword.operator.conditional.negative.regexp punctuation.parenthesis.conditional.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-parentheses-non-capturing - - begin - \(\?: - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.non-capturing.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - single-three-fregexp-parentheses - - begin - \( - end - (\)|(?=\'\'\')) - beginCaptures - - 0 - - name - support.other.parenthesis.regexp punctuation.parenthesis.begin.regexp - - - endCaptures - - 1 - - name - support.other.parenthesis.regexp punctuation.parenthesis.end.regexp - - 2 - - name - invalid.illegal.newline.python - - - patterns - - - include - #single-three-fregexp-expression - - - include - #comments-string-single-three - - - - double-one-fregexp-expression - - patterns - - - include - #fregexp-base-expression - - - include - #double-one-regexp-character-set - - - include - #double-one-regexp-comments - - - include - #regexp-flags - - - include - #double-one-regexp-named-group - - - include - #regexp-backreference - - - include - #double-one-fregexp-lookahead - - - include - #double-one-fregexp-lookahead-negative - - - include - #double-one-fregexp-lookbehind - - - include - #double-one-fregexp-lookbehind-negative - - - include - #double-one-fregexp-conditional - - - include - #double-one-fregexp-parentheses-non-capturing - - - include - #double-one-fregexp-parentheses + + name + constant.character.set.regexp + + + endCaptures + + 1 + + name + punctuation.character.set.end.regexp constant.other.set.regexp + + 2 + + name + invalid.illegal.newline.python + + + patterns + + + include + #regexp-charecter-set-escapes + + + name + constant.character.set.regexp + match + [^\n] + + - double-one-fregexp-named-group + double-one-regexp-named-group name meta.named.regexp @@ -6302,11 +4623,48 @@ indirectly through syntactic constructs include - #double-one-fregexp-expression + #double-one-regexp-expression + + + + double-one-regexp-comments + + name + comment.regexp + begin + \(\?# + end + (\)|(?="))|((?=(?<!\\)\n)) + beginCaptures + + 0 + + name + punctuation.comment.begin.regexp + + + endCaptures + + 1 + + name + punctuation.comment.end.regexp + + 2 + + name + invalid.illegal.newline.python + + + patterns + + + include + #codetags - double-one-fregexp-lookahead + double-one-regexp-lookahead begin (\()\?= @@ -6342,11 +4700,11 @@ indirectly through syntactic constructs include - #double-one-fregexp-expression + #double-one-regexp-expression - double-one-fregexp-lookahead-negative + double-one-regexp-lookahead-negative begin (\()\?! @@ -6382,11 +4740,11 @@ indirectly through syntactic constructs include - #double-one-fregexp-expression + #double-one-regexp-expression - double-one-fregexp-lookbehind + double-one-regexp-lookbehind begin (\()\?<= @@ -6422,11 +4780,11 @@ indirectly through syntactic constructs include - #double-one-fregexp-expression + #double-one-regexp-expression - double-one-fregexp-lookbehind-negative + double-one-regexp-lookbehind-negative begin (\()\?<! @@ -6462,11 +4820,11 @@ indirectly through syntactic constructs include - #double-one-fregexp-expression + #double-one-regexp-expression - double-one-fregexp-conditional + double-one-regexp-conditional begin (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) @@ -6502,11 +4860,11 @@ indirectly through syntactic constructs include - #double-one-fregexp-expression + #double-one-regexp-expression - double-one-fregexp-parentheses-non-capturing + double-one-regexp-parentheses-non-capturing begin \(\?: @@ -6537,11 +4895,11 @@ indirectly through syntactic constructs include - #double-one-fregexp-expression + #double-one-regexp-expression - double-one-fregexp-parentheses + double-one-regexp-parentheses begin \( @@ -6572,17 +4930,17 @@ indirectly through syntactic constructs include - #double-one-fregexp-expression + #double-one-regexp-expression - double-three-fregexp-expression + double-three-regexp-expression patterns include - #fregexp-base-expression + #regexp-base-expression include @@ -6606,31 +4964,31 @@ indirectly through syntactic constructs include - #double-three-fregexp-lookahead + #double-three-regexp-lookahead include - #double-three-fregexp-lookahead-negative + #double-three-regexp-lookahead-negative include - #double-three-fregexp-lookbehind + #double-three-regexp-lookbehind include - #double-three-fregexp-lookbehind-negative + #double-three-regexp-lookbehind-negative include - #double-three-fregexp-conditional + #double-three-regexp-conditional include - #double-three-fregexp-parentheses-non-capturing + #double-three-regexp-parentheses-non-capturing include - #double-three-fregexp-parentheses + #double-three-regexp-parentheses include @@ -6638,7 +4996,71 @@ indirectly through syntactic constructs - double-three-fregexp-named-group + double-three-regexp-character-set + + patterns + + + match + (?x) + \[ \^? \] (?! .*?\]) + + + + name + meta.character.set.regexp + begin + (\[)(\^)?(\])? + end + (\]|(?=""")) + beginCaptures + + 1 + + name + punctuation.character.set.begin.regexp constant.other.set.regexp + + 2 + + name + keyword.operator.negation.regexp + + 3 + + name + constant.character.set.regexp + + + endCaptures + + 1 + + name + punctuation.character.set.end.regexp constant.other.set.regexp + + 2 + + name + invalid.illegal.newline.python + + + patterns + + + include + #regexp-charecter-set-escapes + + + name + constant.character.set.regexp + match + [^\n] + + + + + + double-three-regexp-named-group name meta.named.regexp @@ -6678,7 +5100,7 @@ indirectly through syntactic constructs include - #double-three-fregexp-expression + #double-three-regexp-expression include @@ -6686,7 +5108,44 @@ indirectly through syntactic constructs - double-three-fregexp-lookahead + double-three-regexp-comments + + name + comment.regexp + begin + \(\?# + end + (\)|(?=""")) + beginCaptures + + 0 + + name + punctuation.comment.begin.regexp + + + endCaptures + + 1 + + name + punctuation.comment.end.regexp + + 2 + + name + invalid.illegal.newline.python + + + patterns + + + include + #codetags + + + + double-three-regexp-lookahead begin (\()\?= @@ -6722,7 +5181,7 @@ indirectly through syntactic constructs include - #double-three-fregexp-expression + #double-three-regexp-expression include @@ -6730,7 +5189,7 @@ indirectly through syntactic constructs - double-three-fregexp-lookahead-negative + double-three-regexp-lookahead-negative begin (\()\?! @@ -6766,7 +5225,7 @@ indirectly through syntactic constructs include - #double-three-fregexp-expression + #double-three-regexp-expression include @@ -6774,7 +5233,7 @@ indirectly through syntactic constructs - double-three-fregexp-lookbehind + double-three-regexp-lookbehind begin (\()\?<= @@ -6810,7 +5269,7 @@ indirectly through syntactic constructs include - #double-three-fregexp-expression + #double-three-regexp-expression include @@ -6818,7 +5277,7 @@ indirectly through syntactic constructs - double-three-fregexp-lookbehind-negative + double-three-regexp-lookbehind-negative begin (\()\?<! @@ -6854,7 +5313,7 @@ indirectly through syntactic constructs include - #double-three-fregexp-expression + #double-three-regexp-expression include @@ -6862,7 +5321,7 @@ indirectly through syntactic constructs - double-three-fregexp-conditional + double-three-regexp-conditional begin (\()\?\((\w+(?:\s+[[:alnum:]]+)?|\d+)\) @@ -6898,7 +5357,7 @@ indirectly through syntactic constructs include - #double-three-fregexp-expression + #double-three-regexp-expression include @@ -6906,7 +5365,7 @@ indirectly through syntactic constructs - double-three-fregexp-parentheses-non-capturing + double-three-regexp-parentheses-non-capturing begin \(\?: @@ -6937,7 +5396,7 @@ indirectly through syntactic constructs include - #double-three-fregexp-expression + #double-three-regexp-expression include @@ -6945,7 +5404,7 @@ indirectly through syntactic constructs - double-three-fregexp-parentheses + double-three-regexp-parentheses begin \( @@ -6976,7 +5435,7 @@ indirectly through syntactic constructs include - #double-three-fregexp-expression + #double-three-regexp-expression include @@ -6984,12 +5443,12 @@ indirectly through syntactic constructs - fregexp-single-one-line + regexp-single-one-line name - string.interpolated.python string.regexp.quoted.single.python + string.regexp.quoted.single.python begin - \b(([uU]r)|([fF]r)|(r[fF]?))(\') + \b(([uU]r)|([bB]r)|(r[bB]?))(\') end (\')|(?<!\\)(\n) beginCaptures @@ -7032,16 +5491,16 @@ indirectly through syntactic constructs include - #single-one-fregexp-expression + #single-one-regexp-expression - fregexp-single-three-line + regexp-single-three-line name - string.interpolated.python string.regexp.quoted.multi.python + string.regexp.quoted.multi.python begin - \b(([uU]r)|([fF]r)|(r[fF]?))(\'\'\') + \b(([uU]r)|([bB]r)|(r[bB]?))(\'\'\') end (\'\'\') beginCaptures @@ -7084,16 +5543,16 @@ indirectly through syntactic constructs include - #single-three-fregexp-expression + #single-three-regexp-expression - fregexp-double-one-line + regexp-double-one-line name - string.interpolated.python string.regexp.quoted.single.python + string.regexp.quoted.single.python begin - \b(([uU]r)|([fF]r)|(r[fF]?))(") + \b(([uU]r)|([bB]r)|(r[bB]?))(") end (")|(?<!\\)(\n) beginCaptures @@ -7136,16 +5595,16 @@ indirectly through syntactic constructs include - #double-one-fregexp-expression + #double-one-regexp-expression - fregexp-double-three-line + regexp-double-three-line name - string.interpolated.python string.regexp.quoted.multi.python + string.regexp.quoted.multi.python begin - \b(([uU]r)|([fF]r)|(r[fF]?))(""") + \b(([uU]r)|([bB]r)|(r[bB]?))(""") end (""") beginCaptures @@ -7188,7 +5647,7 @@ indirectly through syntactic constructs include - #double-three-fregexp-expression + #double-three-regexp-expression @@ -7336,7 +5795,7 @@ indirectly through syntactic constructs name string.quoted.single.python begin - (\b[rR](?=[uU]))?([uU])?((['"])) + (?:\b([rR])(?=[uU]))?([uU])?((['"])) end (\3)|((?<!\\)\n) beginCaptures @@ -7636,7 +6095,7 @@ indirectly through syntactic constructs name string.quoted.multi.python begin - (\b[rR](?=[uU]))?([uU])?('''|""") + (?:\b([rR])(?=[uU]))?([uU])?('''|""") end (\3) beginCaptures @@ -7915,7 +6374,7 @@ indirectly through syntactic constructs name meta.fstring.python begin - (\b(?:[R][fF]|[fF][R]))((['"])) + (\b(?:[rR][fF]|[fF][rR]))((['"])) end (\2)|((?<!\\)\n) beginCaptures @@ -8056,12 +6515,18 @@ indirectly through syntactic constructs name storage.type.format.python match - (![rsa])(?=}) + (=(![rsa])?)(?=}) + + + name + storage.type.format.python + match + (=?![rsa])(?=}) match (?x) - (![rsa])? + ( (?: =?) (?: ![rsa])? ) ( : \w? [<>=^]? [-+ ]? \#? \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=}) @@ -8075,7 +6540,7 @@ indirectly through syntactic constructs 2 name - support.other.format.python + storage.type.format.python @@ -8088,7 +6553,7 @@ indirectly through syntactic constructs fstring-terminator-single-tail begin - (![rsa])?(:)(?=.*?{) + ((?:=?)(?:![rsa])?)(:)(?=.*?{) end (?=})|(?=\n) beginCaptures @@ -8101,7 +6566,7 @@ indirectly through syntactic constructs 2 name - support.other.format.python + storage.type.format.python patterns @@ -8116,49 +6581,49 @@ indirectly through syntactic constructs name - support.other.format.python + storage.type.format.python match ([bcdeEfFgGnosxX%])(?=}) name - support.other.format.python + storage.type.format.python match (\.\d+) name - support.other.format.python + storage.type.format.python match (,) name - support.other.format.python + storage.type.format.python match (\d+) name - support.other.format.python + storage.type.format.python match (\#) name - support.other.format.python + storage.type.format.python match ([-+ ]) name - support.other.format.python + storage.type.format.python match ([<>=^]) name - support.other.format.python + storage.type.format.python match (\w) @@ -8287,7 +6752,7 @@ indirectly through syntactic constructs name meta.fstring.python begin - (\b(?:[R][fF]|[fF][R]))('''|""") + (\b(?:[rR][fF]|[fF][rR]))('''|""") end (\2) beginCaptures @@ -8428,12 +6893,18 @@ indirectly through syntactic constructs name storage.type.format.python match - (![rsa])(?=}) + (=(![rsa])?)(?=}) + + + name + storage.type.format.python + match + (=?![rsa])(?=}) match (?x) - (![rsa])? + ( (?: =?) (?: ![rsa])? ) ( : \w? [<>=^]? [-+ ]? \#? \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=}) @@ -8447,7 +6918,7 @@ indirectly through syntactic constructs 2 name - support.other.format.python + storage.type.format.python @@ -8460,7 +6931,7 @@ indirectly through syntactic constructs fstring-terminator-multi-tail begin - (![rsa])?(:)(?=.*?{) + ((?:=?)(?:![rsa])?)(:)(?=.*?{) end (?=}) beginCaptures @@ -8473,7 +6944,7 @@ indirectly through syntactic constructs 2 name - support.other.format.python + storage.type.format.python patterns @@ -8488,49 +6959,49 @@ indirectly through syntactic constructs name - support.other.format.python + storage.type.format.python match ([bcdeEfFgGnosxX%])(?=}) name - support.other.format.python + storage.type.format.python match (\.\d+) name - support.other.format.python + storage.type.format.python match (,) name - support.other.format.python + storage.type.format.python match (\d+) name - support.other.format.python + storage.type.format.python match (\#) name - support.other.format.python + storage.type.format.python match ([-+ ]) name - support.other.format.python + storage.type.format.python match ([<>=^]) name - support.other.format.python + storage.type.format.python match (\w) diff --git a/grammars/MagicRegExp.cson b/grammars/MagicRegExp.cson index 69e0c300..175b80f6 100644 --- a/grammars/MagicRegExp.cson +++ b/grammars/MagicRegExp.cson @@ -35,6 +35,24 @@ repository: include: "#regexp-base-common" } ] + "fstring-formatting-braces": + patterns: [ + { + comment: "empty braces are illegal" + match: "({)(\\s*?)(})" + captures: + "1": + name: "constant.character.format.placeholder.other.python" + "2": + name: "invalid.illegal.brace.python" + "3": + name: "constant.character.format.placeholder.other.python" + } + { + name: "constant.character.escape.python" + match: "({{|}})" + } + ] "regexp-base-common": patterns: [ { @@ -170,6 +188,11 @@ repository: include: "#regexp-escape-catchall" } ] + codetags: + match: "(?:\\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\\b)" + captures: + "1": + name: "keyword.codetag.notation.python" "regexp-expression": patterns: [ { diff --git a/grammars/MagicRegExp.tmLanguage b/grammars/MagicRegExp.tmLanguage index 3bc9d48e..3ddacc70 100644 --- a/grammars/MagicRegExp.tmLanguage +++ b/grammars/MagicRegExp.tmLanguage @@ -1,7 +1,7 @@ - + name MagicRegExp @@ -58,6 +58,42 @@ + fstring-formatting-braces + + patterns + + + comment + empty braces are illegal + match + ({)(\s*?)(}) + captures + + 1 + + name + constant.character.format.placeholder.other.python + + 2 + + name + invalid.illegal.brace.python + + 3 + + name + constant.character.format.placeholder.other.python + + + + + name + constant.character.escape.python + match + ({{|}}) + + + regexp-base-common patterns @@ -268,6 +304,19 @@ + codetags + + match + (?:\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\b) + captures + + 1 + + name + keyword.codetag.notation.python + + + regexp-expression patterns diff --git a/grammars/python-console.cson b/grammars/python-console.cson new file mode 100644 index 00000000..21600219 --- /dev/null +++ b/grammars/python-console.cson @@ -0,0 +1,50 @@ +# Copyright (c) 2014 GitHub Inc. + +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: + +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# -------------------------------------------------------------------- + +# This package was derived from a TextMate bundle located at +# https://github.com/textmate/python.tmbundle and distributed under the following +# license, located in `README.mdown`: + +# Permission to copy, use, modify, sell and distribute this +# software is granted. This software is provided "as is" without +# express or implied warranty, and with no claim as to its +# suitability for any purpose. + +'scopeName': 'text.python.console' +'name': 'Python Console' +'fileTypes': [ + 'doctest' + 'pycon' +] +'patterns': [ + { + 'match': '^(>{3}|\\.{3}|In \\[\\d+\\]:) (.+)$' + 'captures': + '1': + 'name': 'punctuation.separator.prompt.python.console' + '2': + 'patterns': [ + 'include': 'source.python' + ] + } +] diff --git a/grammars/python-traceback.cson b/grammars/python-traceback.cson new file mode 100644 index 00000000..1121bbd2 --- /dev/null +++ b/grammars/python-traceback.cson @@ -0,0 +1,66 @@ +# Copyright (c) 2014 GitHub Inc. + +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: + +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# -------------------------------------------------------------------- + +# This package was derived from a TextMate bundle located at +# https://github.com/textmate/python.tmbundle and distributed under the following +# license, located in `README.mdown`: + +# Permission to copy, use, modify, sell and distribute this +# software is granted. This software is provided "as is" without +# express or implied warranty, and with no claim as to its +# suitability for any purpose. + +'scopeName': 'text.python.traceback' +'name': 'Python Traceback' +'fileTypes': [ + 'pytb' +] +'patterns': [ + { + 'match': '^ File ("[^"]+"), line (\\d+)(?:, in (.+))?$' + 'captures': { + '1': + 'name': 'string.python.traceback' + '2': + 'name': 'constant.numeric.python.traceback' + '3': + 'name': 'entity.name.function.python.traceback' + } + } + { + 'match': '^ (.+)$' + 'captures': + '1': + 'patterns': [ + 'include': 'source.python' + ] + } + { + 'match': '^([^\\s:]+):(?: (.+))?$' + 'captures': + '1': + 'name': 'entity.name.type.class.python.traceback' + '2': + 'name': 'string.python.traceback' + } +] diff --git a/grammars/src/MagicPython.syntax.yaml b/grammars/src/MagicPython.syntax.yaml index 589f7741..ebfccdaf 100644 --- a/grammars/src/MagicPython.syntax.yaml +++ b/grammars/src/MagicPython.syntax.yaml @@ -4,9 +4,8 @@ scopeName: source.python # NOTE: remember to update package.json with VSCode file types. fileTypes: [py, py3, rpy, pyw, cpy, pyi, SConstruct, Sconstruct, sconstruct, SConscript, - gyp, gypi, wsgi] -first_line_match: ^#!/.*\bpython[\d\.]*\b -firstLineMatch: ^#!/.*\bpython[\d\.]*\b + gyp, gypi, wsgi, kv, Snakefile, smk, tac] +firstLineMatch: ^#![ \t]*/.*\bpython[\d\.]*\b uuid: 742deb57-6e38-4192-bed6-410746efd85d patterns: @@ -104,82 +103,6 @@ repository: bquote: '(""")' equote: '(""")' - - file: 'regexp.inc.syntax.yaml' - vars: - prefix: 'single-one-' - basename: 'fregexp' - marker: "|(?=\\')" - nested: '' - guard: "|((?=(?= | <= | < | >) (?# 5) + | (:=) (?# 6) + captures: '1': {name: keyword.operator.logical.python} '2': {name: keyword.control.flow.python} '3': {name: keyword.operator.bitwise.python} '4': {name: keyword.operator.arithmetic.python} '5': {name: keyword.operator.comparison.python} + '6': {name: keyword.operator.assignment.python} punctuation: patterns: @@ -603,7 +583,6 @@ repository: captures: '1': {name: storage.type.imaginary.number.python} '2': {name: invalid.illegal.dec.python} - '3': {name: invalid.illegal.dec.python} number-hex: name: constant.numeric.hex.python @@ -652,10 +631,6 @@ repository: - include: '#regexp-double-three-line' - include: '#regexp-single-one-line' - include: '#regexp-double-one-line' - - include: '#fregexp-single-three-line' - - include: '#fregexp-double-three-line' - - include: '#fregexp-single-one-line' - - include: '#fregexp-double-one-line' string: patterns: @@ -730,7 +705,7 @@ repository: - include: '#impossible' f-expression: - comment: All valid Python expressions, except comments and line cont + comment: All valid Python expressions, except comments and line continuation patterns: - include: '#expression-bare' - include: '#member-access' @@ -763,81 +738,98 @@ repository: match: \\$ string-formatting: - name: constant.character.format.placeholder.other.python + name: meta.format.percent.python match: | (?x) - % (\([\w\s]*\))? - [-+#0 ]* - (\d+|\*)? (\.(\d+|\*))? - ([hlL])? - [diouxXeEfFgGcrsa%] + ( + % (\([\w\s]*\))? + [-+#0 ]* + (\d+|\*)? (\.(\d+|\*))? + ([hlL])? + [diouxXeEfFgGcrsab%] + ) + captures: + '1': {name: constant.character.format.placeholder.other.python} string-brace-formatting: patterns: - - name: constant.character.format.placeholder.other.python + - name: meta.format.brace.python match: | (?x) - (?: + ( {{ | }} | (?: { - \w*? (\.[[:alpha:]_]\w*? | \[[^\]'"]+\])*? + \w* (\.[[:alpha:]_]\w* | \[[^\]'"]+\])* (![rsa])? ( : \w? [<>=^]? [-+ ]? \#? \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )? }) ) captures: - '2': {name: storage.type.format.python} - '3': {name: support.other.format.python} - - name: constant.character.format.placeholder.other.python - begin: | + # We use this awkward scope name instead of perhaps more + # elegant "support.other.format.python" because it appears + # that by default it gives a better visual result in various + # systems (e.g. GitHub). This is due to the fact that + # "storage.type..." is more likely to have special + # highlighting in any given color scheme than + # "support.other..." + '1': {name: constant.character.format.placeholder.other.python} + '3': {name: storage.type.format.python} + '4': {name: storage.type.format.python} + + # The reason for 2 separate rules is to establish which one + # should be matched with a higher priority. The rule below is + # more generic, so it should be matched only if the specific one + # didn't. + - name: meta.format.brace.python + match: | (?x) - \{ - \w*? (\.[[:alpha:]_]\w*? | \[[^\]'"]+\])*? + ( + { + \w* (\.[[:alpha:]_]\w* | \[[^\]'"]+\])* (![rsa])? (:) - (?=[^'"}\n]*\}) - end: \} - beginCaptures: - '2': {name: storage.type.format.python} - '3': {name: support.other.format.python} - patterns: - - match: | - (?x) \{ [^'"}\n]*? \} (?=.*?\}) + [^'"{}\n]* (?: + \{ [^'"}\n]*? \} [^'"{}\n]* + )* + } + ) + captures: + '1': {name: constant.character.format.placeholder.other.python} + '3': {name: storage.type.format.python} + '4': {name: storage.type.format.python} fstring-formatting: patterns: - include: '#fstring-formatting-braces' - include: '#fstring-formatting-singe-brace' - fstring-formatting-braces: - patterns: - - comment: empty braces are illegal - match: ({)(\s*?)(}) - captures: - '1': {name: constant.character.format.placeholder.other.python} - '2': {name: invalid.illegal.brace.python} - '3': {name: constant.character.format.placeholder.other.python} - - name: constant.character.escape.python - match: ({{|}}) - fstring-formatting-singe-brace: name: invalid.illegal.brace.python match: (}(?!})) import: comment: | - Import statements + Import statements used to correctly mark `from`, `import`, and `as` patterns: - - match: | - (?x) - \s* \b(from)\b (\s*\.+\s*) (import)? - captures: + - begin: \b(?=^]? [-+ ]? \#? - \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=}) - captures: - '1': {name: storage.type.format.python} - '2': {name: support.other.format.python} +fstring-terminator-${line}: + patterns: + - name: storage.type.format.python + match: (=(![rsa])?)(?=}) + - name: storage.type.format.python + match: (=?![rsa])(?=}) + - match: | + (?x) + ( (?: =?) (?: ![rsa])? ) + ( : \w? [<>=^]? [-+ ]? \#? + \d* ,? (\.\d+)? [bcdeEfFgGnosxX%]? )(?=}) + captures: + # We use this awkward scope name instead of perhaps more + # elegant "support.other.format.python" because it appears + # that by default it gives a better visual result in various + # systems (e.g. GitHub). This is due to the fact that + # "storage.type..." is more likely to have special + # highlighting in any given color scheme than + # "support.other..." + # + '1': {name: storage.type.format.python} + '2': {name: storage.type.format.python} - - include: '#fstring-terminator-${line}-tail' + - include: '#fstring-terminator-${line}-tail' - fstring-terminator-${line}-tail: - begin: (![rsa])?(:)(?=.*?{) - end: (?=})${fguard} - beginCaptures: - '1': {name: storage.type.format.python} - '2': {name: support.other.format.python} +fstring-terminator-${line}-tail: + begin: ((?:=?)(?:![rsa])?)(:)(?=.*?{) + end: (?=})${fguard} + beginCaptures: + '1': {name: storage.type.format.python} + '2': {name: storage.type.format.python} - patterns: - - include: '#fstring-illegal-${line}-brace' - - include: '#fstring-${line}-brace' - - name: support.other.format.python - match: ([bcdeEfFgGnosxX%])(?=}) - - name: support.other.format.python - match: (\.\d+) - - name: support.other.format.python - match: (,) - - name: support.other.format.python - match: (\d+) - - name: support.other.format.python - match: (\#) - - name: support.other.format.python - match: ([-+ ]) - - name: support.other.format.python - match: ([<>=^]) - - name: support.other.format.python - match: (\w) + patterns: + - include: '#fstring-illegal-${line}-brace' + - include: '#fstring-${line}-brace' + - name: storage.type.format.python + match: ([bcdeEfFgGnosxX%])(?=}) + - name: storage.type.format.python + match: (\.\d+) + - name: storage.type.format.python + match: (,) + - name: storage.type.format.python + match: (\d+) + - name: storage.type.format.python + match: (\#) + - name: storage.type.format.python + match: ([-+ ]) + - name: storage.type.format.python + match: ([<>=^]) + - name: storage.type.format.python + match: (\w) ... diff --git a/grammars/src/pystring.inc.syntax.yaml b/grammars/src/pystring.inc.syntax.yaml index 36e0bd1b..5127ae72 100644 --- a/grammars/src/pystring.inc.syntax.yaml +++ b/grammars/src/pystring.inc.syntax.yaml @@ -1,118 +1,117 @@ --- -repository: - string-raw-quoted-${line}-line: - name: string.quoted.raw.${line}.python - begin: \b(([uU]R)|(R))(${marker}) - end: (\4)${guard} - beginCaptures: - '2': {name: invalid.deprecated.prefix.python} - '3': {name: storage.type.string.python} - '4': {name: punctuation.definition.string.begin.python} - endCaptures: - '1': {name: punctuation.definition.string.end.python} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#string-${line}-bad-brace1-formatting-raw' - - include: '#string-${line}-bad-brace2-formatting-raw' - - include: '#string-raw-guts' +string-raw-quoted-${line}-line: + name: string.quoted.raw.${line}.python + begin: \b(([uU]R)|(R))(${marker}) + end: (\4)${guard} + beginCaptures: + '2': {name: invalid.deprecated.prefix.python} + '3': {name: storage.type.string.python} + '4': {name: punctuation.definition.string.begin.python} + endCaptures: + '1': {name: punctuation.definition.string.end.python} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#string-${line}-bad-brace1-formatting-raw' + - include: '#string-${line}-bad-brace2-formatting-raw' + - include: '#string-raw-guts' - string-bin-quoted-${line}-line: - name: string.quoted.binary.${line}.python - begin: (\b[bB])(${marker}) - end: (\2)${guard} - beginCaptures: - '1': {name: storage.type.string.python} - '2': {name: punctuation.definition.string.begin.python} - endCaptures: - '1': {name: punctuation.definition.string.end.python} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#string-entity' +string-bin-quoted-${line}-line: + name: string.quoted.binary.${line}.python + begin: (\b[bB])(${marker}) + end: (\2)${guard} + beginCaptures: + '1': {name: storage.type.string.python} + '2': {name: punctuation.definition.string.begin.python} + endCaptures: + '1': {name: punctuation.definition.string.end.python} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#string-entity' - string-raw-bin-quoted-${line}-line: - name: string.quoted.raw.binary.${line}.python - begin: (\b(?:R[bB]|[bB]R))(${marker}) - end: (\2)${guard} - beginCaptures: - '1': {name: storage.type.string.python} - '2': {name: punctuation.definition.string.begin.python} - endCaptures: - '1': {name: punctuation.definition.string.end.python} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#string-raw-bin-guts' +string-raw-bin-quoted-${line}-line: + name: string.quoted.raw.binary.${line}.python + begin: (\b(?:R[bB]|[bB]R))(${marker}) + end: (\2)${guard} + beginCaptures: + '1': {name: storage.type.string.python} + '2': {name: punctuation.definition.string.begin.python} + endCaptures: + '1': {name: punctuation.definition.string.end.python} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#string-raw-bin-guts' - string-quoted-${line}-line: - name: string.quoted.${line}.python - begin: (\b[rR](?=[uU]))?([uU])?(${marker}) - end: (\3)${guard} - beginCaptures: - '1': {name: invalid.illegal.prefix.python} - '2': {name: storage.type.string.python} - '3': {name: punctuation.definition.string.begin.python} - endCaptures: - '1': {name: punctuation.definition.string.end.python} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#string-${line}-bad-brace1-formatting-unicode' - - include: '#string-${line}-bad-brace2-formatting-unicode' - - include: '#string-unicode-guts' +string-quoted-${line}-line: + name: string.quoted.${line}.python + begin: (?:\b([rR])(?=[uU]))?([uU])?(${marker}) + end: (\3)${guard} + beginCaptures: + '1': {name: invalid.illegal.prefix.python} + '2': {name: storage.type.string.python} + '3': {name: punctuation.definition.string.begin.python} + endCaptures: + '1': {name: punctuation.definition.string.end.python} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#string-${line}-bad-brace1-formatting-unicode' + - include: '#string-${line}-bad-brace2-formatting-unicode' + - include: '#string-unicode-guts' - string-${line}-bad-brace1-formatting-unicode: - comment: template using {% ... %} - begin: | - (?x) - (?= \{% - ( .*? (?!${marker}${guard}) ) - %\} - ) - end: (?=${marker}${guard}) - patterns: - - include: '#escape-sequence-unicode' - - include: '#escape-sequence' - - include: '#string-line-continuation' +string-${line}-bad-brace1-formatting-unicode: + comment: template using {% ... %} + begin: | + (?x) + (?= \{% + ( .*? (?!${marker}${guard}) ) + %\} + ) + end: (?=${marker}${guard}) + patterns: + - include: '#escape-sequence-unicode' + - include: '#escape-sequence' + - include: '#string-line-continuation' - string-${line}-bad-brace1-formatting-raw: - comment: template using {% ... %} - begin: | - (?x) - (?= \{% - ( .*? (?!${marker}${guard}) ) - %\} - ) - end: (?=${marker}${guard}) - patterns: - - include: '#string-consume-escape' +string-${line}-bad-brace1-formatting-raw: + comment: template using {% ... %} + begin: | + (?x) + (?= \{% + ( .*? (?!${marker}${guard}) ) + %\} + ) + end: (?=${marker}${guard}) + patterns: + - include: '#string-consume-escape' - string-${line}-bad-brace2-formatting-unicode: - comment: odd format or format-like syntax - begin: | - (?x) - (?!\{\{) - (?= \{ ( - \w*? (?!${marker}${guard}) [^!:\.\[}\w] - ) - .*?(?!${marker}${guard}) - \} - ) - end: (?=${marker}${guard}) - patterns: - - include: '#escape-sequence-unicode' - - include: '#string-entity' +string-${line}-bad-brace2-formatting-unicode: + comment: odd format or format-like syntax + begin: | + (?x) + (?!\{\{) + (?= \{ ( + \w*? (?!${marker}${guard}) [^!:\.\[}\w] + ) + .*?(?!${marker}${guard}) + \} + ) + end: (?=${marker}${guard}) + patterns: + - include: '#escape-sequence-unicode' + - include: '#string-entity' - string-${line}-bad-brace2-formatting-raw: - comment: odd format or format-like syntax - begin: | - (?x) - (?!\{\{) - (?= \{ ( - \w*? (?!${marker}${guard}) [^!:\.\[}\w] - ) - .*?(?!${marker}${guard}) - \} - ) - end: (?=${marker}${guard}) - patterns: - - include: '#string-consume-escape' - - include: '#string-formatting' +string-${line}-bad-brace2-formatting-raw: + comment: odd format or format-like syntax + begin: | + (?x) + (?!\{\{) + (?= \{ ( + \w*? (?!${marker}${guard}) [^!:\.\[}\w] + ) + .*?(?!${marker}${guard}) + \} + ) + end: (?=${marker}${guard}) + patterns: + - include: '#string-consume-escape' + - include: '#string-formatting' ... diff --git a/grammars/src/regexp-common.inc.syntax.yaml b/grammars/src/regexp-common.inc.syntax.yaml index d7b02061..828a630b 100644 --- a/grammars/src/regexp-common.inc.syntax.yaml +++ b/grammars/src/regexp-common.inc.syntax.yaml @@ -1,112 +1,127 @@ --- -repository: - regexp-base-expression: - patterns: - - include: '#regexp-quantifier' - - include: '#regexp-base-common' +regexp-base-expression: + patterns: + - include: '#regexp-quantifier' + - include: '#regexp-base-common' - fregexp-base-expression: - patterns: - - include: '#fregexp-quantifier' - - include: '#fstring-formatting-braces' - - match: \{.*?\} - - include: '#regexp-base-common' +fregexp-base-expression: + patterns: + - include: '#fregexp-quantifier' + - include: '#fstring-formatting-braces' + - match: \{.*?\} + - include: '#regexp-base-common' - regexp-base-common: - patterns: - - name: support.other.match.any.regexp - match: \. - - name: support.other.match.begin.regexp - match: \^ - - name: support.other.match.end.regexp - match: \$ - - name: keyword.operator.quantifier.regexp - match: '[+*?]\??' - - name: keyword.operator.disjunction.regexp - match: \| - - include: '#regexp-escape-sequence' +fstring-formatting-braces: + patterns: + - comment: empty braces are illegal + match: ({)(\s*?)(}) + captures: + '1': {name: constant.character.format.placeholder.other.python} + '2': {name: invalid.illegal.brace.python} + '3': {name: constant.character.format.placeholder.other.python} + - name: constant.character.escape.python + match: ({{|}}) - regexp-quantifier: - name: keyword.operator.quantifier.regexp - match: | - (?x) - \{( - \d+ | \d+,(\d+)? | ,\d+ - )\} +regexp-base-common: + patterns: + - name: support.other.match.any.regexp + match: \. + - name: support.other.match.begin.regexp + match: \^ + - name: support.other.match.end.regexp + match: \$ + - name: keyword.operator.quantifier.regexp + match: '[+*?]\??' + - name: keyword.operator.disjunction.regexp + match: \| + - include: '#regexp-escape-sequence' - fregexp-quantifier: - name: keyword.operator.quantifier.regexp - match: | - (?x) - \{\{( - \d+ | \d+,(\d+)? | ,\d+ - )\}\} +regexp-quantifier: + name: keyword.operator.quantifier.regexp + match: | + (?x) + \{( + \d+ | \d+,(\d+)? | ,\d+ + )\} - regexp-backreference-number: - name: meta.backreference.regexp - match: (\\[1-9]\d?) - captures: - '1': {name: entity.name.tag.backreference.regexp} +fregexp-quantifier: + name: keyword.operator.quantifier.regexp + match: | + (?x) + \{\{( + \d+ | \d+,(\d+)? | ,\d+ + )\}\} - regexp-backreference: - name: meta.backreference.named.regexp - match: | - (?x) - (\() (\?P= \w+(?:\s+[[:alnum:]]+)?) (\)) - captures: - '1': {name: punctuation.parenthesis.backreference.named.begin.regexp - support.other.parenthesis.regexp} - '2': {name: entity.name.tag.named.backreference.regexp} - '3': {name: punctuation.parenthesis.backreference.named.end.regexp - support.other.parenthesis.regexp} +regexp-backreference-number: + name: meta.backreference.regexp + match: (\\[1-9]\d?) + captures: + '1': {name: entity.name.tag.backreference.regexp} - regexp-flags: - name: storage.modifier.flag.regexp - match: \(\?[aiLmsux]+\) +regexp-backreference: + name: meta.backreference.named.regexp + match: | + (?x) + (\() (\?P= \w+(?:\s+[[:alnum:]]+)?) (\)) + captures: + '1': {name: punctuation.parenthesis.backreference.named.begin.regexp + support.other.parenthesis.regexp} + '2': {name: entity.name.tag.named.backreference.regexp} + '3': {name: punctuation.parenthesis.backreference.named.end.regexp + support.other.parenthesis.regexp} - regexp-escape-special: - name: support.other.escape.special.regexp - match: \\([AbBdDsSwWZ]) +regexp-flags: + name: storage.modifier.flag.regexp + match: \(\?[aiLmsux]+\) - regexp-escape-character: - name: constant.character.escape.regexp - match: | - (?x) - \\ ( - x[0-9A-Fa-f]{2} - | 0[0-7]{1,2} - | [0-7]{3} - ) +regexp-escape-special: + name: support.other.escape.special.regexp + match: \\([AbBdDsSwWZ]) - regexp-escape-unicode: - name: constant.character.unicode.regexp - match: | - (?x) - \\ ( - u[0-9A-Fa-f]{4} - | U[0-9A-Fa-f]{8} - ) +regexp-escape-character: + name: constant.character.escape.regexp + match: | + (?x) + \\ ( + x[0-9A-Fa-f]{2} + | 0[0-7]{1,2} + | [0-7]{3} + ) - regexp-escape-catchall: - name: constant.character.escape.regexp - match: \\(.|\n) +regexp-escape-unicode: + name: constant.character.unicode.regexp + match: | + (?x) + \\ ( + u[0-9A-Fa-f]{4} + | U[0-9A-Fa-f]{8} + ) - regexp-escape-sequence: - patterns: - - include: '#regexp-escape-special' - - include: '#regexp-escape-character' - - include: '#regexp-escape-unicode' - - include: '#regexp-backreference-number' - - include: '#regexp-escape-catchall' +regexp-escape-catchall: + name: constant.character.escape.regexp + match: \\(.|\n) - regexp-charecter-set-escapes: - patterns: - - name: constant.character.escape.regexp - match: \\[abfnrtv\\] - - include: '#regexp-escape-special' - - name: constant.character.escape.regexp - match: \\([0-7]{1,3}) - - include: '#regexp-escape-character' - - include: '#regexp-escape-unicode' - - include: '#regexp-escape-catchall' +regexp-escape-sequence: + patterns: + - include: '#regexp-escape-special' + - include: '#regexp-escape-character' + - include: '#regexp-escape-unicode' + - include: '#regexp-backreference-number' + - include: '#regexp-escape-catchall' + +regexp-charecter-set-escapes: + patterns: + - name: constant.character.escape.regexp + match: \\[abfnrtv\\] + - include: '#regexp-escape-special' + - name: constant.character.escape.regexp + match: \\([0-7]{1,3}) + - include: '#regexp-escape-character' + - include: '#regexp-escape-unicode' + - include: '#regexp-escape-catchall' + +codetags: + match: (?:\b(NOTE|XXX|HACK|FIXME|BUG|TODO)\b) + captures: + '1': {name: keyword.codetag.notation.python} ... diff --git a/grammars/src/regexp.inc.syntax.yaml b/grammars/src/regexp.inc.syntax.yaml index ea55ff55..aac9e286 100644 --- a/grammars/src/regexp.inc.syntax.yaml +++ b/grammars/src/regexp.inc.syntax.yaml @@ -1,170 +1,169 @@ --- -repository: - ${prefix}${basename}-expression: - patterns: - - include: '#${basename}-base-expression' - - include: '#${prefix}regexp-character-set' - - include: '#${prefix}regexp-comments' - - include: '#regexp-flags' - - include: '#${prefix}regexp-named-group' - - include: '#regexp-backreference' - - include: '#${prefix}${basename}-lookahead' - - include: '#${prefix}${basename}-lookahead-negative' - - include: '#${prefix}${basename}-lookbehind' - - include: '#${prefix}${basename}-lookbehind-negative' - - include: '#${prefix}${basename}-conditional' - - include: '#${prefix}${basename}-parentheses-non-capturing' - - include: '#${prefix}${basename}-parentheses' - ${nested} +${prefix}${basename}-expression: + patterns: + - include: '#${basename}-base-expression' + - include: '#${prefix}regexp-character-set' + - include: '#${prefix}regexp-comments' + - include: '#regexp-flags' + - include: '#${prefix}regexp-named-group' + - include: '#regexp-backreference' + - include: '#${prefix}${basename}-lookahead' + - include: '#${prefix}${basename}-lookahead-negative' + - include: '#${prefix}${basename}-lookbehind' + - include: '#${prefix}${basename}-lookbehind-negative' + - include: '#${prefix}${basename}-conditional' + - include: '#${prefix}${basename}-parentheses-non-capturing' + - include: '#${prefix}${basename}-parentheses' + ${nested} - ${prefix}regexp-character-set: - patterns: - - match: | - (?x) - \[ \^? \] (?! .*?\]) - - name: meta.character.set.regexp - begin: (\[)(\^)?(\])? - end: (\]${marker})${guard} - beginCaptures: - '1': {name: constant.other.set.regexp - punctuation.character.set.begin.regexp} - '2': {name: keyword.operator.negation.regexp} - '3': {name: constant.character.set.regexp} - endCaptures: - '1': {name: constant.other.set.regexp - punctuation.character.set.end.regexp} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#regexp-charecter-set-escapes' - - name: constant.character.set.regexp - match: '[^\n]' +${prefix}regexp-character-set: + patterns: + - match: | + (?x) + \[ \^? \] (?! .*?\]) + - name: meta.character.set.regexp + begin: (\[)(\^)?(\])? + end: (\]${marker})${guard} + beginCaptures: + '1': {name: constant.other.set.regexp + punctuation.character.set.begin.regexp} + '2': {name: keyword.operator.negation.regexp} + '3': {name: constant.character.set.regexp} + endCaptures: + '1': {name: constant.other.set.regexp + punctuation.character.set.end.regexp} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#regexp-charecter-set-escapes' + - name: constant.character.set.regexp + match: '[^\n]' - ${prefix}${basename}-named-group: - name: meta.named.regexp - begin: | - (?x) - (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) - end: (\)${marker})${guard} - beginCaptures: - '1': {name: punctuation.parenthesis.named.begin.regexp - support.other.parenthesis.regexp} - '2': {name: entity.name.tag.named.group.regexp} - endCaptures: - '1': {name: punctuation.parenthesis.named.end.regexp - support.other.parenthesis.regexp} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#${prefix}${basename}-expression' - ${nested} +${prefix}${basename}-named-group: + name: meta.named.regexp + begin: | + (?x) + (\() (\?P <\w+(?:\s+[[:alnum:]]+)?>) + end: (\)${marker})${guard} + beginCaptures: + '1': {name: punctuation.parenthesis.named.begin.regexp + support.other.parenthesis.regexp} + '2': {name: entity.name.tag.named.group.regexp} + endCaptures: + '1': {name: punctuation.parenthesis.named.end.regexp + support.other.parenthesis.regexp} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#${prefix}${basename}-expression' + ${nested} - ${prefix}regexp-comments: - name: comment.regexp - begin: \(\?# - end: (\)${marker})${guard} - beginCaptures: - '0': {name: punctuation.comment.begin.regexp} - endCaptures: - '1': {name: punctuation.comment.end.regexp} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#codetags' +${prefix}regexp-comments: + name: comment.regexp + begin: \(\?# + end: (\)${marker})${guard} + beginCaptures: + '0': {name: punctuation.comment.begin.regexp} + endCaptures: + '1': {name: punctuation.comment.end.regexp} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#codetags' - ${prefix}${basename}-lookahead: - begin: '(\()\?=' - end: (\)${marker})${guard} - beginCaptures: - '0': {name: keyword.operator.lookahead.regexp} - '1': {name: punctuation.parenthesis.lookahead.begin.regexp} - endCaptures: - '1': {name: punctuation.parenthesis.lookahead.end.regexp - keyword.operator.lookahead.regexp} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#${prefix}${basename}-expression' - ${nested} +${prefix}${basename}-lookahead: + begin: '(\()\?=' + end: (\)${marker})${guard} + beginCaptures: + '0': {name: keyword.operator.lookahead.regexp} + '1': {name: punctuation.parenthesis.lookahead.begin.regexp} + endCaptures: + '1': {name: punctuation.parenthesis.lookahead.end.regexp + keyword.operator.lookahead.regexp} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#${prefix}${basename}-expression' + ${nested} - ${prefix}${basename}-lookahead-negative: - begin: '(\()\?!' - end: (\)${marker})${guard} - beginCaptures: - '0': {name: keyword.operator.lookahead.negative.regexp} - '1': {name: punctuation.parenthesis.lookahead.begin.regexp} - endCaptures: - '1': {name: punctuation.parenthesis.lookahead.end.regexp - keyword.operator.lookahead.negative.regexp} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#${prefix}${basename}-expression' - ${nested} +${prefix}${basename}-lookahead-negative: + begin: '(\()\?!' + end: (\)${marker})${guard} + beginCaptures: + '0': {name: keyword.operator.lookahead.negative.regexp} + '1': {name: punctuation.parenthesis.lookahead.begin.regexp} + endCaptures: + '1': {name: punctuation.parenthesis.lookahead.end.regexp + keyword.operator.lookahead.negative.regexp} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#${prefix}${basename}-expression' + ${nested} - ${prefix}${basename}-lookbehind: - begin: '(\()\?<=' - end: (\)${marker})${guard} - beginCaptures: - '0': {name: keyword.operator.lookbehind.regexp} - '1': {name: punctuation.parenthesis.lookbehind.begin.regexp} - endCaptures: - '1': {name: punctuation.parenthesis.lookbehind.end.regexp - keyword.operator.lookbehind.regexp} - '2': {name: invalid.illegal.newline.python} - patterns: - - include: '#${prefix}${basename}-expression' - ${nested} +${prefix}${basename}-lookbehind: + begin: '(\()\?<=' + end: (\)${marker})${guard} + beginCaptures: + '0': {name: keyword.operator.lookbehind.regexp} + '1': {name: punctuation.parenthesis.lookbehind.begin.regexp} + endCaptures: + '1': {name: punctuation.parenthesis.lookbehind.end.regexp + keyword.operator.lookbehind.regexp} + '2': {name: invalid.illegal.newline.python} + patterns: + - include: '#${prefix}${basename}-expression' + ${nested} - ${prefix}${basename}-lookbehind-negative: - begin: '(\()\? - - - - - - - - - - - - - - - diff --git a/misc/example.py b/misc/example.py index 6b820e63..718f3074 100644 --- a/misc/example.py +++ b/misc/example.py @@ -1,7 +1,7 @@ import asyncio -def showcase(): +def showcase(a, b, /, c, d, *, e, f): """Some code to showcase the syntax. Docstrings are recognized and have an additional scope. @@ -21,7 +21,9 @@ async def coroutine(db:aio_db.DatabaseConnection) -> List[str]: async with db.transaction(): result = await db.query(...) - print(f'Result: {result!r}') + print(f'Result: {result!r} {a=} {b=!r}') + print(Rf'data: {c=}') + print(rf'data: {c=}') mapping = None # type: Dict[int, Any] # PEP 484 @@ -41,8 +43,14 @@ async def coroutine(db:aio_db.DatabaseConnection) -> List[str]: # NOTE Numbers with leading zeros are invalid in Python 3, # use 0o... answer = func(0xdeadbeef + 0b00100001 + 0123 + 0o123 + - 1_005_123 + # PEP 515 + 1_005_123 + # PEP 515 # complex numbers .10e12 + 2j) @ mat + # walrus operator + filtered_data = [y for x in data if (y := f(x)) is not None] + + # position-only params + bar = lambda q, w, /, e, r: (q + w + e + r) + return R'''No escapes '\' in this \one''' diff --git a/misc/scopes b/misc/scopes index 85bc470e..8eec651e 100644 --- a/misc/scopes +++ b/misc/scopes @@ -56,23 +56,29 @@ keyword.operator.lookahead.regexp keyword.operator.lookbehind.negative.regexp keyword.operator.lookbehind.regexp keyword.operator.negation.regexp +keyword.operator.positional.parameter.python keyword.operator.python keyword.operator.quantifier.regexp keyword.operator.unpacking.arguments.python keyword.operator.unpacking.parameter.python +meta.attribute.python meta.backreference.named.regexp meta.backreference.regexp meta.character.set.regexp meta.class.inheritance.python meta.class.python +meta.format.brace.python +meta.format.percent.python meta.fstring.python meta.function-call.generic.python meta.function-call.python meta.function.decorator.python meta.function.parameters.python meta.function.python +meta.indexed-name.python meta.item-access.python meta.lambda-function.python +meta.member.access.python meta.named.regexp meta.typehint.comment.python punctuation.character.set.begin.regexp @@ -82,6 +88,7 @@ punctuation.comment.end.regexp punctuation.definition.arguments.begin.python punctuation.definition.arguments.end.python punctuation.definition.comment.python +punctuation.definition.decorator.python punctuation.definition.dict.begin.python punctuation.definition.dict.end.python punctuation.definition.inheritance.begin.python @@ -120,6 +127,7 @@ punctuation.separator.dict.python punctuation.separator.element.python punctuation.separator.inheritance.python punctuation.separator.parameters.python +punctuation.separator.period.python punctuation.separator.slice.python storage.modifier.declaration.python storage.modifier.flag.regexp @@ -149,7 +157,6 @@ string.regexp.quoted.single.python support.function.builtin.python support.function.magic.python support.other.escape.special.regexp -support.other.format.python support.other.match.any.regexp support.other.match.begin.regexp support.other.match.end.regexp diff --git a/package.json b/package.json index b525cf27..a974bd74 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "MagicPython", - "version": "1.0.6", + "version": "1.1.1", "engines": { "atom": "*", "node": "*", @@ -25,7 +25,7 @@ "syntax", "python3" ], - "icon": "icon.svg", + "icon": "icon.png", "galleryBanner": { "color": "#2c2c2c", "theme": "dark" @@ -55,7 +55,11 @@ ".SConscript", ".gyp", ".gypi", - ".wsgi" + ".wsgi", + ".kv", + "Snakefile", + ".smk", + ".tac" ] } ], @@ -66,5 +70,10 @@ "path": "./grammars/MagicPython.tmLanguage" } ] - } + }, + "devDependencies": { + "oniguruma": "^7.0.0", + "syntaxdev": "^0.1.3" + }, + "dependencies": {} } diff --git a/settings/atom.cson b/settings/atom.cson index ef9a58d1..e66d6044 100644 --- a/settings/atom.cson +++ b/settings/atom.cson @@ -35,7 +35,7 @@ 'autoIndentOnPaste': false 'softTabs': true 'tabLength': 4 - 'foldEndPattern': '^\\s*"""\\s*$' + 'foldEndPattern': '^\\s*\\}|^\\s*\\]|^\\s*\\)' 'commentStart': '# ' 'increaseIndentPattern': '^\\s*(class|((async\\s+)?(def|with|for))|elif|else|except|finally|if|try|while)\\b.*:\\s*$' 'decreaseIndentPattern': '^\\s*(elif|else|except|finally)\\b.*:' diff --git a/settings/sublime/Default.sublime-keymap b/settings/sublime/Default.sublime-keymap index 50c1b423..3a0ae806 100644 --- a/settings/sublime/Default.sublime-keymap +++ b/settings/sublime/Default.sublime-keymap @@ -5,7 +5,7 @@ { "key": "setting.auto_match_enabled", "operator": "equal", "operand": true }, { "key": "selection_empty", "operator": "equal", "operand": true, "match_all": true }, { "key": "following_text", "operator": "regex_contains", "operand": "^(?:\t| |\\)|]|\\}|>|$)", "match_all": true }, - { "key": "preceding_text", "operator": "regex_contains", "operand": "(?:[bfruBFRU]|[^'[:alnum:]_])$", "match_all": true }, + { "key": "preceding_text", "operator": "regex_contains", "operand": "(?:[bfruBFRU]|[^\"[:alnum:]_])$", "match_all": true }, { "key": "eol_selector", "operator": "not_equal", "operand": "string.quoted.double - punctuation.definition.string.end", "match_all": true }, { "key": "eol_selector", "operator": "equal", "operand": "source.python", "match_all": true } ] diff --git a/settings/sublime/Symbol List.tmPreferences b/settings/sublime/Symbol List.tmPreferences index e832b863..987e0f72 100644 --- a/settings/sublime/Symbol List.tmPreferences +++ b/settings/sublime/Symbol List.tmPreferences @@ -15,7 +15,7 @@ # transform into 'class Classname' form s/(class)\s+([[:alpha:]_]\w*)\s*.*/$1 $2/g; - # transform into '[async] funciton_name(args,...)' form + # transform into '[async] funciton_name(args,...)' long form s/(?x) def \s+ ([[:alpha:]_]\w*) \s* \( @@ -24,6 +24,13 @@ # remove extra whitespace in arg list (except leading) s/(\S)\s{2,}/$1 /g; + + # transform into short form + s/(?x) + (class\s|async\s)? (\w+\() + (.{80})(.+) + (\))$ + /$1$2$3\.\.\.$5/g; uuid diff --git a/snippets/atom.cson b/snippets/atom.cson index 41827a5e..0f03f31b 100644 --- a/snippets/atom.cson +++ b/snippets/atom.cson @@ -34,9 +34,18 @@ '#!/usr/bin/env python': 'prefix': 'env' 'body': '#!/usr/bin/env python\n' + '#!/usr/bin/env python3': + 'prefix': 'env3' + 'body': '#!/usr/bin/env python3\n' '# coding=utf-8': 'prefix': 'enc' - 'body': '# coding=utf-8\n' + 'body': '# -*- coding: utf-8 -*-\n' + 'Import': + 'prefix': 'im' + 'body': 'import ${1:package/module}' + 'From/Import': + 'prefix': 'fim' + 'body': 'from ${1:package/module} import ${2:names}' 'Assert Equal': 'prefix': 'ase' 'body': 'self.assertEqual(${1:expected}, ${2:actual}${3:, \'${4:message}\'})$0' @@ -97,6 +106,9 @@ 'while': 'prefix': 'while' 'body': 'while ${1:condition}:\n\t${2:pass}' + 'with statement': + 'prefix': 'with' + 'body': 'with ${1:expression} as ${2:target}:\n\t${3:pass}' 'Try/Except/Else/Finally': 'prefix': 'tryef' 'body': 'try:\n\t${1:pass}\nexcept${2: ${3:Exception} as ${4:e}}:\n\t${5:raise}\nelse:\n\t${6:pass}\nfinally:\n\t${7:pass}' @@ -118,12 +130,21 @@ 'Dictionary Comprehension': 'prefix': 'dc' 'body': '{${1:key}: ${2:value} for ${3:key}, ${4:value} in ${5:variable}}' + 'Set Comprehension': + 'prefix': 'sc' + 'body': '{${1:value} for ${2:value} in ${3:variable}}' 'PDB set trace': 'prefix': 'pdb' 'body': 'import pdb; pdb.set_trace()\n' 'iPDB set trace': 'prefix': 'ipdb' 'body': 'import ipdb; ipdb.set_trace()\n' + 'rPDB set trace': + 'prefix': 'rpdb' + 'body': 'import rpdb2; rpdb2.start_embedded_debugger(\'${1:debug_password}\')$0' + 'PuDB set trace': + 'prefix': 'pudb' + 'body': 'import pudb; pudb.set_trace()' '__magic__': 'prefix': '__' 'body': '__${1:init}__' diff --git a/snippets/sublime/Try-Except-Else-Finally.sublime-snippet b/snippets/sublime/Try-Except-Else-Finally.sublime-snippet index 00c41fb9..ff120922 100644 --- a/snippets/sublime/Try-Except-Else-Finally.sublime-snippet +++ b/snippets/sublime/Try-Except-Else-Finally.sublime-snippet @@ -1,7 +1,7 @@ diff --git a/snippets/sublime/Try-Except-Finally.sublime-snippet b/snippets/sublime/Try-Except-Finally.sublime-snippet index 0ee74a40..b088cbe6 100644 --- a/snippets/sublime/Try-Except-Finally.sublime-snippet +++ b/snippets/sublime/Try-Except-Finally.sublime-snippet @@ -1,7 +1,7 @@ diff --git a/snippets/sublime/Try-Except.sublime-snippet b/snippets/sublime/Try-Except.sublime-snippet index bdf36a88..58f38d1b 100644 --- a/snippets/sublime/Try-Except.sublime-snippet +++ b/snippets/sublime/Try-Except.sublime-snippet @@ -1,7 +1,7 @@ try source.python diff --git a/test/atom-spec/README.md b/test/atom-spec/README.md new file mode 100644 index 00000000..2708e928 --- /dev/null +++ b/test/atom-spec/README.md @@ -0,0 +1,2 @@ +This directory contains tests for Atom editor. Spec files prefixed +with `python-` are autogenerated by `make`. diff --git a/test/atom-spec/basics-spec.js b/test/atom-spec/basics-spec.js new file mode 100644 index 00000000..b8b35468 --- /dev/null +++ b/test/atom-spec/basics-spec.js @@ -0,0 +1,25 @@ +describe("MagicPython basic integration tests", function() { + var grammar = null; + + beforeEach(function() { + waitsForPromise(function() { + return atom.packages.activatePackage("MagicPython") + }); + runs(function() { + grammar = atom.grammars.grammarForScopeName("source.python") + }); + }); + + it("recognises shebang on firstline", function() { + expect(grammar.firstLineRegex.scanner.findNextMatchSync( + "#!/usr/bin/env python")).not.toBeNull(); + + expect(grammar.firstLineRegex.scanner.findNextMatchSync( + "#! /usr/bin/env python")).not.toBeNull(); + }); + + it("parses the grammar", function() { + expect(grammar).toBeDefined(); + expect(grammar.scopeName).toBe("source.python"); + }); +}); diff --git a/test/atom-spec/console-spec.js b/test/atom-spec/console-spec.js new file mode 100644 index 00000000..39c13528 --- /dev/null +++ b/test/atom-spec/console-spec.js @@ -0,0 +1,27 @@ +describe("python-console tests", function() { + var grammar = null; + + beforeEach(function() { + waitsForPromise(function() { + return atom.packages.activatePackage("MagicPython") + }); + runs(function() { + grammar = atom.grammars.grammarForScopeName("text.python.console") + }); + }); + + it("highlights >>>", function() { + tokens = grammar.tokenizeLines(">>> print") + + expect(tokens[0][0].value).toBe(">>>"); + expect(tokens[0][0].scopes).toEqual( + ['text.python.console', 'punctuation.separator.prompt.python.console']); + + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(['text.python.console']); + + expect(tokens[0][2].value).toBe("print"); + expect(tokens[0][2].scopes).toEqual( + ['text.python.console', 'support.function.builtin.python']); + }); +}); diff --git a/test/atom-spec/python-re-spec.js b/test/atom-spec/python-re-spec.js new file mode 100644 index 00000000..2874e777 --- /dev/null +++ b/test/atom-spec/python-re-spec.js @@ -0,0 +1,556 @@ +// !!! autogenerated; do not edit !!! + + + +describe("Grammar Tests", function() { + var grammar = null; + beforeEach(function() { + waitsForPromise(function() { + return atom.packages.activatePackage("MagicPython") + }); + runs(function() { + grammar = atom.grammars.grammarForScopeName("source.regexp.python") + }); + }); + + + it("test/regexp/comments1.re", + function() { + tokens = grammar.tokenizeLines("foo(?#comment)bar") + expect(tokens[0][0].value).toBe("foo"); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][1].value).toBe("(?#"); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python","comment.regexp","punctuation.comment.begin.regexp"]); + expect(tokens[0][2].value).toBe("comment"); + expect(tokens[0][2].scopes).toEqual(["source.regexp.python","comment.regexp"]); + expect(tokens[0][3].value).toBe(")"); + expect(tokens[0][3].scopes).toEqual(["source.regexp.python","comment.regexp","punctuation.comment.end.regexp"]); + expect(tokens[0][4].value).toBe("bar"); + expect(tokens[0][4].scopes).toEqual(["source.regexp.python"]); + }); + + it("test/regexp/comments2.re", + function() { + tokens = grammar.tokenizeLines("(?x)\n foo (?#\ncomment 1\n ) bar\n baz (?#comment 2)") + expect(tokens[0][0].value).toBe("(?x)"); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python","storage.modifier.flag.regexp"]); + expect(tokens[1][0].value).toBe(" foo "); + expect(tokens[1][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[1][1].value).toBe("(?#"); + expect(tokens[1][1].scopes).toEqual(["source.regexp.python","comment.regexp","punctuation.comment.begin.regexp"]); + expect(tokens[2][0].value).toBe("comment 1"); + expect(tokens[2][0].scopes).toEqual(["source.regexp.python","comment.regexp"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.regexp.python","comment.regexp"]); + expect(tokens[3][1].value).toBe(")"); + expect(tokens[3][1].scopes).toEqual(["source.regexp.python","comment.regexp","punctuation.comment.end.regexp"]); + expect(tokens[3][2].value).toBe(" bar"); + expect(tokens[3][2].scopes).toEqual(["source.regexp.python"]); + expect(tokens[4][0].value).toBe(" baz "); + expect(tokens[4][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[4][1].value).toBe("(?#"); + expect(tokens[4][1].scopes).toEqual(["source.regexp.python","comment.regexp","punctuation.comment.begin.regexp"]); + expect(tokens[4][2].value).toBe("comment 2"); + expect(tokens[4][2].scopes).toEqual(["source.regexp.python","comment.regexp"]); + expect(tokens[4][3].value).toBe(")"); + expect(tokens[4][3].scopes).toEqual(["source.regexp.python","comment.regexp","punctuation.comment.end.regexp"]); + }); + + it("test/regexp/comments3.re", + function() { + tokens = grammar.tokenizeLines("foo(?#NOTE:comment)bar") + expect(tokens[0][0].value).toBe("foo"); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][1].value).toBe("(?#"); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python","comment.regexp","punctuation.comment.begin.regexp"]); + expect(tokens[0][2].value).toBe("NOTE"); + expect(tokens[0][2].scopes).toEqual(["source.regexp.python","comment.regexp","keyword.codetag.notation.python"]); + expect(tokens[0][3].value).toBe(":comment"); + expect(tokens[0][3].scopes).toEqual(["source.regexp.python","comment.regexp"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.regexp.python","comment.regexp","punctuation.comment.end.regexp"]); + expect(tokens[0][5].value).toBe("bar"); + expect(tokens[0][5].scopes).toEqual(["source.regexp.python"]); + }); + + it("test/regexp/conditional1.re", + function() { + tokens = grammar.tokenizeLines("(<)?(\\w+@\\w+(?:\\.\\w+)+)(?(1)>|$)") + expect(tokens[0][0].value).toBe("("); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[0][1].value).toBe("<"); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][2].value).toBe(")"); + expect(tokens[0][2].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[0][3].value).toBe("?"); + expect(tokens[0][3].scopes).toEqual(["source.regexp.python","keyword.operator.quantifier.regexp"]); + expect(tokens[0][4].value).toBe("("); + expect(tokens[0][4].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[0][5].value).toBe("\\w"); + expect(tokens[0][5].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][6].value).toBe("+"); + expect(tokens[0][6].scopes).toEqual(["source.regexp.python","keyword.operator.quantifier.regexp"]); + expect(tokens[0][7].value).toBe("@"); + expect(tokens[0][7].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][8].value).toBe("\\w"); + expect(tokens[0][8].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][9].value).toBe("+"); + expect(tokens[0][9].scopes).toEqual(["source.regexp.python","keyword.operator.quantifier.regexp"]); + expect(tokens[0][10].value).toBe("(?:"); + expect(tokens[0][10].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.non-capturing.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[0][11].value).toBe("\\."); + expect(tokens[0][11].scopes).toEqual(["source.regexp.python","constant.character.escape.regexp"]); + expect(tokens[0][12].value).toBe("\\w"); + expect(tokens[0][12].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][13].value).toBe("+"); + expect(tokens[0][13].scopes).toEqual(["source.regexp.python","keyword.operator.quantifier.regexp"]); + expect(tokens[0][14].value).toBe(")"); + expect(tokens[0][14].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.non-capturing.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[0][15].value).toBe("+"); + expect(tokens[0][15].scopes).toEqual(["source.regexp.python","keyword.operator.quantifier.regexp"]); + expect(tokens[0][16].value).toBe(")"); + expect(tokens[0][16].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[0][17].value).toBe("("); + expect(tokens[0][17].scopes).toEqual(["source.regexp.python","keyword.operator.conditional.regexp","punctuation.parenthesis.conditional.begin.regexp"]); + expect(tokens[0][18].value).toBe("?(1)"); + expect(tokens[0][18].scopes).toEqual(["source.regexp.python","keyword.operator.conditional.regexp"]); + expect(tokens[0][19].value).toBe(">"); + expect(tokens[0][19].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][20].value).toBe("|"); + expect(tokens[0][20].scopes).toEqual(["source.regexp.python","keyword.operator.disjunction.regexp"]); + expect(tokens[0][21].value).toBe("$"); + expect(tokens[0][21].scopes).toEqual(["source.regexp.python","support.other.match.end.regexp"]); + expect(tokens[0][22].value).toBe(")"); + expect(tokens[0][22].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.conditional.end.regexp keyword.operator.conditional.negative.regexp"]); + }); + + it("test/regexp/escaping1.re", + function() { + tokens = grammar.tokenizeLines("foo\\\nbar") + expect(tokens[0][0].value).toBe("foo"); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][1].value).toBe("\\"); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python","constant.character.escape.regexp"]); + expect(tokens[1][0].value).toBe("bar"); + expect(tokens[1][0].scopes).toEqual(["source.regexp.python"]); + }); + + it("test/regexp/escaping2.re", + function() { + tokens = grammar.tokenizeLines("start\\A\\b\\B\\d\\D\\s\\S\\w\\W\\Z\\\\\\1\\01\\1111\\0011\\xfab\\u123fb\\U1234aaaab\\c") + expect(tokens[0][0].value).toBe("start"); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][1].value).toBe("\\A"); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][2].value).toBe("\\b"); + expect(tokens[0][2].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][3].value).toBe("\\B"); + expect(tokens[0][3].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][4].value).toBe("\\d"); + expect(tokens[0][4].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][5].value).toBe("\\D"); + expect(tokens[0][5].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][6].value).toBe("\\s"); + expect(tokens[0][6].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][7].value).toBe("\\S"); + expect(tokens[0][7].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][8].value).toBe("\\w"); + expect(tokens[0][8].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][9].value).toBe("\\W"); + expect(tokens[0][9].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][10].value).toBe("\\Z"); + expect(tokens[0][10].scopes).toEqual(["source.regexp.python","support.other.escape.special.regexp"]); + expect(tokens[0][11].value).toBe("\\\\"); + expect(tokens[0][11].scopes).toEqual(["source.regexp.python","constant.character.escape.regexp"]); + expect(tokens[0][12].value).toBe("\\1"); + expect(tokens[0][12].scopes).toEqual(["source.regexp.python","meta.backreference.regexp","entity.name.tag.backreference.regexp"]); + expect(tokens[0][13].value).toBe("\\01"); + expect(tokens[0][13].scopes).toEqual(["source.regexp.python","constant.character.escape.regexp"]); + expect(tokens[0][14].value).toBe("\\111"); + expect(tokens[0][14].scopes).toEqual(["source.regexp.python","constant.character.escape.regexp"]); + expect(tokens[0][15].value).toBe("1"); + expect(tokens[0][15].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][16].value).toBe("\\001"); + expect(tokens[0][16].scopes).toEqual(["source.regexp.python","constant.character.escape.regexp"]); + expect(tokens[0][17].value).toBe("1"); + expect(tokens[0][17].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][18].value).toBe("\\xfa"); + expect(tokens[0][18].scopes).toEqual(["source.regexp.python","constant.character.escape.regexp"]); + expect(tokens[0][19].value).toBe("b"); + expect(tokens[0][19].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][20].value).toBe("\\u123f"); + expect(tokens[0][20].scopes).toEqual(["source.regexp.python","constant.character.unicode.regexp"]); + expect(tokens[0][21].value).toBe("b"); + expect(tokens[0][21].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][22].value).toBe("\\U1234aaaa"); + expect(tokens[0][22].scopes).toEqual(["source.regexp.python","constant.character.unicode.regexp"]); + expect(tokens[0][23].value).toBe("b"); + expect(tokens[0][23].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][24].value).toBe("\\c"); + expect(tokens[0][24].scopes).toEqual(["source.regexp.python","constant.character.escape.regexp"]); + }); + + it("test/regexp/escaping3.re", + function() { + tokens = grammar.tokenizeLines("[start\\A\\b\\B\\d\\D\\s\\S\\w\\W\\Z\\\\\\1\\01\\1111\\0011\\xfab\\u123fb\\U1234aaaab\\c]") + expect(tokens[0][0].value).toBe("["); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[0][1].value).toBe("s"); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][2].value).toBe("t"); + expect(tokens[0][2].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][3].value).toBe("a"); + expect(tokens[0][3].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][4].value).toBe("r"); + expect(tokens[0][4].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][5].value).toBe("t"); + expect(tokens[0][5].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][6].value).toBe("\\A"); + expect(tokens[0][6].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][7].value).toBe("\\b"); + expect(tokens[0][7].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][8].value).toBe("\\B"); + expect(tokens[0][8].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][9].value).toBe("\\d"); + expect(tokens[0][9].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][10].value).toBe("\\D"); + expect(tokens[0][10].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][11].value).toBe("\\s"); + expect(tokens[0][11].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][12].value).toBe("\\S"); + expect(tokens[0][12].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][13].value).toBe("\\w"); + expect(tokens[0][13].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][14].value).toBe("\\W"); + expect(tokens[0][14].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][15].value).toBe("\\Z"); + expect(tokens[0][15].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][16].value).toBe("\\\\"); + expect(tokens[0][16].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][17].value).toBe("\\1"); + expect(tokens[0][17].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][18].value).toBe("\\01"); + expect(tokens[0][18].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][19].value).toBe("\\111"); + expect(tokens[0][19].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][20].value).toBe("1"); + expect(tokens[0][20].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][21].value).toBe("\\001"); + expect(tokens[0][21].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][22].value).toBe("1"); + expect(tokens[0][22].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][23].value).toBe("\\xfa"); + expect(tokens[0][23].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][24].value).toBe("b"); + expect(tokens[0][24].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][25].value).toBe("\\u123f"); + expect(tokens[0][25].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.unicode.regexp"]); + expect(tokens[0][26].value).toBe("b"); + expect(tokens[0][26].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][27].value).toBe("\\U1234aaaa"); + expect(tokens[0][27].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.unicode.regexp"]); + expect(tokens[0][28].value).toBe("b"); + expect(tokens[0][28].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][29].value).toBe("\\c"); + expect(tokens[0][29].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][30].value).toBe("]"); + expect(tokens[0][30].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + }); + + it("test/regexp/escaping4.re", + function() { + tokens = grammar.tokenizeLines("[start\\a\\b\\c\\d\\f\\n\\r\\t\\v]") + expect(tokens[0][0].value).toBe("["); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[0][1].value).toBe("s"); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][2].value).toBe("t"); + expect(tokens[0][2].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][3].value).toBe("a"); + expect(tokens[0][3].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][4].value).toBe("r"); + expect(tokens[0][4].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][5].value).toBe("t"); + expect(tokens[0][5].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][6].value).toBe("\\a"); + expect(tokens[0][6].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][7].value).toBe("\\b"); + expect(tokens[0][7].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][8].value).toBe("\\c"); + expect(tokens[0][8].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][9].value).toBe("\\d"); + expect(tokens[0][9].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[0][10].value).toBe("\\f"); + expect(tokens[0][10].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][11].value).toBe("\\n"); + expect(tokens[0][11].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][12].value).toBe("\\r"); + expect(tokens[0][12].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][13].value).toBe("\\t"); + expect(tokens[0][13].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][14].value).toBe("\\v"); + expect(tokens[0][14].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[0][15].value).toBe("]"); + expect(tokens[0][15].scopes).toEqual(["source.regexp.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + }); + + it("test/regexp/flags1.re", + function() { + tokens = grammar.tokenizeLines("(?aiLmsux)foo") + expect(tokens[0][0].value).toBe("(?aiLmsux)"); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python","storage.modifier.flag.regexp"]); + expect(tokens[0][1].value).toBe("foo"); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python"]); + }); + + it("test/regexp/group1.re", + function() { + tokens = grammar.tokenizeLines("(?x)\n (?:not) (foo)\n .* (?PBAR)\n \\1 (?P=bar)") + expect(tokens[0][0].value).toBe("(?x)"); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python","storage.modifier.flag.regexp"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[1][1].value).toBe("(?:"); + expect(tokens[1][1].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.non-capturing.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[1][2].value).toBe("not"); + expect(tokens[1][2].scopes).toEqual(["source.regexp.python"]); + expect(tokens[1][3].value).toBe(")"); + expect(tokens[1][3].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.non-capturing.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.regexp.python"]); + expect(tokens[1][5].value).toBe("("); + expect(tokens[1][5].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[1][6].value).toBe("foo"); + expect(tokens[1][6].scopes).toEqual(["source.regexp.python"]); + expect(tokens[1][7].value).toBe(")"); + expect(tokens[1][7].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[2][1].value).toBe("."); + expect(tokens[2][1].scopes).toEqual(["source.regexp.python","support.other.match.any.regexp"]); + expect(tokens[2][2].value).toBe("*"); + expect(tokens[2][2].scopes).toEqual(["source.regexp.python","keyword.operator.quantifier.regexp"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.regexp.python"]); + expect(tokens[2][4].value).toBe("("); + expect(tokens[2][4].scopes).toEqual(["source.regexp.python","meta.named.regexp","punctuation.parenthesis.named.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[2][5].value).toBe("?P"); + expect(tokens[2][5].scopes).toEqual(["source.regexp.python","meta.named.regexp","entity.name.tag.named.group.regexp"]); + expect(tokens[2][6].value).toBe("BAR"); + expect(tokens[2][6].scopes).toEqual(["source.regexp.python","meta.named.regexp"]); + expect(tokens[2][7].value).toBe(")"); + expect(tokens[2][7].scopes).toEqual(["source.regexp.python","meta.named.regexp","punctuation.parenthesis.named.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[3][1].value).toBe("\\1"); + expect(tokens[3][1].scopes).toEqual(["source.regexp.python","meta.backreference.regexp","entity.name.tag.backreference.regexp"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.regexp.python"]); + expect(tokens[3][3].value).toBe("("); + expect(tokens[3][3].scopes).toEqual(["source.regexp.python","meta.backreference.named.regexp","punctuation.parenthesis.backreference.named.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[3][4].value).toBe("?P=bar"); + expect(tokens[3][4].scopes).toEqual(["source.regexp.python","meta.backreference.named.regexp","entity.name.tag.named.backreference.regexp"]); + expect(tokens[3][5].value).toBe(")"); + expect(tokens[3][5].scopes).toEqual(["source.regexp.python","meta.backreference.named.regexp","punctuation.parenthesis.backreference.named.end.regexp support.other.parenthesis.regexp"]); + }); + + it("test/regexp/lookahead1.re", + function() { + tokens = grammar.tokenizeLines("foo(?=bar)") + expect(tokens[0][0].value).toBe("foo"); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][1].value).toBe("("); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python","keyword.operator.lookahead.regexp","punctuation.parenthesis.lookahead.begin.regexp"]); + expect(tokens[0][2].value).toBe("?="); + expect(tokens[0][2].scopes).toEqual(["source.regexp.python","keyword.operator.lookahead.regexp"]); + expect(tokens[0][3].value).toBe("bar"); + expect(tokens[0][3].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.regexp"]); + }); + + it("test/regexp/lookahead2.re", + function() { + tokens = grammar.tokenizeLines("foo(?!bar)") + expect(tokens[0][0].value).toBe("foo"); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][1].value).toBe("("); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python","keyword.operator.lookahead.negative.regexp","punctuation.parenthesis.lookahead.begin.regexp"]); + expect(tokens[0][2].value).toBe("?!"); + expect(tokens[0][2].scopes).toEqual(["source.regexp.python","keyword.operator.lookahead.negative.regexp"]); + expect(tokens[0][3].value).toBe("bar"); + expect(tokens[0][3].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.lookahead.end.regexp keyword.operator.lookahead.negative.regexp"]); + }); + + it("test/regexp/lookbehind1.re", + function() { + tokens = grammar.tokenizeLines("(?<=foo)bar") + expect(tokens[0][0].value).toBe("("); + expect(tokens[0][0].scopes).toEqual(["source.regexp.python","keyword.operator.lookbehind.regexp","punctuation.parenthesis.lookbehind.begin.regexp"]); + expect(tokens[0][1].value).toBe("?<="); + expect(tokens[0][1].scopes).toEqual(["source.regexp.python","keyword.operator.lookbehind.regexp"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.regexp.python"]); + expect(tokens[0][3].value).toBe(")"); + expect(tokens[0][3].scopes).toEqual(["source.regexp.python","punctuation.parenthesis.lookbehind.end.regexp keyword.operator.lookbehind.regexp"]); + expect(tokens[0][4].value).toBe("bar"); + expect(tokens[0][4].scopes).toEqual(["source.regexp.python"]); + }); + + it("test/regexp/lookbehind2.re", + function() { + tokens = grammar.tokenizeLines("(?>sys.stderr, \"er\"\nprint(\"er\", file=sys.stderr)\nprint (x, y)\nprint((x, y))") + expect(tokens[0][0].value).toBe("print"); + expect(tokens[0][0].scopes).toEqual(["source.python","support.function.builtin.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("\""); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][3].value).toBe("is"); + expect(tokens[0][3].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][4].value).toBe("\""); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][5].value).toBe(","); + expect(tokens[0][5].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("2"); + expect(tokens[0][7].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("*"); + expect(tokens[0][8].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][9].value).toBe("2"); + expect(tokens[0][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][0].value).toBe("print"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[1][1].value).toBe("("); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][2].value).toBe("\""); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][3].value).toBe("is"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[1][4].value).toBe("\""); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][5].value).toBe(","); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][7].value).toBe("2"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[1][8].value).toBe("*"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.arithmetic.python"]); + expect(tokens[1][9].value).toBe("2"); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[1][10].value).toBe(")"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[2][0].value).toBe("print"); + expect(tokens[2][0].scopes).toEqual(["source.python","support.function.builtin.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("x"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe(","); + expect(tokens[2][3].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[3][0].value).toBe("print"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[3][1].value).toBe("("); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][2].value).toBe("x"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][3].value).toBe(","); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][5].value).toBe("end"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][6].value).toBe("="); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[3][7].value).toBe("\""); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][8].value).toBe(" "); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[3][9].value).toBe("\""); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][10].value).toBe(")"); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][0].value).toBe("print"); + expect(tokens[4][0].scopes).toEqual(["source.python","support.function.builtin.python"]); + expect(tokens[5][0].value).toBe("print"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[5][1].value).toBe("("); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][2].value).toBe(")"); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[6][0].value).toBe("print"); + expect(tokens[6][0].scopes).toEqual(["source.python","support.function.builtin.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe(">>"); + expect(tokens[6][2].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[6][3].value).toBe("sys"); + expect(tokens[6][3].scopes).toEqual(["source.python"]); + expect(tokens[6][4].value).toBe("."); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[6][5].value).toBe("stderr"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[6][6].value).toBe(","); + expect(tokens[6][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[6][7].value).toBe(" "); + expect(tokens[6][7].scopes).toEqual(["source.python"]); + expect(tokens[6][8].value).toBe("\""); + expect(tokens[6][8].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][9].value).toBe("er"); + expect(tokens[6][9].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[6][10].value).toBe("\""); + expect(tokens[6][10].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[7][0].value).toBe("print"); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[7][1].value).toBe("("); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[7][2].value).toBe("\""); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("er"); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[7][4].value).toBe("\""); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[7][5].value).toBe(","); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[7][6].value).toBe(" "); + expect(tokens[7][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][7].value).toBe("file"); + expect(tokens[7][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[7][8].value).toBe("="); + expect(tokens[7][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[7][9].value).toBe("sys"); + expect(tokens[7][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][10].value).toBe("."); + expect(tokens[7][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[7][11].value).toBe("stderr"); + expect(tokens[7][11].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[7][12].value).toBe(")"); + expect(tokens[7][12].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[8][0].value).toBe("print"); + expect(tokens[8][0].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[8][1].value).toBe(" "); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.function-call.python"]); + expect(tokens[8][2].value).toBe("("); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[8][3].value).toBe("x"); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[8][4].value).toBe(","); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[8][5].value).toBe(" "); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[8][6].value).toBe("y"); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[8][7].value).toBe(")"); + expect(tokens[8][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[9][0].value).toBe("print"); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[9][1].value).toBe("("); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[9][2].value).toBe("("); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.parenthesis.begin.python"]); + expect(tokens[9][3].value).toBe("x"); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[9][4].value).toBe(","); + expect(tokens[9][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.element.python"]); + expect(tokens[9][5].value).toBe(" "); + expect(tokens[9][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[9][6].value).toBe("y"); + expect(tokens[9][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[9][7].value).toBe(")"); + expect(tokens[9][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.parenthesis.end.python"]); + expect(tokens[9][8].value).toBe(")"); + expect(tokens[9][8].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/classes/class1.py", + function() { + tokens = grammar.tokenizeLines("class Foo -> None: pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("Foo"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("->"); + expect(tokens[0][4].scopes).toEqual(["source.python","invalid.illegal.annotation.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("None"); + expect(tokens[0][6].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[0][7].value).toBe(":"); + expect(tokens[0][7].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("pass"); + expect(tokens[0][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class10.py", + function() { + tokens = grammar.tokenizeLines("class Foo(Bar(q=1) (w=2) (e=3)): pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[0][4].value).toBe("Bar"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","entity.other.inherited-class.python"]); + expect(tokens[0][5].value).toBe("("); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][6].value).toBe("q"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[0][7].value).toBe("="); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][8].value).toBe("1"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][9].value).toBe(")"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[0][11].value).toBe("("); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][12].value).toBe("w"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[0][13].value).toBe("="); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][14].value).toBe("2"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][15].value).toBe(")"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][16].value).toBe(" "); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[0][17].value).toBe("("); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][18].value).toBe("e"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[0][19].value).toBe("="); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][20].value).toBe("3"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][21].value).toBe(")"); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][22].value).toBe(")"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[0][23].value).toBe(":"); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[0][24].value).toBe(" "); + expect(tokens[0][24].scopes).toEqual(["source.python"]); + expect(tokens[0][25].value).toBe("pass"); + expect(tokens[0][25].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class11.py", + function() { + tokens = grammar.tokenizeLines("class F:\n def __init__(self, a, b=1):\n self.a = a\n self.b = b\n print(self)\n self()\n a.self = 1\n a.self.bar = 2\n self[123]") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("F"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][1].value).toBe("def"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][3].value).toBe("__init__"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","support.function.magic.python"]); + expect(tokens[1][4].value).toBe("("); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][5].value).toBe("self"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.self.python"]); + expect(tokens[1][6].value).toBe(","); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][8].value).toBe("a"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][9].value).toBe(","); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][11].value).toBe("b"); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][12].value).toBe("="); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[1][13].value).toBe("1"); + expect(tokens[1][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[1][14].value).toBe(")"); + expect(tokens[1][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][15].value).toBe(":"); + expect(tokens[1][15].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("self"); + expect(tokens[2][1].scopes).toEqual(["source.python","variable.language.special.self.python"]); + expect(tokens[2][2].value).toBe("."); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[2][3].value).toBe("a"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("="); + expect(tokens[2][5].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][6].value).toBe(" "); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe("a"); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("self"); + expect(tokens[3][1].scopes).toEqual(["source.python","variable.language.special.self.python"]); + expect(tokens[3][2].value).toBe("."); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[3][3].value).toBe("b"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[3][5].value).toBe("="); + expect(tokens[3][5].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][6].value).toBe(" "); + expect(tokens[3][6].scopes).toEqual(["source.python"]); + expect(tokens[3][7].value).toBe("b"); + expect(tokens[3][7].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("print"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe("self"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.language.special.self.python"]); + expect(tokens[4][4].value).toBe(")"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("self"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function-call.python","variable.language.special.self.python"]); + expect(tokens[5][2].value).toBe("("); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][3].value).toBe(")"); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("a"); + expect(tokens[6][1].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe("."); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[6][3].value).toBe("self"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[6][4].value).toBe(" "); + expect(tokens[6][4].scopes).toEqual(["source.python"]); + expect(tokens[6][5].value).toBe("="); + expect(tokens[6][5].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[6][6].value).toBe(" "); + expect(tokens[6][6].scopes).toEqual(["source.python"]); + expect(tokens[6][7].value).toBe("1"); + expect(tokens[6][7].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("a"); + expect(tokens[7][1].scopes).toEqual(["source.python"]); + expect(tokens[7][2].value).toBe("."); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[7][3].value).toBe("self"); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[7][4].value).toBe("."); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[7][5].value).toBe("bar"); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[7][6].value).toBe(" "); + expect(tokens[7][6].scopes).toEqual(["source.python"]); + expect(tokens[7][7].value).toBe("="); + expect(tokens[7][7].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[7][8].value).toBe(" "); + expect(tokens[7][8].scopes).toEqual(["source.python"]); + expect(tokens[7][9].value).toBe("2"); + expect(tokens[7][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("self"); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.item-access.python","variable.language.special.self.python"]); + expect(tokens[8][2].value).toBe("["); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[8][3].value).toBe("123"); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.item-access.python","meta.item-access.arguments.python","constant.numeric.dec.python"]); + expect(tokens[8][4].value).toBe("]"); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/classes/class12.py", + function() { + tokens = grammar.tokenizeLines("class F:\n @classmethod\n def meth(cls, a, b=1):\n cls.a = a\n cls.b = b\n print(cls)\n cls()\n cls + 1\n a.cls = 1\n a.cls.__name__\n cls[123]") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("F"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[1][1].value).toBe("@"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[1][2].value).toBe("classmethod"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.decorator.python","support.type.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][1].value).toBe("def"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][3].value).toBe("meth"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][4].value).toBe("("); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][5].value).toBe("cls"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.cls.python"]); + expect(tokens[2][6].value).toBe(","); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[2][8].value).toBe("a"); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][9].value).toBe(","); + expect(tokens[2][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[2][10].value).toBe(" "); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[2][11].value).toBe("b"); + expect(tokens[2][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][12].value).toBe("="); + expect(tokens[2][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[2][13].value).toBe("1"); + expect(tokens[2][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[2][14].value).toBe(")"); + expect(tokens[2][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][15].value).toBe(":"); + expect(tokens[2][15].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("cls"); + expect(tokens[3][1].scopes).toEqual(["source.python","variable.language.special.cls.python"]); + expect(tokens[3][2].value).toBe("."); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[3][3].value).toBe("a"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[3][5].value).toBe("="); + expect(tokens[3][5].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][6].value).toBe(" "); + expect(tokens[3][6].scopes).toEqual(["source.python"]); + expect(tokens[3][7].value).toBe("a"); + expect(tokens[3][7].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("cls"); + expect(tokens[4][1].scopes).toEqual(["source.python","variable.language.special.cls.python"]); + expect(tokens[4][2].value).toBe("."); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[4][3].value).toBe("b"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[4][4].value).toBe(" "); + expect(tokens[4][4].scopes).toEqual(["source.python"]); + expect(tokens[4][5].value).toBe("="); + expect(tokens[4][5].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[4][6].value).toBe(" "); + expect(tokens[4][6].scopes).toEqual(["source.python"]); + expect(tokens[4][7].value).toBe("b"); + expect(tokens[4][7].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("print"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[5][2].value).toBe("("); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][3].value).toBe("cls"); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.language.special.cls.python"]); + expect(tokens[5][4].value).toBe(")"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("cls"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function-call.python","variable.language.special.cls.python"]); + expect(tokens[6][2].value).toBe("("); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[6][3].value).toBe(")"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("cls"); + expect(tokens[7][1].scopes).toEqual(["source.python","variable.language.special.cls.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("+"); + expect(tokens[7][3].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[7][4].value).toBe(" "); + expect(tokens[7][4].scopes).toEqual(["source.python"]); + expect(tokens[7][5].value).toBe("1"); + expect(tokens[7][5].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("a"); + expect(tokens[8][1].scopes).toEqual(["source.python"]); + expect(tokens[8][2].value).toBe("."); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[8][3].value).toBe("cls"); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[8][4].value).toBe(" "); + expect(tokens[8][4].scopes).toEqual(["source.python"]); + expect(tokens[8][5].value).toBe("="); + expect(tokens[8][5].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[8][6].value).toBe(" "); + expect(tokens[8][6].scopes).toEqual(["source.python"]); + expect(tokens[8][7].value).toBe("1"); + expect(tokens[8][7].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("a"); + expect(tokens[9][1].scopes).toEqual(["source.python"]); + expect(tokens[9][2].value).toBe("."); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[9][3].value).toBe("cls"); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[9][4].value).toBe("."); + expect(tokens[9][4].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[9][5].value).toBe("__name__"); + expect(tokens[9][5].scopes).toEqual(["source.python","meta.member.access.python","support.variable.magic.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("cls"); + expect(tokens[10][1].scopes).toEqual(["source.python","meta.item-access.python","variable.language.special.cls.python"]); + expect(tokens[10][2].value).toBe("["); + expect(tokens[10][2].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[10][3].value).toBe("123"); + expect(tokens[10][3].scopes).toEqual(["source.python","meta.item-access.python","meta.item-access.arguments.python","constant.numeric.dec.python"]); + expect(tokens[10][4].value).toBe("]"); + expect(tokens[10][4].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/classes/class13.py", + function() { + tokens = grammar.tokenizeLines("class A(*a, **b): pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("A"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[0][4].value).toBe("*"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.operator.unpacking.arguments.python"]); + expect(tokens[0][5].value).toBe("a"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][6].value).toBe(","); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][8].value).toBe("**"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.operator.unpacking.arguments.python"]); + expect(tokens[0][9].value).toBe("b"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][10].value).toBe(")"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[0][11].value).toBe(":"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python"]); + expect(tokens[0][13].value).toBe("pass"); + expect(tokens[0][13].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class14.py", + function() { + tokens = grammar.tokenizeLines("class F(f.Exception, f.type, Exception, Exception.a, b=Exception):\n pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("F"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][5].value).toBe("."); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][6].value).toBe("Exception"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","entity.other.inherited-class.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][9].value).toBe("f"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][10].value).toBe("."); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][11].value).toBe("type"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","entity.other.inherited-class.python"]); + expect(tokens[0][12].value).toBe(","); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][14].value).toBe("Exception"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","support.type.exception.python"]); + expect(tokens[0][15].value).toBe(","); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][16].value).toBe(" "); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][17].value).toBe("Exception"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","support.type.exception.python"]); + expect(tokens[0][18].value).toBe("."); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][19].value).toBe("a"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","entity.other.inherited-class.python"]); + expect(tokens[0][20].value).toBe(","); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][21].value).toBe(" "); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][22].value).toBe("b"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","variable.parameter.class.python entity.other.inherited-class.python"]); + expect(tokens[0][23].value).toBe("="); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.operator.assignment.python"]); + expect(tokens[0][24].value).toBe("Exception"); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","support.type.exception.python"]); + expect(tokens[0][25].value).toBe(")"); + expect(tokens[0][25].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[0][26].value).toBe(":"); + expect(tokens[0][26].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("pass"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class15.py", + function() { + tokens = grammar.tokenizeLines("class Spam(Foo, from==12):\n pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Spam"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[0][4].value).toBe("Foo"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][5].value).toBe(","); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][7].value).toBe("from"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.control.flow.python"]); + expect(tokens[0][8].value).toBe("=="); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.operator.comparison.python"]); + expect(tokens[0][9].value).toBe("12"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","constant.numeric.dec.python"]); + expect(tokens[0][10].value).toBe(")"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[0][11].value).toBe(":"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("pass"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class16.py", + function() { + tokens = grammar.tokenizeLines("class Foo(Bar, str, type=12, metaclass=FooMeta): pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[0][4].value).toBe("Bar"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][5].value).toBe(","); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][7].value).toBe("str"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","support.type.python"]); + expect(tokens[0][8].value).toBe(","); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][10].value).toBe("type"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","variable.parameter.class.python entity.other.inherited-class.python"]); + expect(tokens[0][11].value).toBe("="); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.operator.assignment.python"]); + expect(tokens[0][12].value).toBe("12"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","constant.numeric.dec.python"]); + expect(tokens[0][13].value).toBe(","); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][14].value).toBe(" "); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][15].value).toBe("metaclass"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","support.type.metaclass.python"]); + expect(tokens[0][16].value).toBe("="); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.operator.assignment.python"]); + expect(tokens[0][17].value).toBe("FooMeta"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][18].value).toBe(")"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[0][19].value).toBe(":"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[0][20].value).toBe(" "); + expect(tokens[0][20].scopes).toEqual(["source.python"]); + expect(tokens[0][21].value).toBe("pass"); + expect(tokens[0][21].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class2.py", + function() { + tokens = grammar.tokenizeLines("@dec\n# Bar.name=... is not legal, but the test is for highlighter not breaking badly\nclass Spam(Foo.Bar, Bar.name={'very': 'odd'}):\n pass") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe("dec"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[1][0].value).toBe("#"); + expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][1].value).toBe(" Bar.name=... is not legal, but the test is for highlighter not breaking badly"); + expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe("class"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[2][2].value).toBe("Spam"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[2][4].value).toBe("Foo"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[2][5].value).toBe("."); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[2][6].value).toBe("Bar"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","entity.other.inherited-class.python"]); + expect(tokens[2][7].value).toBe(","); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[2][8].value).toBe(" "); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[2][9].value).toBe("Bar"); + expect(tokens[2][9].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[2][10].value).toBe("."); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[2][11].value).toBe("name"); + expect(tokens[2][11].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","meta.member.access.python","entity.other.inherited-class.python"]); + expect(tokens[2][12].value).toBe("="); + expect(tokens[2][12].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.operator.assignment.python"]); + expect(tokens[2][13].value).toBe("{"); + expect(tokens[2][13].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.dict.begin.python"]); + expect(tokens[2][14].value).toBe("'"); + expect(tokens[2][14].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][15].value).toBe("very"); + expect(tokens[2][15].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","string.quoted.single.python"]); + expect(tokens[2][16].value).toBe("'"); + expect(tokens[2][16].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][17].value).toBe(":"); + expect(tokens[2][17].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.dict.python"]); + expect(tokens[2][18].value).toBe(" "); + expect(tokens[2][18].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[2][19].value).toBe("'"); + expect(tokens[2][19].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][20].value).toBe("odd"); + expect(tokens[2][20].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","string.quoted.single.python"]); + expect(tokens[2][21].value).toBe("'"); + expect(tokens[2][21].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][22].value).toBe("}"); + expect(tokens[2][22].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.dict.end.python"]); + expect(tokens[2][23].value).toBe(")"); + expect(tokens[2][23].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[2][24].value).toBe(":"); + expect(tokens[2][24].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("pass"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class3.py", + function() { + tokens = grammar.tokenizeLines("class Spam(Foo, from=12):\n pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Spam"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[0][4].value).toBe("Foo"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][5].value).toBe(","); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][7].value).toBe("from"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.control.flow.python"]); + expect(tokens[0][8].value).toBe("="); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.operator.assignment.python"]); + expect(tokens[0][9].value).toBe("12"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","constant.numeric.dec.python"]); + expect(tokens[0][10].value).toBe(")"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[0][11].value).toBe(":"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("pass"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class4.py", + function() { + tokens = grammar.tokenizeLines("class Exception: pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Exception"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","support.type.exception.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("pass"); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class5.py", + function() { + tokens = grammar.tokenizeLines("class Foo(Exception, MyObj): pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[0][4].value).toBe("Exception"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","support.type.exception.python"]); + expect(tokens[0][5].value).toBe(","); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.separator.inheritance.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python"]); + expect(tokens[0][7].value).toBe("MyObj"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][8].value).toBe(")"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[0][9].value).toBe(":"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("pass"); + expect(tokens[0][11].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class6.py", + function() { + tokens = grammar.tokenizeLines("class Foo(metaclass=FooMeta): pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[0][4].value).toBe("metaclass"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","support.type.metaclass.python"]); + expect(tokens[0][5].value).toBe("="); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","keyword.operator.assignment.python"]); + expect(tokens[0][6].value).toBe("FooMeta"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][7].value).toBe(")"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[0][8].value).toBe(":"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("pass"); + expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class7.py", + function() { + tokens = grammar.tokenizeLines("class class:\n pass\n# doesn't break") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("class"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","keyword.control.flow.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("pass"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("#"); + expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][1].value).toBe(" doesn't break"); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/classes/class8.py", + function() { + tokens = grammar.tokenizeLines("class None: pass\nclass True(False): pass\nclass False(object): pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("None"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","keyword.illegal.name.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("pass"); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][0].value).toBe("class"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[1][2].value).toBe("True"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.class.python","keyword.illegal.name.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[1][4].value).toBe("False"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","constant.language.python"]); + expect(tokens[1][5].value).toBe(")"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[1][6].value).toBe(":"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python"]); + expect(tokens[1][8].value).toBe("pass"); + expect(tokens[1][8].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("class"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[2][2].value).toBe("False"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.class.python","keyword.illegal.name.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[2][4].value).toBe("object"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","support.type.python"]); + expect(tokens[2][5].value).toBe(")"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[2][6].value).toBe(":"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe("pass"); + expect(tokens[2][8].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/classes/class9.py", + function() { + tokens = grammar.tokenizeLines("class Foo:\n __slots__ = ()\n __match_args__ = ('key', 'name')") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("__slots__"); + expect(tokens[1][1].scopes).toEqual(["source.python","support.variable.magic.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("="); + expect(tokens[1][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[1][5].value).toBe("("); + expect(tokens[1][5].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[1][6].value).toBe(")"); + expect(tokens[1][6].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("__match_args__"); + expect(tokens[2][1].scopes).toEqual(["source.python","support.variable.magic.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("="); + expect(tokens[2][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("("); + expect(tokens[2][5].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[2][6].value).toBe("'"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][7].value).toBe("key"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][8].value).toBe("'"); + expect(tokens[2][8].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][9].value).toBe(","); + expect(tokens[2][9].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][10].value).toBe(" "); + expect(tokens[2][10].scopes).toEqual(["source.python"]); + expect(tokens[2][11].value).toBe("'"); + expect(tokens[2][11].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][12].value).toBe("name"); + expect(tokens[2][12].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][13].value).toBe("'"); + expect(tokens[2][13].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][14].value).toBe(")"); + expect(tokens[2][14].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + }); + + it("test/classes/super1.py", + function() { + tokens = grammar.tokenizeLines("class Foo:\n def __init__(self):\n super().__init__(foo=1)\n super(). __init__(foo=1)\n super(). \\\n__init__(foo=1)\n __init__(foo=1)\n\n foo.__init__(bar=1)\n __init__(bar=1)\n if:\n __init__(bar=1)") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][1].value).toBe("def"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][3].value).toBe("__init__"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","support.function.magic.python"]); + expect(tokens[1][4].value).toBe("("); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][5].value).toBe("self"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.self.python"]); + expect(tokens[1][6].value).toBe(")"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][7].value).toBe(":"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("super"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function-call.python","support.type.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe(")"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[2][4].value).toBe("."); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[2][5].value).toBe("__init__"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","support.function.magic.python"]); + expect(tokens[2][6].value).toBe("("); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][7].value).toBe("foo"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[2][8].value).toBe("="); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[2][9].value).toBe("1"); + expect(tokens[2][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[2][10].value).toBe(")"); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("super"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function-call.python","support.type.python"]); + expect(tokens[3][2].value).toBe("("); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][3].value).toBe(")"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][4].value).toBe("."); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[3][5].value).toBe(" "); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[3][6].value).toBe("__init__"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","support.function.magic.python"]); + expect(tokens[3][7].value).toBe("("); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][8].value).toBe("foo"); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][9].value).toBe("="); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[3][10].value).toBe("1"); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][11].value).toBe(")"); + expect(tokens[3][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("super"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function-call.python","support.type.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe(")"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][4].value).toBe("."); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[4][5].value).toBe(" "); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[4][6].value).toBe("\\"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.continuation.line.python"]); + expect(tokens[4][7].value).toBe(""); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[5][0].value).toBe("__init__"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","support.function.magic.python"]); + expect(tokens[5][1].value).toBe("("); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][2].value).toBe("foo"); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[5][3].value).toBe("="); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[5][4].value).toBe("1"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[5][5].value).toBe(")"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("__init__"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.magic.python"]); + expect(tokens[6][2].value).toBe("("); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[6][3].value).toBe("foo"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[6][4].value).toBe("="); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[6][5].value).toBe("1"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[6][6].value).toBe(")"); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][0].value).toBe(""); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("foo"); + expect(tokens[8][1].scopes).toEqual(["source.python"]); + expect(tokens[8][2].value).toBe("."); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[8][3].value).toBe("__init__"); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","support.function.magic.python"]); + expect(tokens[8][4].value).toBe("("); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[8][5].value).toBe("bar"); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[8][6].value).toBe("="); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[8][7].value).toBe("1"); + expect(tokens[8][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[8][8].value).toBe(")"); + expect(tokens[8][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("__init__"); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.magic.python"]); + expect(tokens[9][2].value).toBe("("); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[9][3].value).toBe("bar"); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[9][4].value).toBe("="); + expect(tokens[9][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[9][5].value).toBe("1"); + expect(tokens[9][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[9][6].value).toBe(")"); + expect(tokens[9][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("if"); + expect(tokens[10][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[10][2].value).toBe(":"); + expect(tokens[10][2].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[11][0].value).toBe(" "); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[11][1].value).toBe("__init__"); + expect(tokens[11][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.magic.python"]); + expect(tokens[11][2].value).toBe("("); + expect(tokens[11][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[11][3].value).toBe("bar"); + expect(tokens[11][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[11][4].value).toBe("="); + expect(tokens[11][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[11][5].value).toBe("1"); + expect(tokens[11][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[11][6].value).toBe(")"); + expect(tokens[11][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/comments/comment1.py", + function() { + tokens = grammar.tokenizeLines("# XXX foo\n# FIXME: bug\n# NB: XXXx xXXX but XXX!\n# ALSO HACK and NOTE and TODO highlight XXX and FIXME.") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[0][2].value).toBe("XXX"); + expect(tokens[0][2].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[0][3].value).toBe(" foo"); + expect(tokens[0][3].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("#"); + expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][2].value).toBe("FIXME"); + expect(tokens[1][2].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[1][3].value).toBe(": bug"); + expect(tokens[1][3].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe("#"); + expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][1].value).toBe(" NB: XXXx xXXX but "); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][2].value).toBe("XXX"); + expect(tokens[2][2].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[2][3].value).toBe("!"); + expect(tokens[2][3].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][0].value).toBe("#"); + expect(tokens[3][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][1].value).toBe(" ALSO "); + expect(tokens[3][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][2].value).toBe("HACK"); + expect(tokens[3][2].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[3][3].value).toBe(" and "); + expect(tokens[3][3].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][4].value).toBe("NOTE"); + expect(tokens[3][4].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[3][5].value).toBe(" and "); + expect(tokens[3][5].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][6].value).toBe("TODO"); + expect(tokens[3][6].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[3][7].value).toBe(" highlight "); + expect(tokens[3][7].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][8].value).toBe("XXX"); + expect(tokens[3][8].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[3][9].value).toBe(" and "); + expect(tokens[3][9].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][10].value).toBe("FIXME"); + expect(tokens[3][10].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[3][11].value).toBe("."); + expect(tokens[3][11].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/comments/typing1.py", + function() { + tokens = grammar.tokenizeLines("# type: ignore # test\n# type: ignore\n# type:ignore\n#type:ignore\n# type: ignore 1\n# type: 1 ignore\n# type : ignore\n##type: ignore\n#.type: ignore") + expect(tokens[0][0].value).toBe("# "); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[0][1].value).toBe("type:"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[0][2].value).toBe(" "); + expect(tokens[0][2].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[0][3].value).toBe("ignore"); + expect(tokens[0][3].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.ignore.notation.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[0][5].value).toBe("#"); + expect(tokens[0][5].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][6].value).toBe(" test"); + expect(tokens[0][6].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("# "); + expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[1][1].value).toBe("type:"); + expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[1][3].value).toBe("ignore"); + expect(tokens[1][3].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.ignore.notation.python"]); + expect(tokens[2][0].value).toBe("# "); + expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[2][1].value).toBe("type:"); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[2][2].value).toBe("ignore"); + expect(tokens[2][2].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.ignore.notation.python"]); + expect(tokens[3][0].value).toBe("#"); + expect(tokens[3][0].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[3][1].value).toBe("type:"); + expect(tokens[3][1].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[3][2].value).toBe("ignore"); + expect(tokens[3][2].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.ignore.notation.python"]); + expect(tokens[4][0].value).toBe("# "); + expect(tokens[4][0].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[4][1].value).toBe("type:"); + expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[4][3].value).toBe("ignore"); + expect(tokens[4][3].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.variable.notation.python"]); + expect(tokens[4][4].value).toBe(" 1"); + expect(tokens[4][4].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[5][0].value).toBe("# "); + expect(tokens[5][0].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[5][1].value).toBe("type:"); + expect(tokens[5][1].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[5][3].value).toBe("1 "); + expect(tokens[5][3].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[5][4].value).toBe("ignore"); + expect(tokens[5][4].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.variable.notation.python"]); + expect(tokens[6][0].value).toBe("#"); + expect(tokens[6][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[6][1].value).toBe(" type : ignore"); + expect(tokens[6][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[7][0].value).toBe("#"); + expect(tokens[7][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[7][1].value).toBe("#type: ignore"); + expect(tokens[7][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[8][0].value).toBe("#"); + expect(tokens[8][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[8][1].value).toBe(".type: ignore"); + expect(tokens[8][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/comments/typing2.py", + function() { + tokens = grammar.tokenizeLines("a = 1 # type:\na = 1 # type: # 123") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("1"); + expect(tokens[0][4].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("#"); + expect(tokens[0][6].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][7].value).toBe(" type:"); + expect(tokens[0][7].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("1"); + expect(tokens[1][4].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("#"); + expect(tokens[1][6].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][7].value).toBe(" type: # 123"); + expect(tokens[1][7].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/comments/typing3.py", + function() { + tokens = grammar.tokenizeLines("# abc type: def") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" abc type: def"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/comments/typing4.py", + function() { + tokens = grammar.tokenizeLines("x = None # type: List[str, a]\ny = None # type: Dict[int, Any] # int") + expect(tokens[0][0].value).toBe("x"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("None"); + expect(tokens[0][4].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("# "); + expect(tokens[0][6].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[0][7].value).toBe("type:"); + expect(tokens[0][7].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[0][9].value).toBe("List"); + expect(tokens[0][9].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.type.notation.python"]); + expect(tokens[0][10].value).toBe("["); + expect(tokens[0][10].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.punctuation.notation.python"]); + expect(tokens[0][11].value).toBe("str"); + expect(tokens[0][11].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.type.notation.python"]); + expect(tokens[0][12].value).toBe(","); + expect(tokens[0][12].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.punctuation.notation.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[0][14].value).toBe("a"); + expect(tokens[0][14].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.variable.notation.python"]); + expect(tokens[0][15].value).toBe("]"); + expect(tokens[0][15].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.punctuation.notation.python"]); + expect(tokens[1][0].value).toBe("y"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("None"); + expect(tokens[1][4].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("# "); + expect(tokens[1][6].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[1][7].value).toBe("type:"); + expect(tokens[1][7].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[1][8].value).toBe(" "); + expect(tokens[1][8].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[1][9].value).toBe("Dict"); + expect(tokens[1][9].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.type.notation.python"]); + expect(tokens[1][10].value).toBe("["); + expect(tokens[1][10].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.punctuation.notation.python"]); + expect(tokens[1][11].value).toBe("int"); + expect(tokens[1][11].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.type.notation.python"]); + expect(tokens[1][12].value).toBe(","); + expect(tokens[1][12].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.punctuation.notation.python"]); + expect(tokens[1][13].value).toBe(" "); + expect(tokens[1][13].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[1][14].value).toBe("Any"); + expect(tokens[1][14].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.type.notation.python"]); + expect(tokens[1][15].value).toBe("]"); + expect(tokens[1][15].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.punctuation.notation.python"]); + expect(tokens[1][16].value).toBe(" "); + expect(tokens[1][16].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[1][17].value).toBe("#"); + expect(tokens[1][17].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][18].value).toBe(" int"); + expect(tokens[1][18].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/comments/typing5.py", + function() { + tokens = grammar.tokenizeLines("if isinstance(t1, TypeVar): # type: ignore\n continue") + expect(tokens[0][0].value).toBe("if"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("isinstance"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][4].value).toBe("t1"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[0][5].value).toBe(","); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[0][7].value).toBe("TypeVar"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[0][8].value).toBe(")"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][9].value).toBe(":"); + expect(tokens[0][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("# "); + expect(tokens[0][11].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[0][12].value).toBe("type:"); + expect(tokens[0][12].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[0][14].value).toBe("ignore"); + expect(tokens[0][14].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.ignore.notation.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("continue"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/comments/typing6.py", + function() { + tokens = grammar.tokenizeLines("a=1#type:int\na=1#type:int#int") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe("="); + expect(tokens[0][1].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][2].value).toBe("1"); + expect(tokens[0][2].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][3].value).toBe("#"); + expect(tokens[0][3].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[0][4].value).toBe("type:"); + expect(tokens[0][4].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[0][5].value).toBe("int"); + expect(tokens[0][5].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.type.notation.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("="); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][2].value).toBe("1"); + expect(tokens[1][2].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][3].value).toBe("#"); + expect(tokens[1][3].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python"]); + expect(tokens[1][4].value).toBe("type:"); + expect(tokens[1][4].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.directive.notation.python"]); + expect(tokens[1][5].value).toBe("int"); + expect(tokens[1][5].scopes).toEqual(["source.python","comment.line.number-sign.python","meta.typehint.comment.python","comment.typehint.type.notation.python"]); + expect(tokens[1][6].value).toBe("#"); + expect(tokens[1][6].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][7].value).toBe("int"); + expect(tokens[1][7].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/docstrings/class1.py", + function() { + tokens = grammar.tokenizeLines("class Foo:\n '''TEST'''\n\nclass Foo:\n r'''TEST'''\n\nclass Foo:\n R'''TEST'''\n\nclass Foo:\n u'''TEST'''\n\nclass Foo:\n U'''TEST'''\n\nclass Foo:\n b'''TEST'''\n\nclass Foo:\n B'''TEST'''") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("'''"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TEST"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[1][3].value).toBe("'''"); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("class"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[3][2].value).toBe("Foo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[3][3].value).toBe(":"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("r"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[4][2].value).toBe("'''"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][3].value).toBe("TEST"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][4].value).toBe("'''"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("class"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[6][2].value).toBe("Foo"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[6][3].value).toBe(":"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("R"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[7][2].value).toBe("'''"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("TEST"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][4].value).toBe("'''"); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe(""); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[9][0].value).toBe("class"); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[9][1].value).toBe(" "); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[9][2].value).toBe("Foo"); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[9][3].value).toBe(":"); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("u"); + expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[10][2].value).toBe("'''"); + expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][3].value).toBe("TEST"); + expect(tokens[10][3].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[10][4].value).toBe("'''"); + expect(tokens[10][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[11][0].value).toBe(""); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[12][0].value).toBe("class"); + expect(tokens[12][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[12][1].value).toBe(" "); + expect(tokens[12][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[12][2].value).toBe("Foo"); + expect(tokens[12][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[12][3].value).toBe(":"); + expect(tokens[12][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[13][0].value).toBe(" "); + expect(tokens[13][0].scopes).toEqual(["source.python"]); + expect(tokens[13][1].value).toBe("U"); + expect(tokens[13][1].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[13][2].value).toBe("'''"); + expect(tokens[13][2].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[13][3].value).toBe("TEST"); + expect(tokens[13][3].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[13][4].value).toBe("'''"); + expect(tokens[13][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[14][0].value).toBe(""); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[15][0].value).toBe("class"); + expect(tokens[15][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[15][1].value).toBe(" "); + expect(tokens[15][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[15][2].value).toBe("Foo"); + expect(tokens[15][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[15][3].value).toBe(":"); + expect(tokens[15][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[16][0].value).toBe(" "); + expect(tokens[16][0].scopes).toEqual(["source.python"]); + expect(tokens[16][1].value).toBe("b"); + expect(tokens[16][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[16][2].value).toBe("'''"); + expect(tokens[16][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[16][3].value).toBe("TEST"); + expect(tokens[16][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[16][4].value).toBe("'''"); + expect(tokens[16][4].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[17][0].value).toBe(""); + expect(tokens[17][0].scopes).toEqual(["source.python"]); + expect(tokens[18][0].value).toBe("class"); + expect(tokens[18][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[18][1].value).toBe(" "); + expect(tokens[18][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[18][2].value).toBe("Foo"); + expect(tokens[18][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[18][3].value).toBe(":"); + expect(tokens[18][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[19][0].value).toBe(" "); + expect(tokens[19][0].scopes).toEqual(["source.python"]); + expect(tokens[19][1].value).toBe("B"); + expect(tokens[19][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[19][2].value).toBe("'''"); + expect(tokens[19][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[19][3].value).toBe("TEST"); + expect(tokens[19][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[19][4].value).toBe("'''"); + expect(tokens[19][4].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/class2.py", + function() { + tokens = grammar.tokenizeLines("class Foo:\n \"\"\"TEST\"\"\"\n\nclass Foo:\n r\"\"\"TEST\"\"\"\n\nclass Foo:\n R\"\"\"TEST\"\"\"\n\nclass Foo:\n u\"\"\"TEST\"\"\"\n\nclass Foo:\n U\"\"\"TEST\"\"\"\n\nclass Foo:\n b\"\"\"TEST\"\"\"\n\nclass Foo:\n B\"\"\"TEST\"\"\"") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("\"\"\""); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TEST"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[1][3].value).toBe("\"\"\""); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("class"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[3][2].value).toBe("Foo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[3][3].value).toBe(":"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("r"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[4][2].value).toBe("\"\"\""); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][3].value).toBe("TEST"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][4].value).toBe("\"\"\""); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("class"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[6][2].value).toBe("Foo"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[6][3].value).toBe(":"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("R"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[7][2].value).toBe("\"\"\""); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("TEST"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][4].value).toBe("\"\"\""); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe(""); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[9][0].value).toBe("class"); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[9][1].value).toBe(" "); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[9][2].value).toBe("Foo"); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[9][3].value).toBe(":"); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("u"); + expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[10][2].value).toBe("\"\"\""); + expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][3].value).toBe("TEST"); + expect(tokens[10][3].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[10][4].value).toBe("\"\"\""); + expect(tokens[10][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[11][0].value).toBe(""); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[12][0].value).toBe("class"); + expect(tokens[12][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[12][1].value).toBe(" "); + expect(tokens[12][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[12][2].value).toBe("Foo"); + expect(tokens[12][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[12][3].value).toBe(":"); + expect(tokens[12][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[13][0].value).toBe(" "); + expect(tokens[13][0].scopes).toEqual(["source.python"]); + expect(tokens[13][1].value).toBe("U"); + expect(tokens[13][1].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[13][2].value).toBe("\"\"\""); + expect(tokens[13][2].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[13][3].value).toBe("TEST"); + expect(tokens[13][3].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[13][4].value).toBe("\"\"\""); + expect(tokens[13][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[14][0].value).toBe(""); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[15][0].value).toBe("class"); + expect(tokens[15][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[15][1].value).toBe(" "); + expect(tokens[15][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[15][2].value).toBe("Foo"); + expect(tokens[15][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[15][3].value).toBe(":"); + expect(tokens[15][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[16][0].value).toBe(" "); + expect(tokens[16][0].scopes).toEqual(["source.python"]); + expect(tokens[16][1].value).toBe("b"); + expect(tokens[16][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[16][2].value).toBe("\"\"\""); + expect(tokens[16][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[16][3].value).toBe("TEST"); + expect(tokens[16][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[16][4].value).toBe("\"\"\""); + expect(tokens[16][4].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[17][0].value).toBe(""); + expect(tokens[17][0].scopes).toEqual(["source.python"]); + expect(tokens[18][0].value).toBe("class"); + expect(tokens[18][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[18][1].value).toBe(" "); + expect(tokens[18][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[18][2].value).toBe("Foo"); + expect(tokens[18][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[18][3].value).toBe(":"); + expect(tokens[18][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[19][0].value).toBe(" "); + expect(tokens[19][0].scopes).toEqual(["source.python"]); + expect(tokens[19][1].value).toBe("B"); + expect(tokens[19][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[19][2].value).toBe("\"\"\""); + expect(tokens[19][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[19][3].value).toBe("TEST"); + expect(tokens[19][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[19][4].value).toBe("\"\"\""); + expect(tokens[19][4].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/codetag1.py", + function() { + tokens = grammar.tokenizeLines("''' foo bar XXX baz '''\n\ndef foo():\n ''' foo FIXME baz '''") + expect(tokens[0][0].value).toBe("'''"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe(" foo bar "); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[0][2].value).toBe("XXX"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","keyword.codetag.notation.python"]); + expect(tokens[0][3].value).toBe(" baz "); + expect(tokens[0][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[0][4].value).toBe("'''"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][5].value).toBe(":"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("'''"); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe(" foo "); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][3].value).toBe("FIXME"); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","keyword.codetag.notation.python"]); + expect(tokens[3][4].value).toBe(" baz "); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][5].value).toBe("'''"); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/codetag2.py", + function() { + tokens = grammar.tokenizeLines("' foo bar XXX baz '\n\ndef foo():\n ' foo FIXME baz '") + expect(tokens[0][0].value).toBe("'"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe(" foo bar "); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[0][2].value).toBe("XXX"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","keyword.codetag.notation.python"]); + expect(tokens[0][3].value).toBe(" baz "); + expect(tokens[0][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][5].value).toBe(":"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("'"); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe(" foo "); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[3][3].value).toBe("FIXME"); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python","keyword.codetag.notation.python"]); + expect(tokens[3][4].value).toBe(" baz "); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[3][5].value).toBe("'"); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/continuation1.py", + function() { + tokens = grammar.tokenizeLines("# not a docstring\na = \\\nr'''\n>>> print(42)\na[wer]\n'''\n\nb = \\\n# docstring\nr'''\n>>> print()\na[wer]\n'''") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" not a docstring"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("\\"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][5].value).toBe(""); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("r"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","storage.type.string.python"]); + expect(tokens[2][1].value).toBe("'''"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][0].value).toBe(">>> print"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[3][1].value).toBe("("); + expect(tokens[3][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.parenthesis.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[3][2].value).toBe("42"); + expect(tokens[3][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[3][3].value).toBe(")"); + expect(tokens[3][3].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.parenthesis.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[4][1].value).toBe("["); + expect(tokens[4][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[4][2].value).toBe("w"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[4][3].value).toBe("e"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[4][4].value).toBe("r"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[4][5].value).toBe("]"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + expect(tokens[5][0].value).toBe("'''"); + expect(tokens[5][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe(""); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[7][0].value).toBe("b"); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe(" "); + expect(tokens[7][1].scopes).toEqual(["source.python"]); + expect(tokens[7][2].value).toBe("="); + expect(tokens[7][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[7][3].value).toBe(" "); + expect(tokens[7][3].scopes).toEqual(["source.python"]); + expect(tokens[7][4].value).toBe("\\"); + expect(tokens[7][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[7][5].value).toBe(""); + expect(tokens[7][5].scopes).toEqual(["source.python"]); + expect(tokens[8][0].value).toBe("#"); + expect(tokens[8][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[8][1].value).toBe(" docstring"); + expect(tokens[8][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[9][0].value).toBe("r"); + expect(tokens[9][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[9][1].value).toBe("'''"); + expect(tokens[9][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][0].value).toBe(">>> "); + expect(tokens[10][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","keyword.control.flow.python"]); + expect(tokens[10][1].value).toBe("print()"); + expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[11][0].value).toBe("a[wer]"); + expect(tokens[11][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[12][0].value).toBe("'''"); + expect(tokens[12][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/continuation2.py", + function() { + tokens = grammar.tokenizeLines("'\n'\n# comment") + expect(tokens[0][0].value).toBe("'"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe(""); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","invalid.illegal.newline.python"]); + expect(tokens[1][0].value).toBe("'"); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][1].value).toBe(""); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","invalid.illegal.newline.python"]); + expect(tokens[2][0].value).toBe("#"); + expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][1].value).toBe(" comment"); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/docstrings/continuation3.py", + function() { + tokens = grammar.tokenizeLines("# not a docstring\na = \\\nr'>>> print(42)a[wer]'\n\nb = \\\n# docstring\nr'>>> print()a[wer]'") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" not a docstring"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("\\"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][5].value).toBe(""); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("r"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[2][1].value).toBe("'"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe(">>> print"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.parenthesis.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[2][4].value).toBe("42"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[2][5].value).toBe(")"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.parenthesis.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[2][6].value).toBe("a"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[2][7].value).toBe("["); + expect(tokens[2][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[2][8].value).toBe("w"); + expect(tokens[2][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[2][9].value).toBe("e"); + expect(tokens[2][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[2][10].value).toBe("r"); + expect(tokens[2][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[2][11].value).toBe("]"); + expect(tokens[2][11].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + expect(tokens[2][12].value).toBe("'"); + expect(tokens[2][12].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe(""); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe("b"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("="); + expect(tokens[4][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe("\\"); + expect(tokens[4][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[4][5].value).toBe(""); + expect(tokens[4][5].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe("#"); + expect(tokens[5][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][1].value).toBe(" docstring"); + expect(tokens[5][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[6][0].value).toBe("r"); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[6][1].value).toBe("'"); + expect(tokens[6][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][2].value).toBe(">>> print()a[wer]"); + expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[6][3].value).toBe("'"); + expect(tokens[6][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/continuation4.py", + function() { + tokens = grammar.tokenizeLines(" # not a docstring\n a = \\\n r'''\n >>> print(42)\n a[wer]\n '''\n\n b = \\\n # docstring\n r'''\n >>> print()\n a[wer]\n '''") + expect(tokens[0][0].value).toBe(" "); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe("#"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][2].value).toBe(" not a docstring"); + expect(tokens[0][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("a"); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("="); + expect(tokens[1][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[1][5].value).toBe("\\"); + expect(tokens[1][5].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][6].value).toBe(""); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("r"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","storage.type.string.python"]); + expect(tokens[2][2].value).toBe("'''"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][0].value).toBe(" >>> print"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[3][1].value).toBe("("); + expect(tokens[3][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.parenthesis.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[3][2].value).toBe("42"); + expect(tokens[3][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[3][3].value).toBe(")"); + expect(tokens[3][3].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.parenthesis.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[4][0].value).toBe(" a"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[4][1].value).toBe("["); + expect(tokens[4][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[4][2].value).toBe("w"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[4][3].value).toBe("e"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[4][4].value).toBe("r"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[4][5].value).toBe("]"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[5][1].value).toBe("'''"); + expect(tokens[5][1].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe(""); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("b"); + expect(tokens[7][1].scopes).toEqual(["source.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("="); + expect(tokens[7][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[7][4].value).toBe(" "); + expect(tokens[7][4].scopes).toEqual(["source.python"]); + expect(tokens[7][5].value).toBe("\\"); + expect(tokens[7][5].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[7][6].value).toBe(""); + expect(tokens[7][6].scopes).toEqual(["source.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("#"); + expect(tokens[8][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[8][2].value).toBe(" docstring"); + expect(tokens[8][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("r"); + expect(tokens[9][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[9][2].value).toBe("'''"); + expect(tokens[9][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[10][1].value).toBe(">>> "); + expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","keyword.control.flow.python"]); + expect(tokens[10][2].value).toBe("print()"); + expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[11][0].value).toBe(" a[wer]"); + expect(tokens[11][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[12][0].value).toBe(" "); + expect(tokens[12][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[12][1].value).toBe("'''"); + expect(tokens[12][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/continuation5.py", + function() { + tokens = grammar.tokenizeLines("'implicit ' \"concatenation\"\n\n'''implicit\n''' 'concatenation'\n\n'''implicit\n''' \"\"\"\nconcatenation\n\"\"\"\n\n'implicit' '''\nconcatenation\n'''") + expect(tokens[0][0].value).toBe("'"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("implicit "); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[0][2].value).toBe("'"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("\""); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("concatenation"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[0][6].value).toBe("\""); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("'''"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][1].value).toBe("implicit"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][0].value).toBe("'''"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("'"); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][3].value).toBe("concatenation"); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[3][4].value).toBe("'"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe("'''"); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][1].value).toBe("implicit"); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[6][0].value).toBe("'''"); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe("\"\"\""); + expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][0].value).toBe("concatenation"); + expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[8][0].value).toBe("\"\"\""); + expect(tokens[8][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[9][0].value).toBe(""); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[10][0].value).toBe("'"); + expect(tokens[10][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][1].value).toBe("implicit"); + expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[10][2].value).toBe("'"); + expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[10][3].value).toBe(" "); + expect(tokens[10][3].scopes).toEqual(["source.python"]); + expect(tokens[10][4].value).toBe("'''"); + expect(tokens[10][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[11][0].value).toBe("concatenation"); + expect(tokens[11][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[12][0].value).toBe("'''"); + expect(tokens[12][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/def1.py", + function() { + tokens = grammar.tokenizeLines("def foo():\n '''TE\\'''ST'''\n\ndef foo():\n r'''TE\\'''ST'''\n\ndef foo():\n R'''TE\\'''ST'''\n\ndef foo():\n u'''TEST'''\n\ndef foo():\n U'''TEST'''\n\ndef foo():\n b'''TEST'''\n\ndef foo():\n B'''TEST'''") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("'''"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TE"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[1][3].value).toBe("\\'"); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[1][4].value).toBe("''ST"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[1][5].value).toBe("'''"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("def"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[3][2].value).toBe("foo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[3][3].value).toBe("("); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[3][4].value).toBe(")"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[3][5].value).toBe(":"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("r"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[4][2].value).toBe("'''"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][3].value).toBe("TE"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][4].value).toBe("\\'"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][5].value).toBe("''ST"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][6].value).toBe("'''"); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("def"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][2].value).toBe("foo"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[6][3].value).toBe("("); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[6][4].value).toBe(")"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[6][5].value).toBe(":"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("R"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[7][2].value).toBe("'''"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("TE"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][4].value).toBe("\\'"); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][5].value).toBe("''ST"); + expect(tokens[7][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][6].value).toBe("'''"); + expect(tokens[7][6].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe(""); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[9][0].value).toBe("def"); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[9][1].value).toBe(" "); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[9][2].value).toBe("foo"); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[9][3].value).toBe("("); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[9][4].value).toBe(")"); + expect(tokens[9][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[9][5].value).toBe(":"); + expect(tokens[9][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("u"); + expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[10][2].value).toBe("'''"); + expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][3].value).toBe("TEST"); + expect(tokens[10][3].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[10][4].value).toBe("'''"); + expect(tokens[10][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[11][0].value).toBe(""); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[12][0].value).toBe("def"); + expect(tokens[12][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[12][1].value).toBe(" "); + expect(tokens[12][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[12][2].value).toBe("foo"); + expect(tokens[12][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[12][3].value).toBe("("); + expect(tokens[12][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[12][4].value).toBe(")"); + expect(tokens[12][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[12][5].value).toBe(":"); + expect(tokens[12][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[13][0].value).toBe(" "); + expect(tokens[13][0].scopes).toEqual(["source.python"]); + expect(tokens[13][1].value).toBe("U"); + expect(tokens[13][1].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[13][2].value).toBe("'''"); + expect(tokens[13][2].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[13][3].value).toBe("TEST"); + expect(tokens[13][3].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[13][4].value).toBe("'''"); + expect(tokens[13][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[14][0].value).toBe(""); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[15][0].value).toBe("def"); + expect(tokens[15][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[15][1].value).toBe(" "); + expect(tokens[15][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[15][2].value).toBe("foo"); + expect(tokens[15][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[15][3].value).toBe("("); + expect(tokens[15][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[15][4].value).toBe(")"); + expect(tokens[15][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[15][5].value).toBe(":"); + expect(tokens[15][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[16][0].value).toBe(" "); + expect(tokens[16][0].scopes).toEqual(["source.python"]); + expect(tokens[16][1].value).toBe("b"); + expect(tokens[16][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[16][2].value).toBe("'''"); + expect(tokens[16][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[16][3].value).toBe("TEST"); + expect(tokens[16][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[16][4].value).toBe("'''"); + expect(tokens[16][4].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[17][0].value).toBe(""); + expect(tokens[17][0].scopes).toEqual(["source.python"]); + expect(tokens[18][0].value).toBe("def"); + expect(tokens[18][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[18][1].value).toBe(" "); + expect(tokens[18][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[18][2].value).toBe("foo"); + expect(tokens[18][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[18][3].value).toBe("("); + expect(tokens[18][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[18][4].value).toBe(")"); + expect(tokens[18][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[18][5].value).toBe(":"); + expect(tokens[18][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[19][0].value).toBe(" "); + expect(tokens[19][0].scopes).toEqual(["source.python"]); + expect(tokens[19][1].value).toBe("B"); + expect(tokens[19][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[19][2].value).toBe("'''"); + expect(tokens[19][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[19][3].value).toBe("TEST"); + expect(tokens[19][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[19][4].value).toBe("'''"); + expect(tokens[19][4].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/def2.py", + function() { + tokens = grammar.tokenizeLines("def foo():\n \"\"\"TE\\\"\"\"ST\"\"\"\n\ndef foo():\n r\"\"\"TE\\\"\"\"ST\"\"\"\n\ndef foo():\n R\"\"\"TE\\\"\"\"ST\"\"\"\n\ndef foo():\n u\"\"\"TEST\"\"\"\n\ndef foo():\n U\"\"\"TEST\"\"\"\n\ndef foo():\n b\"\"\"TEST\"\"\"\n\ndef foo():\n B\"\"\"TEST\"\"\"") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("\"\"\""); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TE"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[1][3].value).toBe("\\\""); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[1][4].value).toBe("\"\"ST"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[1][5].value).toBe("\"\"\""); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("def"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[3][2].value).toBe("foo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[3][3].value).toBe("("); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[3][4].value).toBe(")"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[3][5].value).toBe(":"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("r"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[4][2].value).toBe("\"\"\""); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][3].value).toBe("TE"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][4].value).toBe("\\\""); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][5].value).toBe("\"\"ST"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][6].value).toBe("\"\"\""); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("def"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][2].value).toBe("foo"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[6][3].value).toBe("("); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[6][4].value).toBe(")"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[6][5].value).toBe(":"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("R"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[7][2].value).toBe("\"\"\""); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("TE"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][4].value).toBe("\\\""); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][5].value).toBe("\"\"ST"); + expect(tokens[7][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][6].value).toBe("\"\"\""); + expect(tokens[7][6].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe(""); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[9][0].value).toBe("def"); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[9][1].value).toBe(" "); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[9][2].value).toBe("foo"); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[9][3].value).toBe("("); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[9][4].value).toBe(")"); + expect(tokens[9][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[9][5].value).toBe(":"); + expect(tokens[9][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("u"); + expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[10][2].value).toBe("\"\"\""); + expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][3].value).toBe("TEST"); + expect(tokens[10][3].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[10][4].value).toBe("\"\"\""); + expect(tokens[10][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[11][0].value).toBe(""); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[12][0].value).toBe("def"); + expect(tokens[12][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[12][1].value).toBe(" "); + expect(tokens[12][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[12][2].value).toBe("foo"); + expect(tokens[12][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[12][3].value).toBe("("); + expect(tokens[12][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[12][4].value).toBe(")"); + expect(tokens[12][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[12][5].value).toBe(":"); + expect(tokens[12][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[13][0].value).toBe(" "); + expect(tokens[13][0].scopes).toEqual(["source.python"]); + expect(tokens[13][1].value).toBe("U"); + expect(tokens[13][1].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[13][2].value).toBe("\"\"\""); + expect(tokens[13][2].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[13][3].value).toBe("TEST"); + expect(tokens[13][3].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[13][4].value).toBe("\"\"\""); + expect(tokens[13][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[14][0].value).toBe(""); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[15][0].value).toBe("def"); + expect(tokens[15][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[15][1].value).toBe(" "); + expect(tokens[15][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[15][2].value).toBe("foo"); + expect(tokens[15][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[15][3].value).toBe("("); + expect(tokens[15][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[15][4].value).toBe(")"); + expect(tokens[15][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[15][5].value).toBe(":"); + expect(tokens[15][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[16][0].value).toBe(" "); + expect(tokens[16][0].scopes).toEqual(["source.python"]); + expect(tokens[16][1].value).toBe("b"); + expect(tokens[16][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[16][2].value).toBe("\"\"\""); + expect(tokens[16][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[16][3].value).toBe("TEST"); + expect(tokens[16][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[16][4].value).toBe("\"\"\""); + expect(tokens[16][4].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[17][0].value).toBe(""); + expect(tokens[17][0].scopes).toEqual(["source.python"]); + expect(tokens[18][0].value).toBe("def"); + expect(tokens[18][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[18][1].value).toBe(" "); + expect(tokens[18][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[18][2].value).toBe("foo"); + expect(tokens[18][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[18][3].value).toBe("("); + expect(tokens[18][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[18][4].value).toBe(")"); + expect(tokens[18][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[18][5].value).toBe(":"); + expect(tokens[18][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[19][0].value).toBe(" "); + expect(tokens[19][0].scopes).toEqual(["source.python"]); + expect(tokens[19][1].value).toBe("B"); + expect(tokens[19][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[19][2].value).toBe("\"\"\""); + expect(tokens[19][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[19][3].value).toBe("TEST"); + expect(tokens[19][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[19][4].value).toBe("\"\"\""); + expect(tokens[19][4].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/def3.py", + function() { + tokens = grammar.tokenizeLines("def foo():\n 'TE\\'ST'\n\ndef foo():\n r'TE\\'ST'\n\ndef foo():\n R'TE\\'ST'\n\ndef foo():\n u'TEST'\n\ndef foo():\n U'TEST'\n\ndef foo():\n b'TEST'\n\ndef foo():\n B'TEST'") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("'"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TE"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[1][3].value).toBe("\\'"); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python","constant.character.escape.python"]); + expect(tokens[1][4].value).toBe("ST"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("def"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[3][2].value).toBe("foo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[3][3].value).toBe("("); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[3][4].value).toBe(")"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[3][5].value).toBe(":"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("r"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[4][2].value).toBe("'"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][3].value).toBe("TE"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[4][4].value).toBe("\\'"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[4][5].value).toBe("ST"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[4][6].value).toBe("'"); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("def"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][2].value).toBe("foo"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[6][3].value).toBe("("); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[6][4].value).toBe(")"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[6][5].value).toBe(":"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("R"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[7][2].value).toBe("'"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("TE"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[7][4].value).toBe("\\'"); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[7][5].value).toBe("ST"); + expect(tokens[7][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[7][6].value).toBe("'"); + expect(tokens[7][6].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe(""); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[9][0].value).toBe("def"); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[9][1].value).toBe(" "); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[9][2].value).toBe("foo"); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[9][3].value).toBe("("); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[9][4].value).toBe(")"); + expect(tokens[9][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[9][5].value).toBe(":"); + expect(tokens[9][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("u"); + expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[10][2].value).toBe("'"); + expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][3].value).toBe("TEST"); + expect(tokens[10][3].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[10][4].value).toBe("'"); + expect(tokens[10][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[11][0].value).toBe(""); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[12][0].value).toBe("def"); + expect(tokens[12][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[12][1].value).toBe(" "); + expect(tokens[12][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[12][2].value).toBe("foo"); + expect(tokens[12][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[12][3].value).toBe("("); + expect(tokens[12][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[12][4].value).toBe(")"); + expect(tokens[12][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[12][5].value).toBe(":"); + expect(tokens[12][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[13][0].value).toBe(" "); + expect(tokens[13][0].scopes).toEqual(["source.python"]); + expect(tokens[13][1].value).toBe("U"); + expect(tokens[13][1].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[13][2].value).toBe("'"); + expect(tokens[13][2].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[13][3].value).toBe("TEST"); + expect(tokens[13][3].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[13][4].value).toBe("'"); + expect(tokens[13][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[14][0].value).toBe(""); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[15][0].value).toBe("def"); + expect(tokens[15][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[15][1].value).toBe(" "); + expect(tokens[15][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[15][2].value).toBe("foo"); + expect(tokens[15][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[15][3].value).toBe("("); + expect(tokens[15][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[15][4].value).toBe(")"); + expect(tokens[15][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[15][5].value).toBe(":"); + expect(tokens[15][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[16][0].value).toBe(" "); + expect(tokens[16][0].scopes).toEqual(["source.python"]); + expect(tokens[16][1].value).toBe("b"); + expect(tokens[16][1].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[16][2].value).toBe("'"); + expect(tokens[16][2].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[16][3].value).toBe("TEST"); + expect(tokens[16][3].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[16][4].value).toBe("'"); + expect(tokens[16][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[17][0].value).toBe(""); + expect(tokens[17][0].scopes).toEqual(["source.python"]); + expect(tokens[18][0].value).toBe("def"); + expect(tokens[18][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[18][1].value).toBe(" "); + expect(tokens[18][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[18][2].value).toBe("foo"); + expect(tokens[18][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[18][3].value).toBe("("); + expect(tokens[18][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[18][4].value).toBe(")"); + expect(tokens[18][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[18][5].value).toBe(":"); + expect(tokens[18][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[19][0].value).toBe(" "); + expect(tokens[19][0].scopes).toEqual(["source.python"]); + expect(tokens[19][1].value).toBe("B"); + expect(tokens[19][1].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[19][2].value).toBe("'"); + expect(tokens[19][2].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[19][3].value).toBe("TEST"); + expect(tokens[19][3].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[19][4].value).toBe("'"); + expect(tokens[19][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/def4.py", + function() { + tokens = grammar.tokenizeLines("def foo():\n \"TE\\\"ST\"\n\ndef foo():\n r\"TE\\\"ST\"\n\ndef foo():\n R\"TE\\\"ST\"\n\ndef foo():\n u\"TEST\"\n\ndef foo():\n U\"TEST\"\n\ndef foo():\n b\"TEST\"\n\ndef foo():\n B\"TEST\"") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("\""); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TE"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[1][3].value).toBe("\\\""); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python","constant.character.escape.python"]); + expect(tokens[1][4].value).toBe("ST"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[1][5].value).toBe("\""); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("def"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[3][2].value).toBe("foo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[3][3].value).toBe("("); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[3][4].value).toBe(")"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[3][5].value).toBe(":"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("r"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[4][2].value).toBe("\""); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][3].value).toBe("TE"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[4][4].value).toBe("\\\""); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[4][5].value).toBe("ST"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[4][6].value).toBe("\""); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("def"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][2].value).toBe("foo"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[6][3].value).toBe("("); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[6][4].value).toBe(")"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[6][5].value).toBe(":"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("R"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[7][2].value).toBe("\""); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("TE"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[7][4].value).toBe("\\\""); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[7][5].value).toBe("ST"); + expect(tokens[7][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[7][6].value).toBe("\""); + expect(tokens[7][6].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe(""); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[9][0].value).toBe("def"); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[9][1].value).toBe(" "); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[9][2].value).toBe("foo"); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[9][3].value).toBe("("); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[9][4].value).toBe(")"); + expect(tokens[9][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[9][5].value).toBe(":"); + expect(tokens[9][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("u"); + expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[10][2].value).toBe("\""); + expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][3].value).toBe("TEST"); + expect(tokens[10][3].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[10][4].value).toBe("\""); + expect(tokens[10][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[11][0].value).toBe(""); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[12][0].value).toBe("def"); + expect(tokens[12][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[12][1].value).toBe(" "); + expect(tokens[12][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[12][2].value).toBe("foo"); + expect(tokens[12][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[12][3].value).toBe("("); + expect(tokens[12][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[12][4].value).toBe(")"); + expect(tokens[12][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[12][5].value).toBe(":"); + expect(tokens[12][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[13][0].value).toBe(" "); + expect(tokens[13][0].scopes).toEqual(["source.python"]); + expect(tokens[13][1].value).toBe("U"); + expect(tokens[13][1].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[13][2].value).toBe("\""); + expect(tokens[13][2].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[13][3].value).toBe("TEST"); + expect(tokens[13][3].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[13][4].value).toBe("\""); + expect(tokens[13][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[14][0].value).toBe(""); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[15][0].value).toBe("def"); + expect(tokens[15][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[15][1].value).toBe(" "); + expect(tokens[15][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[15][2].value).toBe("foo"); + expect(tokens[15][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[15][3].value).toBe("("); + expect(tokens[15][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[15][4].value).toBe(")"); + expect(tokens[15][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[15][5].value).toBe(":"); + expect(tokens[15][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[16][0].value).toBe(" "); + expect(tokens[16][0].scopes).toEqual(["source.python"]); + expect(tokens[16][1].value).toBe("b"); + expect(tokens[16][1].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[16][2].value).toBe("\""); + expect(tokens[16][2].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[16][3].value).toBe("TEST"); + expect(tokens[16][3].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[16][4].value).toBe("\""); + expect(tokens[16][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[17][0].value).toBe(""); + expect(tokens[17][0].scopes).toEqual(["source.python"]); + expect(tokens[18][0].value).toBe("def"); + expect(tokens[18][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[18][1].value).toBe(" "); + expect(tokens[18][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[18][2].value).toBe("foo"); + expect(tokens[18][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[18][3].value).toBe("("); + expect(tokens[18][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[18][4].value).toBe(")"); + expect(tokens[18][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[18][5].value).toBe(":"); + expect(tokens[18][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[19][0].value).toBe(" "); + expect(tokens[19][0].scopes).toEqual(["source.python"]); + expect(tokens[19][1].value).toBe("B"); + expect(tokens[19][1].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[19][2].value).toBe("\""); + expect(tokens[19][2].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[19][3].value).toBe("TEST"); + expect(tokens[19][3].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[19][4].value).toBe("\""); + expect(tokens[19][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/escaping1.py", + function() { + tokens = grammar.tokenizeLines("'''Module docstring\n\n {{ %d simple \\\\ string \\\n foo \\' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\n\n multiline \"unicode\" string \\\n \\xf1 \\u1234aaaa \\U1234aaaa\n \\N{BLACK SPADE SUIT}\n'''") + expect(tokens[0][0].value).toBe("'''"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("Module docstring"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[2][0].value).toBe(" {{ %d simple "); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[2][1].value).toBe("\\\\"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[2][2].value).toBe(" string "); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[2][3].value).toBe("\\"); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.language.python"]); + expect(tokens[3][0].value).toBe(" foo "); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][1].value).toBe("\\'"); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][3].value).toBe("\\\""); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][5].value).toBe("\\a"); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][6].value).toBe(" "); + expect(tokens[3][6].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][7].value).toBe("\\b"); + expect(tokens[3][7].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][8].value).toBe(" \\c "); + expect(tokens[3][8].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][9].value).toBe("\\f"); + expect(tokens[3][9].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][10].value).toBe(" "); + expect(tokens[3][10].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][11].value).toBe("\\n"); + expect(tokens[3][11].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][12].value).toBe(" "); + expect(tokens[3][12].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][13].value).toBe("\\r"); + expect(tokens[3][13].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][14].value).toBe(" "); + expect(tokens[3][14].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][15].value).toBe("\\t"); + expect(tokens[3][15].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][16].value).toBe(" "); + expect(tokens[3][16].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][17].value).toBe("\\v"); + expect(tokens[3][17].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][18].value).toBe(" "); + expect(tokens[3][18].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][19].value).toBe("\\5"); + expect(tokens[3][19].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][20].value).toBe(" "); + expect(tokens[3][20].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][21].value).toBe("\\55"); + expect(tokens[3][21].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][22].value).toBe(" "); + expect(tokens[3][22].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][23].value).toBe("\\555"); + expect(tokens[3][23].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][24].value).toBe(" "); + expect(tokens[3][24].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][25].value).toBe("\\05"); + expect(tokens[3][25].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[3][26].value).toBe(" "); + expect(tokens[3][26].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][27].value).toBe("\\005"); + expect(tokens[3][27].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[5][0].value).toBe(" multiline \"unicode\" string "); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[5][1].value).toBe("\\"); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.language.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[6][1].value).toBe("\\xf1"); + expect(tokens[6][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[6][2].value).toBe(" "); + expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[6][3].value).toBe("\\u1234"); + expect(tokens[6][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[6][4].value).toBe("aaaa "); + expect(tokens[6][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[6][5].value).toBe("\\U1234aaaa"); + expect(tokens[6][5].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[7][1].value).toBe("\\N{BLACK SPADE SUIT}"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[8][0].value).toBe("'''"); + expect(tokens[8][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/escaping2.py", + function() { + tokens = grammar.tokenizeLines("r'''Module docstring\n\n {{ %d simple \\\\ string \\\n foo \\' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\n\n multiline \"unicode\" string \\\n \\xf1 \\u1234aaaa \\U1234aaaa\n \\N{BLACK SPADE SUIT}\n >>> aaa\n'''\n\nR'''Module docstring\n\n {{ %d simple \\\\ string \\\n foo \\' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\n\n multiline \"unicode\" string \\\n \\xf1 \\u1234aaaa \\U1234aaaa\n \\N{BLACK SPADE SUIT}\n >>> aaa\n'''") + expect(tokens[0][0].value).toBe("r"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[0][1].value).toBe("'''"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("Module docstring"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[2][0].value).toBe(" {{ %d simple "); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[2][1].value).toBe("\\\\"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[2][2].value).toBe(" string "); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[2][3].value).toBe("\\"); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[3][0].value).toBe(" foo "); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[3][1].value).toBe("\\'"); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[3][3].value).toBe("\\\""); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[3][4].value).toBe(" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[5][0].value).toBe(" multiline \"unicode\" string "); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[5][1].value).toBe("\\"); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[6][0].value).toBe(" \\xf1 \\u1234aaaa \\U1234aaaa"); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][0].value).toBe(" \\N{BLACK SPADE SUIT}"); + expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[8][1].value).toBe(">>> "); + expect(tokens[8][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","keyword.control.flow.python"]); + expect(tokens[8][2].value).toBe("aaa"); + expect(tokens[8][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[9][0].value).toBe("'''"); + expect(tokens[9][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[10][0].value).toBe(""); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[11][0].value).toBe("R"); + expect(tokens[11][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[11][1].value).toBe("'''"); + expect(tokens[11][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[11][2].value).toBe("Module docstring"); + expect(tokens[11][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[12][0].value).toBe(""); + expect(tokens[12][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[13][0].value).toBe(" {{ %d simple "); + expect(tokens[13][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[13][1].value).toBe("\\\\"); + expect(tokens[13][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[13][2].value).toBe(" string "); + expect(tokens[13][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[13][3].value).toBe("\\"); + expect(tokens[13][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[14][0].value).toBe(" foo "); + expect(tokens[14][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[14][1].value).toBe("\\'"); + expect(tokens[14][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[14][2].value).toBe(" "); + expect(tokens[14][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[14][3].value).toBe("\\\""); + expect(tokens[14][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[14][4].value).toBe(" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005"); + expect(tokens[14][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[15][0].value).toBe(""); + expect(tokens[15][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[16][0].value).toBe(" multiline \"unicode\" string "); + expect(tokens[16][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[16][1].value).toBe("\\"); + expect(tokens[16][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[17][0].value).toBe(" \\xf1 \\u1234aaaa \\U1234aaaa"); + expect(tokens[17][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[18][0].value).toBe(" \\N{BLACK SPADE SUIT}"); + expect(tokens[18][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[19][0].value).toBe(" "); + expect(tokens[19][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[19][1].value).toBe(">>> "); + expect(tokens[19][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","keyword.control.flow.python"]); + expect(tokens[19][2].value).toBe("aaa"); + expect(tokens[19][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[20][0].value).toBe("'''"); + expect(tokens[20][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/escaping3.py", + function() { + tokens = grammar.tokenizeLines("r'''Module docstring\n\n (?x) # not a regexp\n foo[20]{42} # not a comment\n'''") + expect(tokens[0][0].value).toBe("r"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[0][1].value).toBe("'''"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("Module docstring"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[2][0].value).toBe(" (?x) # not a regexp"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[3][0].value).toBe(" foo[20]{42} # not a comment"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][0].value).toBe("'''"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/format1.py", + function() { + tokens = grammar.tokenizeLines("r'''Module docstring\n\n %(language)s has %(number)03d quote types.\n'''") + expect(tokens[0][0].value).toBe("r"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[0][1].value).toBe("'''"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("Module docstring"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[2][0].value).toBe(" %(language)s has %(number)03d quote types."); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[3][0].value).toBe("'''"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/format2.py", + function() { + tokens = grammar.tokenizeLines("'''{foo}'''") + expect(tokens[0][0].value).toBe("'''"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("{foo}"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[0][2].value).toBe("'''"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/mix1.py", + function() { + tokens = grammar.tokenizeLines("'''TEST'''\n\nclass Foo:\n # comment\n R'''TEST'''\n\n def foo(self, a:'''TEST''') -> '''TEST''': #ok\n r'''TEST'''\n with bar:\n pass\n\n def bar(self, a:'''TEST''') -> '''TEST''': pass\n '''TEST''' # additional docstring\n with bar:\n pass") + expect(tokens[0][0].value).toBe("'''"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("TEST"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[0][2].value).toBe("'''"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("class"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[2][2].value).toBe("Foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[2][3].value).toBe(":"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("#"); + expect(tokens[3][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][2].value).toBe(" comment"); + expect(tokens[3][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("R"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[4][2].value).toBe("'''"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][3].value).toBe("TEST"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][4].value).toBe("'''"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][1].value).toBe("def"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[6][2].value).toBe(" "); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][3].value).toBe("foo"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[6][4].value).toBe("("); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[6][5].value).toBe("self"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.self.python"]); + expect(tokens[6][6].value).toBe(","); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[6][7].value).toBe(" "); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[6][8].value).toBe("a"); + expect(tokens[6][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[6][9].value).toBe(":"); + expect(tokens[6][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[6][10].value).toBe("'''"); + expect(tokens[6][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][11].value).toBe("TEST"); + expect(tokens[6][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.multi.python"]); + expect(tokens[6][12].value).toBe("'''"); + expect(tokens[6][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[6][13].value).toBe(")"); + expect(tokens[6][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[6][14].value).toBe(" "); + expect(tokens[6][14].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][15].value).toBe("->"); + expect(tokens[6][15].scopes).toEqual(["source.python","meta.function.python","punctuation.separator.annotation.result.python"]); + expect(tokens[6][16].value).toBe(" "); + expect(tokens[6][16].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][17].value).toBe("'''"); + expect(tokens[6][17].scopes).toEqual(["source.python","meta.function.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][18].value).toBe("TEST"); + expect(tokens[6][18].scopes).toEqual(["source.python","meta.function.python","string.quoted.multi.python"]); + expect(tokens[6][19].value).toBe("'''"); + expect(tokens[6][19].scopes).toEqual(["source.python","meta.function.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[6][20].value).toBe(":"); + expect(tokens[6][20].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[6][21].value).toBe(" "); + expect(tokens[6][21].scopes).toEqual(["source.python"]); + expect(tokens[6][22].value).toBe("#"); + expect(tokens[6][22].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[6][23].value).toBe("ok"); + expect(tokens[6][23].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("r"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[7][2].value).toBe("'''"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("TEST"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][4].value).toBe("'''"); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("with"); + expect(tokens[8][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][2].value).toBe(" "); + expect(tokens[8][2].scopes).toEqual(["source.python"]); + expect(tokens[8][3].value).toBe("bar"); + expect(tokens[8][3].scopes).toEqual(["source.python"]); + expect(tokens[8][4].value).toBe(":"); + expect(tokens[8][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("pass"); + expect(tokens[9][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[10][0].value).toBe(""); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[11][0].value).toBe(" "); + expect(tokens[11][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[11][1].value).toBe("def"); + expect(tokens[11][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[11][2].value).toBe(" "); + expect(tokens[11][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[11][3].value).toBe("bar"); + expect(tokens[11][3].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[11][4].value).toBe("("); + expect(tokens[11][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[11][5].value).toBe("self"); + expect(tokens[11][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.self.python"]); + expect(tokens[11][6].value).toBe(","); + expect(tokens[11][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[11][7].value).toBe(" "); + expect(tokens[11][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[11][8].value).toBe("a"); + expect(tokens[11][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[11][9].value).toBe(":"); + expect(tokens[11][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[11][10].value).toBe("'''"); + expect(tokens[11][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[11][11].value).toBe("TEST"); + expect(tokens[11][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.multi.python"]); + expect(tokens[11][12].value).toBe("'''"); + expect(tokens[11][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[11][13].value).toBe(")"); + expect(tokens[11][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[11][14].value).toBe(" "); + expect(tokens[11][14].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[11][15].value).toBe("->"); + expect(tokens[11][15].scopes).toEqual(["source.python","meta.function.python","punctuation.separator.annotation.result.python"]); + expect(tokens[11][16].value).toBe(" "); + expect(tokens[11][16].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[11][17].value).toBe("'''"); + expect(tokens[11][17].scopes).toEqual(["source.python","meta.function.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[11][18].value).toBe("TEST"); + expect(tokens[11][18].scopes).toEqual(["source.python","meta.function.python","string.quoted.multi.python"]); + expect(tokens[11][19].value).toBe("'''"); + expect(tokens[11][19].scopes).toEqual(["source.python","meta.function.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[11][20].value).toBe(":"); + expect(tokens[11][20].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[11][21].value).toBe(" "); + expect(tokens[11][21].scopes).toEqual(["source.python"]); + expect(tokens[11][22].value).toBe("pass"); + expect(tokens[11][22].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[12][0].value).toBe(" "); + expect(tokens[12][0].scopes).toEqual(["source.python"]); + expect(tokens[12][1].value).toBe("'''"); + expect(tokens[12][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[12][2].value).toBe("TEST"); + expect(tokens[12][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[12][3].value).toBe("'''"); + expect(tokens[12][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[12][4].value).toBe(" "); + expect(tokens[12][4].scopes).toEqual(["source.python"]); + expect(tokens[12][5].value).toBe("#"); + expect(tokens[12][5].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[12][6].value).toBe(" additional docstring"); + expect(tokens[12][6].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[13][0].value).toBe(" "); + expect(tokens[13][0].scopes).toEqual(["source.python"]); + expect(tokens[13][1].value).toBe("with"); + expect(tokens[13][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[13][2].value).toBe(" "); + expect(tokens[13][2].scopes).toEqual(["source.python"]); + expect(tokens[13][3].value).toBe("bar"); + expect(tokens[13][3].scopes).toEqual(["source.python"]); + expect(tokens[13][4].value).toBe(":"); + expect(tokens[13][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[14][0].value).toBe(" "); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[14][1].value).toBe("pass"); + expect(tokens[14][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/docstrings/mix2.py", + function() { + tokens = grammar.tokenizeLines("'TEST'\n\nclass Foo:\n # comment\n R'TEST'\n\n def foo(self, a:'TEST') -> 'TEST': #ok\n r'TEST'\n with bar:\n pass\n\n def bar(self, a:'TEST') -> 'TEST': pass\n 'TEST' # additional docstring\n with bar:\n pass") + expect(tokens[0][0].value).toBe("'"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("TEST"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[0][2].value).toBe("'"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("class"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[2][2].value).toBe("Foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[2][3].value).toBe(":"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("#"); + expect(tokens[3][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][2].value).toBe(" comment"); + expect(tokens[3][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("R"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[4][2].value).toBe("'"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][3].value).toBe("TEST"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[4][4].value).toBe("'"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][1].value).toBe("def"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[6][2].value).toBe(" "); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][3].value).toBe("foo"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[6][4].value).toBe("("); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[6][5].value).toBe("self"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.self.python"]); + expect(tokens[6][6].value).toBe(","); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[6][7].value).toBe(" "); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[6][8].value).toBe("a"); + expect(tokens[6][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[6][9].value).toBe(":"); + expect(tokens[6][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[6][10].value).toBe("'"); + expect(tokens[6][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][11].value).toBe("TEST"); + expect(tokens[6][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[6][12].value).toBe("'"); + expect(tokens[6][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[6][13].value).toBe(")"); + expect(tokens[6][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[6][14].value).toBe(" "); + expect(tokens[6][14].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][15].value).toBe("->"); + expect(tokens[6][15].scopes).toEqual(["source.python","meta.function.python","punctuation.separator.annotation.result.python"]); + expect(tokens[6][16].value).toBe(" "); + expect(tokens[6][16].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][17].value).toBe("'"); + expect(tokens[6][17].scopes).toEqual(["source.python","meta.function.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][18].value).toBe("TEST"); + expect(tokens[6][18].scopes).toEqual(["source.python","meta.function.python","string.quoted.single.python"]); + expect(tokens[6][19].value).toBe("'"); + expect(tokens[6][19].scopes).toEqual(["source.python","meta.function.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[6][20].value).toBe(":"); + expect(tokens[6][20].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[6][21].value).toBe(" "); + expect(tokens[6][21].scopes).toEqual(["source.python"]); + expect(tokens[6][22].value).toBe("#"); + expect(tokens[6][22].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[6][23].value).toBe("ok"); + expect(tokens[6][23].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("r"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[7][2].value).toBe("'"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("TEST"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[7][4].value).toBe("'"); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("with"); + expect(tokens[8][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][2].value).toBe(" "); + expect(tokens[8][2].scopes).toEqual(["source.python"]); + expect(tokens[8][3].value).toBe("bar"); + expect(tokens[8][3].scopes).toEqual(["source.python"]); + expect(tokens[8][4].value).toBe(":"); + expect(tokens[8][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("pass"); + expect(tokens[9][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[10][0].value).toBe(""); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[11][0].value).toBe(" "); + expect(tokens[11][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[11][1].value).toBe("def"); + expect(tokens[11][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[11][2].value).toBe(" "); + expect(tokens[11][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[11][3].value).toBe("bar"); + expect(tokens[11][3].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[11][4].value).toBe("("); + expect(tokens[11][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[11][5].value).toBe("self"); + expect(tokens[11][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.self.python"]); + expect(tokens[11][6].value).toBe(","); + expect(tokens[11][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[11][7].value).toBe(" "); + expect(tokens[11][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[11][8].value).toBe("a"); + expect(tokens[11][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[11][9].value).toBe(":"); + expect(tokens[11][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[11][10].value).toBe("'"); + expect(tokens[11][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[11][11].value).toBe("TEST"); + expect(tokens[11][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[11][12].value).toBe("'"); + expect(tokens[11][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[11][13].value).toBe(")"); + expect(tokens[11][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[11][14].value).toBe(" "); + expect(tokens[11][14].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[11][15].value).toBe("->"); + expect(tokens[11][15].scopes).toEqual(["source.python","meta.function.python","punctuation.separator.annotation.result.python"]); + expect(tokens[11][16].value).toBe(" "); + expect(tokens[11][16].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[11][17].value).toBe("'"); + expect(tokens[11][17].scopes).toEqual(["source.python","meta.function.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[11][18].value).toBe("TEST"); + expect(tokens[11][18].scopes).toEqual(["source.python","meta.function.python","string.quoted.single.python"]); + expect(tokens[11][19].value).toBe("'"); + expect(tokens[11][19].scopes).toEqual(["source.python","meta.function.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[11][20].value).toBe(":"); + expect(tokens[11][20].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[11][21].value).toBe(" "); + expect(tokens[11][21].scopes).toEqual(["source.python"]); + expect(tokens[11][22].value).toBe("pass"); + expect(tokens[11][22].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[12][0].value).toBe(" "); + expect(tokens[12][0].scopes).toEqual(["source.python"]); + expect(tokens[12][1].value).toBe("'"); + expect(tokens[12][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[12][2].value).toBe("TEST"); + expect(tokens[12][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[12][3].value).toBe("'"); + expect(tokens[12][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[12][4].value).toBe(" "); + expect(tokens[12][4].scopes).toEqual(["source.python"]); + expect(tokens[12][5].value).toBe("#"); + expect(tokens[12][5].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[12][6].value).toBe(" additional docstring"); + expect(tokens[12][6].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[13][0].value).toBe(" "); + expect(tokens[13][0].scopes).toEqual(["source.python"]); + expect(tokens[13][1].value).toBe("with"); + expect(tokens[13][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[13][2].value).toBe(" "); + expect(tokens[13][2].scopes).toEqual(["source.python"]); + expect(tokens[13][3].value).toBe("bar"); + expect(tokens[13][3].scopes).toEqual(["source.python"]); + expect(tokens[13][4].value).toBe(":"); + expect(tokens[13][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[14][0].value).toBe(" "); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[14][1].value).toBe("pass"); + expect(tokens[14][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/docstrings/module1.py", + function() { + tokens = grammar.tokenizeLines("'''TEST'''\nr'''TEST'''\nR'''TEST'''\n\nu'''TEST'''\nU'''TEST'''\nb'''TEST'''\nB'''TEST'''") + expect(tokens[0][0].value).toBe("'''"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("TEST"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[0][2].value).toBe("'''"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("r"); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[1][1].value).toBe("'''"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TEST"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[1][3].value).toBe("'''"); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("R"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[2][1].value).toBe("'''"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("TEST"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[2][3].value).toBe("'''"); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe(""); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe("u"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[4][1].value).toBe("'''"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][2].value).toBe("TEST"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[4][3].value).toBe("'''"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("U"); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[5][1].value).toBe("'''"); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][2].value).toBe("TEST"); + expect(tokens[5][2].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[5][3].value).toBe("'''"); + expect(tokens[5][3].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe("b"); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[6][1].value).toBe("'''"); + expect(tokens[6][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][2].value).toBe("TEST"); + expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[6][3].value).toBe("'''"); + expect(tokens[6][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[7][0].value).toBe("B"); + expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[7][1].value).toBe("'''"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][2].value).toBe("TEST"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[7][3].value).toBe("'''"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/module2.py", + function() { + tokens = grammar.tokenizeLines("\"\"\"TEST\"\"\"\nr\"\"\"TEST\"\"\"\nR\"\"\"TEST\"\"\"\n\nu\"\"\"TEST\"\"\"\nU\"\"\"TEST\"\"\"\nb\"\"\"TEST\"\"\"\nB\"\"\"TEST\"\"\"") + expect(tokens[0][0].value).toBe("\"\"\""); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("TEST"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[0][2].value).toBe("\"\"\""); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("r"); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[1][1].value).toBe("\"\"\""); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TEST"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[1][3].value).toBe("\"\"\""); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("R"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[2][1].value).toBe("\"\"\""); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("TEST"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[2][3].value).toBe("\"\"\""); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe(""); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe("u"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[4][1].value).toBe("\"\"\""); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][2].value).toBe("TEST"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[4][3].value).toBe("\"\"\""); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("U"); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.multi.python","storage.type.string.python"]); + expect(tokens[5][1].value).toBe("\"\"\""); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][2].value).toBe("TEST"); + expect(tokens[5][2].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[5][3].value).toBe("\"\"\""); + expect(tokens[5][3].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe("b"); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[6][1].value).toBe("\"\"\""); + expect(tokens[6][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][2].value).toBe("TEST"); + expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[6][3].value).toBe("\"\"\""); + expect(tokens[6][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[7][0].value).toBe("B"); + expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[7][1].value).toBe("\"\"\""); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][2].value).toBe("TEST"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[7][3].value).toBe("\"\"\""); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/module3.py", + function() { + tokens = grammar.tokenizeLines("'TE\\'ST'\nr'TE\\'ST'\nR'TE\\'ST'\n\nu'TEST'\nU'TEST'\nb'TEST'\nB'TEST'") + expect(tokens[0][0].value).toBe("'"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("TE"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[0][2].value).toBe("\\'"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","constant.character.escape.python"]); + expect(tokens[0][3].value).toBe("ST"); + expect(tokens[0][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("r"); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[1][1].value).toBe("'"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TE"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[1][3].value).toBe("\\'"); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[1][4].value).toBe("ST"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("R"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[2][1].value).toBe("'"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("TE"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[2][3].value).toBe("\\'"); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[2][4].value).toBe("ST"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe(""); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe("u"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[4][1].value).toBe("'"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][2].value).toBe("TEST"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][3].value).toBe("'"); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("U"); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[5][1].value).toBe("'"); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][2].value).toBe("TEST"); + expect(tokens[5][2].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[5][3].value).toBe("'"); + expect(tokens[5][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe("b"); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[6][1].value).toBe("'"); + expect(tokens[6][1].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][2].value).toBe("TEST"); + expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[6][3].value).toBe("'"); + expect(tokens[6][3].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[7][0].value).toBe("B"); + expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[7][1].value).toBe("'"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][2].value).toBe("TEST"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[7][3].value).toBe("'"); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/module4.py", + function() { + tokens = grammar.tokenizeLines("\"TE\\\"ST\"\nr\"TE\\\"ST\"\nR\"TE\\\"ST\"\n\nu\"TEST\"\nU\"TEST\"\nb\"TEST\"\nB\"TEST\"") + expect(tokens[0][0].value).toBe("\""); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("TE"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[0][2].value).toBe("\\\""); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","constant.character.escape.python"]); + expect(tokens[0][3].value).toBe("ST"); + expect(tokens[0][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[0][4].value).toBe("\""); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("r"); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[1][1].value).toBe("\""); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("TE"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[1][3].value).toBe("\\\""); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[1][4].value).toBe("ST"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[1][5].value).toBe("\""); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("R"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","storage.type.string.python"]); + expect(tokens[2][1].value).toBe("\""); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("TE"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[2][3].value).toBe("\\\""); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[2][4].value).toBe("ST"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python"]); + expect(tokens[2][5].value).toBe("\""); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.docstring.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe(""); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe("u"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[4][1].value).toBe("\""); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][2].value).toBe("TEST"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][3].value).toBe("\""); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("U"); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[5][1].value).toBe("\""); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][2].value).toBe("TEST"); + expect(tokens[5][2].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[5][3].value).toBe("\""); + expect(tokens[5][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe("b"); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[6][1].value).toBe("\""); + expect(tokens[6][1].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][2].value).toBe("TEST"); + expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[6][3].value).toBe("\""); + expect(tokens[6][3].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[7][0].value).toBe("B"); + expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[7][1].value).toBe("\""); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][2].value).toBe("TEST"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[7][3].value).toBe("\""); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/oneline1.py", + function() { + tokens = grammar.tokenizeLines("'''>>> print(\"\"\"docstring\"\"\")'''\nasync\n\"\"\">>> print('''docstring''')\"\"\"\nawait\n\"\"\"\\n>>> print('''docstring''')\"\"\"\nawait\n\"\"\" >>> print('''docstring''')\"\"\"\nawait\n\"\"\" 1 >>> print('''docstring''')\"\"\"\nawait") + expect(tokens[0][0].value).toBe("'''"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe(">>> "); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","keyword.control.flow.python"]); + expect(tokens[0][2].value).toBe("print(\"\"\"docstring\"\"\")"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[0][3].value).toBe("'''"); + expect(tokens[0][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("async"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("\"\"\""); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][1].value).toBe(">>> "); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","keyword.control.flow.python"]); + expect(tokens[2][2].value).toBe("print('''docstring''')"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[2][3].value).toBe("\"\"\""); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("await"); + expect(tokens[3][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][0].value).toBe("\"\"\""); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][1].value).toBe("\\n"); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","constant.character.escape.python"]); + expect(tokens[4][2].value).toBe(">>> print('''docstring''')"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[4][3].value).toBe("\"\"\""); + expect(tokens[4][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("await"); + expect(tokens[5][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[6][0].value).toBe("\"\"\""); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[6][2].value).toBe(">>> "); + expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","keyword.control.flow.python"]); + expect(tokens[6][3].value).toBe("print('''docstring''')"); + expect(tokens[6][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[6][4].value).toBe("\"\"\""); + expect(tokens[6][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[7][0].value).toBe("await"); + expect(tokens[7][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][0].value).toBe("\"\"\""); + expect(tokens[8][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[8][1].value).toBe(" 1 >>> print('''docstring''')"); + expect(tokens[8][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[8][2].value).toBe("\"\"\""); + expect(tokens[8][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[9][0].value).toBe("await"); + expect(tokens[9][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/docstrings/oneline2.py", + function() { + tokens = grammar.tokenizeLines("def foo():\n '''>>> print(\"\"\"docstring\"\"\")'''\ndef foo():\n \"\"\">>> print('''docstring''')\"\"\"") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("'''"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe(">>> "); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","keyword.control.flow.python"]); + expect(tokens[1][3].value).toBe("print(\"\"\"docstring\"\"\")"); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[1][4].value).toBe("'''"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][5].value).toBe(":"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("\"\"\""); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe(">>> "); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","keyword.control.flow.python"]); + expect(tokens[3][3].value).toBe("print('''docstring''')"); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][4].value).toBe("\"\"\""); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/oneline3.py", + function() { + tokens = grammar.tokenizeLines("def foo():\n '>>> print(\"docstring\")'\ndef foo():\n \">>> print('docstring')\"") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("'"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe(">>> print(\"docstring\")"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[1][3].value).toBe("'"); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][5].value).toBe(":"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("\""); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe(">>> print('docstring')"); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[3][3].value).toBe("\""); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/prompt1.py", + function() { + tokens = grammar.tokenizeLines("r'''Module docstring\n\n Some text followed by code sample:\n >>> for a in foo(2, b=1,\n ... c=3):\n ... print(a)\n 0\n 1\n'''") + expect(tokens[0][0].value).toBe("r"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","storage.type.string.python"]); + expect(tokens[0][1].value).toBe("'''"); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("Module docstring"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[2][0].value).toBe(" Some text followed by code sample:"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[3][1].value).toBe(">>> "); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe("for a in foo(2, b=1,"); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[4][1].value).toBe("... "); + expect(tokens[4][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","keyword.control.flow.python"]); + expect(tokens[4][2].value).toBe(" c=3):"); + expect(tokens[4][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[5][1].value).toBe("... "); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","keyword.control.flow.python"]); + expect(tokens[5][2].value).toBe(" print(a)"); + expect(tokens[5][2].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[6][0].value).toBe(" 0"); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[7][0].value).toBe(" 1"); + expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python"]); + expect(tokens[8][0].value).toBe("'''"); + expect(tokens[8][0].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/prompt2.py", + function() { + tokens = grammar.tokenizeLines("\"\"\"\n def foo():\n ...\n >>>\n\"\"\"") + expect(tokens[0][0].value).toBe("\"\"\""); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][0].value).toBe(" def foo():"); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[2][0].value).toBe(" ..."); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[3][0].value).toBe(" >>>"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[4][0].value).toBe("\"\"\""); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/docstrings/prompt3.py", + function() { + tokens = grammar.tokenizeLines("'''...'''\n...") + expect(tokens[0][0].value).toBe("'''"); + expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][1].value).toBe("..."); + expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[0][2].value).toBe("'''"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("..."); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + }); + + it("test/docstrings/regr1.py", + function() { + tokens = grammar.tokenizeLines("#:\n @asd\n def foo():\n pass") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(":"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[1][1].value).toBe("@"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[1][2].value).toBe("asd"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][1].value).toBe("def"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][3].value).toBe("foo"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][4].value).toBe("("); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][5].value).toBe(")"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][6].value).toBe(":"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("pass"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/expressions/const1.py", + function() { + tokens = grammar.tokenizeLines("QQQQ QQQQ_123 QQQQ123 PROTOCOL_v2 QQQ.bar baz.AA_a _AAA foo._AAA\nQQQq QQQq123 self.FOOO() _ _1 __1 _1A __1A _a __a __ ___ ___a ___1 __aA ___Aa") + expect(tokens[0][0].value).toBe("QQQQ"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("QQQQ_123"); + expect(tokens[0][2].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("QQQQ123"); + expect(tokens[0][4].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("PROTOCOL_v2"); + expect(tokens[0][6].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("QQQ"); + expect(tokens[0][8].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[0][9].value).toBe("."); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][10].value).toBe("bar"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe("baz"); + expect(tokens[0][12].scopes).toEqual(["source.python"]); + expect(tokens[0][13].value).toBe("."); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][14].value).toBe("AA_a"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.member.access.python","constant.other.caps.python"]); + expect(tokens[0][15].value).toBe(" "); + expect(tokens[0][15].scopes).toEqual(["source.python"]); + expect(tokens[0][16].value).toBe("_AAA"); + expect(tokens[0][16].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[0][17].value).toBe(" "); + expect(tokens[0][17].scopes).toEqual(["source.python"]); + expect(tokens[0][18].value).toBe("foo"); + expect(tokens[0][18].scopes).toEqual(["source.python"]); + expect(tokens[0][19].value).toBe("."); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][20].value).toBe("_AAA"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.member.access.python","constant.other.caps.python"]); + expect(tokens[1][0].value).toBe("QQQq"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("QQQq123"); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("self"); + expect(tokens[1][4].scopes).toEqual(["source.python","variable.language.special.self.python"]); + expect(tokens[1][5].value).toBe("."); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[1][6].value).toBe("FOOO"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][7].value).toBe("("); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][8].value).toBe(")"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][9].value).toBe(" "); + expect(tokens[1][9].scopes).toEqual(["source.python"]); + expect(tokens[1][10].value).toBe("_"); + expect(tokens[1][10].scopes).toEqual(["source.python"]); + expect(tokens[1][11].value).toBe(" "); + expect(tokens[1][11].scopes).toEqual(["source.python"]); + expect(tokens[1][12].value).toBe("_1"); + expect(tokens[1][12].scopes).toEqual(["source.python"]); + expect(tokens[1][13].value).toBe(" "); + expect(tokens[1][13].scopes).toEqual(["source.python"]); + expect(tokens[1][14].value).toBe("__1"); + expect(tokens[1][14].scopes).toEqual(["source.python"]); + expect(tokens[1][15].value).toBe(" "); + expect(tokens[1][15].scopes).toEqual(["source.python"]); + expect(tokens[1][16].value).toBe("_1A"); + expect(tokens[1][16].scopes).toEqual(["source.python"]); + expect(tokens[1][17].value).toBe(" "); + expect(tokens[1][17].scopes).toEqual(["source.python"]); + expect(tokens[1][18].value).toBe("__1A"); + expect(tokens[1][18].scopes).toEqual(["source.python"]); + expect(tokens[1][19].value).toBe(" "); + expect(tokens[1][19].scopes).toEqual(["source.python"]); + expect(tokens[1][20].value).toBe("_a"); + expect(tokens[1][20].scopes).toEqual(["source.python"]); + expect(tokens[1][21].value).toBe(" "); + expect(tokens[1][21].scopes).toEqual(["source.python"]); + expect(tokens[1][22].value).toBe("__a"); + expect(tokens[1][22].scopes).toEqual(["source.python"]); + expect(tokens[1][23].value).toBe(" "); + expect(tokens[1][23].scopes).toEqual(["source.python"]); + expect(tokens[1][24].value).toBe("__"); + expect(tokens[1][24].scopes).toEqual(["source.python"]); + expect(tokens[1][25].value).toBe(" "); + expect(tokens[1][25].scopes).toEqual(["source.python"]); + expect(tokens[1][26].value).toBe("___"); + expect(tokens[1][26].scopes).toEqual(["source.python"]); + expect(tokens[1][27].value).toBe(" "); + expect(tokens[1][27].scopes).toEqual(["source.python"]); + expect(tokens[1][28].value).toBe("___a"); + expect(tokens[1][28].scopes).toEqual(["source.python"]); + expect(tokens[1][29].value).toBe(" "); + expect(tokens[1][29].scopes).toEqual(["source.python"]); + expect(tokens[1][30].value).toBe("___1"); + expect(tokens[1][30].scopes).toEqual(["source.python"]); + expect(tokens[1][31].value).toBe(" "); + expect(tokens[1][31].scopes).toEqual(["source.python"]); + expect(tokens[1][32].value).toBe("__aA"); + expect(tokens[1][32].scopes).toEqual(["source.python"]); + expect(tokens[1][33].value).toBe(" "); + expect(tokens[1][33].scopes).toEqual(["source.python"]); + expect(tokens[1][34].value).toBe("___Aa"); + expect(tokens[1][34].scopes).toEqual(["source.python"]); + }); + + it("test/expressions/const2.py", + function() { + tokens = grammar.tokenizeLines("_AA __AB ___AA\n_A __A ___A A1 A_1 _A_1 A_foo") + expect(tokens[0][0].value).toBe("_AA"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("__AB"); + expect(tokens[0][2].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("___AA"); + expect(tokens[0][4].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[1][0].value).toBe("_A"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("__A"); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("___A"); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("A1"); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python"]); + expect(tokens[1][8].value).toBe("A_1"); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe(" "); + expect(tokens[1][9].scopes).toEqual(["source.python"]); + expect(tokens[1][10].value).toBe("_A_1"); + expect(tokens[1][10].scopes).toEqual(["source.python"]); + expect(tokens[1][11].value).toBe(" "); + expect(tokens[1][11].scopes).toEqual(["source.python"]); + expect(tokens[1][12].value).toBe("A_foo"); + expect(tokens[1][12].scopes).toEqual(["source.python"]); + }); + + it("test/expressions/const3.py", + function() { + tokens = grammar.tokenizeLines("T61STRING = 20\nT61_STRING\nT_STRING\n_T_S_T_R_I_N_G_\nA_CLASS\n\n# not enough upper-case letters in the beginning\n_T_s_TRING\nA_Class") + expect(tokens[0][0].value).toBe("T61STRING"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("20"); + expect(tokens[0][4].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][0].value).toBe("T61_STRING"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[2][0].value).toBe("T_STRING"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[3][0].value).toBe("_T_S_T_R_I_N_G_"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[4][0].value).toBe("A_CLASS"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("#"); + expect(tokens[6][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[6][1].value).toBe(" not enough upper-case letters in the beginning"); + expect(tokens[6][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[7][0].value).toBe("_T_s_TRING"); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[8][0].value).toBe("A_Class"); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + }); + + it("test/expressions/expr1.py", + function() { + tokens = grammar.tokenizeLines("~a + b @ c ^ d // e % f & e and not g or h") + expect(tokens[0][0].value).toBe("~"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[0][1].value).toBe("a"); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe(" "); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe("+"); + expect(tokens[0][3].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("b"); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("@"); + expect(tokens[0][7].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("c"); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("^"); + expect(tokens[0][11].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python"]); + expect(tokens[0][13].value).toBe("d"); + expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[0][14].value).toBe(" "); + expect(tokens[0][14].scopes).toEqual(["source.python"]); + expect(tokens[0][15].value).toBe("//"); + expect(tokens[0][15].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][16].value).toBe(" "); + expect(tokens[0][16].scopes).toEqual(["source.python"]); + expect(tokens[0][17].value).toBe("e"); + expect(tokens[0][17].scopes).toEqual(["source.python"]); + expect(tokens[0][18].value).toBe(" "); + expect(tokens[0][18].scopes).toEqual(["source.python"]); + expect(tokens[0][19].value).toBe("%"); + expect(tokens[0][19].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][20].value).toBe(" "); + expect(tokens[0][20].scopes).toEqual(["source.python"]); + expect(tokens[0][21].value).toBe("f"); + expect(tokens[0][21].scopes).toEqual(["source.python"]); + expect(tokens[0][22].value).toBe(" "); + expect(tokens[0][22].scopes).toEqual(["source.python"]); + expect(tokens[0][23].value).toBe("&"); + expect(tokens[0][23].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[0][24].value).toBe(" "); + expect(tokens[0][24].scopes).toEqual(["source.python"]); + expect(tokens[0][25].value).toBe("e"); + expect(tokens[0][25].scopes).toEqual(["source.python"]); + expect(tokens[0][26].value).toBe(" "); + expect(tokens[0][26].scopes).toEqual(["source.python"]); + expect(tokens[0][27].value).toBe("and"); + expect(tokens[0][27].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[0][28].value).toBe(" "); + expect(tokens[0][28].scopes).toEqual(["source.python"]); + expect(tokens[0][29].value).toBe("not"); + expect(tokens[0][29].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[0][30].value).toBe(" "); + expect(tokens[0][30].scopes).toEqual(["source.python"]); + expect(tokens[0][31].value).toBe("g"); + expect(tokens[0][31].scopes).toEqual(["source.python"]); + expect(tokens[0][32].value).toBe(" "); + expect(tokens[0][32].scopes).toEqual(["source.python"]); + expect(tokens[0][33].value).toBe("or"); + expect(tokens[0][33].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[0][34].value).toBe(" "); + expect(tokens[0][34].scopes).toEqual(["source.python"]); + expect(tokens[0][35].value).toBe("h"); + expect(tokens[0][35].scopes).toEqual(["source.python"]); + }); + + it("test/expressions/expr10.py", + function() { + tokens = grammar.tokenizeLines("a = ...\na(..., c=...)\na = ((...), ...)\n....__class__") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("..."); + expect(tokens[0][4].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][1].value).toBe("("); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][2].value).toBe("..."); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.other.ellipsis.python"]); + expect(tokens[1][3].value).toBe(","); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][5].value).toBe("c"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[1][6].value).toBe("="); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[1][7].value).toBe("..."); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.other.ellipsis.python"]); + expect(tokens[1][8].value).toBe(")"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("("); + expect(tokens[2][4].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[2][5].value).toBe("("); + expect(tokens[2][5].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[2][6].value).toBe("..."); + expect(tokens[2][6].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[2][7].value).toBe(")"); + expect(tokens[2][7].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[2][8].value).toBe(","); + expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("..."); + expect(tokens[2][10].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[2][11].value).toBe(")"); + expect(tokens[2][11].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[3][0].value).toBe("..."); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[3][1].value).toBe("."); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[3][2].value).toBe("__class__"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.member.access.python","support.variable.magic.python"]); + }); + + it("test/expressions/expr11.py", + function() { + tokens = grammar.tokenizeLines("a = self.some_list[1:2]") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("self"); + expect(tokens[0][4].scopes).toEqual(["source.python","variable.language.special.self.python"]); + expect(tokens[0][5].value).toBe("."); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][6].value).toBe("some_list"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.member.access.python","meta.item-access.python","meta.indexed-name.python"]); + expect(tokens[0][7].value).toBe("["); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.member.access.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][8].value).toBe("1"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.member.access.python","meta.item-access.python","meta.item-access.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][9].value).toBe(":"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.member.access.python","meta.item-access.python","meta.item-access.arguments.python","punctuation.separator.slice.python"]); + expect(tokens[0][10].value).toBe("2"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.member.access.python","meta.item-access.python","meta.item-access.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][11].value).toBe("]"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.member.access.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/expressions/expr12.py", + function() { + tokens = grammar.tokenizeLines("print []\nprint {}\nprint 1") + expect(tokens[0][0].value).toBe("print"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.item-access.python","support.function.builtin.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.item-access.python"]); + expect(tokens[0][2].value).toBe("["); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][3].value).toBe("]"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][0].value).toBe("print"); + expect(tokens[1][0].scopes).toEqual(["source.python","support.function.builtin.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("{"); + expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]); + expect(tokens[1][3].value).toBe("}"); + expect(tokens[1][3].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]); + expect(tokens[2][0].value).toBe("print"); + expect(tokens[2][0].scopes).toEqual(["source.python","support.function.builtin.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("1"); + expect(tokens[2][2].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/expressions/expr13.py", + function() { + tokens = grammar.tokenizeLines("super()") + expect(tokens[0][0].value).toBe("super"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function-call.python","support.type.python"]); + expect(tokens[0][1].value).toBe("("); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][2].value).toBe(")"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/expressions/expr14.py", + function() { + tokens = grammar.tokenizeLines("a <> b") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("<"); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[0][3].value).toBe(">"); + expect(tokens[0][3].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("b"); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + }); + + it("test/expressions/expr15.py", + function() { + tokens = grammar.tokenizeLines("foofrom.something") + expect(tokens[0][0].value).toBe("foofrom"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe("."); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("something"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + }); + + it("test/expressions/expr16.py", + function() { + tokens = grammar.tokenizeLines("foo(djsgfjs==123)") + expect(tokens[0][0].value).toBe("foo"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][1].value).toBe("("); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][2].value).toBe("djsgfjs"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[0][3].value).toBe("=="); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.comparison.python"]); + expect(tokens[0][4].value).toBe("123"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][5].value).toBe(")"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/expressions/expr17.py", + function() { + tokens = grammar.tokenizeLines("a++\nb--\n++a\n--b\na++c\nc--b\na(--a)") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe("++"); + expect(tokens[0][1].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[1][0].value).toBe("b"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("--"); + expect(tokens[1][1].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[2][0].value).toBe("++"); + expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[2][1].value).toBe("a"); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("--"); + expect(tokens[3][0].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[3][1].value).toBe("b"); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("++"); + expect(tokens[4][1].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[4][2].value).toBe("c"); + expect(tokens[4][2].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe("c"); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("--"); + expect(tokens[5][1].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[5][2].value).toBe("b"); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("a"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[6][1].value).toBe("("); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[6][2].value).toBe("--"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","invalid.illegal.operator.python"]); + expect(tokens[6][3].value).toBe("a"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[6][4].value).toBe(")"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/expressions/expr18.py", + function() { + tokens = grammar.tokenizeLines("a.Exception\nException.a") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe("."); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("Exception"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + expect(tokens[1][0].value).toBe("Exception"); + expect(tokens[1][0].scopes).toEqual(["source.python","support.type.exception.python"]); + expect(tokens[1][1].value).toBe("."); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[1][2].value).toBe("a"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.member.access.python","meta.attribute.python"]); + }); + + it("test/expressions/expr19.py", + function() { + tokens = grammar.tokenizeLines("a. #foo\na.\n#foo\na. \\\n#foo\na. 'bar'\na.\n'bar'\na. \\\n'bar'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe("."); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe(" "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[0][3].value).toBe("#"); + expect(tokens[0][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][4].value).toBe("foo"); + expect(tokens[0][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("."); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[1][2].value).toBe(""); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[2][0].value).toBe("#"); + expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][1].value).toBe("foo"); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("."); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[3][3].value).toBe("\\"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.continuation.line.python"]); + expect(tokens[3][4].value).toBe(""); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[4][0].value).toBe("#"); + expect(tokens[4][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[4][1].value).toBe("foo"); + expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[5][0].value).toBe("a"); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("."); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[5][3].value).toBe("'"); + expect(tokens[5][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][4].value).toBe("bar"); + expect(tokens[5][4].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[5][5].value).toBe("'"); + expect(tokens[5][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe("a"); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("."); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[6][2].value).toBe(""); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[7][0].value).toBe("'"); + expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][1].value).toBe("bar"); + expect(tokens[7][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[7][2].value).toBe("'"); + expect(tokens[7][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe("a"); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("."); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[8][2].value).toBe(" "); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[8][3].value).toBe("\\"); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.continuation.line.python"]); + expect(tokens[8][4].value).toBe(""); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.member.access.python"]); + expect(tokens[9][0].value).toBe("'"); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.member.access.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[9][1].value).toBe("bar"); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.member.access.python","string.quoted.single.python"]); + expect(tokens[9][2].value).toBe("'"); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.member.access.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/expressions/expr2.py", + function() { + tokens = grammar.tokenizeLines("a @= b\na -= c\na ^= d") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("@="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("b"); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("-="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("c"); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("^="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("d"); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + }); + + it("test/expressions/expr20.py", + function() { + tokens = grammar.tokenizeLines("a == b\na != b\na < b\na <= b\na > b\na >= b") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("=="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("b"); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("!="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("b"); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("<"); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("b"); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("<="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("b"); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe(">"); + expect(tokens[4][2].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe("b"); + expect(tokens[4][4].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe("a"); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe(">="); + expect(tokens[5][2].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[5][4].value).toBe("b"); + expect(tokens[5][4].scopes).toEqual(["source.python"]); + }); + + it("test/expressions/expr21.py", + function() { + tokens = grammar.tokenizeLines("while chunk := file.read(8192):\n process(chunk)\n y0 = (y1 := f(x))") + expect(tokens[0][0].value).toBe("while"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("chunk"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe(":="); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("file"); + expect(tokens[0][6].scopes).toEqual(["source.python","variable.legacy.builtin.python"]); + expect(tokens[0][7].value).toBe("."); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][8].value).toBe("read"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][9].value).toBe("("); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][10].value).toBe("8192"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][11].value).toBe(")"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][12].value).toBe(":"); + expect(tokens[0][12].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("process"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][2].value).toBe("("); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][3].value).toBe("chunk"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("y0"); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("="); + expect(tokens[2][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("("); + expect(tokens[2][5].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[2][6].value).toBe("y1"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe(":="); + expect(tokens[2][8].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("f"); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[2][11].value).toBe("("); + expect(tokens[2][11].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][12].value).toBe("x"); + expect(tokens[2][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[2][13].value).toBe(")"); + expect(tokens[2][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[2][14].value).toBe(")"); + expect(tokens[2][14].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + }); + + it("test/expressions/expr22.py", + function() { + tokens = grammar.tokenizeLines("match = 13\ncase = 12\nif case == 4:\n return match * 5") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("13"); + expect(tokens[0][4].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][0].value).toBe("case"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("12"); + expect(tokens[1][4].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][0].value).toBe("if"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("case"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("=="); + expect(tokens[2][4].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("4"); + expect(tokens[2][6].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][7].value).toBe(":"); + expect(tokens[2][7].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("return"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("match"); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[3][5].value).toBe("*"); + expect(tokens[3][5].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[3][6].value).toBe(" "); + expect(tokens[3][6].scopes).toEqual(["source.python"]); + expect(tokens[3][7].value).toBe("5"); + expect(tokens[3][7].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/expressions/expr3.py", + function() { + tokens = grammar.tokenizeLines("(a, *rest, b) = range(5)") + expect(tokens[0][0].value).toBe("("); + expect(tokens[0][0].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][1].value).toBe("a"); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe(","); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("*"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][5].value).toBe("rest"); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe(","); + expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("b"); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe(")"); + expect(tokens[0][9].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("="); + expect(tokens[0][11].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python"]); + expect(tokens[0][13].value).toBe("range"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[0][14].value).toBe("("); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][15].value).toBe("5"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][16].value).toBe(")"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/expressions/expr4.py", + function() { + tokens = grammar.tokenizeLines("arr2 = [i for i in range(7) if i != 3]") + expect(tokens[0][0].value).toBe("arr2"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("["); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[0][5].value).toBe("i"); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("for"); + expect(tokens[0][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("i"); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("in"); + expect(tokens[0][11].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python"]); + expect(tokens[0][13].value).toBe("range"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[0][14].value).toBe("("); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][15].value).toBe("7"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][16].value).toBe(")"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][17].value).toBe(" "); + expect(tokens[0][17].scopes).toEqual(["source.python"]); + expect(tokens[0][18].value).toBe("if"); + expect(tokens[0][18].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][19].value).toBe(" "); + expect(tokens[0][19].scopes).toEqual(["source.python"]); + expect(tokens[0][20].value).toBe("i"); + expect(tokens[0][20].scopes).toEqual(["source.python"]); + expect(tokens[0][21].value).toBe(" "); + expect(tokens[0][21].scopes).toEqual(["source.python"]); + expect(tokens[0][22].value).toBe("!="); + expect(tokens[0][22].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[0][23].value).toBe(" "); + expect(tokens[0][23].scopes).toEqual(["source.python"]); + expect(tokens[0][24].value).toBe("3"); + expect(tokens[0][24].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][25].value).toBe("]"); + expect(tokens[0][25].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + }); + + it("test/expressions/expr5.py", + function() { + tokens = grammar.tokenizeLines("a = (from, a)\nb = [from, b]\nc = {from: {import: a}}") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("("); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][5].value).toBe("from"); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][6].value).toBe(","); + expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("a"); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe(")"); + expect(tokens[0][9].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[1][0].value).toBe("b"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("["); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][5].value).toBe("from"); + expect(tokens[1][5].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][6].value).toBe(","); + expect(tokens[1][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python"]); + expect(tokens[1][8].value).toBe("b"); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe("]"); + expect(tokens[1][9].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[2][0].value).toBe("c"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("{"); + expect(tokens[2][4].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]); + expect(tokens[2][5].value).toBe("from"); + expect(tokens[2][5].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][6].value).toBe(":"); + expect(tokens[2][6].scopes).toEqual(["source.python","punctuation.separator.dict.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe("{"); + expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]); + expect(tokens[2][9].value).toBe("import"); + expect(tokens[2][9].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][10].value).toBe(":"); + expect(tokens[2][10].scopes).toEqual(["source.python","punctuation.separator.dict.python"]); + expect(tokens[2][11].value).toBe(" "); + expect(tokens[2][11].scopes).toEqual(["source.python"]); + expect(tokens[2][12].value).toBe("a"); + expect(tokens[2][12].scopes).toEqual(["source.python"]); + expect(tokens[2][13].value).toBe("}"); + expect(tokens[2][13].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]); + expect(tokens[2][14].value).toBe("}"); + expect(tokens[2][14].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]); + }); + + it("test/expressions/expr6.py", + function() { + tokens = grammar.tokenizeLines("a = (a, b(a=1), {c: d(b=1), e: [a, b(z=1)]})") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("("); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][5].value).toBe("a"); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe(","); + expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("b"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][9].value).toBe("("); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][10].value).toBe("a"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[0][11].value).toBe("="); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][12].value).toBe("1"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][13].value).toBe(")"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][14].value).toBe(","); + expect(tokens[0][14].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][15].value).toBe(" "); + expect(tokens[0][15].scopes).toEqual(["source.python"]); + expect(tokens[0][16].value).toBe("{"); + expect(tokens[0][16].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]); + expect(tokens[0][17].value).toBe("c"); + expect(tokens[0][17].scopes).toEqual(["source.python"]); + expect(tokens[0][18].value).toBe(":"); + expect(tokens[0][18].scopes).toEqual(["source.python","punctuation.separator.dict.python"]); + expect(tokens[0][19].value).toBe(" "); + expect(tokens[0][19].scopes).toEqual(["source.python"]); + expect(tokens[0][20].value).toBe("d"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][21].value).toBe("("); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][22].value).toBe("b"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[0][23].value).toBe("="); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][24].value).toBe("1"); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][25].value).toBe(")"); + expect(tokens[0][25].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][26].value).toBe(","); + expect(tokens[0][26].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][27].value).toBe(" "); + expect(tokens[0][27].scopes).toEqual(["source.python"]); + expect(tokens[0][28].value).toBe("e"); + expect(tokens[0][28].scopes).toEqual(["source.python"]); + expect(tokens[0][29].value).toBe(":"); + expect(tokens[0][29].scopes).toEqual(["source.python","punctuation.separator.dict.python"]); + expect(tokens[0][30].value).toBe(" "); + expect(tokens[0][30].scopes).toEqual(["source.python"]); + expect(tokens[0][31].value).toBe("["); + expect(tokens[0][31].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[0][32].value).toBe("a"); + expect(tokens[0][32].scopes).toEqual(["source.python"]); + expect(tokens[0][33].value).toBe(","); + expect(tokens[0][33].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][34].value).toBe(" "); + expect(tokens[0][34].scopes).toEqual(["source.python"]); + expect(tokens[0][35].value).toBe("b"); + expect(tokens[0][35].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][36].value).toBe("("); + expect(tokens[0][36].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][37].value).toBe("z"); + expect(tokens[0][37].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[0][38].value).toBe("="); + expect(tokens[0][38].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][39].value).toBe("1"); + expect(tokens[0][39].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][40].value).toBe(")"); + expect(tokens[0][40].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][41].value).toBe("]"); + expect(tokens[0][41].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[0][42].value).toBe("}"); + expect(tokens[0][42].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]); + expect(tokens[0][43].value).toBe(")"); + expect(tokens[0][43].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + }); + + it("test/expressions/expr7.py", + function() { + tokens = grammar.tokenizeLines("a.True = b.False = d.None") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe("."); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("True"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.member.access.python","keyword.illegal.name.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("="); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("b"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("."); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][8].value).toBe("False"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.member.access.python","keyword.illegal.name.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("="); + expect(tokens[0][10].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe("d"); + expect(tokens[0][12].scopes).toEqual(["source.python"]); + expect(tokens[0][13].value).toBe("."); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][14].value).toBe("None"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.member.access.python","keyword.illegal.name.python"]); + }); + + it("test/expressions/expr8.py", + function() { + tokens = grammar.tokenizeLines("assert a or b, 'aaa'") + expect(tokens[0][0].value).toBe("assert"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("a"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("or"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("b"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("'"); + expect(tokens[0][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][10].value).toBe("aaa"); + expect(tokens[0][10].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][11].value).toBe("'"); + expect(tokens[0][11].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/expressions/expr9.py", + function() { + tokens = grammar.tokenizeLines("yield from a\nyield a") + expect(tokens[0][0].value).toBe("yield from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("a"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("yield"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("a"); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + }); + + it("test/expressions/keywords.py", + function() { + tokens = grammar.tokenizeLines("as async await continue del assert break finally\nfrom elif else if import except pass raise\nreturn try while with\n\nnonlocal global class def\n\nfor") + expect(tokens[0][0].value).toBe("as"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("async"); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("await"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("continue"); + expect(tokens[0][6].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("del"); + expect(tokens[0][8].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("assert"); + expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe("break"); + expect(tokens[0][12].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[0][14].value).toBe("finally"); + expect(tokens[0][14].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("elif"); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("else"); + expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("if"); + expect(tokens[1][6].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python"]); + expect(tokens[1][8].value).toBe("import"); + expect(tokens[1][8].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][9].value).toBe(" "); + expect(tokens[1][9].scopes).toEqual(["source.python"]); + expect(tokens[1][10].value).toBe("except"); + expect(tokens[1][10].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][11].value).toBe(" "); + expect(tokens[1][11].scopes).toEqual(["source.python"]); + expect(tokens[1][12].value).toBe("pass"); + expect(tokens[1][12].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][13].value).toBe(" "); + expect(tokens[1][13].scopes).toEqual(["source.python"]); + expect(tokens[1][14].value).toBe("raise"); + expect(tokens[1][14].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("return"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("try"); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("while"); + expect(tokens[2][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("with"); + expect(tokens[2][6].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][0].value).toBe(""); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe("nonlocal"); + expect(tokens[4][0].scopes).toEqual(["source.python","storage.modifier.declaration.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("global"); + expect(tokens[4][2].scopes).toEqual(["source.python","storage.modifier.declaration.python"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe("class"); + expect(tokens[4][4].scopes).toEqual(["source.python","storage.type.class.python"]); + expect(tokens[4][5].value).toBe(" def"); + expect(tokens[4][5].scopes).toEqual(["source.python","storage.type.function.python"]); + expect(tokens[5][0].value).toBe(""); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("for"); + expect(tokens[6][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/expressions/special1.py", + function() { + tokens = grammar.tokenizeLines("__version__ __doc__ __file__\n__author__") + expect(tokens[0][0].value).toBe("__version__"); + expect(tokens[0][0].scopes).toEqual(["source.python","support.variable.magic.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("__doc__"); + expect(tokens[0][2].scopes).toEqual(["source.python","support.variable.magic.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("__file__"); + expect(tokens[0][4].scopes).toEqual(["source.python","support.variable.magic.python"]); + expect(tokens[1][0].value).toBe("__author__"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + }); + + it("test/expressions/special2.py", + function() { + tokens = grammar.tokenizeLines("__post_init__\ndef __class_getitem__(): pass\n__mro_entries__") + expect(tokens[0][0].value).toBe("__post_init__"); + expect(tokens[0][0].scopes).toEqual(["source.python","support.variable.magic.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("__class_getitem__"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","support.function.magic.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("__mro_entries__"); + expect(tokens[2][0].scopes).toEqual(["source.python","support.variable.magic.python"]); + }); + + it("test/fstrings/comment1.py", + function() { + tokens = grammar.tokenizeLines("f'prefix{10 # comment, making the string technically illegal\ndef foo(): pass") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("prefix"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("10"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][5].value).toBe(" # "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][6].value).toBe("comment"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","punctuation.separator.element.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][9].value).toBe("making"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][11].value).toBe("the"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][13].value).toBe("string"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][14].value).toBe(" "); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][15].value).toBe("technically"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][16].value).toBe(" "); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][17].value).toBe("illegal"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][18].value).toBe(""); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.newline.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/fstrings/comment2.py", + function() { + tokens = grammar.tokenizeLines("f'''\n prefix{10\n + 32} suffix'''") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'''"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe(""); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][0].value).toBe(" prefix"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("{"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][2].value).toBe("10"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][1].value).toBe("+"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][3].value).toBe("32"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[2][4].value).toBe("}"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][5].value).toBe(" suffix"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][6].value).toBe("'''"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/comment3.py", + function() { + tokens = grammar.tokenizeLines("f'''\n prefix{10 # comment is still illegal here\n } suffix'''") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'''"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe(""); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][0].value).toBe(" prefix"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("{"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][2].value).toBe("10"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[1][3].value).toBe(" # "); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][4].value).toBe("comment"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][6].value).toBe("is"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","keyword.operator.logical.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][8].value).toBe("still"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][9].value).toBe(" "); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][10].value).toBe("illegal"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][11].value).toBe(" "); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][12].value).toBe("here"); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][1].value).toBe("}"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][2].value).toBe(" suffix"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][3].value).toBe("'''"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/comment4.py", + function() { + tokens = grammar.tokenizeLines("self.assertEqual(f'{10:#{3 != {4:5} and width}x}', ' 0xa')") + expect(tokens[0][0].value).toBe("self"); + expect(tokens[0][0].scopes).toEqual(["source.python","variable.language.special.self.python"]); + expect(tokens[0][1].value).toBe("."); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("assertEqual"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("{"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("10"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe(":"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][9].value).toBe("#"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][10].value).toBe("{"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("3"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[0][13].value).toBe("!="); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","keyword.operator.comparison.python"]); + expect(tokens[0][14].value).toBe(" "); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[0][15].value).toBe("{"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","punctuation.definition.dict.begin.python"]); + expect(tokens[0][16].value).toBe("4"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][17].value).toBe(":"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","punctuation.separator.dict.python"]); + expect(tokens[0][18].value).toBe("5"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][19].value).toBe("}"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","punctuation.definition.dict.end.python"]); + expect(tokens[0][20].value).toBe(" "); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[0][21].value).toBe("and"); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","keyword.operator.logical.python"]); + expect(tokens[0][22].value).toBe(" "); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[0][23].value).toBe("width"); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[0][24].value).toBe("}"); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][25].value).toBe("x"); + expect(tokens[0][25].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][26].value).toBe("}"); + expect(tokens[0][26].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][27].value).toBe("'"); + expect(tokens[0][27].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[0][28].value).toBe(","); + expect(tokens[0][28].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[0][29].value).toBe(" "); + expect(tokens[0][29].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[0][30].value).toBe("'"); + expect(tokens[0][30].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][31].value).toBe(" 0xa"); + expect(tokens[0][31].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[0][32].value).toBe("'"); + expect(tokens[0][32].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][33].value).toBe(")"); + expect(tokens[0][33].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/fstrings/empty1.py", + function() { + tokens = grammar.tokenizeLines("f\"{} { }\"\nf\"\"\"{}\n{ }\n\"\"\"") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("\""); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("{"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][3].value).toBe("}"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("{"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); + expect(tokens[0][7].value).toBe("}"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][8].value).toBe("\""); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("f"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("\"\"\""); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("{"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][3].value).toBe("}"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe(""); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][0].value).toBe("{"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); + expect(tokens[2][2].value).toBe("}"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][3].value).toBe(""); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[3][0].value).toBe("\"\"\""); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/empty2.py", + function() { + tokens = grammar.tokenizeLines("rf\"{} { }\"\nrf\"\"\"{}\n{ }\n\"\"\"") + expect(tokens[0][0].value).toBe("rf"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("\""); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("{"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][3].value).toBe("}"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("{"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); + expect(tokens[0][7].value).toBe("}"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][8].value).toBe("\""); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("rf"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("\"\"\""); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("{"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][3].value).toBe("}"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe(""); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][0].value).toBe("{"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); + expect(tokens[2][2].value).toBe("}"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][3].value).toBe(""); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][0].value).toBe("\"\"\""); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/fraw1.py", + function() { + tokens = grammar.tokenizeLines("a = fr'[a-z]'\na = Fr'[a-z]'\na = rf'[a-z]'\na = rF'[a-z]'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("fr"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("[a-z]"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("Fr"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("[a-z]"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("rf"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("[a-z]"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("rF"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][5].value).toBe("'"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[3][6].value).toBe("[a-z]"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][7].value).toBe("'"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/fraw2.py", + function() { + tokens = grammar.tokenizeLines("rf'fo{{2}}'\nrf\"fo{{2}}\"\nrf'''fo{{2}}'''\nrf\"\"\"fo{{2}}\"\"\"") + expect(tokens[0][0].value).toBe("rf"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("fo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][4].value).toBe("2"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("}}"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("rf"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("\""); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("fo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][3].value).toBe("{{"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[1][4].value).toBe("2"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][5].value).toBe("}}"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[1][6].value).toBe("\""); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("rf"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("'''"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("fo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][3].value).toBe("{{"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[2][4].value).toBe("2"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][5].value).toBe("}}"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[2][6].value).toBe("'''"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("rf"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("\"\"\""); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("fo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][3].value).toBe("{{"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[3][4].value).toBe("2"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][5].value).toBe("}}"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[3][6].value).toBe("\"\"\""); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/fraw3.py", + function() { + tokens = grammar.tokenizeLines("rf'fo{2}'\nrf\"fo{2}\"\nrf'''fo{2}'''\nrf\"\"\"fo{2}\"\"\"") + expect(tokens[0][0].value).toBe("rf"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("fo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("2"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][5].value).toBe("}"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("rf"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("\""); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("fo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][3].value).toBe("{"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe("2"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[1][5].value).toBe("}"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][6].value).toBe("\""); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("rf"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("'''"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("fo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[2][3].value).toBe("{"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][4].value).toBe("2"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[2][5].value).toBe("}"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][6].value).toBe("'''"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("rf"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("\"\"\""); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("fo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python"]); + expect(tokens[3][3].value).toBe("{"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][4].value).toBe("2"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[3][5].value).toBe("}"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][6].value).toBe("\"\"\""); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/fraw4.py", + function() { + tokens = grammar.tokenizeLines("a = rf'fo{{{2}}}'\na = rf'fo{{{bar}}}'\na = rf'fo{{2}}'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("rf"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("fo"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][7].value).toBe("{{"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][8].value).toBe("{"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("2"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][10].value).toBe("}"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("}}"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][12].value).toBe("'"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("rf"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("fo"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][7].value).toBe("{{"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[1][8].value).toBe("{"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][9].value).toBe("bar"); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][10].value).toBe("}"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][11].value).toBe("}}"); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[1][12].value).toBe("'"); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("rf"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("fo"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][7].value).toBe("{{"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[2][8].value).toBe("2"); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][9].value).toBe("}}"); + expect(tokens[2][9].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[2][10].value).toBe("'"); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/nested1.py", + function() { + tokens = grammar.tokenizeLines("f'''\n prefix {\n foo(f\"\"\"\n inner prefix\n { bar[\"q\"] + f'insane{42 + 9000}stuff{def aaa(): pass}111'}\n inner suffix\n \"\"\")\n } suffix\n'''") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'''"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe(""); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][0].value).toBe(" prefix "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("{"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][1].value).toBe("foo"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("f"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][4].value).toBe("\"\"\""); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe(""); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[3][0].value).toBe(" inner prefix"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[4][1].value).toBe("{"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[4][3].value).toBe("bar"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.indexed-name.python"]); + expect(tokens[4][4].value).toBe("["); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][5].value).toBe("\""); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.item-access.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][6].value).toBe("q"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.item-access.arguments.python","string.quoted.single.python"]); + expect(tokens[4][7].value).toBe("\""); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","meta.item-access.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][8].value).toBe("]"); + expect(tokens[4][8].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][9].value).toBe(" "); + expect(tokens[4][9].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[4][10].value).toBe("+"); + expect(tokens[4][10].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[4][11].value).toBe(" "); + expect(tokens[4][11].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python"]); + expect(tokens[4][12].value).toBe("f"); + expect(tokens[4][12].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][13].value).toBe("'"); + expect(tokens[4][13].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][14].value).toBe("insane"); + expect(tokens[4][14].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][15].value).toBe("{"); + expect(tokens[4][15].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][16].value).toBe("42"); + expect(tokens[4][16].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[4][17].value).toBe(" "); + expect(tokens[4][17].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); + expect(tokens[4][18].value).toBe("+"); + expect(tokens[4][18].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[4][19].value).toBe(" "); + expect(tokens[4][19].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); + expect(tokens[4][20].value).toBe("9000"); + expect(tokens[4][20].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[4][21].value).toBe("}"); + expect(tokens[4][21].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][22].value).toBe("stuff"); + expect(tokens[4][22].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][23].value).toBe("{"); + expect(tokens[4][23].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][24].value).toBe("def"); + expect(tokens[4][24].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","keyword.control.flow.python"]); + expect(tokens[4][25].value).toBe(" "); + expect(tokens[4][25].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); + expect(tokens[4][26].value).toBe("aaa"); + expect(tokens[4][26].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[4][27].value).toBe("("); + expect(tokens[4][27].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][28].value).toBe(")"); + expect(tokens[4][28].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][29].value).toBe(":"); + expect(tokens[4][29].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","punctuation.separator.colon.python"]); + expect(tokens[4][30].value).toBe(" "); + expect(tokens[4][30].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python"]); + expect(tokens[4][31].value).toBe("pass"); + expect(tokens[4][31].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","keyword.control.flow.python"]); + expect(tokens[4][32].value).toBe("}"); + expect(tokens[4][32].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][33].value).toBe("111"); + expect(tokens[4][33].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][34].value).toBe("'"); + expect(tokens[4][34].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][35].value).toBe("}"); + expect(tokens[4][35].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][36].value).toBe(""); + expect(tokens[4][36].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[5][0].value).toBe(" inner suffix"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[6][1].value).toBe("\"\"\""); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][2].value).toBe(")"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[7][1].value).toBe("}"); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[7][2].value).toBe(" suffix"); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[8][0].value).toBe("'''"); + expect(tokens[8][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/nested2.py", + function() { + tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'[2]}}\"") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("\""); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("result: "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("value"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][6].value).toBe("{"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("60"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("}"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("."); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe("{"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("16"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][12].value).toBe("!s"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][13].value).toBe(":2"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][14].value).toBe("}"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][15].value).toBe("{"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][16].value).toBe("'"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][17].value).toBe("qwerty"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python"]); + expect(tokens[0][18].value).toBe("'"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][19].value).toBe("["); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.begin.python"]); + expect(tokens[0][20].value).toBe("2"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][21].value).toBe("]"); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.end.python"]); + expect(tokens[0][22].value).toBe("}"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][23].value).toBe("}"); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][24].value).toBe("\""); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/nested3.py", + function() { + tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"\n# comment") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("\""); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("result: "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("value"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][6].value).toBe("{"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("60"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("}"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("."); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe("{"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("16"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][12].value).toBe("!s"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][13].value).toBe(":2"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][14].value).toBe("}"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][15].value).toBe("{"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][16].value).toBe("'"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][17].value).toBe("qwerty"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python"]); + expect(tokens[0][18].value).toBe("'"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][19].value).toBe(""); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.newline.python"]); + expect(tokens[1][0].value).toBe("["); + expect(tokens[1][0].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][1].value).toBe("2"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][2].value).toBe("]"); + expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[1][3].value).toBe("}}"); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("\""); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe(""); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python","invalid.illegal.newline.python"]); + expect(tokens[2][0].value).toBe("#"); + expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][1].value).toBe(" comment"); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/fstrings/nested4.py", + function() { + tokens = grammar.tokenizeLines("f\"\"\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"\"\"\ndef foo(): pass") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("\"\"\""); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("result: "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("value"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][6].value).toBe("{"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("60"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("}"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("."); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe("{"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("16"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][12].value).toBe("!s"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][13].value).toBe(":2"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][14].value).toBe("}"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][15].value).toBe("{"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][16].value).toBe("'"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][17].value).toBe("qwerty"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python"]); + expect(tokens[0][18].value).toBe("'"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("["); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][1].value).toBe("2"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[1][2].value).toBe("]"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.end.python"]); + expect(tokens[1][3].value).toBe("}"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe("}"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][5].value).toBe("\"\"\""); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][5].value).toBe(":"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][6].value).toBe(" "); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe("pass"); + expect(tokens[2][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/fstrings/prefixes1.py", + function() { + tokens = grammar.tokenizeLines("a = f's t r'\na = f\"s t r\"\na = F's t r'\na = F\"s t r\"\na = f'''s t r'''\na = F\"\"\"s t r\"\"\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("s t r"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("f"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][5].value).toBe("\""); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("s t r"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][7].value).toBe("\""); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("F"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("s t r"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("F"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][5].value).toBe("\""); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[3][6].value).toBe("s t r"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][7].value).toBe("\""); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("="); + expect(tokens[4][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe("f"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[4][5].value).toBe("'''"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][6].value).toBe("s t r"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[4][7].value).toBe("'''"); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("a"); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe("="); + expect(tokens[5][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[5][4].value).toBe("F"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[5][5].value).toBe("\"\"\""); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[5][6].value).toBe("s t r"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[5][7].value).toBe("\"\"\""); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/prefixes2.py", + function() { + tokens = grammar.tokenizeLines("f'some {obj}'\nF'some {obj}'\nrf'some {obj}'\nrF'some {obj}'\nRf'some {obj}'\nRF'some {obj}'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("some "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("obj"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe("}"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("F"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("'"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("some "); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][3].value).toBe("{"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe("obj"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][5].value).toBe("}"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][6].value).toBe("'"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("rf"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("'"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("some "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][3].value).toBe("{"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][4].value).toBe("obj"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][5].value).toBe("}"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][6].value).toBe("'"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("rF"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("'"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("some "); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][3].value).toBe("{"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][4].value).toBe("obj"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[3][5].value).toBe("}"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][6].value).toBe("'"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe("Rf"); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[4][1].value).toBe("'"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[4][2].value).toBe("some "); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[4][3].value).toBe("{"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][4].value).toBe("obj"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[4][5].value).toBe("}"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][6].value).toBe("'"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("RF"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[5][1].value).toBe("'"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[5][2].value).toBe("some "); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[5][3].value).toBe("{"); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][4].value).toBe("obj"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[5][5].value).toBe("}"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][6].value).toBe("'"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/prefixes3.py", + function() { + tokens = grammar.tokenizeLines("fr'some {obj}'\nFr'some {obj}'\nfR'some {obj}'\nFR'some {obj}'") + expect(tokens[0][0].value).toBe("fr"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("some "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("obj"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe("}"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("Fr"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("'"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("some "); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[1][3].value).toBe("{"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][4].value).toBe("obj"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][5].value).toBe("}"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][6].value).toBe("'"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("fR"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("'"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("some "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[2][3].value).toBe("{"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][4].value).toBe("obj"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][5].value).toBe("}"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][6].value).toBe("'"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("FR"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("'"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("some "); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python"]); + expect(tokens[3][3].value).toBe("{"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][4].value).toBe("obj"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[3][5].value).toBe("}"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][6].value).toBe("'"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.raw.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/prefixes4.py", + function() { + tokens = grammar.tokenizeLines("fu'some {obj}'\nFu'some {obj}'\nfU'some {obj}'\nFU'some {obj}'\n\nuf'some {obj}'\nuF'some {obj}'\nUf'some {obj}'\nUF'some {obj}'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("u"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[0][2].value).toBe("'"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][3].value).toBe("some "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][4].value).toBe("{"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][5].value).toBe("obj"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][6].value).toBe("}"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("F"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("u"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[1][2].value).toBe("'"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[1][3].value).toBe("some "); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][4].value).toBe("{"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][5].value).toBe("obj"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][6].value).toBe("}"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("f"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("U"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[2][2].value).toBe("'"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][3].value).toBe("some "); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][4].value).toBe("{"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][5].value).toBe("obj"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][6].value).toBe("}"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("F"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("U"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[3][2].value).toBe("'"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[3][3].value).toBe("some "); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][4].value).toBe("{"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][5].value).toBe("obj"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[3][6].value).toBe("}"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][7].value).toBe("'"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe("u"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[5][1].value).toBe("f"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[5][2].value).toBe("'"); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); + expect(tokens[5][3].value).toBe("some "); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[5][4].value).toBe("{"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][5].value).toBe("obj"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[5][6].value).toBe("}"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][7].value).toBe("'"); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe("u"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[6][1].value).toBe("F"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][2].value).toBe("'"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); + expect(tokens[6][3].value).toBe("some "); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][4].value).toBe("{"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[6][5].value).toBe("obj"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[6][6].value).toBe("}"); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[6][7].value).toBe("'"); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[7][0].value).toBe("U"); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[7][1].value).toBe("f"); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[7][2].value).toBe("'"); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("some "); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[7][4].value).toBe("{"); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[7][5].value).toBe("obj"); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[7][6].value).toBe("}"); + expect(tokens[7][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[7][7].value).toBe("'"); + expect(tokens[7][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe("U"); + expect(tokens[8][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[8][1].value).toBe("F"); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[8][2].value).toBe("'"); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); + expect(tokens[8][3].value).toBe("some "); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[8][4].value).toBe("{"); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[8][5].value).toBe("obj"); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[8][6].value).toBe("}"); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[8][7].value).toBe("'"); + expect(tokens[8][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/prefixes5.py", + function() { + tokens = grammar.tokenizeLines("fb'some {obj}'\nFb'some {obj}'\nfB'some {obj}'\nFB'some {obj}'\n\nbf'some {obj}'\nbF'some {obj}'\nBf'some {obj}'\nBF'some {obj}'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("b"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[0][2].value).toBe("'"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][3].value).toBe("some "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][4].value).toBe("{"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][5].value).toBe("obj"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][6].value).toBe("}"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("F"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("b"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[1][2].value).toBe("'"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[1][3].value).toBe("some "); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][4].value).toBe("{"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][5].value).toBe("obj"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][6].value).toBe("}"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("f"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("B"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[2][2].value).toBe("'"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][3].value).toBe("some "); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][4].value).toBe("{"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][5].value).toBe("obj"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][6].value).toBe("}"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("F"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("B"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[3][2].value).toBe("'"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[3][3].value).toBe("some "); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[3][4].value).toBe("{"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][5].value).toBe("obj"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[3][6].value).toBe("}"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][7].value).toBe("'"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe("b"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[5][1].value).toBe("f"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[5][2].value).toBe("'"); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); + expect(tokens[5][3].value).toBe("some "); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[5][4].value).toBe("{"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][5].value).toBe("obj"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[5][6].value).toBe("}"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[5][7].value).toBe("'"); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe("b"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[6][1].value).toBe("F"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][2].value).toBe("'"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); + expect(tokens[6][3].value).toBe("some "); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][4].value).toBe("{"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[6][5].value).toBe("obj"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[6][6].value).toBe("}"); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[6][7].value).toBe("'"); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[7][0].value).toBe("B"); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[7][1].value).toBe("f"); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[7][2].value).toBe("'"); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); + expect(tokens[7][3].value).toBe("some "); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[7][4].value).toBe("{"); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[7][5].value).toBe("obj"); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[7][6].value).toBe("}"); + expect(tokens[7][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[7][7].value).toBe("'"); + expect(tokens[7][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[8][0].value).toBe("B"); + expect(tokens[8][0].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.prefix.python"]); + expect(tokens[8][1].value).toBe("F"); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[8][2].value).toBe("'"); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python punctuation.definition.string.begin.python"]); + expect(tokens[8][3].value).toBe("some "); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[8][4].value).toBe("{"); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[8][5].value).toBe("obj"); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[8][6].value).toBe("}"); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[8][7].value).toBe("'"); + expect(tokens[8][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/simple1.py", + function() { + tokens = grammar.tokenizeLines("a = f\"{0[ ]:X>+10d}\"\na = f\"{0[ ]!s:X>+10d}\"\na = f\"{0[ ]:Xd>+10d}\" #invalid") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("\""); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("{"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("0"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("["); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.begin.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe("]"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.end.python"]); + expect(tokens[0][11].value).toBe(":X>+10d"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][12].value).toBe("}"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][13].value).toBe("\""); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("f"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[1][5].value).toBe("\""); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("{"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][7].value).toBe("0"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[1][8].value).toBe("["); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][9].value).toBe(" "); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][10].value).toBe("]"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.end.python"]); + expect(tokens[1][11].value).toBe("!s"); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[1][12].value).toBe(":X>+10d"); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[1][13].value).toBe("}"); + expect(tokens[1][13].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][14].value).toBe("\""); + expect(tokens[1][14].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("f"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][5].value).toBe("\""); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("{"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][7].value).toBe("0"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[2][8].value).toBe("["); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.begin.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][10].value).toBe("]"); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.fstring.python","punctuation.definition.list.end.python"]); + expect(tokens[2][11].value).toBe(":"); + expect(tokens[2][11].scopes).toEqual(["source.python","meta.fstring.python","punctuation.separator.colon.python"]); + expect(tokens[2][12].value).toBe("Xd"); + expect(tokens[2][12].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][13].value).toBe(">"); + expect(tokens[2][13].scopes).toEqual(["source.python","meta.fstring.python","keyword.operator.comparison.python"]); + expect(tokens[2][14].value).toBe("+"); + expect(tokens[2][14].scopes).toEqual(["source.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[2][15].value).toBe("10d"); + expect(tokens[2][15].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.name.python"]); + expect(tokens[2][16].value).toBe("}"); + expect(tokens[2][16].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][17].value).toBe("\""); + expect(tokens[2][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][18].value).toBe(" "); + expect(tokens[2][18].scopes).toEqual(["source.python"]); + expect(tokens[2][19].value).toBe("#"); + expect(tokens[2][19].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][20].value).toBe("invalid"); + expect(tokens[2][20].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/fstrings/simple10.py", + function() { + tokens = grammar.tokenizeLines("f'values: {a=} {b=!r}'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("values: "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("a"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][5].value).toBe("="); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][6].value).toBe("}"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][8].value).toBe("{"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("b"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][10].value).toBe("=!r"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][11].value).toBe("}"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][12].value).toBe("'"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/simple2.py", + function() { + tokens = grammar.tokenizeLines("a = f\"normal {{ normal }} normal } {10!r} normal {fo.__add__!s}\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("\""); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("normal "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][7].value).toBe("{{"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][8].value).toBe(" normal "); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][9].value).toBe("}}"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][10].value).toBe(" normal "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][11].value).toBe("}"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][13].value).toBe("{"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][14].value).toBe("10"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][15].value).toBe("!r"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][16].value).toBe("}"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][17].value).toBe(" normal "); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][18].value).toBe("{"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][19].value).toBe("fo"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][20].value).toBe("."); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.fstring.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][21].value).toBe("__add__"); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.fstring.python","meta.member.access.python","support.function.magic.python"]); + expect(tokens[0][22].value).toBe("!s"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][23].value).toBe("}"); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][24].value).toBe("\""); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/simple3.py", + function() { + tokens = grammar.tokenizeLines("a = f'hello { foo(\"bar\")/23 !r:f} times'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("hello "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][7].value).toBe("{"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][9].value).toBe("foo"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][10].value).toBe("("); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][11].value).toBe("\""); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][12].value).toBe("bar"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[0][13].value).toBe("\""); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][14].value).toBe(")"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][15].value).toBe("/"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][16].value).toBe("23"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][17].value).toBe(" "); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][18].value).toBe("!r"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][19].value).toBe(":f"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][20].value).toBe("}"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][21].value).toBe(" times"); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][22].value).toBe("'"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/simple4.py", + function() { + tokens = grammar.tokenizeLines("a = f'''hello { foo(\"bar\")/23 !r:f} times'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("'''"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("hello "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][7].value).toBe("{"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][9].value).toBe("foo"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][10].value).toBe("("); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][11].value).toBe("\""); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][12].value).toBe("bar"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[0][13].value).toBe("\""); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][14].value).toBe(")"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][15].value).toBe("/"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][16].value).toBe("23"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][17].value).toBe(" "); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][18].value).toBe("!r"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][19].value).toBe(":f"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[0][20].value).toBe("}"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][21].value).toBe(" times"); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][22].value).toBe("'''"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/simple5.py", + function() { + tokens = grammar.tokenizeLines("a = f'''\nhello { foo(\"bar\")/23 !r:f} times\n'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("'''"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe(""); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][0].value).toBe("hello "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("{"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][3].value).toBe("foo"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][4].value).toBe("("); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][5].value).toBe("\""); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("bar"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[1][7].value).toBe("\""); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][8].value).toBe(")"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][9].value).toBe("/"); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[1][10].value).toBe("23"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[1][11].value).toBe(" "); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[1][12].value).toBe("!r"); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[1][13].value).toBe(":f"); + expect(tokens[1][13].scopes).toEqual(["source.python","meta.fstring.python","storage.type.format.python"]); + expect(tokens[1][14].value).toBe("}"); + expect(tokens[1][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][15].value).toBe(" times"); + expect(tokens[1][15].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][0].value).toBe("'''"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/simple6.py", + function() { + tokens = grammar.tokenizeLines("f'insane{42 + 9000}stuff{def aaa(): pass}'\n# def aaa() must not be parsed as a valid declaration") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("insane"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][4].value).toBe("42"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][6].value).toBe("+"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][8].value).toBe("9000"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[0][9].value).toBe("}"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][10].value).toBe("stuff"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][11].value).toBe("{"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][12].value).toBe("def"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","keyword.control.flow.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][14].value).toBe("aaa"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][15].value).toBe("("); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][16].value).toBe(")"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][17].value).toBe(":"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","punctuation.separator.colon.python"]); + expect(tokens[0][18].value).toBe(" "); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][19].value).toBe("pass"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.fstring.python","keyword.control.flow.python"]); + expect(tokens[0][20].value).toBe("}"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][21].value).toBe("'"); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("#"); + expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][1].value).toBe(" def aaa() must not be parsed as a valid declaration"); + expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/fstrings/simple7.py", + function() { + tokens = grammar.tokenizeLines("f'abc \\ efg'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("abc "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("\\ efg"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/simple8.py", + function() { + tokens = grammar.tokenizeLines("f'abc \\} }} }}} }}}} }}}}} efg'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("abc "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("\\"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][4].value).toBe("}"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][6].value).toBe("}}"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][8].value).toBe("}}"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][9].value).toBe("}"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][11].value).toBe("}}"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][12].value).toBe("}}"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][14].value).toBe("}}"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][15].value).toBe("}}"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][16].value).toBe("}"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.fstring.python","invalid.illegal.brace.python"]); + expect(tokens[0][17].value).toBe(" efg"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][18].value).toBe("'"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/simple9.py", + function() { + tokens = grammar.tokenizeLines("f'foo {{{bar}}}'") + expect(tokens[0][0].value).toBe("f"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][1].value).toBe("'"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][2].value).toBe("foo "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[0][3].value).toBe("{{"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][4].value).toBe("{"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][5].value).toBe("bar"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[0][6].value).toBe("}"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("}}"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[0][8].value).toBe("'"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/unicode1.py", + function() { + tokens = grammar.tokenizeLines("a = f\"\"\"\nmultiline \"unicode\" string \\\n\n \\xf1 \\u1234aaaa \\U1234aaaa\n\n \\N{BLACK SPADE SUIT}\n\"\"\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("\"\"\""); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe(""); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][0].value).toBe("multiline \"unicode\" string "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][1].value).toBe("\\"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.fstring.python","constant.language.python"]); + expect(tokens[1][2].value).toBe(""); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[3][1].value).toBe("\\xf1"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[3][3].value).toBe("\\u1234"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[3][4].value).toBe("aaaa "); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[3][5].value).toBe("\\U1234aaaa"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[3][6].value).toBe(""); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[5][1].value).toBe("\\N{BLACK SPADE SUIT}"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[5][2].value).toBe(""); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[6][0].value).toBe("\"\"\""); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/fstrings/unicode2.py", + function() { + tokens = grammar.tokenizeLines("a = f\"\"\"\nmultiline \"unicode\" string\n \\N{BLACK SPADE SUIT} {foo+2}\n\"\"\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("f"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.multi.python string.interpolated.python"]); + expect(tokens[0][5].value).toBe("\"\"\""); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe(""); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[1][0].value).toBe("multiline \"unicode\" string"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][1].value).toBe("\\N{BLACK SPADE SUIT}"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.fstring.python","constant.character.escape.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[2][3].value).toBe("{"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][4].value).toBe("foo"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[2][5].value).toBe("+"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.fstring.python","keyword.operator.arithmetic.python"]); + expect(tokens[2][6].value).toBe("2"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.fstring.python","constant.numeric.dec.python"]); + expect(tokens[2][7].value).toBe("}"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][8].value).toBe(""); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python"]); + expect(tokens[3][0].value).toBe("\"\"\""); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.multi.python string.interpolated.python punctuation.definition.string.end.python"]); + }); + + it("test/functions/async1.py", + function() { + tokens = grammar.tokenizeLines("@foo\nasync def foo():\n a = 1\n async for a, b, c in b:\n async with b as d, c:\n await func(a, b=1)") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe("foo"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[1][0].value).toBe("async"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.async.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("def"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][4].value).toBe("foo"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][5].value).toBe("("); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][6].value).toBe(")"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][7].value).toBe(":"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("a"); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("="); + expect(tokens[2][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("1"); + expect(tokens[2][5].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("async"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("for"); + expect(tokens[3][3].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[3][5].value).toBe("a"); + expect(tokens[3][5].scopes).toEqual(["source.python"]); + expect(tokens[3][6].value).toBe(","); + expect(tokens[3][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[3][7].value).toBe(" "); + expect(tokens[3][7].scopes).toEqual(["source.python"]); + expect(tokens[3][8].value).toBe("b"); + expect(tokens[3][8].scopes).toEqual(["source.python"]); + expect(tokens[3][9].value).toBe(","); + expect(tokens[3][9].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[3][10].value).toBe(" "); + expect(tokens[3][10].scopes).toEqual(["source.python"]); + expect(tokens[3][11].value).toBe("c"); + expect(tokens[3][11].scopes).toEqual(["source.python"]); + expect(tokens[3][12].value).toBe(" "); + expect(tokens[3][12].scopes).toEqual(["source.python"]); + expect(tokens[3][13].value).toBe("in"); + expect(tokens[3][13].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][14].value).toBe(" "); + expect(tokens[3][14].scopes).toEqual(["source.python"]); + expect(tokens[3][15].value).toBe("b"); + expect(tokens[3][15].scopes).toEqual(["source.python"]); + expect(tokens[3][16].value).toBe(":"); + expect(tokens[3][16].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("async"); + expect(tokens[4][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python"]); + expect(tokens[4][3].value).toBe("with"); + expect(tokens[4][3].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][4].value).toBe(" "); + expect(tokens[4][4].scopes).toEqual(["source.python"]); + expect(tokens[4][5].value).toBe("b"); + expect(tokens[4][5].scopes).toEqual(["source.python"]); + expect(tokens[4][6].value).toBe(" "); + expect(tokens[4][6].scopes).toEqual(["source.python"]); + expect(tokens[4][7].value).toBe("as"); + expect(tokens[4][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][8].value).toBe(" "); + expect(tokens[4][8].scopes).toEqual(["source.python"]); + expect(tokens[4][9].value).toBe("d"); + expect(tokens[4][9].scopes).toEqual(["source.python"]); + expect(tokens[4][10].value).toBe(","); + expect(tokens[4][10].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][11].value).toBe(" "); + expect(tokens[4][11].scopes).toEqual(["source.python"]); + expect(tokens[4][12].value).toBe("c"); + expect(tokens[4][12].scopes).toEqual(["source.python"]); + expect(tokens[4][13].value).toBe(":"); + expect(tokens[4][13].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("await"); + expect(tokens[5][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("func"); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[5][4].value).toBe("("); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][5].value).toBe("a"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][6].value).toBe(","); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[5][7].value).toBe(" "); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][8].value).toBe("b"); + expect(tokens[5][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[5][9].value).toBe("="); + expect(tokens[5][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[5][10].value).toBe("1"); + expect(tokens[5][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[5][11].value).toBe(")"); + expect(tokens[5][11].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/functions/async2.py", + function() { + tokens = grammar.tokenizeLines("async = await = 2") + expect(tokens[0][0].value).toBe("async"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("await"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("="); + expect(tokens[0][6].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("2"); + expect(tokens[0][8].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/functions/async3.py", + function() { + tokens = grammar.tokenizeLines("casync def foo(): pass") + expect(tokens[0][0].value).toBe("casync"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("def"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][4].value).toBe("foo"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][5].value).toBe("("); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][6].value).toBe(")"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][7].value).toBe(":"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("pass"); + expect(tokens[0][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl1.py", + function() { + tokens = grammar.tokenizeLines("def myfunc(self, # gotta have self\n param1=\"value\", # values are cool\n param2=True, # or False, whatever\n **kwargs): # you never know\n pass") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("myfunc"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("self"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.self.python"]); + expect(tokens[0][5].value).toBe(","); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][7].value).toBe("#"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][8].value).toBe(" gotta have self"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][1].value).toBe("param1"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[1][3].value).toBe("\""); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][4].value).toBe("value"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[1][5].value).toBe("\""); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][6].value).toBe(","); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][8].value).toBe("#"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][9].value).toBe(" values are cool"); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[2][1].value).toBe("param2"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[2][3].value).toBe("True"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.language.python"]); + expect(tokens[2][4].value).toBe(","); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[2][6].value).toBe("#"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][7].value).toBe(" or False, whatever"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][1].value).toBe("**"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[3][2].value).toBe("kwargs"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[3][3].value).toBe(")"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[3][4].value).toBe(":"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[3][5].value).toBe(" "); + expect(tokens[3][5].scopes).toEqual(["source.python"]); + expect(tokens[3][6].value).toBe("#"); + expect(tokens[3][6].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][7].value).toBe(" you never know"); + expect(tokens[3][7].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("pass"); + expect(tokens[4][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl10.py", + function() { + tokens = grammar.tokenizeLines("def foo(True): True") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("True"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.illegal.name.python"]); + expect(tokens[0][5].value).toBe(")"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][6].value).toBe(":"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("True"); + expect(tokens[0][8].scopes).toEqual(["source.python","constant.language.python"]); + }); + + it("test/functions/decl11.py", + function() { + tokens = grammar.tokenizeLines("# testing annotations split over multiple lines\ndef foo(a:('abc' 'def')==123, boo: 'abc'\n\n 'def' == foo(n(m=0)))") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" testing annotations split over multiple lines"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe("a"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[1][6].value).toBe("("); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.parenthesis.begin.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][8].value).toBe("abc"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[1][9].value).toBe("'"); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][11].value).toBe("'"); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][12].value).toBe("def"); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[1][13].value).toBe("'"); + expect(tokens[1][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][14].value).toBe(")"); + expect(tokens[1][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.parenthesis.end.python"]); + expect(tokens[1][15].value).toBe("=="); + expect(tokens[1][15].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.comparison.python"]); + expect(tokens[1][16].value).toBe("123"); + expect(tokens[1][16].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[1][17].value).toBe(","); + expect(tokens[1][17].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[1][18].value).toBe(" "); + expect(tokens[1][18].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][19].value).toBe("boo"); + expect(tokens[1][19].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][20].value).toBe(":"); + expect(tokens[1][20].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[1][21].value).toBe(" "); + expect(tokens[1][21].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][22].value).toBe("'"); + expect(tokens[1][22].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][23].value).toBe("abc"); + expect(tokens[1][23].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[1][24].value).toBe("'"); + expect(tokens[1][24].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][1].value).toBe("'"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("def"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[3][3].value).toBe("'"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][5].value).toBe("=="); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.comparison.python"]); + expect(tokens[3][6].value).toBe(" "); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][7].value).toBe("foo"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][8].value).toBe("("); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][9].value).toBe("n"); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][10].value).toBe("("); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][11].value).toBe("m"); + expect(tokens[3][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][12].value).toBe("="); + expect(tokens[3][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[3][13].value).toBe("0"); + expect(tokens[3][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][14].value).toBe(")"); + expect(tokens[3][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][15].value).toBe(")"); + expect(tokens[3][15].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][16].value).toBe(")"); + expect(tokens[3][16].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + }); + + it("test/functions/decl12.py", + function() { + tokens = grammar.tokenizeLines("def get_streaks(s) \\\n -> 'spam': pass") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("get_streaks"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("s"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][5].value).toBe(")"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][7].value).toBe("\\"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][8].value).toBe(""); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][1].value).toBe("->"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python","punctuation.separator.annotation.result.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][3].value).toBe("'"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][4].value).toBe("spam"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","string.quoted.single.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][6].value).toBe(":"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python"]); + expect(tokens[1][8].value).toBe("pass"); + expect(tokens[1][8].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl13.py", + function() { + tokens = grammar.tokenizeLines("def foo()\n -> notOK:\n pass") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("->"); + expect(tokens[1][1].scopes).toEqual(["source.python","invalid.illegal.annotation.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("notOK"); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe(":"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("pass"); + expect(tokens[2][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl14.py", + function() { + tokens = grammar.tokenizeLines("# testing comments in function definition\ndef foo( # before args\n a=42, # between\n # args\n b= # in args\n 24,\n d # before '='\n =99,\n e\n ) # incomplete definition, missing COLON, you're probably typing it\n # pre docstring\n '''Docstring'''\n # post docstring\n\ndef bar(): return 1") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" testing comments in function definition"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][5].value).toBe("#"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][6].value).toBe(" before args"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[2][1].value).toBe("a"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[2][3].value).toBe("42"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[2][4].value).toBe(","); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[2][6].value).toBe("#"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][7].value).toBe(" between"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][1].value).toBe("#"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][2].value).toBe(" args"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[4][1].value).toBe("b"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[4][2].value).toBe("="); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[4][4].value).toBe("#"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[4][5].value).toBe(" in args"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][1].value).toBe("24"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[5][2].value).toBe(","); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[6][1].value).toBe("d"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[6][2].value).toBe(" "); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[6][3].value).toBe("#"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[6][4].value).toBe(" before '='"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","comment.line.number-sign.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[7][1].value).toBe("="); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[7][2].value).toBe("99"); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[7][3].value).toBe(","); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[8][1].value).toBe("e"); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[9][1].value).toBe(")"); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[9][2].value).toBe(" "); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[9][3].value).toBe("#"); + expect(tokens[9][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[9][4].value).toBe(" incomplete definition, missing COLON, you're probably typing it"); + expect(tokens[9][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("#"); + expect(tokens[10][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[10][2].value).toBe(" pre docstring"); + expect(tokens[10][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[11][0].value).toBe(" "); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[11][1].value).toBe("'''"); + expect(tokens[11][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[11][2].value).toBe("Docstring"); + expect(tokens[11][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]); + expect(tokens[11][3].value).toBe("'''"); + expect(tokens[11][3].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[12][0].value).toBe(" "); + expect(tokens[12][0].scopes).toEqual(["source.python"]); + expect(tokens[12][1].value).toBe("#"); + expect(tokens[12][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[12][2].value).toBe(" post docstring"); + expect(tokens[12][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[13][0].value).toBe(""); + expect(tokens[13][0].scopes).toEqual(["source.python"]); + expect(tokens[14][0].value).toBe("def"); + expect(tokens[14][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[14][1].value).toBe(" "); + expect(tokens[14][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[14][2].value).toBe("bar"); + expect(tokens[14][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[14][3].value).toBe("("); + expect(tokens[14][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[14][4].value).toBe(")"); + expect(tokens[14][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[14][5].value).toBe(":"); + expect(tokens[14][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[14][6].value).toBe(" "); + expect(tokens[14][6].scopes).toEqual(["source.python"]); + expect(tokens[14][7].value).toBe("return"); + expect(tokens[14][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[14][8].value).toBe(" "); + expect(tokens[14][8].scopes).toEqual(["source.python"]); + expect(tokens[14][9].value).toBe("1"); + expect(tokens[14][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/functions/decl15.py", + function() { + tokens = grammar.tokenizeLines("def showcase(a, b, /, c, d, *, e, f):\n return") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("showcase"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("a"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][5].value).toBe(","); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][7].value).toBe("b"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][8].value).toBe(","); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][10].value).toBe("/"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.positional.parameter.python"]); + expect(tokens[0][11].value).toBe(", "); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][12].value).toBe("c"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][13].value).toBe(","); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][14].value).toBe(" "); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][15].value).toBe("d"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][16].value).toBe(","); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][17].value).toBe(" "); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][18].value).toBe("*"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[0][19].value).toBe(", "); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][20].value).toBe("e"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][21].value).toBe(","); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][22].value).toBe(" "); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][23].value).toBe("f"); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][24].value).toBe(")"); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][25].value).toBe(":"); + expect(tokens[0][25].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("return"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl2.py", + function() { + tokens = grammar.tokenizeLines("def result_annot(lambda, lambda=) -> qqq[None]:\n pass") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("result_annot"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(", "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][6].value).toBe("lambda"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.control.flow.python"]); + expect(tokens[0][7].value).toBe("="); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[0][8].value).toBe(")"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][10].value).toBe("->"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.function.python","punctuation.separator.annotation.result.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][12].value).toBe("qqq"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.function.python","meta.item-access.python","meta.indexed-name.python"]); + expect(tokens[0][13].value).toBe("["); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.function.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][14].value).toBe("None"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.function.python","meta.item-access.python","meta.item-access.arguments.python","constant.language.python"]); + expect(tokens[0][15].value).toBe("]"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.function.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][16].value).toBe(":"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("pass"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl3.py", + function() { + tokens = grammar.tokenizeLines("# testing annotations split over multiple lines\ndef __init__(self, a:('abc' 'def')=123, boo: 'abc'\n\n 'def' = foo(n(m=0), baz=\n 13)) -> 123 : 123") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" testing annotations split over multiple lines"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("__init__"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","support.function.magic.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe("self"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.self.python"]); + expect(tokens[1][5].value).toBe(","); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][7].value).toBe("a"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][8].value).toBe(":"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[1][9].value).toBe("("); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.parenthesis.begin.python"]); + expect(tokens[1][10].value).toBe("'"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][11].value).toBe("abc"); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[1][12].value).toBe("'"); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][13].value).toBe(" "); + expect(tokens[1][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][14].value).toBe("'"); + expect(tokens[1][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][15].value).toBe("def"); + expect(tokens[1][15].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[1][16].value).toBe("'"); + expect(tokens[1][16].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][17].value).toBe(")"); + expect(tokens[1][17].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.parenthesis.end.python"]); + expect(tokens[1][18].value).toBe("="); + expect(tokens[1][18].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.assignment.python"]); + expect(tokens[1][19].value).toBe("123"); + expect(tokens[1][19].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[1][20].value).toBe(","); + expect(tokens[1][20].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[1][21].value).toBe(" "); + expect(tokens[1][21].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][22].value).toBe("boo"); + expect(tokens[1][22].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][23].value).toBe(":"); + expect(tokens[1][23].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[1][24].value).toBe(" "); + expect(tokens[1][24].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[1][25].value).toBe("'"); + expect(tokens[1][25].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][26].value).toBe("abc"); + expect(tokens[1][26].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[1][27].value).toBe("'"); + expect(tokens[1][27].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][1].value).toBe("'"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][2].value).toBe("def"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python"]); + expect(tokens[3][3].value).toBe("'"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][5].value).toBe("="); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.assignment.python"]); + expect(tokens[3][6].value).toBe(" "); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][7].value).toBe("foo"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][8].value).toBe("("); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][9].value).toBe("n"); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][10].value).toBe("("); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][11].value).toBe("m"); + expect(tokens[3][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][12].value).toBe("="); + expect(tokens[3][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[3][13].value).toBe("0"); + expect(tokens[3][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][14].value).toBe(")"); + expect(tokens[3][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][15].value).toBe(","); + expect(tokens[3][15].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[3][16].value).toBe(" "); + expect(tokens[3][16].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][17].value).toBe("baz"); + expect(tokens[3][17].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][18].value).toBe("="); + expect(tokens[3][18].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[4][1].value).toBe("13"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[4][2].value).toBe(")"); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][3].value).toBe(")"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[4][4].value).toBe(" "); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[4][5].value).toBe("->"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function.python","punctuation.separator.annotation.result.python"]); + expect(tokens[4][6].value).toBe(" "); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[4][7].value).toBe("123"); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.function.python","constant.numeric.dec.python"]); + expect(tokens[4][8].value).toBe(" "); + expect(tokens[4][8].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[4][9].value).toBe(":"); + expect(tokens[4][9].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[4][10].value).toBe(" "); + expect(tokens[4][10].scopes).toEqual(["source.python"]); + expect(tokens[4][11].value).toBe("123"); + expect(tokens[4][11].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/functions/decl4.py", + function() { + tokens = grammar.tokenizeLines("# testing annotations split over multiple lines\ndef some_func(a:\n lambda x=None:\n {key: val\n for key, val in\n (x if x is not None else [])\n }=42):") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" testing annotations split over multiple lines"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("some_func"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe("a"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[2][1].value).toBe("lambda"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][3].value).toBe("x"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][4].value).toBe("="); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[2][5].value).toBe("None"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.language.python"]); + expect(tokens[2][6].value).toBe(":"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][1].value).toBe("{"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.dict.begin.python"]); + expect(tokens[3][2].value).toBe("key"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][3].value).toBe(":"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.dict.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][5].value).toBe("val"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[4][1].value).toBe("for"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.control.flow.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[4][3].value).toBe("key"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[4][4].value).toBe(","); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.element.python"]); + expect(tokens[4][5].value).toBe(" "); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[4][6].value).toBe("val"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[4][7].value).toBe(" "); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[4][8].value).toBe("in"); + expect(tokens[4][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.control.flow.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][1].value).toBe("("); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.parenthesis.begin.python"]); + expect(tokens[5][2].value).toBe("x"); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][4].value).toBe("if"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.control.flow.python"]); + expect(tokens[5][5].value).toBe(" "); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][6].value).toBe("x"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][7].value).toBe(" "); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][8].value).toBe("is"); + expect(tokens[5][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.logical.python"]); + expect(tokens[5][9].value).toBe(" "); + expect(tokens[5][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][10].value).toBe("not"); + expect(tokens[5][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.logical.python"]); + expect(tokens[5][11].value).toBe(" "); + expect(tokens[5][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][12].value).toBe("None"); + expect(tokens[5][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.language.python"]); + expect(tokens[5][13].value).toBe(" "); + expect(tokens[5][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][14].value).toBe("else"); + expect(tokens[5][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.control.flow.python"]); + expect(tokens[5][15].value).toBe(" "); + expect(tokens[5][15].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[5][16].value).toBe("["); + expect(tokens[5][16].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.list.begin.python"]); + expect(tokens[5][17].value).toBe("]"); + expect(tokens[5][17].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.list.end.python"]); + expect(tokens[5][18].value).toBe(")"); + expect(tokens[5][18].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.parenthesis.end.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[6][1].value).toBe("}"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.dict.end.python"]); + expect(tokens[6][2].value).toBe("="); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.assignment.python"]); + expect(tokens[6][3].value).toBe("42"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[6][4].value).toBe(")"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[6][5].value).toBe(":"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + }); + + it("test/functions/decl5.py", + function() { + tokens = grammar.tokenizeLines("def f()->123:pass") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("f"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][5].value).toBe("->"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","punctuation.separator.annotation.result.python"]); + expect(tokens[0][6].value).toBe("123"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","constant.numeric.dec.python"]); + expect(tokens[0][7].value).toBe(":"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[0][8].value).toBe("pass"); + expect(tokens[0][8].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl6.py", + function() { + tokens = grammar.tokenizeLines("def True(): pass\ndef None(): pass\ndef False(): pass") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("True"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","keyword.illegal.name.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("pass"); + expect(tokens[0][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("None"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","keyword.illegal.name.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("False"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","keyword.illegal.name.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][5].value).toBe(":"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][6].value).toBe(" "); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe("pass"); + expect(tokens[2][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl7.py", + function() { + tokens = grammar.tokenizeLines("def foo(*, a): pass\ndef foo(*a): pass\ndef foo(**a): pass") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("*"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[0][5].value).toBe(", "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][7].value).toBe(")"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][8].value).toBe(":"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("pass"); + expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe("*"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[1][5].value).toBe("a"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][6].value).toBe(")"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][7].value).toBe(":"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][8].value).toBe(" "); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe("pass"); + expect(tokens[1][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe("**"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[2][5].value).toBe("a"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][6].value).toBe(")"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][7].value).toBe(":"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][8].value).toBe(" "); + expect(tokens[2][8].scopes).toEqual(["source.python"]); + expect(tokens[2][9].value).toBe("pass"); + expect(tokens[2][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl8.py", + function() { + tokens = grammar.tokenizeLines("def foo(a=1**1, *b:3*2=1*2, **a=1*2**3): pass") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("a"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][5].value).toBe("="); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[0][6].value).toBe("1"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][7].value).toBe("**"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][8].value).toBe("1"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][9].value).toBe(","); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][11].value).toBe("*"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[0][12].value).toBe("b"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][13].value).toBe(":"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.annotation.python"]); + expect(tokens[0][14].value).toBe("3"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][15].value).toBe("*"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][16].value).toBe("2"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][17].value).toBe("="); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.assignment.python"]); + expect(tokens[0][18].value).toBe("1"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][19].value).toBe("*"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][20].value).toBe("2"); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][21].value).toBe(","); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][22].value).toBe(" "); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[0][23].value).toBe("**"); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[0][24].value).toBe("a"); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][25].value).toBe("="); + expect(tokens[0][25].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[0][26].value).toBe("1"); + expect(tokens[0][26].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][27].value).toBe("*"); + expect(tokens[0][27].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][28].value).toBe("2"); + expect(tokens[0][28].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][29].value).toBe("**"); + expect(tokens[0][29].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][30].value).toBe("3"); + expect(tokens[0][30].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][31].value).toBe(")"); + expect(tokens[0][31].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][32].value).toBe(":"); + expect(tokens[0][32].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[0][33].value).toBe(" "); + expect(tokens[0][33].scopes).toEqual(["source.python"]); + expect(tokens[0][34].value).toBe("pass"); + expect(tokens[0][34].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decl9.py", + function() { + tokens = grammar.tokenizeLines("cdef foo(): pass") + expect(tokens[0][0].value).toBe("cdef"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][4].value).toBe(")"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("pass"); + expect(tokens[0][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decorators1.py", + function() { + tokens = grammar.tokenizeLines("@some_decorator # with comment\n@some.class.decorator\n@some_decorator(1)\n@some.decorator (1, 3)\n@some_decorator(a=2, b={'q': 42}, **kwargs)\n@classmethod\ndef decorated(a): pass") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe("some_decorator"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][2].value).toBe(" "); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][3].value).toBe("#"); + expect(tokens[0][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][4].value).toBe(" with comment"); + expect(tokens[0][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("@"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[1][1].value).toBe("some"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[1][2].value).toBe("."); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[1][3].value).toBe("class"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.decorator.python","keyword.control.flow.python"]); + expect(tokens[1][4].value).toBe("."); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[1][5].value).toBe("decorator"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[2][0].value).toBe("@"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[2][1].value).toBe("some_decorator"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("1"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][0].value).toBe("@"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[3][1].value).toBe("some"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[3][2].value).toBe("."); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[3][3].value).toBe("decorator"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[3][5].value).toBe("("); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][6].value).toBe("1"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][7].value).toBe(","); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[3][8].value).toBe(" "); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python"]); + expect(tokens[3][9].value).toBe("3"); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][10].value).toBe(")"); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][0].value).toBe("@"); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[4][1].value).toBe("some_decorator"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe("a"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[4][4].value).toBe("="); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[4][5].value).toBe("2"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[4][6].value).toBe(","); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[4][7].value).toBe(" "); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python"]); + expect(tokens[4][8].value).toBe("b"); + expect(tokens[4][8].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[4][9].value).toBe("="); + expect(tokens[4][9].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[4][10].value).toBe("{"); + expect(tokens[4][10].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","punctuation.definition.dict.begin.python"]); + expect(tokens[4][11].value).toBe("'"); + expect(tokens[4][11].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][12].value).toBe("q"); + expect(tokens[4][12].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[4][13].value).toBe("'"); + expect(tokens[4][13].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][14].value).toBe(":"); + expect(tokens[4][14].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","punctuation.separator.dict.python"]); + expect(tokens[4][15].value).toBe(" "); + expect(tokens[4][15].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python"]); + expect(tokens[4][16].value).toBe("42"); + expect(tokens[4][16].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[4][17].value).toBe("}"); + expect(tokens[4][17].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","punctuation.definition.dict.end.python"]); + expect(tokens[4][18].value).toBe(","); + expect(tokens[4][18].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[4][19].value).toBe(" "); + expect(tokens[4][19].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python"]); + expect(tokens[4][20].value).toBe("**"); + expect(tokens[4][20].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","keyword.operator.unpacking.arguments.python"]); + expect(tokens[4][21].value).toBe("kwargs"); + expect(tokens[4][21].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python"]); + expect(tokens[4][22].value).toBe(")"); + expect(tokens[4][22].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][0].value).toBe("@"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[5][1].value).toBe("classmethod"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function.decorator.python","support.type.python"]); + expect(tokens[6][0].value).toBe("def"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[6][2].value).toBe("decorated"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[6][3].value).toBe("("); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[6][4].value).toBe("a"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[6][5].value).toBe(")"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[6][6].value).toBe(":"); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[6][7].value).toBe(" "); + expect(tokens[6][7].scopes).toEqual(["source.python"]); + expect(tokens[6][8].value).toBe("pass"); + expect(tokens[6][8].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decorators2.py", + function() { + tokens = grammar.tokenizeLines("@ f . bar . a . b\ndef foo(): pass") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][2].value).toBe("f"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][4].value).toBe("."); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][6].value).toBe("bar"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][8].value).toBe("."); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][10].value).toBe("a"); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][12].value).toBe("."); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][14].value).toBe("b"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decorators3.py", + function() { + tokens = grammar.tokenizeLines("@ f . bar (baz = 1)\ndef foo(): pass") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][2].value).toBe("f"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][4].value).toBe("."); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][6].value).toBe("bar"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][8].value).toBe("("); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][9].value).toBe("baz"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python"]); + expect(tokens[0][11].value).toBe("="); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python"]); + expect(tokens[0][13].value).toBe("1"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][14].value).toBe(")"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decorators4.py", + function() { + tokens = grammar.tokenizeLines("@a.b.c.None.z\ndef foo(): pass") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe("a"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][2].value).toBe("."); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe("b"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][4].value).toBe("."); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][5].value).toBe("c"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][6].value).toBe("."); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][7].value).toBe("None"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.decorator.python","keyword.illegal.name.python"]); + expect(tokens[0][8].value).toBe("."); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][9].value).toBe("z"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decorators5.py", + function() { + tokens = grammar.tokenizeLines("@a. \\\n b . \\\n c.None.z \\\n baz(q=1)\n@foo.ok\ndef foo(): pass") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe("a"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][2].value).toBe("."); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][4].value).toBe("\\"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][5].value).toBe(""); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[1][1].value).toBe("b"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[1][3].value).toBe("."); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[1][5].value).toBe("\\"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][6].value).toBe(""); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[2][1].value).toBe("c"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[2][2].value).toBe("."); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[2][3].value).toBe("None"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.decorator.python","keyword.illegal.name.python"]); + expect(tokens[2][4].value).toBe("."); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[2][5].value).toBe("z"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[2][6].value).toBe(" "); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[2][7].value).toBe("\\"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.separator.continuation.line.python"]); + expect(tokens[2][8].value).toBe(""); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[3][1].value).toBe("baz"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[3][2].value).toBe("("); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][3].value).toBe("q"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][4].value).toBe("="); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[3][5].value).toBe("1"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][6].value).toBe(")"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[4][0].value).toBe("@"); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[4][1].value).toBe("foo"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[4][2].value).toBe("."); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[4][3].value).toBe("ok"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[5][0].value).toBe("def"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[5][2].value).toBe("foo"); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[5][3].value).toBe("("); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[5][4].value).toBe(")"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[5][5].value).toBe(":"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[5][6].value).toBe(" "); + expect(tokens[5][6].scopes).toEqual(["source.python"]); + expect(tokens[5][7].value).toBe("pass"); + expect(tokens[5][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decorators6.py", + function() { + tokens = grammar.tokenizeLines("@a. b . \\\n c.None.z(foo=BAR). \\\n baz[1:2]\n@foo.class.bar[]\n@foo.ok '''\ndef foo(): pass") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe("a"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][2].value).toBe("."); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][4].value).toBe("b"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][6].value).toBe("."); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[0][8].value).toBe("\\"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][9].value).toBe(""); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[1][1].value).toBe("c"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[1][2].value).toBe("."); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[1][3].value).toBe("None"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.decorator.python","keyword.illegal.name.python"]); + expect(tokens[1][4].value).toBe("."); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[1][5].value).toBe("z"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[1][6].value).toBe("("); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][7].value).toBe("foo"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[1][8].value).toBe("="); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[1][9].value).toBe("BAR"); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function.decorator.python","meta.function-call.arguments.python","constant.other.caps.python"]); + expect(tokens[1][10].value).toBe(")"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][11].value).toBe(". \\"); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.function.decorator.python","invalid.illegal.decorator.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("baz"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.item-access.python","meta.indexed-name.python"]); + expect(tokens[2][2].value).toBe("["); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("1"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.item-access.python","meta.item-access.arguments.python","constant.numeric.dec.python"]); + expect(tokens[2][4].value).toBe(":"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.item-access.python","meta.item-access.arguments.python","punctuation.separator.slice.python"]); + expect(tokens[2][5].value).toBe("2"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.item-access.python","meta.item-access.arguments.python","constant.numeric.dec.python"]); + expect(tokens[2][6].value).toBe("]"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][0].value).toBe("@"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[3][1].value).toBe("foo"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[3][2].value).toBe("."); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[3][3].value).toBe("class"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.decorator.python","keyword.control.flow.python"]); + expect(tokens[3][4].value).toBe("."); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[3][5].value).toBe("bar"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[3][6].value).toBe("[]"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function.decorator.python","invalid.illegal.decorator.python","invalid.illegal.decorator.python"]); + expect(tokens[4][0].value).toBe("@"); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[4][1].value).toBe("foo"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[4][2].value).toBe("."); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.separator.period.python"]); + expect(tokens[4][3].value).toBe("ok"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[4][4].value).toBe(" "); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function.decorator.python","invalid.illegal.decorator.python"]); + expect(tokens[4][5].value).toBe("'''"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function.decorator.python","invalid.illegal.decorator.python","invalid.illegal.decorator.python"]); + expect(tokens[5][0].value).toBe("def"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[5][2].value).toBe("foo"); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[5][3].value).toBe("("); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[5][4].value).toBe(")"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[5][5].value).toBe(":"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[5][6].value).toBe(" "); + expect(tokens[5][6].scopes).toEqual(["source.python"]); + expect(tokens[5][7].value).toBe("pass"); + expect(tokens[5][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decorators7.py", + function() { + tokens = grammar.tokenizeLines("# this is testing trailing whitespace after the decorator\n# DO NOT DELETE TRAILING WHITESTAPCE IN THIS FILE\n@foo \n@foo() \n@bar\t\n@bar()\t\n@bar() illegal # legal\n@bar(): \ndef baz(): pass") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" this is testing trailing whitespace after the decorator"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("#"); + expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][1].value).toBe(" DO NOT DELETE TRAILING WHITESTAPCE IN THIS FILE"); + expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe("@"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[2][1].value).toBe("foo"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[3][0].value).toBe("@"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[3][1].value).toBe("foo"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[3][2].value).toBe("("); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][3].value).toBe(")"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe("@"); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[4][1].value).toBe("bar"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[4][2].value).toBe("\t"); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[5][0].value).toBe("@"); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[5][1].value).toBe("bar"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[5][2].value).toBe("("); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][3].value).toBe(")"); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][4].value).toBe("\t"); + expect(tokens[5][4].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe("@"); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[6][1].value).toBe("bar"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[6][2].value).toBe("("); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[6][3].value).toBe(")"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[6][4].value).toBe(" illegal"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function.decorator.python","invalid.illegal.decorator.python"]); + expect(tokens[6][5].value).toBe(" "); + expect(tokens[6][5].scopes).toEqual(["source.python"]); + expect(tokens[6][6].value).toBe("#"); + expect(tokens[6][6].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[6][7].value).toBe(" legal"); + expect(tokens[6][7].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[7][0].value).toBe("@"); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[7][1].value).toBe("bar"); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[7][2].value).toBe("("); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[7][3].value).toBe(")"); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][4].value).toBe(":"); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.function.decorator.python","invalid.illegal.decorator.python"]); + expect(tokens[7][5].value).toBe(" "); + expect(tokens[7][5].scopes).toEqual(["source.python"]); + expect(tokens[8][0].value).toBe("def"); + expect(tokens[8][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[8][1].value).toBe(" "); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[8][2].value).toBe("baz"); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[8][3].value).toBe("("); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[8][4].value).toBe(")"); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[8][5].value).toBe(":"); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[8][6].value).toBe(" "); + expect(tokens[8][6].scopes).toEqual(["source.python"]); + expect(tokens[8][7].value).toBe("pass"); + expect(tokens[8][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/decorators8.py", + function() { + tokens = grammar.tokenizeLines("@deco().abc # SyntaxError: invalid syntax\ndef foo(): pass") + expect(tokens[0][0].value).toBe("@"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[0][1].value).toBe("deco"); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[0][2].value).toBe("("); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][3].value).toBe(")"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.decorator.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][4].value).toBe(".abc"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.decorator.python","invalid.illegal.decorator.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("#"); + expect(tokens[0][6].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][7].value).toBe(" SyntaxError: invalid syntax"); + expect(tokens[0][7].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/lambda1.py", + function() { + tokens = grammar.tokenizeLines("# That's how we want it:\nlll(lambda)") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" That's how we want it:"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("lll"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][1].value).toBe("("); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][2].value).toBe("lambda"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","storage.type.function.lambda.python"]); + expect(tokens[1][3].value).toBe(")"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/functions/lambda10.py", + function() { + tokens = grammar.tokenizeLines("showcase = lambda a, /, b, *, c: (a + b + c)") + expect(tokens[0][0].value).toBe("showcase"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][9].value).toBe("/"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.positional.parameter.python"]); + expect(tokens[0][10].value).toBe(", "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][11].value).toBe("b"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][12].value).toBe(","); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][14].value).toBe("*"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[0][15].value).toBe(", "); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][16].value).toBe("c"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][17].value).toBe(":"); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][18].value).toBe(" "); + expect(tokens[0][18].scopes).toEqual(["source.python"]); + expect(tokens[0][19].value).toBe("("); + expect(tokens[0][19].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][20].value).toBe("a"); + expect(tokens[0][20].scopes).toEqual(["source.python"]); + expect(tokens[0][21].value).toBe(" "); + expect(tokens[0][21].scopes).toEqual(["source.python"]); + expect(tokens[0][22].value).toBe("+"); + expect(tokens[0][22].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][23].value).toBe(" "); + expect(tokens[0][23].scopes).toEqual(["source.python"]); + expect(tokens[0][24].value).toBe("b"); + expect(tokens[0][24].scopes).toEqual(["source.python"]); + expect(tokens[0][25].value).toBe(" "); + expect(tokens[0][25].scopes).toEqual(["source.python"]); + expect(tokens[0][26].value).toBe("+"); + expect(tokens[0][26].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][27].value).toBe(" "); + expect(tokens[0][27].scopes).toEqual(["source.python"]); + expect(tokens[0][28].value).toBe("c"); + expect(tokens[0][28].scopes).toEqual(["source.python"]); + expect(tokens[0][29].value).toBe(")"); + expect(tokens[0][29].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + }); + + it("test/functions/lambda2.py", + function() { + tokens = grammar.tokenizeLines("lll(lambda=1)") + expect(tokens[0][0].value).toBe("lll"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][1].value).toBe("("); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][2].value).toBe("lambda"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.control.flow.python"]); + expect(tokens[0][3].value).toBe("="); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][4].value).toBe("1"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][5].value).toBe(")"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/functions/lambda3.py", + function() { + tokens = grammar.tokenizeLines("anon = lambda lambda: 42") + expect(tokens[0][0].value).toBe("anon"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("lambda"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","storage.type.function.lambda.python"]); + expect(tokens[0][7].value).toBe(":"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("42"); + expect(tokens[0][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/functions/lambda4.py", + function() { + tokens = grammar.tokenizeLines("lambda a : 1") + expect(tokens[0][0].value).toBe("lambda"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][2].value).toBe("a"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][4].value).toBe(":"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("1"); + expect(tokens[0][6].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/functions/lambda5.py", + function() { + tokens = grammar.tokenizeLines("anon = lambda -> qqq[None]: None\ndef f(): return 1 # this line should not break") + expect(tokens[0][0].value).toBe("anon"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("->"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.illegal.annotation.python"]); + expect(tokens[0][7].value).toBe(" qqq[None]"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][8].value).toBe(":"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("None"); + expect(tokens[0][10].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("f"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("return"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][8].value).toBe(" "); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe("1"); + expect(tokens[1][9].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python"]); + expect(tokens[1][11].value).toBe("#"); + expect(tokens[1][11].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][12].value).toBe(" this line should not break"); + expect(tokens[1][12].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/functions/lambda6.py", + function() { + tokens = grammar.tokenizeLines("anon = lambda a=123, c={'': 555}, \\\n d=toow(24): None") + expect(tokens[0][0].value).toBe("anon"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][7].value).toBe("="); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[0][8].value).toBe("123"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][9].value).toBe(","); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][11].value).toBe("c"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][12].value).toBe("="); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[0][13].value).toBe("{"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.begin.python"]); + expect(tokens[0][14].value).toBe("'"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][15].value).toBe("'"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][16].value).toBe(":"); + expect(tokens[0][16].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.dict.python"]); + expect(tokens[0][17].value).toBe(" "); + expect(tokens[0][17].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][18].value).toBe("555"); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.numeric.dec.python"]); + expect(tokens[0][19].value).toBe("}"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.end.python"]); + expect(tokens[0][20].value).toBe(","); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][21].value).toBe(" "); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][22].value).toBe("\\"); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][23].value).toBe(""); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[1][1].value).toBe("d"); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[1][3].value).toBe("toow"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][4].value).toBe("("); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][5].value).toBe("24"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[1][6].value).toBe(")"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][7].value).toBe(":"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[1][8].value).toBe(" "); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe("None"); + expect(tokens[1][9].scopes).toEqual(["source.python","constant.language.python"]); + }); + + it("test/functions/lambda7.py", + function() { + tokens = grammar.tokenizeLines("anon = lambda a, c={'key':\n\n 555}, e=fff: None") + expect(tokens[0][0].value).toBe("anon"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][9].value).toBe("c"); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][10].value).toBe("="); + expect(tokens[0][10].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[0][11].value).toBe("{"); + expect(tokens[0][11].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.begin.python"]); + expect(tokens[0][12].value).toBe("'"); + expect(tokens[0][12].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][13].value).toBe("key"); + expect(tokens[0][13].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python"]); + expect(tokens[0][14].value).toBe("'"); + expect(tokens[0][14].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][15].value).toBe(":"); + expect(tokens[0][15].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.dict.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][1].value).toBe("555"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","constant.numeric.dec.python"]); + expect(tokens[2][2].value).toBe("}"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.definition.dict.end.python"]); + expect(tokens[2][3].value).toBe(","); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][5].value).toBe("e"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][6].value).toBe("="); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.python"]); + expect(tokens[2][7].value).toBe("fff"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][8].value).toBe(":"); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("None"); + expect(tokens[2][10].scopes).toEqual(["source.python","constant.language.python"]); + }); + + it("test/functions/lambda8.py", + function() { + tokens = grammar.tokenizeLines("anon = lambda a,\n d=1: None\ndef foo(): pass") + expect(tokens[0][0].value).toBe("anon"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[0][8].value).toBe(""); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("d"); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe("1"); + expect(tokens[1][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][4].value).toBe(":"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("None"); + expect(tokens[1][6].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe(")"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][5].value).toBe(":"); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][6].value).toBe(" "); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe("pass"); + expect(tokens[2][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/functions/lambda9.py", + function() { + tokens = grammar.tokenizeLines("lambda *, b: b\nlambda *b: b\nlambda **b: b") + expect(tokens[0][0].value).toBe("lambda"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][2].value).toBe("*"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[0][3].value).toBe(", "); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][4].value).toBe("b"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][5].value).toBe(":"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("b"); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("lambda"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[1][2].value).toBe("*"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[1][3].value).toBe("b"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[1][4].value).toBe(":"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("b"); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("lambda"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[2][2].value).toBe("**"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","keyword.operator.unpacking.parameter.python"]); + expect(tokens[2][3].value).toBe("b"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][4].value).toBe(":"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("b"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + }); + + it("test/illegals/backticks1.py", + function() { + tokens = grammar.tokenizeLines("a = `(1, `2`)`") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("`"); + expect(tokens[0][4].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][5].value).toBe("("); + expect(tokens[0][5].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][6].value).toBe("1"); + expect(tokens[0][6].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.separator.element.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][9].value).toBe("`"); + expect(tokens[0][9].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][10].value).toBe("2"); + expect(tokens[0][10].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); + expect(tokens[0][11].value).toBe("`"); + expect(tokens[0][11].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][12].value).toBe(")"); + expect(tokens[0][12].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.parenthesis.end.python"]); + expect(tokens[0][13].value).toBe("`"); + expect(tokens[0][13].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + }); + + it("test/illegals/backticks2.py", + function() { + tokens = grammar.tokenizeLines("a = `\\\n123`\nprint a") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("`"); + expect(tokens[0][4].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][5].value).toBe("\\"); + expect(tokens[0][5].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][6].value).toBe(""); + expect(tokens[0][6].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[1][0].value).toBe("123"); + expect(tokens[1][0].scopes).toEqual(["source.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); + expect(tokens[1][1].value).toBe("`"); + expect(tokens[1][1].scopes).toEqual(["source.python","invalid.deprecated.backtick.python"]); + expect(tokens[2][0].value).toBe("print"); + expect(tokens[2][0].scopes).toEqual(["source.python","support.function.builtin.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("a"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + }); + + it("test/illegals/backticks3.py", + function() { + tokens = grammar.tokenizeLines("a = lambda `123`\n# comment") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("lambda"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python"]); + expect(tokens[0][6].value).toBe("`"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][7].value).toBe("123"); + expect(tokens[0][7].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python","constant.numeric.dec.python"]); + expect(tokens[0][8].value).toBe("`"); + expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python"]); + expect(tokens[0][9].value).toBe(""); + expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python"]); + expect(tokens[1][0].value).toBe("#"); + expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][1].value).toBe(" comment"); + expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/illegals/illegal1.py", + function() { + tokens = grammar.tokenizeLines("->\ndef f(): pass\n$\n?\na=$('.class').fuuuu(baz=1)\n# we recover just fine\nb = !some_ruby?\n# hey ;)") + expect(tokens[0][0].value).toBe("->"); + expect(tokens[0][0].scopes).toEqual(["source.python","invalid.illegal.annotation.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("f"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("$"); + expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[3][0].value).toBe("?"); + expect(tokens[3][0].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("="); + expect(tokens[4][1].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[4][2].value).toBe("$"); + expect(tokens[4][2].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[4][3].value).toBe("("); + expect(tokens[4][3].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[4][4].value).toBe("'"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][5].value).toBe(".class"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][6].value).toBe("'"); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][7].value).toBe(")"); + expect(tokens[4][7].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[4][8].value).toBe("."); + expect(tokens[4][8].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[4][9].value).toBe("fuuuu"); + expect(tokens[4][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[4][10].value).toBe("("); + expect(tokens[4][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][11].value).toBe("baz"); + expect(tokens[4][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[4][12].value).toBe("="); + expect(tokens[4][12].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[4][13].value).toBe("1"); + expect(tokens[4][13].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[4][14].value).toBe(")"); + expect(tokens[4][14].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][0].value).toBe("#"); + expect(tokens[5][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][1].value).toBe(" we recover just fine"); + expect(tokens[5][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[6][0].value).toBe("b"); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe("="); + expect(tokens[6][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[6][3].value).toBe(" "); + expect(tokens[6][3].scopes).toEqual(["source.python"]); + expect(tokens[6][4].value).toBe("!"); + expect(tokens[6][4].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[6][5].value).toBe("some_ruby"); + expect(tokens[6][5].scopes).toEqual(["source.python"]); + expect(tokens[6][6].value).toBe("?"); + expect(tokens[6][6].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[7][0].value).toBe("#"); + expect(tokens[7][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[7][1].value).toBe(" hey ;)"); + expect(tokens[7][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/illegals/illegal2.py", + function() { + tokens = grammar.tokenizeLines("a&&b||c\na &&= a\nb ||= c") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe("&&"); + expect(tokens[0][1].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[0][2].value).toBe("b"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe("||"); + expect(tokens[0][3].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[0][4].value).toBe("c"); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("&&"); + expect(tokens[1][2].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[1][3].value).toBe("="); + expect(tokens[1][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[1][5].value).toBe("a"); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("b"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("||"); + expect(tokens[2][2].scopes).toEqual(["source.python","invalid.illegal.operator.python"]); + expect(tokens[2][3].value).toBe("="); + expect(tokens[2][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("c"); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + }); + + it("test/linecont/linecont1.py", + function() { + tokens = grammar.tokenizeLines("1 + \\ sdgfsdf\ndef foo(): pass") + expect(tokens[0][0].value).toBe("1"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("+"); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("\\"); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("sdgfsdf"); + expect(tokens[0][6].scopes).toEqual(["source.python","invalid.illegal.line.continuation.python"]); + expect(tokens[1][0].value).toBe("def"); + expect(tokens[1][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[1][4].value).toBe(")"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("pass"); + expect(tokens[1][7].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/linecont/linecont2.py", + function() { + tokens = grammar.tokenizeLines("1 + \\\n 3") + expect(tokens[0][0].value).toBe("1"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("+"); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("\\"); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][5].value).toBe(""); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("3"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/numbers/complex1.py", + function() { + tokens = grammar.tokenizeLines("3.14j\n10.j\n10j\n.001j\n1e100j\n3.14e-10j") + expect(tokens[0][0].value).toBe("3.14"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[0][1].value).toBe("j"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[1][0].value).toBe("10."); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[1][1].value).toBe("j"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[2][0].value).toBe("10"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][1].value).toBe("j"); + expect(tokens[2][1].scopes).toEqual(["source.python","constant.numeric.dec.python","storage.type.imaginary.number.python"]); + expect(tokens[3][0].value).toBe(".001"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[3][1].value).toBe("j"); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[4][0].value).toBe("1e100"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[4][1].value).toBe("j"); + expect(tokens[4][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[5][0].value).toBe("3.14e-10"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[5][1].value).toBe("j"); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + }); + + it("test/numbers/complex2.py", + function() { + tokens = grammar.tokenizeLines("3.141_592j\n10_000.j\n10_000j\n.001_123j\n1e10_000j\n3.141_592e-10_000j") + expect(tokens[0][0].value).toBe("3.141_592"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[0][1].value).toBe("j"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[1][0].value).toBe("10_000."); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[1][1].value).toBe("j"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[2][0].value).toBe("10_000"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][1].value).toBe("j"); + expect(tokens[2][1].scopes).toEqual(["source.python","constant.numeric.dec.python","storage.type.imaginary.number.python"]); + expect(tokens[3][0].value).toBe(".001_123"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[3][1].value).toBe("j"); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[4][0].value).toBe("1e10_000"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[4][1].value).toBe("j"); + expect(tokens[4][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + expect(tokens[5][0].value).toBe("3.141_592e-10_000"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[5][1].value).toBe("j"); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.float.python","storage.type.imaginary.number.python"]); + }); + + it("test/numbers/dec.py", + function() { + tokens = grammar.tokenizeLines("123\n000\n123_456\n1_2_3_4_5_6") + expect(tokens[0][0].value).toBe("123"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][0].value).toBe("000"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][0].value).toBe("123_456"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[3][0].value).toBe("1_2_3_4_5_6"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/numbers/floats1.py", + function() { + tokens = grammar.tokenizeLines("123.456\n0.456\n000.0001\n.01234\n123e5\n123e-5\n000123e-005\n123.456e+5\n0.456e-5") + expect(tokens[0][0].value).toBe("123.456"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[1][0].value).toBe("0.456"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[2][0].value).toBe("000.0001"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[3][0].value).toBe(".01234"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[4][0].value).toBe("123e5"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[5][0].value).toBe("123e-5"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[6][0].value).toBe("000123e-005"); + expect(tokens[6][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[7][0].value).toBe("123.456e+5"); + expect(tokens[7][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[8][0].value).toBe("0.456e-5"); + expect(tokens[8][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + }); + + it("test/numbers/floats2.py", + function() { + tokens = grammar.tokenizeLines("1_234.567_890\n0.456_789\n000.000_1\n.012_34\n1_234e5_000\n1_234e-5_000\n000_123e-000_5\n1_234.567_8e+5_000\n0.456_78e-5_000") + expect(tokens[0][0].value).toBe("1_234.567_890"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[1][0].value).toBe("0.456_789"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[2][0].value).toBe("000.000_1"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[3][0].value).toBe(".012_34"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[4][0].value).toBe("1_234e5_000"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[5][0].value).toBe("1_234e-5_000"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[6][0].value).toBe("000_123e-000_5"); + expect(tokens[6][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[7][0].value).toBe("1_234.567_8e+5_000"); + expect(tokens[7][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + expect(tokens[8][0].value).toBe("0.456_78e-5_000"); + expect(tokens[8][0].scopes).toEqual(["source.python","constant.numeric.float.python"]); + }); + + it("test/numbers/invalid.py", + function() { + tokens = grammar.tokenizeLines("0123\n0123l\n123f\n123d\n123A\n123__456\n123_") + expect(tokens[0][0].value).toBe("0"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][1].value).toBe("123"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.dec.python","invalid.illegal.dec.python"]); + expect(tokens[1][0].value).toBe("0123l"); + expect(tokens[1][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[2][0].value).toBe("123f"); + expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[3][0].value).toBe("123d"); + expect(tokens[3][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[4][0].value).toBe("123A"); + expect(tokens[4][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[5][0].value).toBe("123__456"); + expect(tokens[5][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[6][0].value).toBe("123_"); + expect(tokens[6][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + }); + + it("test/numbers/nondec.py", + function() { + tokens = grammar.tokenizeLines("0o1234567\n0O1234567\n0o1_234_567\n0b0000\n0B0001\n0b_0011_1001\n0xabcdef01234567890\n0XFF12\n0xab_cd_ef_01_23_45_67_89_00\n\n0o\n0b\n0x\n0z\n\n0b02\n0x0z\n0o90") + expect(tokens[0][0].value).toBe("0o"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.oct.python","storage.type.number.python"]); + expect(tokens[0][1].value).toBe("1234567"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.oct.python"]); + expect(tokens[1][0].value).toBe("0O"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.oct.python","storage.type.number.python"]); + expect(tokens[1][1].value).toBe("1234567"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.oct.python"]); + expect(tokens[2][0].value).toBe("0o"); + expect(tokens[2][0].scopes).toEqual(["source.python","constant.numeric.oct.python","storage.type.number.python"]); + expect(tokens[2][1].value).toBe("1_234_567"); + expect(tokens[2][1].scopes).toEqual(["source.python","constant.numeric.oct.python"]); + expect(tokens[3][0].value).toBe("0b"); + expect(tokens[3][0].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[3][1].value).toBe("0000"); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[4][0].value).toBe("0B"); + expect(tokens[4][0].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[4][1].value).toBe("0001"); + expect(tokens[4][1].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[5][0].value).toBe("0b"); + expect(tokens[5][0].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[5][1].value).toBe("_0011_1001"); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[6][0].value).toBe("0x"); + expect(tokens[6][0].scopes).toEqual(["source.python","constant.numeric.hex.python","storage.type.number.python"]); + expect(tokens[6][1].value).toBe("abcdef01234567890"); + expect(tokens[6][1].scopes).toEqual(["source.python","constant.numeric.hex.python"]); + expect(tokens[7][0].value).toBe("0X"); + expect(tokens[7][0].scopes).toEqual(["source.python","constant.numeric.hex.python","storage.type.number.python"]); + expect(tokens[7][1].value).toBe("FF12"); + expect(tokens[7][1].scopes).toEqual(["source.python","constant.numeric.hex.python"]); + expect(tokens[8][0].value).toBe("0x"); + expect(tokens[8][0].scopes).toEqual(["source.python","constant.numeric.hex.python","storage.type.number.python"]); + expect(tokens[8][1].value).toBe("ab_cd_ef_01_23_45_67_89_00"); + expect(tokens[8][1].scopes).toEqual(["source.python","constant.numeric.hex.python"]); + expect(tokens[9][0].value).toBe(""); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[10][0].value).toBe("0o"); + expect(tokens[10][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[11][0].value).toBe("0b"); + expect(tokens[11][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[12][0].value).toBe("0x"); + expect(tokens[12][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[13][0].value).toBe("0z"); + expect(tokens[13][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[14][0].value).toBe(""); + expect(tokens[14][0].scopes).toEqual(["source.python"]); + expect(tokens[15][0].value).toBe("0b02"); + expect(tokens[15][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[16][0].value).toBe("0x0z"); + expect(tokens[16][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + expect(tokens[17][0].value).toBe("0o90"); + expect(tokens[17][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + }); + + it("test/numbers/python2.py", + function() { + tokens = grammar.tokenizeLines("123l\n123L\n1_234l") + expect(tokens[0][0].value).toBe("123"); + expect(tokens[0][0].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[0][1].value).toBe("l"); + expect(tokens[0][1].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[1][0].value).toBe("123"); + expect(tokens[1][0].scopes).toEqual(["source.python","constant.numeric.bin.python"]); + expect(tokens[1][1].value).toBe("L"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.bin.python","storage.type.number.python"]); + expect(tokens[2][0].value).toBe("1_234l"); + expect(tokens[2][0].scopes).toEqual(["source.python","invalid.illegal.name.python"]); + }); + + it("test/regexp/comments1.py", + function() { + tokens = grammar.tokenizeLines("a = r'''foo[abc] # comment'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("r"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'''"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("foo"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[0][7].value).toBe("["); + expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[0][8].value).toBe("a"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][9].value).toBe("b"); + expect(tokens[0][9].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][10].value).toBe("c"); + expect(tokens[0][10].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][11].value).toBe("]"); + expect(tokens[0][11].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python","string.regexp.quoted.multi.python"]); + expect(tokens[0][13].value).toBe("#"); + expect(tokens[0][13].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][14].value).toBe(" comment"); + expect(tokens[0][14].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[0][15].value).toBe("'''"); + expect(tokens[0][15].scopes).toEqual(["source.python","string.regexp.quoted.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/regexp/python1.py", + function() { + tokens = grammar.tokenizeLines("a = r'[a-z]'\na = R'[a-z]'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("r"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("["); + expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[0][7].value).toBe("a"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][8].value).toBe("-"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][9].value).toBe("z"); + expect(tokens[0][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[0][10].value).toBe("]"); + expect(tokens[0][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + expect(tokens[0][11].value).toBe("'"); + expect(tokens[0][11].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("R"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.raw.single.python","storage.type.string.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("[a-z]"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/regexp/python10.py", + function() { + tokens = grammar.tokenizeLines("a = r'(<\\' for\na = r\"(<\\\" for\na = r'[<\\' for\na = r\"[<\\\" for\na = r'(?=\\' for\na = r\"(?=\\\" for\na = r'(?P\\' for\na = r\"(?P\\\" for\na = r'(?"); + expect(tokens[6][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.named.regexp","entity.name.tag.named.group.regexp"]); + expect(tokens[6][8].value).toBe("\\'"); + expect(tokens[6][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.named.regexp","constant.character.escape.regexp"]); + expect(tokens[6][9].value).toBe(" for"); + expect(tokens[6][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.named.regexp"]); + expect(tokens[6][10].value).toBe(""); + expect(tokens[6][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python","invalid.illegal.newline.python"]); + expect(tokens[7][0].value).toBe("a"); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe(" "); + expect(tokens[7][1].scopes).toEqual(["source.python"]); + expect(tokens[7][2].value).toBe("="); + expect(tokens[7][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[7][3].value).toBe(" "); + expect(tokens[7][3].scopes).toEqual(["source.python"]); + expect(tokens[7][4].value).toBe("r"); + expect(tokens[7][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[7][5].value).toBe("\""); + expect(tokens[7][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][6].value).toBe("("); + expect(tokens[7][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.named.regexp","punctuation.parenthesis.named.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[7][7].value).toBe("?P"); + expect(tokens[7][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.named.regexp","entity.name.tag.named.group.regexp"]); + expect(tokens[7][8].value).toBe("\\\""); + expect(tokens[7][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.named.regexp","constant.character.escape.regexp"]); + expect(tokens[7][9].value).toBe(" for"); + expect(tokens[7][9].scopes).toEqual(["source.python","string.regexp.quoted.single.python","meta.named.regexp"]); + expect(tokens[7][10].value).toBe(""); + expect(tokens[7][10].scopes).toEqual(["source.python","string.regexp.quoted.single.python","invalid.illegal.newline.python"]); + expect(tokens[8][0].value).toBe("a"); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe(" "); + expect(tokens[8][1].scopes).toEqual(["source.python"]); + expect(tokens[8][2].value).toBe("="); + expect(tokens[8][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[8][3].value).toBe(" "); + expect(tokens[8][3].scopes).toEqual(["source.python"]); + expect(tokens[8][4].value).toBe("r"); + expect(tokens[8][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[8][5].value).toBe("'"); + expect(tokens[8][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[8][6].value).toBe("("); + expect(tokens[8][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python","keyword.operator.lookbehind.negative.regexp","punctuation.parenthesis.lookbehind.begin.regexp"]); + expect(tokens[8][7].value).toBe("? # Start of group 'key'\n [\"\"\" + _LegalKeyChars + r\"\"\"]+? # Any word of at least one letter\n ) # End of group 'key'\n ( # Optional group: there may not be a value.\n \\s*=\\s* # Equal Sign\n (?P # Start of group 'val'\n \"(?:[^\\\\\"]|\\\\.)*\" # Any doublequoted string\n | # or\n \\w{3},\\s[\\w\\d\\s-]{9,11}\\s[\\d:]{8}\\sGMT # Special case for \"expires\" attr\n | # or\n [\"\"\" + _LegalValueChars + r\"\"\"]* # Any word or empty string\n ) # End of group 'val'\n )? # End of optional value group\n \\s* # Any number of spaces.\n (\\s+|;|$) # Ending either at space, semicolon, or EOS.\n \"\"\", re.ASCII) # May be removed if safe.") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" And now something fun!"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe(""); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("CookiePattern"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("re"); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("."); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[2][6].value).toBe("compile"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[2][7].value).toBe("("); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][8].value).toBe("r"); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","storage.type.string.python"]); + expect(tokens[2][9].value).toBe("\"\"\""); + expect(tokens[2][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[3][1].value).toBe("(?x)"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","storage.modifier.flag.regexp"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[3][3].value).toBe("#"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][4].value).toBe(" This is a verbose pattern"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[4][1].value).toBe("\\s"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","support.other.escape.special.regexp"]); + expect(tokens[4][2].value).toBe("*"); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.quantifier.regexp"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[4][4].value).toBe("#"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[4][5].value).toBe(" Optional whitespace at start of cookie"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[5][1].value).toBe("("); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","punctuation.parenthesis.named.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[5][2].value).toBe("?P"); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","entity.name.tag.named.group.regexp"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[5][4].value).toBe("#"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][5].value).toBe(" Start of group 'key'"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[6][1].value).toBe("["); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[6][2].value).toBe("\"\"\""); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[6][3].value).toBe(" "); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[6][4].value).toBe("+"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.arithmetic.python"]); + expect(tokens[6][5].value).toBe(" "); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[6][6].value).toBe("_LegalKeyChars"); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[6][7].value).toBe(" "); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[6][8].value).toBe("+"); + expect(tokens[6][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.arithmetic.python"]); + expect(tokens[6][9].value).toBe(" "); + expect(tokens[6][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[6][10].value).toBe("r"); + expect(tokens[6][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","storage.type.string.python"]); + expect(tokens[6][11].value).toBe("\"\"\""); + expect(tokens[6][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][12].value).toBe("]"); + expect(tokens[6][12].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[6][13].value).toBe("+?"); + expect(tokens[6][13].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.quantifier.regexp"]); + expect(tokens[6][14].value).toBe(" "); + expect(tokens[6][14].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[6][15].value).toBe("#"); + expect(tokens[6][15].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[6][16].value).toBe(" Any word of at least one letter"); + expect(tokens[6][16].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[7][0].value).toBe(" ) "); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[7][1].value).toBe("#"); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[7][2].value).toBe(" End of group 'key'"); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[8][1].value).toBe("("); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","punctuation.parenthesis.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[8][2].value).toBe(" "); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[8][3].value).toBe("#"); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[8][4].value).toBe(" Optional group: there may not be a value."); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[9][1].value).toBe("\\s"); + expect(tokens[9][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","support.other.escape.special.regexp"]); + expect(tokens[9][2].value).toBe("*"); + expect(tokens[9][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.quantifier.regexp"]); + expect(tokens[9][3].value).toBe("="); + expect(tokens[9][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[9][4].value).toBe("\\s"); + expect(tokens[9][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","support.other.escape.special.regexp"]); + expect(tokens[9][5].value).toBe("*"); + expect(tokens[9][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.quantifier.regexp"]); + expect(tokens[9][6].value).toBe(" "); + expect(tokens[9][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[9][7].value).toBe("#"); + expect(tokens[9][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[9][8].value).toBe(" Equal Sign"); + expect(tokens[9][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[10][1].value).toBe("("); + expect(tokens[10][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","punctuation.parenthesis.named.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[10][2].value).toBe("?P"); + expect(tokens[10][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","entity.name.tag.named.group.regexp"]); + expect(tokens[10][3].value).toBe(" "); + expect(tokens[10][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[10][4].value).toBe("#"); + expect(tokens[10][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[10][5].value).toBe(" Start of group 'val'"); + expect(tokens[10][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python"]); + expect(tokens[11][0].value).toBe(" \""); + expect(tokens[11][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[11][1].value).toBe("(?:"); + expect(tokens[11][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","punctuation.parenthesis.non-capturing.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[11][2].value).toBe("["); + expect(tokens[11][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[11][3].value).toBe("^"); + expect(tokens[11][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","keyword.operator.negation.regexp"]); + expect(tokens[11][4].value).toBe("\\\\"); + expect(tokens[11][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.character.escape.regexp"]); + expect(tokens[11][5].value).toBe("\""); + expect(tokens[11][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[11][6].value).toBe("]"); + expect(tokens[11][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + expect(tokens[11][7].value).toBe("|"); + expect(tokens[11][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","keyword.operator.disjunction.regexp"]); + expect(tokens[11][8].value).toBe("\\\\"); + expect(tokens[11][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","constant.character.escape.regexp"]); + expect(tokens[11][9].value).toBe("."); + expect(tokens[11][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","support.other.match.any.regexp"]); + expect(tokens[11][10].value).toBe(")"); + expect(tokens[11][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","punctuation.parenthesis.non-capturing.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[11][11].value).toBe("*"); + expect(tokens[11][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","keyword.operator.quantifier.regexp"]); + expect(tokens[11][12].value).toBe("\" "); + expect(tokens[11][12].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[11][13].value).toBe("#"); + expect(tokens[11][13].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[11][14].value).toBe(" Any doublequoted string"); + expect(tokens[11][14].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python"]); + expect(tokens[12][0].value).toBe(" "); + expect(tokens[12][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[12][1].value).toBe("|"); + expect(tokens[12][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","keyword.operator.disjunction.regexp"]); + expect(tokens[12][2].value).toBe(" "); + expect(tokens[12][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[12][3].value).toBe("#"); + expect(tokens[12][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[12][4].value).toBe(" or"); + expect(tokens[12][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python"]); + expect(tokens[13][0].value).toBe(" "); + expect(tokens[13][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[13][1].value).toBe("\\w"); + expect(tokens[13][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","support.other.escape.special.regexp"]); + expect(tokens[13][2].value).toBe("{3}"); + expect(tokens[13][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","keyword.operator.quantifier.regexp"]); + expect(tokens[13][3].value).toBe(","); + expect(tokens[13][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[13][4].value).toBe("\\s"); + expect(tokens[13][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","support.other.escape.special.regexp"]); + expect(tokens[13][5].value).toBe("["); + expect(tokens[13][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[13][6].value).toBe("\\w"); + expect(tokens[13][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[13][7].value).toBe("\\d"); + expect(tokens[13][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[13][8].value).toBe("\\s"); + expect(tokens[13][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[13][9].value).toBe("-"); + expect(tokens[13][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[13][10].value).toBe("]"); + expect(tokens[13][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + expect(tokens[13][11].value).toBe("{9,11}"); + expect(tokens[13][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","keyword.operator.quantifier.regexp"]); + expect(tokens[13][12].value).toBe("\\s"); + expect(tokens[13][12].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","support.other.escape.special.regexp"]); + expect(tokens[13][13].value).toBe("["); + expect(tokens[13][13].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[13][14].value).toBe("\\d"); + expect(tokens[13][14].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","support.other.escape.special.regexp"]); + expect(tokens[13][15].value).toBe(":"); + expect(tokens[13][15].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.character.set.regexp"]); + expect(tokens[13][16].value).toBe("]"); + expect(tokens[13][16].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.end.regexp"]); + expect(tokens[13][17].value).toBe("{8}"); + expect(tokens[13][17].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","keyword.operator.quantifier.regexp"]); + expect(tokens[13][18].value).toBe("\\s"); + expect(tokens[13][18].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","support.other.escape.special.regexp"]); + expect(tokens[13][19].value).toBe("GMT "); + expect(tokens[13][19].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[13][20].value).toBe("#"); + expect(tokens[13][20].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[13][21].value).toBe(" Special case for \"expires\" attr"); + expect(tokens[13][21].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python"]); + expect(tokens[14][0].value).toBe(" "); + expect(tokens[14][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[14][1].value).toBe("|"); + expect(tokens[14][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","keyword.operator.disjunction.regexp"]); + expect(tokens[14][2].value).toBe(" "); + expect(tokens[14][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[14][3].value).toBe("#"); + expect(tokens[14][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[14][4].value).toBe(" or"); + expect(tokens[14][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","comment.line.number-sign.python"]); + expect(tokens[15][0].value).toBe(" "); + expect(tokens[15][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp"]); + expect(tokens[15][1].value).toBe("["); + expect(tokens[15][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","meta.named.regexp","meta.character.set.regexp","constant.other.set.regexp punctuation.character.set.begin.regexp"]); + expect(tokens[15][2].value).toBe("\"\"\""); + expect(tokens[15][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[15][3].value).toBe(" "); + expect(tokens[15][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[15][4].value).toBe("+"); + expect(tokens[15][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.arithmetic.python"]); + expect(tokens[15][5].value).toBe(" "); + expect(tokens[15][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[15][6].value).toBe("_LegalValueChars"); + expect(tokens[15][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[15][7].value).toBe(" "); + expect(tokens[15][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[15][8].value).toBe("+"); + expect(tokens[15][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.arithmetic.python"]); + expect(tokens[15][9].value).toBe(" "); + expect(tokens[15][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[15][10].value).toBe("r"); + expect(tokens[15][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","storage.type.string.python"]); + expect(tokens[15][11].value).toBe("\"\"\""); + expect(tokens[15][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[15][12].value).toBe("]"); + expect(tokens[15][12].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[15][13].value).toBe("*"); + expect(tokens[15][13].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.quantifier.regexp"]); + expect(tokens[15][14].value).toBe(" "); + expect(tokens[15][14].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[15][15].value).toBe("#"); + expect(tokens[15][15].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[15][16].value).toBe(" Any word or empty string"); + expect(tokens[15][16].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[16][0].value).toBe(" ) "); + expect(tokens[16][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[16][1].value).toBe("#"); + expect(tokens[16][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[16][2].value).toBe(" End of group 'val'"); + expect(tokens[16][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[17][0].value).toBe(" )"); + expect(tokens[17][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[17][1].value).toBe("?"); + expect(tokens[17][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.quantifier.regexp"]); + expect(tokens[17][2].value).toBe(" "); + expect(tokens[17][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[17][3].value).toBe("#"); + expect(tokens[17][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[17][4].value).toBe(" End of optional value group"); + expect(tokens[17][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[18][0].value).toBe(" "); + expect(tokens[18][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[18][1].value).toBe("\\s"); + expect(tokens[18][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","support.other.escape.special.regexp"]); + expect(tokens[18][2].value).toBe("*"); + expect(tokens[18][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.quantifier.regexp"]); + expect(tokens[18][3].value).toBe(" "); + expect(tokens[18][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[18][4].value).toBe("#"); + expect(tokens[18][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[18][5].value).toBe(" Any number of spaces."); + expect(tokens[18][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[19][0].value).toBe(" "); + expect(tokens[19][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[19][1].value).toBe("("); + expect(tokens[19][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","punctuation.parenthesis.begin.regexp support.other.parenthesis.regexp"]); + expect(tokens[19][2].value).toBe("\\s"); + expect(tokens[19][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","support.other.escape.special.regexp"]); + expect(tokens[19][3].value).toBe("+"); + expect(tokens[19][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.quantifier.regexp"]); + expect(tokens[19][4].value).toBe("|"); + expect(tokens[19][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.disjunction.regexp"]); + expect(tokens[19][5].value).toBe(";"); + expect(tokens[19][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[19][6].value).toBe("|"); + expect(tokens[19][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","keyword.operator.disjunction.regexp"]); + expect(tokens[19][7].value).toBe("$"); + expect(tokens[19][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","support.other.match.end.regexp"]); + expect(tokens[19][8].value).toBe(")"); + expect(tokens[19][8].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","punctuation.parenthesis.end.regexp support.other.parenthesis.regexp"]); + expect(tokens[19][9].value).toBe(" "); + expect(tokens[19][9].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[19][10].value).toBe("#"); + expect(tokens[19][10].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[19][11].value).toBe(" Ending either at space, semicolon, or EOS."); + expect(tokens[19][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","comment.line.number-sign.python"]); + expect(tokens[20][0].value).toBe(" "); + expect(tokens[20][0].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python"]); + expect(tokens[20][1].value).toBe("\"\"\""); + expect(tokens[20][1].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","string.regexp.quoted.multi.python","punctuation.definition.string.end.python"]); + expect(tokens[20][2].value).toBe(","); + expect(tokens[20][2].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[20][3].value).toBe(" "); + expect(tokens[20][3].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[20][4].value).toBe("re"); + expect(tokens[20][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[20][5].value).toBe("."); + expect(tokens[20][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[20][6].value).toBe("ASCII"); + expect(tokens[20][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","meta.member.access.python","constant.other.caps.python"]); + expect(tokens[20][7].value).toBe(")"); + expect(tokens[20][7].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[20][8].value).toBe(" "); + expect(tokens[20][8].scopes).toEqual(["source.python"]); + expect(tokens[20][9].value).toBe("#"); + expect(tokens[20][9].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[20][10].value).toBe(" May be removed if safe."); + expect(tokens[20][10].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/regexp/python9.py", + function() { + tokens = grammar.tokenizeLines("a = r'foo[^]'\na = r'foo[]'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("r"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("foo"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[0][7].value).toBe("[^]"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[0][8].value).toBe("'"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("r"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("foo"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[1][7].value).toBe("[]"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[1][8].value).toBe("'"); + expect(tokens[1][8].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/semicolon/semi1.py", + function() { + tokens = grammar.tokenizeLines("a;\n;\na=1; b") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(";"); + expect(tokens[0][1].scopes).toEqual(["source.python","invalid.deprecated.semicolon.python"]); + expect(tokens[1][0].value).toBe(";"); + expect(tokens[1][0].scopes).toEqual(["source.python","invalid.deprecated.semicolon.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("="); + expect(tokens[2][1].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][2].value).toBe("1"); + expect(tokens[2][2].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][3].value).toBe("; "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("b"); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + }); + + it("test/statements/annotation1.py", + function() { + tokens = grammar.tokenizeLines("some_number: int # variable without initial value\nsome_list: List[int] = [] # variable with initial value") + expect(tokens[0][0].value).toBe("some_number"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(":"); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[0][2].value).toBe(" "); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe("int"); + expect(tokens[0][3].scopes).toEqual(["source.python","support.type.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("#"); + expect(tokens[0][5].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][6].value).toBe(" variable without initial value"); + expect(tokens[0][6].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("some_list"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(":"); + expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("List"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.item-access.python","meta.indexed-name.python"]); + expect(tokens[1][4].value).toBe("["); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][5].value).toBe("int"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.item-access.python","meta.item-access.arguments.python","support.type.python"]); + expect(tokens[1][6].value).toBe("]"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python"]); + expect(tokens[1][8].value).toBe("="); + expect(tokens[1][8].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][9].value).toBe(" "); + expect(tokens[1][9].scopes).toEqual(["source.python"]); + expect(tokens[1][10].value).toBe("["); + expect(tokens[1][10].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][11].value).toBe("]"); + expect(tokens[1][11].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[1][12].value).toBe(" "); + expect(tokens[1][12].scopes).toEqual(["source.python"]); + expect(tokens[1][13].value).toBe("#"); + expect(tokens[1][13].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][14].value).toBe(" variable with initial value"); + expect(tokens[1][14].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/statements/for1.py", + function() { + tokens = grammar.tokenizeLines("for a, b, c in b:\n pass\nelse:\n 1/0") + expect(tokens[0][0].value).toBe("for"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("a"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(","); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("b"); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe(","); + expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("c"); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("in"); + expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe("b"); + expect(tokens[0][12].scopes).toEqual(["source.python"]); + expect(tokens[0][13].value).toBe(":"); + expect(tokens[0][13].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("pass"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][0].value).toBe("else"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(":"); + expect(tokens[2][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("1"); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[3][2].value).toBe("/"); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[3][3].value).toBe("0"); + expect(tokens[3][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/statements/for2.py", + function() { + tokens = grammar.tokenizeLines("forvariable = None\nfor a, b, c, invariable in [2 in q, 2 in w]:\n pass") + expect(tokens[0][0].value).toBe("forvariable"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("None"); + expect(tokens[0][4].scopes).toEqual(["source.python","constant.language.python"]); + expect(tokens[1][0].value).toBe("for"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("a"); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe(","); + expect(tokens[1][3].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[1][5].value).toBe("b"); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe(","); + expect(tokens[1][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python"]); + expect(tokens[1][8].value).toBe("c"); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe(","); + expect(tokens[1][9].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python"]); + expect(tokens[1][11].value).toBe("invariable"); + expect(tokens[1][11].scopes).toEqual(["source.python"]); + expect(tokens[1][12].value).toBe(" "); + expect(tokens[1][12].scopes).toEqual(["source.python"]); + expect(tokens[1][13].value).toBe("in"); + expect(tokens[1][13].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][14].value).toBe(" "); + expect(tokens[1][14].scopes).toEqual(["source.python"]); + expect(tokens[1][15].value).toBe("["); + expect(tokens[1][15].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][16].value).toBe("2"); + expect(tokens[1][16].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][17].value).toBe(" "); + expect(tokens[1][17].scopes).toEqual(["source.python"]); + expect(tokens[1][18].value).toBe("in"); + expect(tokens[1][18].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[1][19].value).toBe(" "); + expect(tokens[1][19].scopes).toEqual(["source.python"]); + expect(tokens[1][20].value).toBe("q"); + expect(tokens[1][20].scopes).toEqual(["source.python"]); + expect(tokens[1][21].value).toBe(","); + expect(tokens[1][21].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[1][22].value).toBe(" "); + expect(tokens[1][22].scopes).toEqual(["source.python"]); + expect(tokens[1][23].value).toBe("2"); + expect(tokens[1][23].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][24].value).toBe(" "); + expect(tokens[1][24].scopes).toEqual(["source.python"]); + expect(tokens[1][25].value).toBe("in"); + expect(tokens[1][25].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[1][26].value).toBe(" "); + expect(tokens[1][26].scopes).toEqual(["source.python"]); + expect(tokens[1][27].value).toBe("w"); + expect(tokens[1][27].scopes).toEqual(["source.python"]); + expect(tokens[1][28].value).toBe("]"); + expect(tokens[1][28].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[1][29].value).toBe(":"); + expect(tokens[1][29].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("pass"); + expect(tokens[2][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/statements/for3.py", + function() { + tokens = grammar.tokenizeLines("for(a, b), c, invariable in[2 in q, 2 in w]:\n pass") + expect(tokens[0][0].value).toBe("for"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe("("); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][2].value).toBe("a"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(","); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("b"); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe(")"); + expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("c"); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe(","); + expect(tokens[0][10].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe("invariable"); + expect(tokens[0][12].scopes).toEqual(["source.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[0][14].value).toBe("in"); + expect(tokens[0][14].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][15].value).toBe("["); + expect(tokens[0][15].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[0][16].value).toBe("2"); + expect(tokens[0][16].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][17].value).toBe(" "); + expect(tokens[0][17].scopes).toEqual(["source.python"]); + expect(tokens[0][18].value).toBe("in"); + expect(tokens[0][18].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[0][19].value).toBe(" "); + expect(tokens[0][19].scopes).toEqual(["source.python"]); + expect(tokens[0][20].value).toBe("q"); + expect(tokens[0][20].scopes).toEqual(["source.python"]); + expect(tokens[0][21].value).toBe(","); + expect(tokens[0][21].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][22].value).toBe(" "); + expect(tokens[0][22].scopes).toEqual(["source.python"]); + expect(tokens[0][23].value).toBe("2"); + expect(tokens[0][23].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][24].value).toBe(" "); + expect(tokens[0][24].scopes).toEqual(["source.python"]); + expect(tokens[0][25].value).toBe("in"); + expect(tokens[0][25].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[0][26].value).toBe(" "); + expect(tokens[0][26].scopes).toEqual(["source.python"]); + expect(tokens[0][27].value).toBe("w"); + expect(tokens[0][27].scopes).toEqual(["source.python"]); + expect(tokens[0][28].value).toBe("]"); + expect(tokens[0][28].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[0][29].value).toBe(":"); + expect(tokens[0][29].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("pass"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/statements/from1.py", + function() { + tokens = grammar.tokenizeLines("from ... import foo as bar\nraise Exception('done') from exc\nyield from foo") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("..."); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("import"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("foo"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("as"); + expect(tokens[0][8].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("bar"); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("raise"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("Exception"); + expect(tokens[1][2].scopes).toEqual(["source.python","meta.function-call.python","support.type.exception.python"]); + expect(tokens[1][3].value).toBe("("); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][4].value).toBe("'"); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe("done"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[1][6].value).toBe("'"); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][7].value).toBe(")"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][8].value).toBe(" "); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe("from"); + expect(tokens[1][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python"]); + expect(tokens[1][11].value).toBe("exc"); + expect(tokens[1][11].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("yield from"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + }); + + it("test/statements/global1.py", + function() { + tokens = grammar.tokenizeLines("global a, b, c") + expect(tokens[0][0].value).toBe("global"); + expect(tokens[0][0].scopes).toEqual(["source.python","storage.modifier.declaration.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("a"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(","); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("b"); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe(","); + expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("c"); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + }); + + it("test/statements/if1.py", + function() { + tokens = grammar.tokenizeLines("if (a if b else c):\n 1\nelif b or c and d:\n 2\nelse:\n 3") + expect(tokens[0][0].value).toBe("if"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("("); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][3].value).toBe("a"); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("if"); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("b"); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("else"); + expect(tokens[0][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("c"); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe(")"); + expect(tokens[0][12].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[0][13].value).toBe(":"); + expect(tokens[0][13].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("1"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][0].value).toBe("elif"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("b"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("or"); + expect(tokens[2][4].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("c"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe("and"); + expect(tokens[2][8].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("d"); + expect(tokens[2][10].scopes).toEqual(["source.python"]); + expect(tokens[2][11].value).toBe(":"); + expect(tokens[2][11].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("2"); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[4][0].value).toBe("else"); + expect(tokens[4][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][1].value).toBe(":"); + expect(tokens[4][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("3"); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/statements/import1.py", + function() { + tokens = grammar.tokenizeLines("from ...foo import bar as spam, baz\nimport time as ham, datetime") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("..."); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe("foo"); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("import"); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("bar"); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("as"); + expect(tokens[0][9].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("spam"); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe(","); + expect(tokens[0][12].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[0][14].value).toBe("baz"); + expect(tokens[0][14].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("import"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("time"); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("as"); + expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("ham"); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe(","); + expect(tokens[1][7].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[1][8].value).toBe(" "); + expect(tokens[1][8].scopes).toEqual(["source.python"]); + expect(tokens[1][9].value).toBe("datetime"); + expect(tokens[1][9].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import2.py", + function() { + tokens = grammar.tokenizeLines("from .... import a\nfrom ... import b\nfrom .. import c") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("...."); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("import"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("..."); + expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("import"); + expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("b"); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("from"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe(".."); + expect(tokens[2][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("import"); + expect(tokens[2][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("c"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import3.py", + function() { + tokens = grammar.tokenizeLines("from....import a\nfrom...import b\nfrom..import c\nfrom.import d") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe("...."); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("import"); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("a"); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[1][2].value).toBe("import"); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("b"); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("from"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][1].value).toBe(".."); + expect(tokens[2][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[2][2].value).toBe("import"); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("c"); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("from"); + expect(tokens[3][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][1].value).toBe("."); + expect(tokens[3][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[3][2].value).toBe("import"); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("d"); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import4.py", + function() { + tokens = grammar.tokenizeLines("from....foo import a\nfrom...foo import b\nfrom..foo import c\nfrom.foo import d") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe("...."); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("import"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("a"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[1][2].value).toBe("foo"); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("import"); + expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("b"); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe("from"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][1].value).toBe(".."); + expect(tokens[2][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("import"); + expect(tokens[2][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("c"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("from"); + expect(tokens[3][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][1].value).toBe("."); + expect(tokens[3][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[3][2].value).toBe("foo"); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("import"); + expect(tokens[3][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][5].value).toBe(" "); + expect(tokens[3][5].scopes).toEqual(["source.python"]); + expect(tokens[3][6].value).toBe("d"); + expect(tokens[3][6].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import5.py", + function() { + tokens = grammar.tokenizeLines("from.foo import d\nfrom.import a\n\nfoo.import\n\nraise Exception from Foo\n\ndef bar():\n yield from baz") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe("."); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("import"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("d"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe("."); + expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[1][2].value).toBe("import"); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("a"); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("foo"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("."); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[3][2].value).toBe("import"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.member.access.python","keyword.control.import.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[5][0].value).toBe("raise"); + expect(tokens[5][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe("Exception"); + expect(tokens[5][2].scopes).toEqual(["source.python","support.type.exception.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[5][4].value).toBe("from"); + expect(tokens[5][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][5].value).toBe(" "); + expect(tokens[5][5].scopes).toEqual(["source.python"]); + expect(tokens[5][6].value).toBe("Foo"); + expect(tokens[5][6].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe(""); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[7][0].value).toBe("def"); + expect(tokens[7][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[7][1].value).toBe(" "); + expect(tokens[7][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[7][2].value).toBe("bar"); + expect(tokens[7][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[7][3].value).toBe("("); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[7][4].value).toBe(")"); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[7][5].value).toBe(":"); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("yield from"); + expect(tokens[8][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][2].value).toBe(" "); + expect(tokens[8][2].scopes).toEqual(["source.python"]); + expect(tokens[8][3].value).toBe("baz"); + expect(tokens[8][3].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import6.py", + function() { + tokens = grammar.tokenizeLines("from __future__ import generator_stop") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("__future__"); + expect(tokens[0][2].scopes).toEqual(["source.python","support.variable.magic.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("import"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("generator_stop"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + }); + + it("test/statements/import7.py", + function() { + tokens = grammar.tokenizeLines("from .importing import *\nfrom importing import *") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("."); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe("importing"); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("import"); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("*"); + expect(tokens[0][7].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[1][0].value).toBe("from"); + expect(tokens[1][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("importing"); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("import"); + expect(tokens[1][4].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][5].value).toBe(" "); + expect(tokens[1][5].scopes).toEqual(["source.python"]); + expect(tokens[1][6].value).toBe("*"); + expect(tokens[1][6].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + }); + + it("test/statements/import8.py", + function() { + tokens = grammar.tokenizeLines("try:\n import \\\n time as ham, \\\n datetime \\\n # XXX: comment at the end of import\nexcept Exception as exc:\n pass") + expect(tokens[0][0].value).toBe("try"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(":"); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("import"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("\\"); + expect(tokens[1][3].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][4].value).toBe(""); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("time"); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("as"); + expect(tokens[2][3].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("ham"); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe(","); + expect(tokens[2][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe("\\"); + expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[2][9].value).toBe(""); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("datetime"); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("\\"); + expect(tokens[3][3].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[3][4].value).toBe(""); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("#"); + expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][3].value).toBe("XXX"); + expect(tokens[4][3].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[4][4].value).toBe(": comment at the end of import"); + expect(tokens[4][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[5][0].value).toBe("except"); + expect(tokens[5][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe("Exception"); + expect(tokens[5][2].scopes).toEqual(["source.python","support.type.exception.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[5][4].value).toBe("as"); + expect(tokens[5][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][5].value).toBe(" "); + expect(tokens[5][5].scopes).toEqual(["source.python"]); + expect(tokens[5][6].value).toBe("exc"); + expect(tokens[5][6].scopes).toEqual(["source.python"]); + expect(tokens[5][7].value).toBe(":"); + expect(tokens[5][7].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("pass"); + expect(tokens[6][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/statements/import9.py", + function() { + tokens = grammar.tokenizeLines("from . . . foo import \\\n (\n # XXX: legal comment inside import\n time as bar,\n # another comment\n baz,\n datetime as ham\n )\nraise Exception('!') from None") + expect(tokens[0][0].value).toBe("from"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("."); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("."); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("."); + expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.period.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("foo"); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("import"); + expect(tokens[0][10].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe("\\"); + expect(tokens[0][12].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[0][13].value).toBe(""); + expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("("); + expect(tokens[1][1].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("#"); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][3].value).toBe("XXX"); + expect(tokens[2][3].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[2][4].value).toBe(": legal comment inside import"); + expect(tokens[2][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("time"); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("as"); + expect(tokens[3][3].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[3][4].value).toBe(" "); + expect(tokens[3][4].scopes).toEqual(["source.python"]); + expect(tokens[3][5].value).toBe("bar"); + expect(tokens[3][5].scopes).toEqual(["source.python"]); + expect(tokens[3][6].value).toBe(","); + expect(tokens[3][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("#"); + expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[4][2].value).toBe(" another comment"); + expect(tokens[4][2].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("baz"); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe(","); + expect(tokens[5][2].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("datetime"); + expect(tokens[6][1].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe(" "); + expect(tokens[6][2].scopes).toEqual(["source.python"]); + expect(tokens[6][3].value).toBe("as"); + expect(tokens[6][3].scopes).toEqual(["source.python","keyword.control.import.python"]); + expect(tokens[6][4].value).toBe(" "); + expect(tokens[6][4].scopes).toEqual(["source.python"]); + expect(tokens[6][5].value).toBe("ham"); + expect(tokens[6][5].scopes).toEqual(["source.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe(")"); + expect(tokens[7][1].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[8][0].value).toBe("raise"); + expect(tokens[8][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][1].value).toBe(" "); + expect(tokens[8][1].scopes).toEqual(["source.python"]); + expect(tokens[8][2].value).toBe("Exception"); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.function-call.python","support.type.exception.python"]); + expect(tokens[8][3].value).toBe("("); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[8][4].value).toBe("'"); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[8][5].value).toBe("!"); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[8][6].value).toBe("'"); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[8][7].value).toBe(")"); + expect(tokens[8][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[8][8].value).toBe(" "); + expect(tokens[8][8].scopes).toEqual(["source.python"]); + expect(tokens[8][9].value).toBe("from"); + expect(tokens[8][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][10].value).toBe(" "); + expect(tokens[8][10].scopes).toEqual(["source.python"]); + expect(tokens[8][11].value).toBe("None"); + expect(tokens[8][11].scopes).toEqual(["source.python","constant.language.python"]); + }); + + it("test/statements/match1.py", + function() { + tokens = grammar.tokenizeLines("def foo(status):\n match status:\n case 404:\n return \"Not found\"\n case 401 | 403:\n return \"Not allowed\"\n case _:\n return \"Something's wrong with the internet\"") + expect(tokens[0][0].value).toBe("def"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[0][2].value).toBe("foo"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[0][4].value).toBe("status"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[0][5].value).toBe(")"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[0][6].value).toBe(":"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("match"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("status"); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe(":"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("case"); + expect(tokens[2][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("404"); + expect(tokens[2][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][4].value).toBe(":"); + expect(tokens[2][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("return"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("\""); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][4].value).toBe("Not found"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[3][5].value).toBe("\""); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("case"); + expect(tokens[4][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python"]); + expect(tokens[4][3].value).toBe("401"); + expect(tokens[4][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[4][4].value).toBe(" "); + expect(tokens[4][4].scopes).toEqual(["source.python"]); + expect(tokens[4][5].value).toBe("|"); + expect(tokens[4][5].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[4][6].value).toBe(" "); + expect(tokens[4][6].scopes).toEqual(["source.python"]); + expect(tokens[4][7].value).toBe("403"); + expect(tokens[4][7].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[4][8].value).toBe(":"); + expect(tokens[4][8].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("return"); + expect(tokens[5][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("\""); + expect(tokens[5][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][4].value).toBe("Not allowed"); + expect(tokens[5][4].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[5][5].value).toBe("\""); + expect(tokens[5][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("case"); + expect(tokens[6][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[6][2].value).toBe(" "); + expect(tokens[6][2].scopes).toEqual(["source.python"]); + expect(tokens[6][3].value).toBe("_"); + expect(tokens[6][3].scopes).toEqual(["source.python"]); + expect(tokens[6][4].value).toBe(":"); + expect(tokens[6][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("return"); + expect(tokens[7][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("\""); + expect(tokens[7][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[7][4].value).toBe("Something's wrong with the internet"); + expect(tokens[7][4].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[7][5].value).toBe("\""); + expect(tokens[7][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/statements/match2.py", + function() { + tokens = grammar.tokenizeLines("match point:\n case Point(x=0, y=0):\n print(\"Origin is the point's location.\")\n case Point(x=0, y=y):\n print(f\"The point is on the y-axis.\")\n case Point(x=x, y=0):\n print(f\"The point is on the x-axis.\")\n case Point():\n print(\"The point is located somewhere else on the plane.\")\n case _:\n print(\"Not a point\")") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("point"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("case"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("Point"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][4].value).toBe("("); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][5].value).toBe("x"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[1][6].value).toBe("="); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[1][7].value).toBe("0"); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[1][8].value).toBe(","); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[1][9].value).toBe(" "); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][10].value).toBe("y"); + expect(tokens[1][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[1][11].value).toBe("="); + expect(tokens[1][11].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[1][12].value).toBe("0"); + expect(tokens[1][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[1][13].value).toBe(")"); + expect(tokens[1][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][14].value).toBe(":"); + expect(tokens[1][14].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("print"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("\""); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][4].value).toBe("Origin is the point's location."); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[2][5].value).toBe("\""); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][6].value).toBe(")"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("case"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("Point"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][4].value).toBe("("); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][5].value).toBe("x"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][6].value).toBe("="); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[3][7].value).toBe("0"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][8].value).toBe(","); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[3][9].value).toBe(" "); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][10].value).toBe("y"); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[3][11].value).toBe("="); + expect(tokens[3][11].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[3][12].value).toBe("y"); + expect(tokens[3][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][13].value).toBe(")"); + expect(tokens[3][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][14].value).toBe(":"); + expect(tokens[3][14].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("print"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe("f"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][4].value).toBe("\""); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][5].value).toBe("The point is on the y-axis."); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][6].value).toBe("\""); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][7].value).toBe(")"); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("case"); + expect(tokens[5][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("Point"); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[5][4].value).toBe("("); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][5].value).toBe("x"); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[5][6].value).toBe("="); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[5][7].value).toBe("x"); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][8].value).toBe(","); + expect(tokens[5][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[5][9].value).toBe(" "); + expect(tokens[5][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][10].value).toBe("y"); + expect(tokens[5][10].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[5][11].value).toBe("="); + expect(tokens[5][11].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[5][12].value).toBe("0"); + expect(tokens[5][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[5][13].value).toBe(")"); + expect(tokens[5][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][14].value).toBe(":"); + expect(tokens[5][14].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("print"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[6][2].value).toBe("("); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[6][3].value).toBe("f"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][4].value).toBe("\""); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[6][5].value).toBe("The point is on the x-axis."); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][6].value).toBe("\""); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][7].value).toBe(")"); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("case"); + expect(tokens[7][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("Point"); + expect(tokens[7][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[7][4].value).toBe("("); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[7][5].value).toBe(")"); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][6].value).toBe(":"); + expect(tokens[7][6].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("print"); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[8][2].value).toBe("("); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[8][3].value).toBe("\""); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[8][4].value).toBe("The point is located somewhere else on the plane."); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[8][5].value).toBe("\""); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[8][6].value).toBe(")"); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("case"); + expect(tokens[9][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[9][2].value).toBe(" "); + expect(tokens[9][2].scopes).toEqual(["source.python"]); + expect(tokens[9][3].value).toBe("_"); + expect(tokens[9][3].scopes).toEqual(["source.python"]); + expect(tokens[9][4].value).toBe(":"); + expect(tokens[9][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("print"); + expect(tokens[10][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[10][2].value).toBe("("); + expect(tokens[10][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[10][3].value).toBe("\""); + expect(tokens[10][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][4].value).toBe("Not a point"); + expect(tokens[10][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[10][5].value).toBe("\""); + expect(tokens[10][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[10][6].value).toBe(")"); + expect(tokens[10][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/statements/match3.py", + function() { + tokens = grammar.tokenizeLines("match points:\n case []:\n print(\"No points in the list.\")\n case [Point(0, 0)]:\n print(\"The origin is the only point in the list.\")\n case [Point(x, y)]:\n print(f\"A single point is in the list.\")\n case [Point(0, y1), Point(0, y2)]:\n print(f\"Two points on the Y axis are in the list.\")\n case _:\n print(\"Something else is found in the list.\")") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("points"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("case"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("["); + expect(tokens[1][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[1][4].value).toBe("]"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("print"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("\""); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][4].value).toBe("No points in the list."); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[2][5].value).toBe("\""); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][6].value).toBe(")"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("case"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("["); + expect(tokens[3][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[3][4].value).toBe("Point"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][5].value).toBe("("); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][6].value).toBe("0"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][7].value).toBe(","); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[3][8].value).toBe(" "); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][9].value).toBe("0"); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[3][10].value).toBe(")"); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][11].value).toBe("]"); + expect(tokens[3][11].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[3][12].value).toBe(":"); + expect(tokens[3][12].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("print"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe("\""); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][4].value).toBe("The origin is the only point in the list."); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[4][5].value).toBe("\""); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][6].value).toBe(")"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("case"); + expect(tokens[5][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("["); + expect(tokens[5][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[5][4].value).toBe("Point"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[5][5].value).toBe("("); + expect(tokens[5][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][6].value).toBe("x"); + expect(tokens[5][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][7].value).toBe(","); + expect(tokens[5][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[5][8].value).toBe(" "); + expect(tokens[5][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][9].value).toBe("y"); + expect(tokens[5][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][10].value).toBe(")"); + expect(tokens[5][10].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][11].value).toBe("]"); + expect(tokens[5][11].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[5][12].value).toBe(":"); + expect(tokens[5][12].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("print"); + expect(tokens[6][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[6][2].value).toBe("("); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[6][3].value).toBe("f"); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][4].value).toBe("\""); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[6][5].value).toBe("A single point is in the list."); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][6].value).toBe("\""); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][7].value).toBe(")"); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("case"); + expect(tokens[7][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("["); + expect(tokens[7][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[7][4].value).toBe("Point"); + expect(tokens[7][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[7][5].value).toBe("("); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[7][6].value).toBe("0"); + expect(tokens[7][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[7][7].value).toBe(","); + expect(tokens[7][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[7][8].value).toBe(" "); + expect(tokens[7][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][9].value).toBe("y1"); + expect(tokens[7][9].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][10].value).toBe(")"); + expect(tokens[7][10].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][11].value).toBe(","); + expect(tokens[7][11].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[7][12].value).toBe(" "); + expect(tokens[7][12].scopes).toEqual(["source.python"]); + expect(tokens[7][13].value).toBe("Point"); + expect(tokens[7][13].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[7][14].value).toBe("("); + expect(tokens[7][14].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[7][15].value).toBe("0"); + expect(tokens[7][15].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[7][16].value).toBe(","); + expect(tokens[7][16].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[7][17].value).toBe(" "); + expect(tokens[7][17].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][18].value).toBe("y2"); + expect(tokens[7][18].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[7][19].value).toBe(")"); + expect(tokens[7][19].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[7][20].value).toBe("]"); + expect(tokens[7][20].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[7][21].value).toBe(":"); + expect(tokens[7][21].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python"]); + expect(tokens[8][1].value).toBe("print"); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[8][2].value).toBe("("); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[8][3].value).toBe("f"); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[8][4].value).toBe("\""); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[8][5].value).toBe("Two points on the Y axis are in the list."); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[8][6].value).toBe("\""); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[8][7].value).toBe(")"); + expect(tokens[8][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("case"); + expect(tokens[9][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[9][2].value).toBe(" "); + expect(tokens[9][2].scopes).toEqual(["source.python"]); + expect(tokens[9][3].value).toBe("_"); + expect(tokens[9][3].scopes).toEqual(["source.python"]); + expect(tokens[9][4].value).toBe(":"); + expect(tokens[9][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[10][0].value).toBe(" "); + expect(tokens[10][0].scopes).toEqual(["source.python"]); + expect(tokens[10][1].value).toBe("print"); + expect(tokens[10][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[10][2].value).toBe("("); + expect(tokens[10][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[10][3].value).toBe("\""); + expect(tokens[10][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[10][4].value).toBe("Something else is found in the list."); + expect(tokens[10][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[10][5].value).toBe("\""); + expect(tokens[10][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[10][6].value).toBe(")"); + expect(tokens[10][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/statements/match4.py", + function() { + tokens = grammar.tokenizeLines("match point:\n case Point(x, y) if x == y:\n print(f\"The point is located on the diagonal Y=X.\")\n case Point(x, y):\n print(f\"Point is not on the diagonal.\")") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("point"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("case"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("Point"); + expect(tokens[1][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[1][4].value).toBe("("); + expect(tokens[1][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[1][5].value).toBe("x"); + expect(tokens[1][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][6].value).toBe(","); + expect(tokens[1][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[1][7].value).toBe(" "); + expect(tokens[1][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][8].value).toBe("y"); + expect(tokens[1][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[1][9].value).toBe(")"); + expect(tokens[1][9].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python"]); + expect(tokens[1][11].value).toBe("if"); + expect(tokens[1][11].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[1][12].value).toBe(" "); + expect(tokens[1][12].scopes).toEqual(["source.python"]); + expect(tokens[1][13].value).toBe("x"); + expect(tokens[1][13].scopes).toEqual(["source.python"]); + expect(tokens[1][14].value).toBe(" "); + expect(tokens[1][14].scopes).toEqual(["source.python"]); + expect(tokens[1][15].value).toBe("=="); + expect(tokens[1][15].scopes).toEqual(["source.python","keyword.operator.comparison.python"]); + expect(tokens[1][16].value).toBe(" "); + expect(tokens[1][16].scopes).toEqual(["source.python"]); + expect(tokens[1][17].value).toBe("y"); + expect(tokens[1][17].scopes).toEqual(["source.python"]); + expect(tokens[1][18].value).toBe(":"); + expect(tokens[1][18].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("print"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[2][2].value).toBe("("); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][3].value).toBe("f"); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][4].value).toBe("\""); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe("The point is located on the diagonal Y=X."); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[2][6].value).toBe("\""); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[2][7].value).toBe(")"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("case"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("Point"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[3][4].value).toBe("("); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[3][5].value).toBe("x"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][6].value).toBe(","); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","punctuation.separator.arguments.python"]); + expect(tokens[3][7].value).toBe(" "); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][8].value).toBe("y"); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[3][9].value).toBe(")"); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[3][10].value).toBe(":"); + expect(tokens[3][10].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("print"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe("f"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][4].value).toBe("\""); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][5].value).toBe("Point is not on the diagonal."); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][6].value).toBe("\""); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][7].value).toBe(")"); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/statements/match5.py", + function() { + tokens = grammar.tokenizeLines("match command.split() if command else ['default']:\n ... # Other cases\n case [\"north\"] | [\"go\", \"north\"]:\n ... # handle case\n case [\"get\", obj] | [\"pick\", \"up\", *other] | [\"pick\", obj, \"up\"]:\n ... # handle case") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("command"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe("."); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][4].value).toBe("split"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][5].value).toBe("("); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][6].value).toBe(")"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("if"); + expect(tokens[0][8].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("command"); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe(" "); + expect(tokens[0][11].scopes).toEqual(["source.python"]); + expect(tokens[0][12].value).toBe("else"); + expect(tokens[0][12].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][13].value).toBe(" "); + expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[0][14].value).toBe("["); + expect(tokens[0][14].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[0][15].value).toBe("'"); + expect(tokens[0][15].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][16].value).toBe("default"); + expect(tokens[0][16].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][17].value).toBe("'"); + expect(tokens[0][17].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][18].value).toBe("]"); + expect(tokens[0][18].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[0][19].value).toBe(":"); + expect(tokens[0][19].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("#"); + expect(tokens[1][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][4].value).toBe(" Other cases"); + expect(tokens[1][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("case"); + expect(tokens[2][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][2].value).toBe(" "); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe("["); + expect(tokens[2][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[2][4].value).toBe("\""); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe("north"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][6].value).toBe("\""); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][7].value).toBe("]"); + expect(tokens[2][7].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[2][8].value).toBe(" "); + expect(tokens[2][8].scopes).toEqual(["source.python"]); + expect(tokens[2][9].value).toBe("|"); + expect(tokens[2][9].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[2][10].value).toBe(" "); + expect(tokens[2][10].scopes).toEqual(["source.python"]); + expect(tokens[2][11].value).toBe("["); + expect(tokens[2][11].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[2][12].value).toBe("\""); + expect(tokens[2][12].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][13].value).toBe("go"); + expect(tokens[2][13].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][14].value).toBe("\""); + expect(tokens[2][14].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][15].value).toBe(","); + expect(tokens[2][15].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][16].value).toBe(" "); + expect(tokens[2][16].scopes).toEqual(["source.python"]); + expect(tokens[2][17].value).toBe("\""); + expect(tokens[2][17].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][18].value).toBe("north"); + expect(tokens[2][18].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][19].value).toBe("\""); + expect(tokens[2][19].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][20].value).toBe("]"); + expect(tokens[2][20].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[2][21].value).toBe(":"); + expect(tokens[2][21].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("..."); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("#"); + expect(tokens[3][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][4].value).toBe(" handle case"); + expect(tokens[3][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("case"); + expect(tokens[4][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][2].value).toBe(" "); + expect(tokens[4][2].scopes).toEqual(["source.python"]); + expect(tokens[4][3].value).toBe("["); + expect(tokens[4][3].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[4][4].value).toBe("\""); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][5].value).toBe("get"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][6].value).toBe("\""); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][7].value).toBe(","); + expect(tokens[4][7].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][8].value).toBe(" "); + expect(tokens[4][8].scopes).toEqual(["source.python"]); + expect(tokens[4][9].value).toBe("obj"); + expect(tokens[4][9].scopes).toEqual(["source.python"]); + expect(tokens[4][10].value).toBe("]"); + expect(tokens[4][10].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[4][11].value).toBe(" "); + expect(tokens[4][11].scopes).toEqual(["source.python"]); + expect(tokens[4][12].value).toBe("|"); + expect(tokens[4][12].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[4][13].value).toBe(" "); + expect(tokens[4][13].scopes).toEqual(["source.python"]); + expect(tokens[4][14].value).toBe("["); + expect(tokens[4][14].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[4][15].value).toBe("\""); + expect(tokens[4][15].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][16].value).toBe("pick"); + expect(tokens[4][16].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][17].value).toBe("\""); + expect(tokens[4][17].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][18].value).toBe(","); + expect(tokens[4][18].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][19].value).toBe(" "); + expect(tokens[4][19].scopes).toEqual(["source.python"]); + expect(tokens[4][20].value).toBe("\""); + expect(tokens[4][20].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][21].value).toBe("up"); + expect(tokens[4][21].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][22].value).toBe("\""); + expect(tokens[4][22].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][23].value).toBe(","); + expect(tokens[4][23].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][24].value).toBe(" "); + expect(tokens[4][24].scopes).toEqual(["source.python"]); + expect(tokens[4][25].value).toBe("*"); + expect(tokens[4][25].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[4][26].value).toBe("other"); + expect(tokens[4][26].scopes).toEqual(["source.python"]); + expect(tokens[4][27].value).toBe("]"); + expect(tokens[4][27].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[4][28].value).toBe(" "); + expect(tokens[4][28].scopes).toEqual(["source.python"]); + expect(tokens[4][29].value).toBe("|"); + expect(tokens[4][29].scopes).toEqual(["source.python","keyword.operator.bitwise.python"]); + expect(tokens[4][30].value).toBe(" "); + expect(tokens[4][30].scopes).toEqual(["source.python"]); + expect(tokens[4][31].value).toBe("["); + expect(tokens[4][31].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[4][32].value).toBe("\""); + expect(tokens[4][32].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][33].value).toBe("pick"); + expect(tokens[4][33].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][34].value).toBe("\""); + expect(tokens[4][34].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][35].value).toBe(","); + expect(tokens[4][35].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][36].value).toBe(" "); + expect(tokens[4][36].scopes).toEqual(["source.python"]); + expect(tokens[4][37].value).toBe("obj"); + expect(tokens[4][37].scopes).toEqual(["source.python"]); + expect(tokens[4][38].value).toBe(","); + expect(tokens[4][38].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][39].value).toBe(" "); + expect(tokens[4][39].scopes).toEqual(["source.python"]); + expect(tokens[4][40].value).toBe("\""); + expect(tokens[4][40].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][41].value).toBe("up"); + expect(tokens[4][41].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][42].value).toBe("\""); + expect(tokens[4][42].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][43].value).toBe("]"); + expect(tokens[4][43].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[4][44].value).toBe(":"); + expect(tokens[4][44].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("..."); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("#"); + expect(tokens[5][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][4].value).toBe(" handle case"); + expect(tokens[5][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/statements/match6.py", + function() { + tokens = grammar.tokenizeLines("match (foo + bar):\n ... # cases\nmatch [foo, bar]:\n ... # cases\nmatch {foo, bar}:\n ... # cases") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("("); + expect(tokens[0][2].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[0][3].value).toBe("foo"); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("+"); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("bar"); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe(")"); + expect(tokens[0][8].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[0][9].value).toBe(":"); + expect(tokens[0][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("#"); + expect(tokens[1][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][4].value).toBe(" cases"); + expect(tokens[1][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe("match"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("["); + expect(tokens[2][2].scopes).toEqual(["source.python","punctuation.definition.list.begin.python"]); + expect(tokens[2][3].value).toBe("foo"); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe(","); + expect(tokens[2][4].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("bar"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe("]"); + expect(tokens[2][7].scopes).toEqual(["source.python","punctuation.definition.list.end.python"]); + expect(tokens[2][8].value).toBe(":"); + expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("..."); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("#"); + expect(tokens[3][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][4].value).toBe(" cases"); + expect(tokens[3][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe("match"); + expect(tokens[4][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("{"); + expect(tokens[4][2].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]); + expect(tokens[4][3].value).toBe("foo"); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe(","); + expect(tokens[4][4].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][5].value).toBe(" "); + expect(tokens[4][5].scopes).toEqual(["source.python"]); + expect(tokens[4][6].value).toBe("bar"); + expect(tokens[4][6].scopes).toEqual(["source.python"]); + expect(tokens[4][7].value).toBe("}"); + expect(tokens[4][7].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]); + expect(tokens[4][8].value).toBe(":"); + expect(tokens[4][8].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("..."); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("#"); + expect(tokens[5][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][4].value).toBe(" cases"); + expect(tokens[5][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/statements/match7.py", + function() { + tokens = grammar.tokenizeLines("match 'prefix' + foo:\n ... # cases\nmatch \"prefix\" + foo:\n ... # cases\nmatch f'prefix{foo}':\n ... # cases\nmatch f\"prefix{foo}\":\n ... # cases\nmatch -foo:\n ... # cases\nmatch not foo:\n ... # cases") + expect(tokens[0][0].value).toBe("match"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("'"); + expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][3].value).toBe("prefix"); + expect(tokens[0][3].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("+"); + expect(tokens[0][6].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("foo"); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe(":"); + expect(tokens[0][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("..."); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python"]); + expect(tokens[1][3].value).toBe("#"); + expect(tokens[1][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][4].value).toBe(" cases"); + expect(tokens[1][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe("match"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("\""); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][3].value).toBe("prefix"); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][4].value).toBe("\""); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("+"); + expect(tokens[2][6].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe("foo"); + expect(tokens[2][8].scopes).toEqual(["source.python"]); + expect(tokens[2][9].value).toBe(":"); + expect(tokens[2][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("..."); + expect(tokens[3][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python"]); + expect(tokens[3][3].value).toBe("#"); + expect(tokens[3][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][4].value).toBe(" cases"); + expect(tokens[3][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[4][0].value).toBe("match"); + expect(tokens[4][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("f"); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][3].value).toBe("'"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[4][4].value).toBe("prefix"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[4][5].value).toBe("{"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][6].value).toBe("foo"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[4][7].value).toBe("}"); + expect(tokens[4][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[4][8].value).toBe("'"); + expect(tokens[4][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[4][9].value).toBe(":"); + expect(tokens[4][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("..."); + expect(tokens[5][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("#"); + expect(tokens[5][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[5][4].value).toBe(" cases"); + expect(tokens[5][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[6][0].value).toBe("match"); + expect(tokens[6][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe("f"); + expect(tokens[6][2].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][3].value).toBe("\""); + expect(tokens[6][3].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.begin.python"]); + expect(tokens[6][4].value).toBe("prefix"); + expect(tokens[6][4].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python"]); + expect(tokens[6][5].value).toBe("{"); + expect(tokens[6][5].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[6][6].value).toBe("foo"); + expect(tokens[6][6].scopes).toEqual(["source.python","meta.fstring.python"]); + expect(tokens[6][7].value).toBe("}"); + expect(tokens[6][7].scopes).toEqual(["source.python","meta.fstring.python","constant.character.format.placeholder.other.python"]); + expect(tokens[6][8].value).toBe("\""); + expect(tokens[6][8].scopes).toEqual(["source.python","meta.fstring.python","string.quoted.single.python string.interpolated.python punctuation.definition.string.end.python"]); + expect(tokens[6][9].value).toBe(":"); + expect(tokens[6][9].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("..."); + expect(tokens[7][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("#"); + expect(tokens[7][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[7][4].value).toBe(" cases"); + expect(tokens[7][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[8][0].value).toBe("match"); + expect(tokens[8][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][1].value).toBe(" "); + expect(tokens[8][1].scopes).toEqual(["source.python"]); + expect(tokens[8][2].value).toBe("-"); + expect(tokens[8][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[8][3].value).toBe("foo"); + expect(tokens[8][3].scopes).toEqual(["source.python"]); + expect(tokens[8][4].value).toBe(":"); + expect(tokens[8][4].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("..."); + expect(tokens[9][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[9][2].value).toBe(" "); + expect(tokens[9][2].scopes).toEqual(["source.python"]); + expect(tokens[9][3].value).toBe("#"); + expect(tokens[9][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[9][4].value).toBe(" cases"); + expect(tokens[9][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[10][0].value).toBe("match"); + expect(tokens[10][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[10][1].value).toBe(" "); + expect(tokens[10][1].scopes).toEqual(["source.python"]); + expect(tokens[10][2].value).toBe("not"); + expect(tokens[10][2].scopes).toEqual(["source.python","keyword.operator.logical.python"]); + expect(tokens[10][3].value).toBe(" "); + expect(tokens[10][3].scopes).toEqual(["source.python"]); + expect(tokens[10][4].value).toBe("foo"); + expect(tokens[10][4].scopes).toEqual(["source.python"]); + expect(tokens[10][5].value).toBe(":"); + expect(tokens[10][5].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[11][0].value).toBe(" "); + expect(tokens[11][0].scopes).toEqual(["source.python"]); + expect(tokens[11][1].value).toBe("..."); + expect(tokens[11][1].scopes).toEqual(["source.python","constant.other.ellipsis.python"]); + expect(tokens[11][2].value).toBe(" "); + expect(tokens[11][2].scopes).toEqual(["source.python"]); + expect(tokens[11][3].value).toBe("#"); + expect(tokens[11][3].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[11][4].value).toBe(" cases"); + expect(tokens[11][4].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/statements/nonlocal1.py", + function() { + tokens = grammar.tokenizeLines("nonlocal a, b, c") + expect(tokens[0][0].value).toBe("nonlocal"); + expect(tokens[0][0].scopes).toEqual(["source.python","storage.modifier.declaration.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("a"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(","); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("b"); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe(","); + expect(tokens[0][6].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("c"); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + }); + + it("test/statements/try1.py", + function() { + tokens = grammar.tokenizeLines("try:\n 1/0\nexcept AbcError as ex:\n pass\nexcept (ZeroDivisionError, GhiError) as ex:\n print(ex)\nelse:\n 1\nfinally:\n 2") + expect(tokens[0][0].value).toBe("try"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(":"); + expect(tokens[0][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("1"); + expect(tokens[1][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][2].value).toBe("/"); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[1][3].value).toBe("0"); + expect(tokens[1][3].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[2][0].value).toBe("except"); + expect(tokens[2][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("AbcError"); + expect(tokens[2][2].scopes).toEqual(["source.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("as"); + expect(tokens[2][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("ex"); + expect(tokens[2][6].scopes).toEqual(["source.python"]); + expect(tokens[2][7].value).toBe(":"); + expect(tokens[2][7].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe("pass"); + expect(tokens[3][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][0].value).toBe("except"); + expect(tokens[4][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[4][3].value).toBe("ZeroDivisionError"); + expect(tokens[4][3].scopes).toEqual(["source.python","support.type.exception.python"]); + expect(tokens[4][4].value).toBe(","); + expect(tokens[4][4].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[4][5].value).toBe(" "); + expect(tokens[4][5].scopes).toEqual(["source.python"]); + expect(tokens[4][6].value).toBe("GhiError"); + expect(tokens[4][6].scopes).toEqual(["source.python"]); + expect(tokens[4][7].value).toBe(")"); + expect(tokens[4][7].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + expect(tokens[4][8].value).toBe(" "); + expect(tokens[4][8].scopes).toEqual(["source.python"]); + expect(tokens[4][9].value).toBe("as"); + expect(tokens[4][9].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[4][10].value).toBe(" "); + expect(tokens[4][10].scopes).toEqual(["source.python"]); + expect(tokens[4][11].value).toBe("ex"); + expect(tokens[4][11].scopes).toEqual(["source.python"]); + expect(tokens[4][12].value).toBe(":"); + expect(tokens[4][12].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("print"); + expect(tokens[5][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[5][2].value).toBe("("); + expect(tokens[5][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[5][3].value).toBe("ex"); + expect(tokens[5][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[5][4].value).toBe(")"); + expect(tokens[5][4].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[6][0].value).toBe("else"); + expect(tokens[6][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[6][1].value).toBe(":"); + expect(tokens[6][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("1"); + expect(tokens[7][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[8][0].value).toBe("finally"); + expect(tokens[8][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[8][1].value).toBe(":"); + expect(tokens[8][1].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[9][0].value).toBe(" "); + expect(tokens[9][0].scopes).toEqual(["source.python"]); + expect(tokens[9][1].value).toBe("2"); + expect(tokens[9][1].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + }); + + it("test/statements/while1.py", + function() { + tokens = grammar.tokenizeLines("while 1:\n pass") + expect(tokens[0][0].value).toBe("while"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("1"); + expect(tokens[0][2].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[0][3].value).toBe(":"); + expect(tokens[0][3].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("pass"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/statements/with1.py", + function() { + tokens = grammar.tokenizeLines("with a as b, c as d:\n pass") + expect(tokens[0][0].value).toBe("with"); + expect(tokens[0][0].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("a"); + expect(tokens[0][2].scopes).toEqual(["source.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("as"); + expect(tokens[0][4].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][5].value).toBe(" "); + expect(tokens[0][5].scopes).toEqual(["source.python"]); + expect(tokens[0][6].value).toBe("b"); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe(","); + expect(tokens[0][7].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("c"); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("as"); + expect(tokens[0][11].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[0][12].value).toBe(" "); + expect(tokens[0][12].scopes).toEqual(["source.python"]); + expect(tokens[0][13].value).toBe("d"); + expect(tokens[0][13].scopes).toEqual(["source.python"]); + expect(tokens[0][14].value).toBe(":"); + expect(tokens[0][14].scopes).toEqual(["source.python","punctuation.separator.colon.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("pass"); + expect(tokens[1][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/strings/bad1.py", + function() { + tokens = grammar.tokenizeLines("a = \"bad \\\\ string\nfoo \\' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\"\ndef foo(a=1): pass") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("\""); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("bad "); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][6].value).toBe("\\\\"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[0][7].value).toBe(" string"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][8].value).toBe(""); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.single.python","invalid.illegal.newline.python"]); + expect(tokens[1][0].value).toBe("foo"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("\\"); + expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][3].value).toBe("' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\""); + expect(tokens[1][3].scopes).toEqual(["source.python","invalid.illegal.line.continuation.python"]); + expect(tokens[2][0].value).toBe("def"); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[2][2].value).toBe("foo"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[2][3].value).toBe("("); + expect(tokens[2][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[2][4].value).toBe("a"); + expect(tokens[2][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[2][5].value).toBe("="); + expect(tokens[2][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[2][6].value).toBe("1"); + expect(tokens[2][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[2][7].value).toBe(")"); + expect(tokens[2][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[2][8].value).toBe(":"); + expect(tokens[2][8].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("pass"); + expect(tokens[2][10].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/strings/bad2.py", + function() { + tokens = grammar.tokenizeLines("a = b\"bad \\\\ string\nfoo \\' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\"\n\ndef foo(a=1): pass") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("b"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("\""); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("bad "); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[0][7].value).toBe("\\\\"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.binary.single.python","constant.character.escape.python"]); + expect(tokens[0][8].value).toBe(" string"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[0][9].value).toBe(""); + expect(tokens[0][9].scopes).toEqual(["source.python","string.quoted.binary.single.python","invalid.illegal.newline.python"]); + expect(tokens[1][0].value).toBe("foo"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("\\"); + expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][3].value).toBe("' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\""); + expect(tokens[1][3].scopes).toEqual(["source.python","invalid.illegal.line.continuation.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("def"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[3][2].value).toBe("foo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[3][3].value).toBe("("); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[3][4].value).toBe("a"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[3][5].value).toBe("="); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[3][6].value).toBe("1"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[3][7].value).toBe(")"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[3][8].value).toBe(":"); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[3][9].value).toBe(" "); + expect(tokens[3][9].scopes).toEqual(["source.python"]); + expect(tokens[3][10].value).toBe("pass"); + expect(tokens[3][10].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); + + it("test/strings/bad3.py", + function() { + tokens = grammar.tokenizeLines("a = r\"bad string\nfoo \\' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\"\n\ndef foo(a=1): pass # doesn't break!") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("r"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("\""); + expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("bad string"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[0][7].value).toBe(""); + expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","invalid.illegal.newline.python"]); + expect(tokens[1][0].value).toBe("foo"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("\\"); + expect(tokens[1][2].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][3].value).toBe("' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\""); + expect(tokens[1][3].scopes).toEqual(["source.python","invalid.illegal.line.continuation.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[3][0].value).toBe("def"); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[3][2].value).toBe("foo"); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[3][3].value).toBe("("); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[3][4].value).toBe("a"); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[3][5].value).toBe("="); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[3][6].value).toBe("1"); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[3][7].value).toBe(")"); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[3][8].value).toBe(":"); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[3][9].value).toBe(" "); + expect(tokens[3][9].scopes).toEqual(["source.python"]); + expect(tokens[3][10].value).toBe("pass"); + expect(tokens[3][10].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[3][11].value).toBe(" "); + expect(tokens[3][11].scopes).toEqual(["source.python"]); + expect(tokens[3][12].value).toBe("#"); + expect(tokens[3][12].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[3][13].value).toBe(" doesn't break!"); + expect(tokens[3][13].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/strings/bug1.py", + function() { + tokens = grammar.tokenizeLines("# issue 150\nrecord = {\n \"a\": {k: str(v) for k, v in foo if \"\"}\n}") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" issue 150"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("record"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("{"); + expect(tokens[1][4].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("\""); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("a"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][3].value).toBe("\""); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][4].value).toBe(":"); + expect(tokens[2][4].scopes).toEqual(["source.python","punctuation.separator.dict.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("{"); + expect(tokens[2][6].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]); + expect(tokens[2][7].value).toBe("k"); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe(":"); + expect(tokens[2][8].scopes).toEqual(["source.python","punctuation.separator.dict.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("str"); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.function-call.python","support.type.python"]); + expect(tokens[2][11].value).toBe("("); + expect(tokens[2][11].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[2][12].value).toBe("v"); + expect(tokens[2][12].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python"]); + expect(tokens[2][13].value).toBe(")"); + expect(tokens[2][13].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[2][14].value).toBe(" "); + expect(tokens[2][14].scopes).toEqual(["source.python"]); + expect(tokens[2][15].value).toBe("for"); + expect(tokens[2][15].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][16].value).toBe(" "); + expect(tokens[2][16].scopes).toEqual(["source.python"]); + expect(tokens[2][17].value).toBe("k"); + expect(tokens[2][17].scopes).toEqual(["source.python"]); + expect(tokens[2][18].value).toBe(","); + expect(tokens[2][18].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][19].value).toBe(" "); + expect(tokens[2][19].scopes).toEqual(["source.python"]); + expect(tokens[2][20].value).toBe("v"); + expect(tokens[2][20].scopes).toEqual(["source.python"]); + expect(tokens[2][21].value).toBe(" "); + expect(tokens[2][21].scopes).toEqual(["source.python"]); + expect(tokens[2][22].value).toBe("in"); + expect(tokens[2][22].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][23].value).toBe(" "); + expect(tokens[2][23].scopes).toEqual(["source.python"]); + expect(tokens[2][24].value).toBe("foo"); + expect(tokens[2][24].scopes).toEqual(["source.python"]); + expect(tokens[2][25].value).toBe(" "); + expect(tokens[2][25].scopes).toEqual(["source.python"]); + expect(tokens[2][26].value).toBe("if"); + expect(tokens[2][26].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[2][27].value).toBe(" "); + expect(tokens[2][27].scopes).toEqual(["source.python"]); + expect(tokens[2][28].value).toBe("\""); + expect(tokens[2][28].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][29].value).toBe("\""); + expect(tokens[2][29].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][30].value).toBe("}"); + expect(tokens[2][30].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]); + expect(tokens[3][0].value).toBe("}"); + expect(tokens[3][0].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]); + }); + + it("test/strings/bug2.py", + function() { + tokens = grammar.tokenizeLines("# issue 150\ncmd = \"git-clang-format --style=\\\"{{BasedOnStyle: Google, ColumnLimit: 100, IndentWidth: 2, \" \\\n \"AlignConsecutiveAssignments: true}}\\\" {COMMIT_SHA} -- ./**/*.proto > {OUTPUT}\".format(") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" issue 150"); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("cmd"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("\""); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe("git-clang-format --style="); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][6].value).toBe("\\\""); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][7].value).toBe("{{"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][8].value).toBe("BasedOnStyle: Google, ColumnLimit: 100, IndentWidth: 2, "); + expect(tokens[1][8].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][9].value).toBe("\""); + expect(tokens[1][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python"]); + expect(tokens[1][11].value).toBe("\\"); + expect(tokens[1][11].scopes).toEqual(["source.python","punctuation.separator.continuation.line.python"]); + expect(tokens[1][12].value).toBe(""); + expect(tokens[1][12].scopes).toEqual(["source.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("\""); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("AlignConsecutiveAssignments: true"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][3].value).toBe("}}"); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][4].value).toBe("\\\""); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][6].value).toBe("{COMMIT_SHA}"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][7].value).toBe(" -- ./**/*.proto > "); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][8].value).toBe("{OUTPUT}"); + expect(tokens[2][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][9].value).toBe("\""); + expect(tokens[2][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][10].value).toBe("."); + expect(tokens[2][10].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[2][11].value).toBe("format"); + expect(tokens[2][11].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[2][12].value).toBe("("); + expect(tokens[2][12].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + }); + + it("test/strings/bytes1.py", + function() { + tokens = grammar.tokenizeLines("a = b\"\"\"\nmultiline \"binary\" string \\\n\n \\xf1 \\u1234aaaa \\U1234aaaa\n\n \\N{BLACK SPADE SUIT}\n\"\"\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("b"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("\"\"\""); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][0].value).toBe("multiline \"binary\" string "); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[1][1].value).toBe("\\"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","constant.language.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[3][1].value).toBe("\\xf1"); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","constant.character.escape.python"]); + expect(tokens[3][2].value).toBe(" \\u1234aaaa \\U1234aaaa"); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[5][0].value).toBe(" \\N{BLACK SPADE SUIT}"); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[6][0].value).toBe("\"\"\""); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/bytes2.py", + function() { + tokens = grammar.tokenizeLines("a[1] = b'''\nmultiline 'binary' string \\\n\n \\xf1 \\u1234aaaa \\U1234aaaa\n\n \\N{BLACK SPADE SUIT}\n'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.item-access.python","meta.indexed-name.python"]); + expect(tokens[0][1].value).toBe("["); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][2].value).toBe("1"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.item-access.python","meta.item-access.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][3].value).toBe("]"); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.item-access.python","punctuation.definition.arguments.end.python"]); + expect(tokens[0][4].value).toBe(" "); + expect(tokens[0][4].scopes).toEqual(["source.python"]); + expect(tokens[0][5].value).toBe("="); + expect(tokens[0][5].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][6].value).toBe(" "); + expect(tokens[0][6].scopes).toEqual(["source.python"]); + expect(tokens[0][7].value).toBe("b"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.binary.multi.python","storage.type.string.python"]); + expect(tokens[0][8].value).toBe("'''"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][0].value).toBe("multiline 'binary' string "); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[1][1].value).toBe("\\"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","constant.language.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[3][1].value).toBe("\\xf1"); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.binary.multi.python","constant.character.escape.python"]); + expect(tokens[3][2].value).toBe(" \\u1234aaaa \\U1234aaaa"); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[5][0].value).toBe(" \\N{BLACK SPADE SUIT}"); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python"]); + expect(tokens[6][0].value).toBe("'''"); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.binary.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/escaping1.py", + function() { + tokens = grammar.tokenizeLines("a = \"simple \\\\ string \\\nfoo \\' \\\" \\a \\b \\c \\f \\n \\r \\t \\v \\5 \\55 \\555 \\05 \\005\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("\""); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("simple "); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][6].value).toBe("\\\\"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[0][7].value).toBe(" string "); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][8].value).toBe("\\"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.single.python","constant.language.python"]); + expect(tokens[1][0].value).toBe("foo "); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][1].value).toBe("\\'"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][2].value).toBe(" "); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][3].value).toBe("\\\""); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][5].value).toBe("\\a"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][7].value).toBe("\\b"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][8].value).toBe(" \\c "); + expect(tokens[1][8].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][9].value).toBe("\\f"); + expect(tokens[1][9].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][10].value).toBe(" "); + expect(tokens[1][10].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][11].value).toBe("\\n"); + expect(tokens[1][11].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][12].value).toBe(" "); + expect(tokens[1][12].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][13].value).toBe("\\r"); + expect(tokens[1][13].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][14].value).toBe(" "); + expect(tokens[1][14].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][15].value).toBe("\\t"); + expect(tokens[1][15].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][16].value).toBe(" "); + expect(tokens[1][16].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][17].value).toBe("\\v"); + expect(tokens[1][17].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][18].value).toBe(" "); + expect(tokens[1][18].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][19].value).toBe("\\5"); + expect(tokens[1][19].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][20].value).toBe(" "); + expect(tokens[1][20].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][21].value).toBe("\\55"); + expect(tokens[1][21].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][22].value).toBe(" "); + expect(tokens[1][22].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][23].value).toBe("\\555"); + expect(tokens[1][23].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][24].value).toBe(" "); + expect(tokens[1][24].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][25].value).toBe("\\05"); + expect(tokens[1][25].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][26].value).toBe(" "); + expect(tokens[1][26].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][27].value).toBe("\\005"); + expect(tokens[1][27].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[1][28].value).toBe("\""); + expect(tokens[1][28].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/escaping2.py", + function() { + tokens = grammar.tokenizeLines("replace = {'\"' : R'\\\"',\n \"'\" : R'\\'',\n '\\\\': R'\\\\'}") + expect(tokens[0][0].value).toBe("replace"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("{"); + expect(tokens[0][4].scopes).toEqual(["source.python","punctuation.definition.dict.begin.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("\""); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe(":"); + expect(tokens[0][9].scopes).toEqual(["source.python","punctuation.separator.dict.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("R"); + expect(tokens[0][11].scopes).toEqual(["source.python","string.quoted.raw.single.python","storage.type.string.python"]); + expect(tokens[0][12].value).toBe("'"); + expect(tokens[0][12].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][13].value).toBe("\\\""); + expect(tokens[0][13].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[0][14].value).toBe("'"); + expect(tokens[0][14].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][15].value).toBe(","); + expect(tokens[0][15].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("\""); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("'"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][3].value).toBe("\""); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][4].value).toBe(" "); + expect(tokens[1][4].scopes).toEqual(["source.python"]); + expect(tokens[1][5].value).toBe(":"); + expect(tokens[1][5].scopes).toEqual(["source.python","punctuation.separator.dict.python"]); + expect(tokens[1][6].value).toBe(" "); + expect(tokens[1][6].scopes).toEqual(["source.python"]); + expect(tokens[1][7].value).toBe("R"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.raw.single.python","storage.type.string.python"]); + expect(tokens[1][8].value).toBe("'"); + expect(tokens[1][8].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][9].value).toBe("\\'"); + expect(tokens[1][9].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[1][10].value).toBe("'"); + expect(tokens[1][10].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][11].value).toBe(","); + expect(tokens[1][11].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe("'"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][2].value).toBe("\\\\"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.single.python","constant.character.escape.python"]); + expect(tokens[2][3].value).toBe("'"); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][4].value).toBe(":"); + expect(tokens[2][4].scopes).toEqual(["source.python","punctuation.separator.dict.python"]); + expect(tokens[2][5].value).toBe(" "); + expect(tokens[2][5].scopes).toEqual(["source.python"]); + expect(tokens[2][6].value).toBe("R"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.raw.single.python","storage.type.string.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][8].value).toBe("\\\\"); + expect(tokens[2][8].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[2][9].value).toBe("'"); + expect(tokens[2][9].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][10].value).toBe("}"); + expect(tokens[2][10].scopes).toEqual(["source.python","punctuation.definition.dict.end.python"]); + }); + + it("test/strings/format1.py", + function() { + tokens = grammar.tokenizeLines("a = \"{0[ ]:X>+10d}\"\na = \"{0[ ]!s:X>+10d}\"\na = \"{0[ ]:Xd>+10d}\" #invalid") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("\""); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("{0[ ]"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe(":X>+10d"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[0][7].value).toBe("}"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][8].value).toBe("\""); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("\""); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe("{0[ ]"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][6].value).toBe("!s"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[1][7].value).toBe(":X>+10d"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[1][8].value).toBe("}"); + expect(tokens[1][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][9].value).toBe("\""); + expect(tokens[1][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("\""); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe("{0[ ]"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][6].value).toBe(":"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[2][7].value).toBe("Xd>+10d}"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][8].value).toBe("\""); + expect(tokens[2][8].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("#"); + expect(tokens[2][10].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][11].value).toBe("invalid"); + expect(tokens[2][11].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + }); + + it("test/strings/format10.py", + function() { + tokens = grammar.tokenizeLines("a = '''blah {foo-bar %d\n blah {foo-bar %d}\n blah {foo-bar %d //insane {}}\n {}blah {foo-bar %d //insane {}}'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'''"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("blah {foo-bar "); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[0][6].value).toBe("%d"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.multi.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][0].value).toBe(" blah "); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[1][1].value).toBe("{foo-bar "); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[1][2].value).toBe("%d"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.multi.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][3].value).toBe("}"); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[2][0].value).toBe(" blah {foo-bar "); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[2][1].value).toBe("%d"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.multi.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][2].value).toBe(" //insane {}}"); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[3][0].value).toBe(" {}blah {foo-bar "); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[3][1].value).toBe("%d"); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.multi.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][2].value).toBe(" //insane {}}"); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[3][3].value).toBe("'''"); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format11.py", + function() { + tokens = grammar.tokenizeLines("a = R'''\\fr{still_ok}ac{m_{j \\rightarrow i}(\\mathrm{good})}\n {not_ok} %d\n'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("R"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.raw.multi.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'''"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("\\fr"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[0][7].value).toBe("{still_ok}"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.raw.multi.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][8].value).toBe("ac"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[0][9].value).toBe("{m_{j \\rightarrow i}(\\mathrm{good})}"); + expect(tokens[0][9].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[1][0].value).toBe(" {not_ok} "); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[1][1].value).toBe("%d"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.raw.multi.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][0].value).toBe("'''"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format12.py", + function() { + tokens = grammar.tokenizeLines("a = R'$\\frac{m_{j \\%srightarrow i}(\\mathrm{%sgood})}{\\su%m{m_{j \\rightarrow i}}}$'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("R"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.raw.single.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("$\\frac"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[0][7].value).toBe("{m_{j \\"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[0][8].value).toBe("%s"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.raw.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("rightarrow i}(\\mathrm{"); + expect(tokens[0][9].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[0][10].value).toBe("%s"); + expect(tokens[0][10].scopes).toEqual(["source.python","string.quoted.raw.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("good})}{\\su%m{m_{j \\rightarrow i}}}$"); + expect(tokens[0][11].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[0][12].value).toBe("'"); + expect(tokens[0][12].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format13.py", + function() { + tokens = grammar.tokenizeLines("a = '''\\n\n{% for item in seq %}\n \\n {{ item }} \\n \\N{BLACK SPADE SUIT}\n{% endfor %}\n'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'''"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("\\n"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.multi.python","constant.character.escape.python"]); + expect(tokens[1][0].value).toBe("{% for item in seq %}"); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[2][1].value).toBe("\\n"); + expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.multi.python","constant.character.escape.python"]); + expect(tokens[2][2].value).toBe(" {{ item }} "); + expect(tokens[2][2].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[2][3].value).toBe("\\n"); + expect(tokens[2][3].scopes).toEqual(["source.python","string.quoted.multi.python","constant.character.escape.python"]); + expect(tokens[2][4].value).toBe(" "); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[2][5].value).toBe("\\N{BLACK SPADE SUIT}"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.multi.python","constant.character.escape.python"]); + expect(tokens[3][0].value).toBe("{% endfor %}"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[4][0].value).toBe("'''"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format14.py", + function() { + tokens = grammar.tokenizeLines("a = R'''\\n\n{% for item in seq %}\n \\n {{ item }} \\n \\N{BLACK SPADE SUIT}\n{% endfor %}\n'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("R"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.raw.multi.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'''"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("\\n"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[1][0].value).toBe("{% for item in seq %}"); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[2][0].value).toBe(" \\n {{ item }} \\n \\N{BLACK SPADE SUIT}"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[3][0].value).toBe("{% endfor %}"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[4][0].value).toBe("'''"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format15.py", + function() { + tokens = grammar.tokenizeLines("a = '''\n{{ before detection }}\n{# jinja comment #}\n{{ after detection }}\n'''") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'''"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][0].value).toBe("{{"); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.multi.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][1].value).toBe(" before detection "); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[1][2].value).toBe("}}"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.multi.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][0].value).toBe("{# jinja comment #}"); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[3][0].value).toBe("{{ after detection }}"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[4][0].value).toBe("'''"); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format16.py", + function() { + tokens = grammar.tokenizeLines("a = b'%b' % b'foo'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("b"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("%b"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.binary.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][8].value).toBe(" "); + expect(tokens[0][8].scopes).toEqual(["source.python"]); + expect(tokens[0][9].value).toBe("%"); + expect(tokens[0][9].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][10].value).toBe(" "); + expect(tokens[0][10].scopes).toEqual(["source.python"]); + expect(tokens[0][11].value).toBe("b"); + expect(tokens[0][11].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[0][12].value).toBe("'"); + expect(tokens[0][12].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][13].value).toBe("foo"); + expect(tokens[0][13].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[0][14].value).toBe("'"); + expect(tokens[0][14].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format2.py", + function() { + tokens = grammar.tokenizeLines("a = \"normal {{ normal }} normal {10!r} normal {fo.__add__!s}\".format(fo=1)") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("\""); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("normal "); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][6].value).toBe("{{"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe(" normal "); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][8].value).toBe("}}"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe(" normal "); + expect(tokens[0][9].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][10].value).toBe("{10"); + expect(tokens[0][10].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][11].value).toBe("!r"); + expect(tokens[0][11].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[0][12].value).toBe("}"); + expect(tokens[0][12].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][13].value).toBe(" normal "); + expect(tokens[0][13].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][14].value).toBe("{fo.__add__"); + expect(tokens[0][14].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][15].value).toBe("!s"); + expect(tokens[0][15].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[0][16].value).toBe("}"); + expect(tokens[0][16].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][17].value).toBe("\""); + expect(tokens[0][17].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][18].value).toBe("."); + expect(tokens[0][18].scopes).toEqual(["source.python","meta.member.access.python","punctuation.separator.period.python"]); + expect(tokens[0][19].value).toBe("format"); + expect(tokens[0][19].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.generic.python"]); + expect(tokens[0][20].value).toBe("("); + expect(tokens[0][20].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[0][21].value).toBe("fo"); + expect(tokens[0][21].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","variable.parameter.function-call.python"]); + expect(tokens[0][22].value).toBe("="); + expect(tokens[0][22].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","keyword.operator.assignment.python"]); + expect(tokens[0][23].value).toBe("1"); + expect(tokens[0][23].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","meta.function-call.arguments.python","constant.numeric.dec.python"]); + expect(tokens[0][24].value).toBe(")"); + expect(tokens[0][24].scopes).toEqual(["source.python","meta.member.access.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + }); + + it("test/strings/format3.py", + function() { + tokens = grammar.tokenizeLines("a = '%i' % 42\na = \"%(language)s has %(number)03d quote types.\"\na = b\"%(language)s has %(number)03d quote types.\"\na = R\"%(language)s has %(number)03d quote types.\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("%i"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[0][7].value).toBe(" "); + expect(tokens[0][7].scopes).toEqual(["source.python"]); + expect(tokens[0][8].value).toBe("%"); + expect(tokens[0][8].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[0][9].value).toBe(" "); + expect(tokens[0][9].scopes).toEqual(["source.python"]); + expect(tokens[0][10].value).toBe("42"); + expect(tokens[0][10].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("\""); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe("%(language)s"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][6].value).toBe(" has "); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][7].value).toBe("%(number)03d"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][8].value).toBe(" quote types."); + expect(tokens[1][8].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][9].value).toBe("\""); + expect(tokens[1][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("b"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[2][5].value).toBe("\""); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("%(language)s"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.binary.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][7].value).toBe(" has "); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[2][8].value).toBe("%(number)03d"); + expect(tokens[2][8].scopes).toEqual(["source.python","string.quoted.binary.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][9].value).toBe(" quote types."); + expect(tokens[2][9].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[2][10].value).toBe("\""); + expect(tokens[2][10].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("R"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.raw.single.python","storage.type.string.python"]); + expect(tokens[3][5].value).toBe("\""); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][6].value).toBe("%(language)s"); + expect(tokens[3][6].scopes).toEqual(["source.python","string.quoted.raw.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][7].value).toBe(" has "); + expect(tokens[3][7].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[3][8].value).toBe("%(number)03d"); + expect(tokens[3][8].scopes).toEqual(["source.python","string.quoted.raw.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][9].value).toBe(" quote types."); + expect(tokens[3][9].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[3][10].value).toBe("\""); + expect(tokens[3][10].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format4.py", + function() { + tokens = grammar.tokenizeLines("a = 'qqq{:%Y-%m-%d %H:%M:%S}www'\na = 'qqq{0:{fill}{align}16}www'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("qqq"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][6].value).toBe("{"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe(":"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[0][8].value).toBe("%Y-%m-%d %H:%M:%S}"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("www"); + expect(tokens[0][9].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][10].value).toBe("'"); + expect(tokens[0][10].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("'"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe("qqq"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][6].value).toBe("{0"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][7].value).toBe(":"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[1][8].value).toBe("{fill}{align}16}"); + expect(tokens[1][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][9].value).toBe("www"); + expect(tokens[1][9].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][10].value).toBe("'"); + expect(tokens[1][10].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format5.py", + function() { + tokens = grammar.tokenizeLines("a = 'qqq{0:{width}{base}}www'\na = 'qqq{0:$20}www'\na = 'qqq{0}www'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("qqq"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][6].value).toBe("{0"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe(":"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[0][8].value).toBe("{width}{base}}"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][9].value).toBe("www"); + expect(tokens[0][9].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][10].value).toBe("'"); + expect(tokens[0][10].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("'"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe("qqq"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][6].value).toBe("{0"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][7].value).toBe(":"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python","storage.type.format.python"]); + expect(tokens[1][8].value).toBe("$20}"); + expect(tokens[1][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][9].value).toBe("www"); + expect(tokens[1][9].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][10].value).toBe("'"); + expect(tokens[1][10].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("'"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe("qqq"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][6].value).toBe("{0}"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][7].value).toBe("www"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][8].value).toBe("'"); + expect(tokens[2][8].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format6.py", + function() { + tokens = grammar.tokenizeLines("a = 'qqqwww'\na = ' { '\na = 'qqq{0:{widt' + 'h}{base}}www'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("qqqwww"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("'"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe(" { "); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][6].value).toBe("'"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("'"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe("qqq{0:{widt"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][6].value).toBe("'"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][7].value).toBe(" "); + expect(tokens[2][7].scopes).toEqual(["source.python"]); + expect(tokens[2][8].value).toBe("+"); + expect(tokens[2][8].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[2][9].value).toBe(" "); + expect(tokens[2][9].scopes).toEqual(["source.python"]); + expect(tokens[2][10].value).toBe("'"); + expect(tokens[2][10].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][11].value).toBe("h}"); + expect(tokens[2][11].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][12].value).toBe("{base}"); + expect(tokens[2][12].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][13].value).toBe("}www"); + expect(tokens[2][13].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][14].value).toBe("'"); + expect(tokens[2][14].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format7.py", + function() { + tokens = grammar.tokenizeLines("# XXX we have to highlight '% o' here, as it is a valid python\n# format spec. Otherwise, it would be hard to spot an error in\n# the code below.\na = '12% of %s' % ('name',)") + expect(tokens[0][0].value).toBe("#"); + expect(tokens[0][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[0][2].value).toBe("XXX"); + expect(tokens[0][2].scopes).toEqual(["source.python","comment.line.number-sign.python","keyword.codetag.notation.python"]); + expect(tokens[0][3].value).toBe(" we have to highlight '% o' here, as it is a valid python"); + expect(tokens[0][3].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[1][0].value).toBe("#"); + expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[1][1].value).toBe(" format spec. Otherwise, it would be hard to spot an error in"); + expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[2][0].value).toBe("#"); + expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]); + expect(tokens[2][1].value).toBe(" the code below."); + expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("'"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][5].value).toBe("12"); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[3][6].value).toBe("% o"); + expect(tokens[3][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][7].value).toBe("f "); + expect(tokens[3][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[3][8].value).toBe("%s"); + expect(tokens[3][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][9].value).toBe("'"); + expect(tokens[3][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][10].value).toBe(" "); + expect(tokens[3][10].scopes).toEqual(["source.python"]); + expect(tokens[3][11].value).toBe("%"); + expect(tokens[3][11].scopes).toEqual(["source.python","keyword.operator.arithmetic.python"]); + expect(tokens[3][12].value).toBe(" "); + expect(tokens[3][12].scopes).toEqual(["source.python"]); + expect(tokens[3][13].value).toBe("("); + expect(tokens[3][13].scopes).toEqual(["source.python","punctuation.parenthesis.begin.python"]); + expect(tokens[3][14].value).toBe("'"); + expect(tokens[3][14].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][15].value).toBe("name"); + expect(tokens[3][15].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[3][16].value).toBe("'"); + expect(tokens[3][16].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][17].value).toBe(","); + expect(tokens[3][17].scopes).toEqual(["source.python","punctuation.separator.element.python"]); + expect(tokens[3][18].value).toBe(")"); + expect(tokens[3][18].scopes).toEqual(["source.python","punctuation.parenthesis.end.python"]); + }); + + it("test/strings/format8.py", + function() { + tokens = grammar.tokenizeLines("a = R'$\\frac{m_{j \\rightarrow i}(\\mathrm{good})}{\\sum{m_{j \\rightarrow i}}}$'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("R"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.raw.single.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("$\\frac"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[0][7].value).toBe("{m_{j \\rightarrow i}(\\mathrm{good})}{\\sum{m_{j \\rightarrow i}}}$"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[0][8].value).toBe("'"); + expect(tokens[0][8].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/format9.py", + function() { + tokens = grammar.tokenizeLines("a = 'blah {foo-bar %d'\na = 'blah {foo-bar %d}'\na = 'blah {foo-bar %d //insane {}}'\na = '{}blah {foo-bar %d //insane {}}'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("blah {foo-bar "); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][6].value).toBe("%d"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("'"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][5].value).toBe("blah "); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][6].value).toBe("{foo-bar "); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][7].value).toBe("%d"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[1][8].value).toBe("}"); + expect(tokens[1][8].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][9].value).toBe("'"); + expect(tokens[1][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("'"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][5].value).toBe("blah "); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][6].value).toBe("{foo-bar "); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][7].value).toBe("%d"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[2][8].value).toBe(" //insane {}}"); + expect(tokens[2][8].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][9].value).toBe("'"); + expect(tokens[2][9].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("'"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][5].value).toBe("{}"); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.brace.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][6].value).toBe("blah "); + expect(tokens[3][6].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[3][7].value).toBe("{foo-bar "); + expect(tokens[3][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[3][8].value).toBe("%d"); + expect(tokens[3][8].scopes).toEqual(["source.python","string.quoted.single.python","meta.format.percent.python","constant.character.format.placeholder.other.python"]); + expect(tokens[3][9].value).toBe(" //insane {}}"); + expect(tokens[3][9].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[3][10].value).toBe("'"); + expect(tokens[3][10].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/note1.py", + function() { + tokens = grammar.tokenizeLines("a = 'XXX FIXME NB TODO'\na = r'XXX FIXME NB TODO'\na = b'XXX FIXME NB TODO'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("XXX FIXME NB TODO"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("r"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("XXX FIXME NB TODO"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("b"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("XXX FIXME NB TODO"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/prefixes1.py", + function() { + tokens = grammar.tokenizeLines("a = 's t r'\na = u's t r'\na = b's t r'\na = r's t r'\na = br's t r'\na = rb's t r'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("'"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][5].value).toBe("s t r"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][6].value).toBe("'"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("u"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("s t r"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("b"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("s t r"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("r"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[3][5].value).toBe("'"); + expect(tokens[3][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][6].value).toBe("s t r"); + expect(tokens[3][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[3][7].value).toBe("'"); + expect(tokens[3][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("="); + expect(tokens[4][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe("br"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[4][5].value).toBe("'"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][6].value).toBe("s t r"); + expect(tokens[4][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[4][7].value).toBe("'"); + expect(tokens[4][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("a"); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe("="); + expect(tokens[5][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[5][4].value).toBe("rb"); + expect(tokens[5][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","storage.type.string.python"]); + expect(tokens[5][5].value).toBe("'"); + expect(tokens[5][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][6].value).toBe("s t r"); + expect(tokens[5][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[5][7].value).toBe("'"); + expect(tokens[5][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/prefixes2.py", + function() { + tokens = grammar.tokenizeLines("a = U'S T R'\na = B'S T R'\na = R'S T R'\na = BR'S T R'\na = RB'S T R'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("U"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("S T R"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("B"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.binary.single.python","storage.type.string.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("S T R"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.binary.single.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("R"); + expect(tokens[2][4].scopes).toEqual(["source.python","string.quoted.raw.single.python","storage.type.string.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("S T R"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("BR"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","storage.type.string.python"]); + expect(tokens[3][5].value).toBe("'"); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][6].value).toBe("S T R"); + expect(tokens[3][6].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python"]); + expect(tokens[3][7].value).toBe("'"); + expect(tokens[3][7].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("="); + expect(tokens[4][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe("RB"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","storage.type.string.python"]); + expect(tokens[4][5].value).toBe("'"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][6].value).toBe("S T R"); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python"]); + expect(tokens[4][7].value).toBe("'"); + expect(tokens[4][7].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/prefixes3.py", + function() { + tokens = grammar.tokenizeLines("a = Ur'S T R'\na = UR'S T R'\na = uB'S T R'\na = Ru'S T R'\na = RU'S T R'\na = bR'S T R'\na = Rb'S T R'") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("Ur"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.regexp.quoted.single.python","invalid.deprecated.prefix.python"]); + expect(tokens[0][5].value).toBe("'"); + expect(tokens[0][5].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[0][6].value).toBe("S T R"); + expect(tokens[0][6].scopes).toEqual(["source.python","string.regexp.quoted.single.python"]); + expect(tokens[0][7].value).toBe("'"); + expect(tokens[0][7].scopes).toEqual(["source.python","string.regexp.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[1][0].value).toBe("a"); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe(" "); + expect(tokens[1][1].scopes).toEqual(["source.python"]); + expect(tokens[1][2].value).toBe("="); + expect(tokens[1][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[1][3].value).toBe(" "); + expect(tokens[1][3].scopes).toEqual(["source.python"]); + expect(tokens[1][4].value).toBe("UR"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.raw.single.python","invalid.deprecated.prefix.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][6].value).toBe("S T R"); + expect(tokens[1][6].scopes).toEqual(["source.python","string.quoted.raw.single.python"]); + expect(tokens[1][7].value).toBe("'"); + expect(tokens[1][7].scopes).toEqual(["source.python","string.quoted.raw.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe("a"); + expect(tokens[2][0].scopes).toEqual(["source.python"]); + expect(tokens[2][1].value).toBe(" "); + expect(tokens[2][1].scopes).toEqual(["source.python"]); + expect(tokens[2][2].value).toBe("="); + expect(tokens[2][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[2][3].value).toBe(" "); + expect(tokens[2][3].scopes).toEqual(["source.python"]); + expect(tokens[2][4].value).toBe("uB"); + expect(tokens[2][4].scopes).toEqual(["source.python"]); + expect(tokens[2][5].value).toBe("'"); + expect(tokens[2][5].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[2][6].value).toBe("S T R"); + expect(tokens[2][6].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[2][7].value).toBe("'"); + expect(tokens[2][7].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[3][0].value).toBe("a"); + expect(tokens[3][0].scopes).toEqual(["source.python"]); + expect(tokens[3][1].value).toBe(" "); + expect(tokens[3][1].scopes).toEqual(["source.python"]); + expect(tokens[3][2].value).toBe("="); + expect(tokens[3][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[3][3].value).toBe(" "); + expect(tokens[3][3].scopes).toEqual(["source.python"]); + expect(tokens[3][4].value).toBe("R"); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.single.python","invalid.illegal.prefix.python"]); + expect(tokens[3][5].value).toBe("u"); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[3][6].value).toBe("'"); + expect(tokens[3][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[3][7].value).toBe("S T R"); + expect(tokens[3][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[3][8].value).toBe("'"); + expect(tokens[3][8].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][0].value).toBe("a"); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe(" "); + expect(tokens[4][1].scopes).toEqual(["source.python"]); + expect(tokens[4][2].value).toBe("="); + expect(tokens[4][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[4][3].value).toBe(" "); + expect(tokens[4][3].scopes).toEqual(["source.python"]); + expect(tokens[4][4].value).toBe("R"); + expect(tokens[4][4].scopes).toEqual(["source.python","string.quoted.single.python","invalid.illegal.prefix.python"]); + expect(tokens[4][5].value).toBe("U"); + expect(tokens[4][5].scopes).toEqual(["source.python","string.quoted.single.python","storage.type.string.python"]); + expect(tokens[4][6].value).toBe("'"); + expect(tokens[4][6].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][7].value).toBe("S T R"); + expect(tokens[4][7].scopes).toEqual(["source.python","string.quoted.single.python"]); + expect(tokens[4][8].value).toBe("'"); + expect(tokens[4][8].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[5][0].value).toBe("a"); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe(" "); + expect(tokens[5][1].scopes).toEqual(["source.python"]); + expect(tokens[5][2].value).toBe("="); + expect(tokens[5][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[5][3].value).toBe(" "); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[5][4].value).toBe("bR"); + expect(tokens[5][4].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","storage.type.string.python"]); + expect(tokens[5][5].value).toBe("'"); + expect(tokens[5][5].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[5][6].value).toBe("S T R"); + expect(tokens[5][6].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python"]); + expect(tokens[5][7].value).toBe("'"); + expect(tokens[5][7].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","punctuation.definition.string.end.python"]); + expect(tokens[6][0].value).toBe("a"); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe(" "); + expect(tokens[6][1].scopes).toEqual(["source.python"]); + expect(tokens[6][2].value).toBe("="); + expect(tokens[6][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[6][3].value).toBe(" "); + expect(tokens[6][3].scopes).toEqual(["source.python"]); + expect(tokens[6][4].value).toBe("Rb"); + expect(tokens[6][4].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","storage.type.string.python"]); + expect(tokens[6][5].value).toBe("'"); + expect(tokens[6][5].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[6][6].value).toBe("S T R"); + expect(tokens[6][6].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python"]); + expect(tokens[6][7].value).toBe("'"); + expect(tokens[6][7].scopes).toEqual(["source.python","string.quoted.raw.binary.single.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/raw1.py", + function() { + tokens = grammar.tokenizeLines("a = R\"\"\"\nmultiline \"raw\" string \\\n\n \\xf1 \\u1234aaaa \\U1234aaaa\n\n \\N{BLACK SPADE SUIT}\n\"\"\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("R"); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.raw.multi.python","storage.type.string.python"]); + expect(tokens[0][5].value).toBe("\"\"\""); + expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.raw.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][0].value).toBe("multiline \"raw\" string "); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[1][1].value).toBe("\\"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[3][0].value).toBe(" \\xf1 \\u1234aaaa \\U1234aaaa"); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[5][0].value).toBe(" \\N"); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[5][1].value).toBe("{BLACK SPADE SUIT}"); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.raw.multi.python"]); + expect(tokens[6][0].value).toBe("\"\"\""); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.raw.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/strings/unicode1.py", + function() { + tokens = grammar.tokenizeLines("a = \"\"\"\nmultiline \"unicode\" string \\\n\n \\xf1 \\u1234aaaa \\U1234aaaa\n\n \\N{BLACK SPADE SUIT}\n\"\"\"") + expect(tokens[0][0].value).toBe("a"); + expect(tokens[0][0].scopes).toEqual(["source.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python"]); + expect(tokens[0][2].value).toBe("="); + expect(tokens[0][2].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[0][3].value).toBe(" "); + expect(tokens[0][3].scopes).toEqual(["source.python"]); + expect(tokens[0][4].value).toBe("\"\"\""); + expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][0].value).toBe("multiline \"unicode\" string "); + expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[1][1].value).toBe("\\"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.multi.python","constant.language.python"]); + expect(tokens[2][0].value).toBe(""); + expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[3][1].value).toBe("\\xf1"); + expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.multi.python","constant.character.escape.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[3][3].value).toBe("\\u1234"); + expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.multi.python","constant.character.escape.python"]); + expect(tokens[3][4].value).toBe("aaaa "); + expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[3][5].value).toBe("\\U1234aaaa"); + expect(tokens[3][5].scopes).toEqual(["source.python","string.quoted.multi.python","constant.character.escape.python"]); + expect(tokens[4][0].value).toBe(""); + expect(tokens[4][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.multi.python"]); + expect(tokens[5][1].value).toBe("\\N{BLACK SPADE SUIT}"); + expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.multi.python","constant.character.escape.python"]); + expect(tokens[6][0].value).toBe("\"\"\""); + expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.multi.python","punctuation.definition.string.end.python"]); + }); + + it("test/unicode/unicode1.py", + function() { + tokens = grammar.tokenizeLines("class Üa(Êa):\n 'Œ\\nŃ'\n @æaœ\n def ŌÏŒĘ(self, Ú=1):\n print('превед 你好')\n return Ù\n ÜBER = 1\n 你好 = lambda: 你好\n def 你好(): pass") + expect(tokens[0][0].value).toBe("class"); + expect(tokens[0][0].scopes).toEqual(["source.python","meta.class.python","storage.type.class.python"]); + expect(tokens[0][1].value).toBe(" "); + expect(tokens[0][1].scopes).toEqual(["source.python","meta.class.python"]); + expect(tokens[0][2].value).toBe("Üa"); + expect(tokens[0][2].scopes).toEqual(["source.python","meta.class.python","entity.name.type.class.python"]); + expect(tokens[0][3].value).toBe("("); + expect(tokens[0][3].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.begin.python"]); + expect(tokens[0][4].value).toBe("Êa"); + expect(tokens[0][4].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","entity.other.inherited-class.python"]); + expect(tokens[0][5].value).toBe(")"); + expect(tokens[0][5].scopes).toEqual(["source.python","meta.class.python","meta.class.inheritance.python","punctuation.definition.inheritance.end.python"]); + expect(tokens[0][6].value).toBe(":"); + expect(tokens[0][6].scopes).toEqual(["source.python","meta.class.python","punctuation.section.class.begin.python"]); + expect(tokens[1][0].value).toBe(" "); + expect(tokens[1][0].scopes).toEqual(["source.python"]); + expect(tokens[1][1].value).toBe("'"); + expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[1][2].value).toBe("Œ"); + expect(tokens[1][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[1][3].value).toBe("\\n"); + expect(tokens[1][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python","constant.character.escape.python"]); + expect(tokens[1][4].value).toBe("Ń"); + expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]); + expect(tokens[1][5].value).toBe("'"); + expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]); + expect(tokens[2][0].value).toBe(" "); + expect(tokens[2][0].scopes).toEqual(["source.python","meta.function.decorator.python"]); + expect(tokens[2][1].value).toBe("@"); + expect(tokens[2][1].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python","punctuation.definition.decorator.python"]); + expect(tokens[2][2].value).toBe("æaœ"); + expect(tokens[2][2].scopes).toEqual(["source.python","meta.function.decorator.python","entity.name.function.decorator.python"]); + expect(tokens[3][0].value).toBe(" "); + expect(tokens[3][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[3][1].value).toBe("def"); + expect(tokens[3][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[3][2].value).toBe(" "); + expect(tokens[3][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[3][3].value).toBe("ŌÏŒĘ"); + expect(tokens[3][3].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[3][4].value).toBe("("); + expect(tokens[3][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[3][5].value).toBe("self"); + expect(tokens[3][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python","variable.parameter.function.language.special.self.python"]); + expect(tokens[3][6].value).toBe(","); + expect(tokens[3][6].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.separator.parameters.python"]); + expect(tokens[3][7].value).toBe(" "); + expect(tokens[3][7].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python"]); + expect(tokens[3][8].value).toBe("Ú"); + expect(tokens[3][8].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","variable.parameter.function.language.python"]); + expect(tokens[3][9].value).toBe("="); + expect(tokens[3][9].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","keyword.operator.python"]); + expect(tokens[3][10].value).toBe("1"); + expect(tokens[3][10].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","constant.numeric.dec.python"]); + expect(tokens[3][11].value).toBe(")"); + expect(tokens[3][11].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[3][12].value).toBe(":"); + expect(tokens[3][12].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[4][0].value).toBe(" "); + expect(tokens[4][0].scopes).toEqual(["source.python"]); + expect(tokens[4][1].value).toBe("print"); + expect(tokens[4][1].scopes).toEqual(["source.python","meta.function-call.python","support.function.builtin.python"]); + expect(tokens[4][2].value).toBe("("); + expect(tokens[4][2].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.begin.python"]); + expect(tokens[4][3].value).toBe("'"); + expect(tokens[4][3].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.begin.python"]); + expect(tokens[4][4].value).toBe("превед 你好"); + expect(tokens[4][4].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python"]); + expect(tokens[4][5].value).toBe("'"); + expect(tokens[4][5].scopes).toEqual(["source.python","meta.function-call.python","meta.function-call.arguments.python","string.quoted.single.python","punctuation.definition.string.end.python"]); + expect(tokens[4][6].value).toBe(")"); + expect(tokens[4][6].scopes).toEqual(["source.python","meta.function-call.python","punctuation.definition.arguments.end.python"]); + expect(tokens[5][0].value).toBe(" "); + expect(tokens[5][0].scopes).toEqual(["source.python"]); + expect(tokens[5][1].value).toBe("return"); + expect(tokens[5][1].scopes).toEqual(["source.python","keyword.control.flow.python"]); + expect(tokens[5][2].value).toBe(" "); + expect(tokens[5][2].scopes).toEqual(["source.python"]); + expect(tokens[5][3].value).toBe("Ù"); + expect(tokens[5][3].scopes).toEqual(["source.python"]); + expect(tokens[6][0].value).toBe(" "); + expect(tokens[6][0].scopes).toEqual(["source.python"]); + expect(tokens[6][1].value).toBe("ÜBER"); + expect(tokens[6][1].scopes).toEqual(["source.python","constant.other.caps.python"]); + expect(tokens[6][2].value).toBe(" "); + expect(tokens[6][2].scopes).toEqual(["source.python"]); + expect(tokens[6][3].value).toBe("="); + expect(tokens[6][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[6][4].value).toBe(" "); + expect(tokens[6][4].scopes).toEqual(["source.python"]); + expect(tokens[6][5].value).toBe("1"); + expect(tokens[6][5].scopes).toEqual(["source.python","constant.numeric.dec.python"]); + expect(tokens[7][0].value).toBe(" "); + expect(tokens[7][0].scopes).toEqual(["source.python"]); + expect(tokens[7][1].value).toBe("你好"); + expect(tokens[7][1].scopes).toEqual(["source.python"]); + expect(tokens[7][2].value).toBe(" "); + expect(tokens[7][2].scopes).toEqual(["source.python"]); + expect(tokens[7][3].value).toBe("="); + expect(tokens[7][3].scopes).toEqual(["source.python","keyword.operator.assignment.python"]); + expect(tokens[7][4].value).toBe(" "); + expect(tokens[7][4].scopes).toEqual(["source.python"]); + expect(tokens[7][5].value).toBe("lambda"); + expect(tokens[7][5].scopes).toEqual(["source.python","meta.lambda-function.python","storage.type.function.lambda.python"]); + expect(tokens[7][6].value).toBe(":"); + expect(tokens[7][6].scopes).toEqual(["source.python","meta.lambda-function.python","punctuation.section.function.lambda.begin.python"]); + expect(tokens[7][7].value).toBe(" "); + expect(tokens[7][7].scopes).toEqual(["source.python"]); + expect(tokens[7][8].value).toBe("你好"); + expect(tokens[7][8].scopes).toEqual(["source.python"]); + expect(tokens[8][0].value).toBe(" "); + expect(tokens[8][0].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[8][1].value).toBe("def"); + expect(tokens[8][1].scopes).toEqual(["source.python","meta.function.python","storage.type.function.python"]); + expect(tokens[8][2].value).toBe(" "); + expect(tokens[8][2].scopes).toEqual(["source.python","meta.function.python"]); + expect(tokens[8][3].value).toBe("你好"); + expect(tokens[8][3].scopes).toEqual(["source.python","meta.function.python","entity.name.function.python"]); + expect(tokens[8][4].value).toBe("("); + expect(tokens[8][4].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.begin.python"]); + expect(tokens[8][5].value).toBe(")"); + expect(tokens[8][5].scopes).toEqual(["source.python","meta.function.python","meta.function.parameters.python","punctuation.definition.parameters.end.python"]); + expect(tokens[8][6].value).toBe(":"); + expect(tokens[8][6].scopes).toEqual(["source.python","meta.function.python","punctuation.section.function.begin.python"]); + expect(tokens[8][7].value).toBe(" "); + expect(tokens[8][7].scopes).toEqual(["source.python"]); + expect(tokens[8][8].value).toBe("pass"); + expect(tokens[8][8].scopes).toEqual(["source.python","keyword.control.flow.python"]); + }); +}); + diff --git a/test/atom-spec/tb-spec.js b/test/atom-spec/tb-spec.js new file mode 100644 index 00000000..bb53fed2 --- /dev/null +++ b/test/atom-spec/tb-spec.js @@ -0,0 +1,46 @@ +describe("python-traceback tests", function() { + var grammar = null; + + beforeEach(function() { + waitsForPromise(function() { + return atom.packages.activatePackage("MagicPython") + }); + runs(function() { + grammar = atom.grammars.grammarForScopeName("text.python.traceback") + }); + }); + + it("highlights a traceback line", function() { + tokens = grammar.tokenizeLines( + ' File "t.py", line 1, in \n' + + ' a = 1/0') + + expect(tokens[0][0].value).toBe(" File "); + expect(tokens[0][0].scopes).toEqual(['text.python.traceback']); + + expect(tokens[0][1].value).toBe('"t.py"'); + expect(tokens[0][1].scopes).toEqual( + ['text.python.traceback', 'string.python.traceback']); + + expect(tokens[0][2].value).toBe(", line "); + expect(tokens[0][2].scopes).toEqual(['text.python.traceback']); + + expect(tokens[0][3].value).toBe("1"); + expect(tokens[0][3].scopes).toEqual( + ['text.python.traceback', 'constant.numeric.python.traceback']); + + expect(tokens[0][4].value).toBe(", in "); + expect(tokens[0][4].scopes).toEqual(['text.python.traceback']); + + expect(tokens[0][5].value).toBe(""); + expect(tokens[0][5].scopes).toEqual( + ['text.python.traceback', 'entity.name.function.python.traceback']); + + expect(tokens[1][1].value).toBe("a"); + expect(tokens[1][1].scopes).toEqual(['text.python.traceback']); + + expect(tokens[1][3].value).toBe("="); + expect(tokens[1][3].scopes).toEqual( + ['text.python.traceback', 'keyword.operator.assignment.python']); + }); +}); diff --git a/test/builtins/builtins1.py b/test/builtins/builtins1.py index 03c18cd6..36866c2a 100644 --- a/test/builtins/builtins1.py +++ b/test/builtins/builtins1.py @@ -5,7 +5,7 @@ NotImplemented True None False Ellipsis Warning Exception BaseException -sum oct abs type object print +sum oct abs type object print exec @@ -39,3 +39,5 @@ object : source.python, support.type.python : source.python print : source.python, support.function.builtin.python + : source.python +exec : source.python, support.function.builtin.python diff --git a/test/builtins/builtins3.py b/test/builtins/builtins3.py index 68036b64..f1f90d9c 100644 --- a/test/builtins/builtins3.py +++ b/test/builtins/builtins3.py @@ -39,6 +39,10 @@ some.__path__ some.__prepare__ some.__package__ +some.__traceback__ +some.__closure__ +some.__globals__ +some.__match_args__ some.__notspecial__ @@ -59,124 +63,136 @@ ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python ] : punctuation.definition.list.end.python, source.python some : source.python -. : source.python -__bases__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__bases__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__class__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__class__ : meta.member.access.python, source.python, support.variable.magic.python assert : keyword.control.flow.python, source.python : source.python __debug__ : source.python, support.variable.magic.python __builtins__ : source.python, support.variable.magic.python __builtins__ : source.python, support.variable.magic.python -. : source.python -len : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +len : meta.attribute.python, meta.member.access.python, source.python print : meta.function-call.python, source.python, support.function.builtin.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python __builtins__ : meta.function-call.arguments.python, meta.function-call.python, source.python, support.variable.magic.python ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python some : source.python -. : source.python -__dict__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__dict__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__doc__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__doc__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__file__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__file__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__members__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__members__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__metaclass__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__metaclass__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__methods__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__methods__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__module__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__module__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__mro__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__mro__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__name__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__name__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__slots__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__slots__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__subclasses__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__subclasses__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__version__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__version__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__weakref__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__weakref__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__qualname__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__qualname__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__code__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__code__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__wrapped__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__wrapped__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__signature__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__signature__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__defaults__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__defaults__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__func__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__func__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__self__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__self__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__kwdefaults__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__kwdefaults__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__matmul__ : source.python, support.function.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__matmul__ : meta.member.access.python, source.python, support.function.magic.python some : source.python -. : source.python -__imatmul__ : source.python, support.function.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__imatmul__ : meta.member.access.python, source.python, support.function.magic.python some : source.python -. : source.python -__rmatmul__ : source.python, support.function.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__rmatmul__ : meta.member.access.python, source.python, support.function.magic.python some : source.python -. : source.python -__annotations__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__annotations__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__init_subclass__ : source.python, support.function.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__init_subclass__ : meta.member.access.python, source.python, support.function.magic.python some : source.python -. : source.python -__set_name__ : source.python, support.function.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__set_name__ : meta.member.access.python, source.python, support.function.magic.python some : source.python -. : source.python -__fspath__ : source.python, support.function.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__fspath__ : meta.member.access.python, source.python, support.function.magic.python some : source.python -. : source.python -__classcell__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__classcell__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__bytes__ : source.python, support.function.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__bytes__ : meta.member.access.python, source.python, support.function.magic.python some : source.python -. : source.python -__spec__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__spec__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__path__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__path__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__prepare__ : source.python, support.function.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__prepare__ : meta.member.access.python, source.python, support.function.magic.python some : source.python -. : source.python -__package__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__package__ : meta.member.access.python, source.python, support.variable.magic.python some : source.python -. : source.python -__notspecial__ : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__traceback__ : meta.member.access.python, source.python, support.variable.magic.python +some : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__closure__ : meta.member.access.python, source.python, support.variable.magic.python +some : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__globals__ : meta.member.access.python, source.python, support.variable.magic.python +some : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__match_args__ : meta.member.access.python, source.python, support.variable.magic.python +some : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__notspecial__ : meta.attribute.python, meta.member.access.python, source.python diff --git a/test/builtins/builtins4.py b/test/builtins/builtins4.py index b78b96fb..d126c0bf 100644 --- a/test/builtins/builtins4.py +++ b/test/builtins/builtins4.py @@ -7,21 +7,22 @@ + some : source.python -. : source.python -int : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +int : meta.attribute.python, meta.member.access.python, source.python some : source.python -. : source.python -sum : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +sum : meta.attribute.python, meta.member.access.python, source.python some : source.python -. : source.python -super : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +super : meta.attribute.python, meta.member.access.python, source.python some : source.python -. : source.python -unicode : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +unicode : meta.attribute.python, meta.member.access.python, source.python some : source.python -. : source.python -foo : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +foo : meta.attribute.python, meta.member.access.python, source.python some : source.python -. : source.python -Exception : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +Exception : meta.attribute.python, meta.member.access.python, source.python diff --git a/test/builtins/builtins5.py b/test/builtins/builtins5.py index ecabf1c5..af3db3e5 100644 --- a/test/builtins/builtins5.py +++ b/test/builtins/builtins5.py @@ -7,17 +7,21 @@ + some : source.python -. : source.python -True : keyword.illegal.name.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +True : keyword.illegal.name.python, meta.member.access.python, source.python some : source.python -. : source.python -\ : punctuation.separator.continuation.line.python, source.python - : source.python - : source.python -True : keyword.illegal.name.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +\ : meta.member.access.python, punctuation.separator.continuation.line.python, source.python + : meta.member.access.python, source.python + : meta.member.access.python, source.python +True : keyword.illegal.name.python, meta.member.access.python, source.python True : constant.language.python, source.python some : source.python -. : source.python - : source.python -True : keyword.illegal.name.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python + : meta.member.access.python, source.python +True : keyword.illegal.name.python, meta.member.access.python, source.python diff --git a/test/builtins/builtins6.py b/test/builtins/builtins6.py new file mode 100644 index 00000000..c110fa4a --- /dev/null +++ b/test/builtins/builtins6.py @@ -0,0 +1,13 @@ +exec() +exec(code_obj) + + + + +exec : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +exec : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +code_obj : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/builtins/builtins7.py b/test/builtins/builtins7.py new file mode 100644 index 00000000..6d23b2b7 --- /dev/null +++ b/test/builtins/builtins7.py @@ -0,0 +1,16 @@ +breakpoint() +aiter() +anext() + + + + +breakpoint : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +aiter : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +anext : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/calls/call1.py b/test/calls/call1.py index aab3f9f7..86f32acb 100644 --- a/test/calls/call1.py +++ b/test/calls/call1.py @@ -2,6 +2,7 @@ + some_call : meta.function-call.generic.python, meta.function-call.python, source.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python A : meta.function-call.arguments.python, meta.function-call.python, source.python @@ -10,7 +11,7 @@ b : meta.function-call.arguments.python, meta.function-call.python, source.python , : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python : meta.function-call.arguments.python, meta.function-call.python, source.python -c : meta.function-call.arguments.python, meta.function-call.python, meta.item-access.python, source.python +c : meta.function-call.arguments.python, meta.function-call.python, meta.indexed-name.python, meta.item-access.python, source.python [ : meta.function-call.arguments.python, meta.function-call.python, meta.item-access.python, punctuation.definition.arguments.begin.python, source.python 1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, meta.item-access.arguments.python, meta.item-access.python, source.python ] : meta.function-call.arguments.python, meta.function-call.python, meta.item-access.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/calls/call10.py b/test/calls/call10.py new file mode 100644 index 00000000..98b52f2d --- /dev/null +++ b/test/calls/call10.py @@ -0,0 +1,32 @@ +x = foo(True, + 3 * 4, + *a, + **bar) + + + + + +x : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +foo : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +True : constant.language.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +3 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +* : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +4 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +* : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function-call.python, source.python +a : meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +** : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function-call.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/calls/call11.py b/test/calls/call11.py new file mode 100644 index 00000000..3695fc57 --- /dev/null +++ b/test/calls/call11.py @@ -0,0 +1,44 @@ +id = Field[uuid.UUID] ( + uuid.UUID, + inheritable=False, + simpledelta=False, + allow_ddl_set=True, +) + + + + + +id : source.python, support.function.builtin.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +Field : meta.indexed-name.python, meta.item-access.python, source.python +[ : meta.item-access.python, punctuation.definition.arguments.begin.python, source.python +uuid : meta.item-access.arguments.python, meta.item-access.python, source.python +. : meta.item-access.arguments.python, meta.item-access.python, meta.member.access.python, punctuation.separator.period.python, source.python +UUID : constant.other.caps.python, meta.item-access.arguments.python, meta.item-access.python, meta.member.access.python, source.python +] : meta.item-access.python, punctuation.definition.arguments.end.python, source.python + : source.python +( : punctuation.definition.arguments.begin.python, source.python + : meta.function-call.arguments.python, source.python +uuid : meta.function-call.arguments.python, source.python +. : meta.function-call.arguments.python, meta.member.access.python, punctuation.separator.period.python, source.python +UUID : constant.other.caps.python, meta.function-call.arguments.python, meta.member.access.python, source.python +, : meta.function-call.arguments.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, source.python +inheritable : meta.function-call.arguments.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, source.python +False : constant.language.python, meta.function-call.arguments.python, source.python +, : meta.function-call.arguments.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, source.python +simpledelta : meta.function-call.arguments.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, source.python +False : constant.language.python, meta.function-call.arguments.python, source.python +, : meta.function-call.arguments.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, source.python +allow_ddl_set : meta.function-call.arguments.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, source.python +True : constant.language.python, meta.function-call.arguments.python, source.python +, : meta.function-call.arguments.python, punctuation.separator.arguments.python, source.python +) : punctuation.definition.arguments.end.python, source.python diff --git a/test/calls/call3.py b/test/calls/call3.py index 3818b51c..3eb5ff59 100644 --- a/test/calls/call3.py +++ b/test/calls/call3.py @@ -4,17 +4,18 @@ + foo : source.python -. : source.python -class : keyword.control.flow.python, meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -a : meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +class : keyword.control.flow.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +a : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python foo : source.python -. : source.python -and : keyword.control.flow.python, meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +and : keyword.control.flow.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python foo : source.python -. : source.python -if : keyword.control.flow.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +if : keyword.control.flow.python, meta.member.access.python, source.python diff --git a/test/calls/call4.py b/test/calls/call4.py index 50f561c0..6da33186 100644 --- a/test/calls/call4.py +++ b/test/calls/call4.py @@ -4,9 +4,10 @@ + foo : source.python -. : source.python -1 : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +1 : meta.member.access.python, source.python foo : meta.function-call.generic.python, meta.function-call.python, source.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python bar : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python @@ -19,13 +20,13 @@ sam : meta.function-call.arguments.python, meta.function-call.python, source.python ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python foo : source.python -. : source.python -None : keyword.illegal.name.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +None : keyword.illegal.name.python, meta.member.access.python, source.python : source.python and : keyword.operator.logical.python, source.python : source.python foo : source.python -. : source.python -None : keyword.illegal.name.python, source.python -. : source.python -baz : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +None : keyword.illegal.name.python, meta.member.access.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +baz : meta.attribute.python, meta.member.access.python, source.python diff --git a/test/calls/call5.py b/test/calls/call5.py index f18470c4..4ea20afa 100644 --- a/test/calls/call5.py +++ b/test/calls/call5.py @@ -5,11 +5,11 @@ f : meta.function-call.generic.python, meta.function-call.python, source.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -* : keyword.operator.unpacking.arguments.python, meta.function-call.python, source.python +* : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function-call.python, source.python a : meta.function-call.arguments.python, meta.function-call.python, source.python ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python f : meta.function-call.generic.python, meta.function-call.python, source.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -** : keyword.operator.unpacking.arguments.python, meta.function-call.python, source.python +** : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function-call.python, source.python a : meta.function-call.arguments.python, meta.function-call.python, source.python ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/calls/call6.py b/test/calls/call6.py index 8cdcd6d4..afc87850 100644 --- a/test/calls/call6.py +++ b/test/calls/call6.py @@ -5,8 +5,8 @@ f : meta.function-call.generic.python, meta.function-call.python, source.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python - : meta.function-call.python, source.python -* : keyword.operator.unpacking.arguments.python, meta.function-call.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +* : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function-call.python, source.python a : meta.function-call.arguments.python, meta.function-call.python, source.python : meta.function-call.arguments.python, meta.function-call.python, source.python , : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python @@ -17,8 +17,8 @@ ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python f : meta.function-call.generic.python, meta.function-call.python, source.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python - : meta.function-call.python, source.python -** : keyword.operator.unpacking.arguments.python, meta.function-call.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +** : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function-call.python, source.python a : meta.function-call.arguments.python, meta.function-call.python, source.python : meta.function-call.arguments.python, meta.function-call.python, source.python , : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python diff --git a/test/calls/call7.py b/test/calls/call7.py index 87242cc7..97cf7ee9 100644 --- a/test/calls/call7.py +++ b/test/calls/call7.py @@ -7,43 +7,44 @@ + foo : source.python -. : source.python -__class__ : meta.function-call.python, source.python, support.variable.magic.python - : meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -bar : meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__class__ : meta.function-call.python, meta.member.access.python, source.python, support.variable.magic.python + : meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python foo : source.python -. : source.python -__class__ : meta.function-call.python, source.python, support.variable.magic.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -bar : meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__class__ : meta.function-call.python, meta.member.access.python, source.python, support.variable.magic.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python foo : source.python -. : source.python -__add__ : meta.function-call.python, source.python, support.function.magic.python - : meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -bar : meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__add__ : meta.function-call.python, meta.member.access.python, source.python, support.function.magic.python + : meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python foo : source.python -. : source.python -__add__ : meta.function-call.python, source.python, support.function.magic.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -bar : meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__add__ : meta.function-call.python, meta.member.access.python, source.python, support.function.magic.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python : source.python foo : source.python -. : source.python -__class__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__class__ : meta.member.access.python, source.python, support.variable.magic.python : source.python 1 : constant.numeric.dec.python, source.python diff --git a/test/calls/call8.py b/test/calls/call8.py index 062a527f..0b78f3d3 100644 --- a/test/calls/call8.py +++ b/test/calls/call8.py @@ -5,37 +5,42 @@ + foo : source.python -. : source.python -__class__ : meta.function-call.python, source.python, support.variable.magic.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -bar : meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +__class__ : meta.function-call.python, meta.member.access.python, source.python, support.variable.magic.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python foo : source.python -. : source.python -__class__ : meta.function-call.python, source.python, support.variable.magic.python - : meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -bar : meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +__class__ : meta.function-call.python, meta.member.access.python, source.python, support.variable.magic.python + : meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python foo : source.python -. : source.python -__add__ : meta.function-call.python, source.python, support.function.magic.python - : meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -bar : meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +__add__ : meta.function-call.python, meta.member.access.python, source.python, support.function.magic.python + : meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python foo : source.python -. : source.python -__add__ : meta.function-call.python, source.python, support.function.magic.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -bar : meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +__add__ : meta.function-call.python, meta.member.access.python, source.python, support.function.magic.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/calls/print1.py b/test/calls/print1.py index 6b1848c5..9339060e 100644 --- a/test/calls/print1.py +++ b/test/calls/print1.py @@ -11,6 +11,7 @@ + print : source.python, support.function.builtin.python : source.python " : punctuation.definition.string.begin.python, source.python, string.quoted.single.python @@ -55,8 +56,8 @@ : source.python >> : keyword.operator.bitwise.python, source.python sys : source.python -. : source.python -stderr : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +stderr : meta.attribute.python, meta.member.access.python, source.python , : punctuation.separator.element.python, source.python : source.python " : punctuation.definition.string.begin.python, source.python, string.quoted.single.python @@ -72,8 +73,8 @@ file : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python = : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python sys : meta.function-call.arguments.python, meta.function-call.python, source.python -. : meta.function-call.arguments.python, meta.function-call.python, source.python -stderr : meta.function-call.arguments.python, meta.function-call.python, source.python +. : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.separator.period.python, source.python +stderr : meta.attribute.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python print : meta.function-call.python, source.python, support.function.builtin.python : meta.function-call.python, source.python diff --git a/test/classes/class11.py b/test/classes/class11.py index 6083dc95..4cb0b927 100644 --- a/test/classes/class11.py +++ b/test/classes/class11.py @@ -10,6 +10,7 @@ def __init__(self, a, b=1): + class : meta.class.python, source.python, storage.type.class.python : meta.class.python, source.python F : entity.name.type.class.python, meta.class.python, source.python @@ -32,16 +33,16 @@ def : meta.function.python, source.python, storage.type.function.pytho : : meta.function.python, punctuation.section.function.begin.python, source.python : source.python self : source.python, variable.language.special.self.python -. : source.python -a : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +a : meta.attribute.python, meta.member.access.python, source.python : source.python = : keyword.operator.assignment.python, source.python : source.python a : source.python : source.python self : source.python, variable.language.special.self.python -. : source.python -b : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +b : meta.attribute.python, meta.member.access.python, source.python : source.python = : keyword.operator.assignment.python, source.python : source.python @@ -57,18 +58,18 @@ def : meta.function.python, source.python, storage.type.function.pytho ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python : source.python a : source.python -. : source.python -self : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +self : meta.attribute.python, meta.member.access.python, source.python : source.python = : keyword.operator.assignment.python, source.python : source.python 1 : constant.numeric.dec.python, source.python : source.python a : source.python -. : source.python -self : source.python -. : source.python -bar : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +self : meta.attribute.python, meta.member.access.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +bar : meta.attribute.python, meta.member.access.python, source.python : source.python = : keyword.operator.assignment.python, source.python : source.python diff --git a/test/classes/class12.py b/test/classes/class12.py index 90e656bf..838a1bb2 100644 --- a/test/classes/class12.py +++ b/test/classes/class12.py @@ -12,12 +12,13 @@ def meth(cls, a, b=1): + class : meta.class.python, source.python, storage.type.class.python : meta.class.python, source.python F : entity.name.type.class.python, meta.class.python, source.python : : meta.class.python, punctuation.section.class.begin.python, source.python : meta.function.decorator.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python classmethod : meta.function.decorator.python, source.python, support.type.python : meta.function.python, source.python def : meta.function.python, source.python, storage.type.function.python @@ -37,16 +38,16 @@ def : meta.function.python, source.python, storage.type.function.pytho : : meta.function.python, punctuation.section.function.begin.python, source.python : source.python cls : source.python, variable.language.special.cls.python -. : source.python -a : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +a : meta.attribute.python, meta.member.access.python, source.python : source.python = : keyword.operator.assignment.python, source.python : source.python a : source.python : source.python cls : source.python, variable.language.special.cls.python -. : source.python -b : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +b : meta.attribute.python, meta.member.access.python, source.python : source.python = : keyword.operator.assignment.python, source.python : source.python @@ -68,18 +69,18 @@ def : meta.function.python, source.python, storage.type.function.pytho 1 : constant.numeric.dec.python, source.python : source.python a : source.python -. : source.python -cls : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +cls : meta.attribute.python, meta.member.access.python, source.python : source.python = : keyword.operator.assignment.python, source.python : source.python 1 : constant.numeric.dec.python, source.python : source.python a : source.python -. : source.python -cls : source.python -. : source.python -__name__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +cls : meta.attribute.python, meta.member.access.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__name__ : meta.member.access.python, source.python, support.variable.magic.python : source.python cls : meta.item-access.python, source.python, variable.language.special.cls.python [ : meta.item-access.python, punctuation.definition.arguments.begin.python, source.python diff --git a/test/classes/class14.py b/test/classes/class14.py index 036543ed..45455895 100644 --- a/test/classes/class14.py +++ b/test/classes/class14.py @@ -3,26 +3,27 @@ class F(f.Exception, f.type, Exception, Exception.a, b=Exception): + class : meta.class.python, source.python, storage.type.class.python : meta.class.python, source.python F : entity.name.type.class.python, meta.class.python, source.python ( : meta.class.inheritance.python, meta.class.python, punctuation.definition.inheritance.begin.python, source.python f : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python -. : meta.class.inheritance.python, meta.class.python, source.python -Exception : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python +. : meta.class.inheritance.python, meta.class.python, meta.member.access.python, punctuation.separator.period.python, source.python +Exception : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, meta.member.access.python, source.python , : meta.class.inheritance.python, meta.class.python, punctuation.separator.inheritance.python, source.python : meta.class.inheritance.python, meta.class.python, source.python f : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python -. : meta.class.inheritance.python, meta.class.python, source.python -type : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python +. : meta.class.inheritance.python, meta.class.python, meta.member.access.python, punctuation.separator.period.python, source.python +type : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, meta.member.access.python, source.python , : meta.class.inheritance.python, meta.class.python, punctuation.separator.inheritance.python, source.python : meta.class.inheritance.python, meta.class.python, source.python Exception : meta.class.inheritance.python, meta.class.python, source.python, support.type.exception.python , : meta.class.inheritance.python, meta.class.python, punctuation.separator.inheritance.python, source.python : meta.class.inheritance.python, meta.class.python, source.python Exception : meta.class.inheritance.python, meta.class.python, source.python, support.type.exception.python -. : meta.class.inheritance.python, meta.class.python, source.python -a : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python +. : meta.class.inheritance.python, meta.class.python, meta.member.access.python, punctuation.separator.period.python, source.python +a : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, meta.member.access.python, source.python , : meta.class.inheritance.python, meta.class.python, punctuation.separator.inheritance.python, source.python : meta.class.inheritance.python, meta.class.python, source.python b : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python, variable.parameter.class.python diff --git a/test/classes/class2.py b/test/classes/class2.py index bba5e52d..1f15e252 100644 --- a/test/classes/class2.py +++ b/test/classes/class2.py @@ -5,7 +5,8 @@ class Spam(Foo.Bar, Bar.name={'very': 'odd'}): -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python + +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python dec : entity.name.function.decorator.python, meta.function.decorator.python, source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python Bar.name=... is not legal, but the test is for highlighter not breaking badly : comment.line.number-sign.python, source.python @@ -14,13 +15,13 @@ class : meta.class.python, source.python, storage.type.class.python Spam : entity.name.type.class.python, meta.class.python, source.python ( : meta.class.inheritance.python, meta.class.python, punctuation.definition.inheritance.begin.python, source.python Foo : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python -. : meta.class.inheritance.python, meta.class.python, source.python -Bar : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python +. : meta.class.inheritance.python, meta.class.python, meta.member.access.python, punctuation.separator.period.python, source.python +Bar : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, meta.member.access.python, source.python , : meta.class.inheritance.python, meta.class.python, punctuation.separator.inheritance.python, source.python : meta.class.inheritance.python, meta.class.python, source.python Bar : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python -. : meta.class.inheritance.python, meta.class.python, source.python -name : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, source.python +. : meta.class.inheritance.python, meta.class.python, meta.member.access.python, punctuation.separator.period.python, source.python +name : entity.other.inherited-class.python, meta.class.inheritance.python, meta.class.python, meta.member.access.python, source.python = : keyword.operator.assignment.python, meta.class.inheritance.python, meta.class.python, source.python { : meta.class.inheritance.python, meta.class.python, punctuation.definition.dict.begin.python, source.python ' : meta.class.inheritance.python, meta.class.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python diff --git a/test/classes/class9.py b/test/classes/class9.py index bbf24feb..a4ea943f 100644 --- a/test/classes/class9.py +++ b/test/classes/class9.py @@ -1,5 +1,6 @@ class Foo: __slots__ = () + __match_args__ = ('key', 'name') @@ -13,4 +14,19 @@ class : meta.class.python, source.python, storage.type.class.python = : keyword.operator.assignment.python, source.python : source.python ( : punctuation.parenthesis.begin.python, source.python +) : punctuation.parenthesis.end.python, source.python + : source.python +__match_args__ : source.python, support.variable.magic.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +( : punctuation.parenthesis.begin.python, source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +key : source.python, string.quoted.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +name : source.python, string.quoted.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.single.python ) : punctuation.parenthesis.end.python, source.python diff --git a/test/classes/super1.py b/test/classes/super1.py index 5172f365..709205c4 100644 --- a/test/classes/super1.py +++ b/test/classes/super1.py @@ -30,37 +30,39 @@ def : meta.function.python, source.python, storage.type.function.pytho super : meta.function-call.python, source.python, support.type.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python -. : source.python -__init__ : meta.function-call.python, source.python, support.function.magic.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__init__ : meta.function-call.python, meta.member.access.python, source.python, support.function.magic.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python : source.python super : meta.function-call.python, source.python, support.type.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python -. : source.python -__init__ : meta.function-call.python, source.python, support.function.magic.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +__init__ : meta.function-call.python, meta.member.access.python, source.python, support.function.magic.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python : source.python super : meta.function-call.python, source.python, support.type.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python ) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python -. : source.python -\ : punctuation.separator.continuation.line.python, source.python - : source.python -__init__ : meta.function-call.python, source.python, support.function.magic.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -foo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +\ : meta.member.access.python, punctuation.separator.continuation.line.python, source.python + : meta.member.access.python, source.python +__init__ : meta.function-call.python, meta.member.access.python, source.python, support.function.magic.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +foo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python : source.python __init__ : meta.function-call.python, source.python, support.function.magic.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python @@ -71,13 +73,13 @@ def : meta.function.python, source.python, storage.type.function.pytho : source.python : source.python foo : source.python -. : source.python -__init__ : meta.function-call.python, source.python, support.function.magic.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -bar : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__init__ : meta.function-call.python, meta.member.access.python, source.python, support.function.magic.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +bar : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python : source.python __init__ : meta.function-call.python, source.python, support.function.magic.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python diff --git a/test/docstrings/continuation2.py b/test/docstrings/continuation2.py index 6219f6fb..b9580e19 100644 --- a/test/docstrings/continuation2.py +++ b/test/docstrings/continuation2.py @@ -1,5 +1,6 @@ ' ' +# comment @@ -7,3 +8,5 @@ : invalid.illegal.newline.python, source.python, string.quoted.docstring.single.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python : invalid.illegal.newline.python, source.python, string.quoted.docstring.single.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + comment : comment.line.number-sign.python, source.python diff --git a/test/docstrings/continuation5.py b/test/docstrings/continuation5.py new file mode 100644 index 00000000..2e1f96ce --- /dev/null +++ b/test/docstrings/continuation5.py @@ -0,0 +1,47 @@ +'implicit ' "concatenation" + +'''implicit +''' 'concatenation' + +'''implicit +''' """ +concatenation +""" + +'implicit' ''' +concatenation +''' + + + +' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python +implicit : source.python, string.quoted.docstring.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python +concatenation : source.python, string.quoted.docstring.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python + : source.python +''' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python +implicit : source.python, string.quoted.docstring.multi.python +''' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python + : source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python +concatenation : source.python, string.quoted.docstring.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python + : source.python +''' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python +implicit : source.python, string.quoted.docstring.multi.python +''' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python + : source.python +""" : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python +concatenation : source.python, string.quoted.docstring.multi.python +""" : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python + : source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python +implicit : source.python, string.quoted.docstring.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python + : source.python +''' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.multi.python +concatenation : source.python, string.quoted.docstring.multi.python +''' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.multi.python diff --git a/test/docstrings/regr1.py b/test/docstrings/regr1.py index 7d3d5dcc..8b5a65e9 100644 --- a/test/docstrings/regr1.py +++ b/test/docstrings/regr1.py @@ -8,7 +8,7 @@ def foo(): # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python : : comment.line.number-sign.python, source.python : meta.function.decorator.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python asd : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.python, source.python def : meta.function.python, source.python, storage.type.function.python diff --git a/test/expressions/const1.py b/test/expressions/const1.py index 8e19a502..5288d71a 100644 --- a/test/expressions/const1.py +++ b/test/expressions/const1.py @@ -3,6 +3,7 @@ + QQQQ : constant.other.caps.python, source.python : source.python QQQQ_123 : constant.other.caps.python, source.python @@ -12,27 +13,27 @@ PROTOCOL_v2 : constant.other.caps.python, source.python : source.python QQQ : constant.other.caps.python, source.python -. : source.python -bar : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +bar : meta.attribute.python, meta.member.access.python, source.python : source.python baz : source.python -. : source.python -AA_a : constant.other.caps.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +AA_a : constant.other.caps.python, meta.member.access.python, source.python : source.python _AAA : constant.other.caps.python, source.python : source.python foo : source.python -. : source.python -_AAA : constant.other.caps.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +_AAA : constant.other.caps.python, meta.member.access.python, source.python QQQq : source.python : source.python QQQq123 : source.python : source.python self : source.python, variable.language.special.self.python -. : source.python -FOOO : meta.function-call.generic.python, meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +FOOO : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python : source.python _ : source.python : source.python diff --git a/test/expressions/const3.py b/test/expressions/const3.py new file mode 100644 index 00000000..252f8f3b --- /dev/null +++ b/test/expressions/const3.py @@ -0,0 +1,27 @@ +T61STRING = 20 +T61_STRING +T_STRING +_T_S_T_R_I_N_G_ +A_CLASS + +# not enough upper-case letters in the beginning +_T_s_TRING +A_Class + + + + +T61STRING : constant.other.caps.python, source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +20 : constant.numeric.dec.python, source.python +T61_STRING : constant.other.caps.python, source.python +T_STRING : constant.other.caps.python, source.python +_T_S_T_R_I_N_G_ : constant.other.caps.python, source.python +A_CLASS : constant.other.caps.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + not enough upper-case letters in the beginning : comment.line.number-sign.python, source.python +_T_s_TRING : source.python +A_Class : source.python diff --git a/test/expressions/expr10.py b/test/expressions/expr10.py index 0a14b7df..df5f1165 100644 --- a/test/expressions/expr10.py +++ b/test/expressions/expr10.py @@ -5,6 +5,7 @@ + a : source.python : source.python = : keyword.operator.assignment.python, source.python @@ -32,5 +33,5 @@ ... : constant.other.ellipsis.python, source.python ) : punctuation.parenthesis.end.python, source.python ... : constant.other.ellipsis.python, source.python -. : source.python -__class__ : source.python, support.variable.magic.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +__class__ : meta.member.access.python, source.python, support.variable.magic.python diff --git a/test/expressions/expr11.py b/test/expressions/expr11.py index 584174eb..de8e348c 100644 --- a/test/expressions/expr11.py +++ b/test/expressions/expr11.py @@ -2,15 +2,16 @@ + a : source.python : source.python = : keyword.operator.assignment.python, source.python : source.python self : source.python, variable.language.special.self.python -. : source.python -some_list : meta.item-access.python, source.python -[ : meta.item-access.python, punctuation.definition.arguments.begin.python, source.python -1 : constant.numeric.dec.python, meta.item-access.arguments.python, meta.item-access.python, source.python -: : meta.item-access.arguments.python, meta.item-access.python, punctuation.separator.slice.python, source.python -2 : constant.numeric.dec.python, meta.item-access.arguments.python, meta.item-access.python, source.python -] : meta.item-access.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +some_list : meta.indexed-name.python, meta.item-access.python, meta.member.access.python, source.python +[ : meta.item-access.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +1 : constant.numeric.dec.python, meta.item-access.arguments.python, meta.item-access.python, meta.member.access.python, source.python +: : meta.item-access.arguments.python, meta.item-access.python, meta.member.access.python, punctuation.separator.slice.python, source.python +2 : constant.numeric.dec.python, meta.item-access.arguments.python, meta.item-access.python, meta.member.access.python, source.python +] : meta.item-access.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/expressions/expr15.py b/test/expressions/expr15.py index 80558f9d..4038218a 100644 --- a/test/expressions/expr15.py +++ b/test/expressions/expr15.py @@ -2,6 +2,7 @@ + foofrom : source.python -. : source.python -something : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +something : meta.attribute.python, meta.member.access.python, source.python diff --git a/test/expressions/expr18.py b/test/expressions/expr18.py index 41ed45db..6e9413bf 100644 --- a/test/expressions/expr18.py +++ b/test/expressions/expr18.py @@ -3,9 +3,10 @@ + a : source.python -. : source.python -Exception : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +Exception : meta.attribute.python, meta.member.access.python, source.python Exception : source.python, support.type.exception.python -. : source.python -a : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +a : meta.attribute.python, meta.member.access.python, source.python diff --git a/test/expressions/expr19.py b/test/expressions/expr19.py index 604b9b84..4a4a3a5b 100644 --- a/test/expressions/expr19.py +++ b/test/expressions/expr19.py @@ -11,34 +11,41 @@ + a : source.python -. : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python foo : comment.line.number-sign.python, source.python a : source.python -. : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python foo : comment.line.number-sign.python, source.python a : source.python -. : source.python -\ : punctuation.separator.continuation.line.python, source.python - : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +\ : meta.member.access.python, punctuation.separator.continuation.line.python, source.python + : meta.member.access.python, source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python foo : comment.line.number-sign.python, source.python a : source.python -. : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python bar : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python -. : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python bar : source.python, string.quoted.docstring.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python a : source.python -. : source.python -\ : punctuation.separator.continuation.line.python, source.python - : source.python -' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python -bar : source.python, string.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.quoted.single.python +. : meta.member.access.python, punctuation.separator.period.python, source.python + : meta.member.access.python, source.python +\ : meta.member.access.python, punctuation.separator.continuation.line.python, source.python + : meta.member.access.python, source.python +' : meta.member.access.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +bar : meta.member.access.python, source.python, string.quoted.single.python +' : meta.member.access.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python diff --git a/test/expressions/expr21.py b/test/expressions/expr21.py new file mode 100644 index 00000000..59556710 --- /dev/null +++ b/test/expressions/expr21.py @@ -0,0 +1,39 @@ +while chunk := file.read(8192): + process(chunk) + y0 = (y1 := f(x)) + + + +while : keyword.control.flow.python, source.python + : source.python +chunk : source.python + : source.python +:= : keyword.operator.assignment.python, source.python + : source.python +file : source.python, variable.legacy.builtin.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +read : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +8192 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +process : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +chunk : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +y0 : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +( : punctuation.parenthesis.begin.python, source.python +y1 : source.python + : source.python +:= : keyword.operator.assignment.python, source.python + : source.python +f : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +) : punctuation.parenthesis.end.python, source.python diff --git a/test/expressions/expr22.py b/test/expressions/expr22.py new file mode 100644 index 00000000..7c03f542 --- /dev/null +++ b/test/expressions/expr22.py @@ -0,0 +1,33 @@ +match = 13 +case = 12 +if case == 4: + return match * 5 + + + +match : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +13 : constant.numeric.dec.python, source.python +case : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +12 : constant.numeric.dec.python, source.python +if : keyword.control.flow.python, source.python + : source.python +case : source.python + : source.python +== : keyword.operator.comparison.python, source.python + : source.python +4 : constant.numeric.dec.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +return : keyword.control.flow.python, source.python + : source.python +match : source.python + : source.python +* : keyword.operator.arithmetic.python, source.python + : source.python +5 : constant.numeric.dec.python, source.python diff --git a/test/expressions/expr4.py b/test/expressions/expr4.py index 6795c76d..ac5e5ddc 100644 --- a/test/expressions/expr4.py +++ b/test/expressions/expr4.py @@ -13,7 +13,7 @@ : source.python i : source.python : source.python -in : keyword.operator.logical.python, source.python +in : keyword.control.flow.python, source.python : source.python range : meta.function-call.python, source.python, support.function.builtin.python ( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python diff --git a/test/expressions/expr5.py b/test/expressions/expr5.py index 0e19a719..9c734c46 100644 --- a/test/expressions/expr5.py +++ b/test/expressions/expr5.py @@ -33,7 +33,7 @@ : : punctuation.separator.dict.python, source.python : source.python { : punctuation.definition.dict.begin.python, source.python -import : keyword.control.flow.python, source.python +import : keyword.control.import.python, source.python : : punctuation.separator.dict.python, source.python : source.python a : source.python diff --git a/test/expressions/expr7.py b/test/expressions/expr7.py index eef5c56e..8bfa1704 100644 --- a/test/expressions/expr7.py +++ b/test/expressions/expr7.py @@ -2,18 +2,19 @@ + a : source.python -. : source.python -True : keyword.illegal.name.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +True : keyword.illegal.name.python, meta.member.access.python, source.python : source.python = : keyword.operator.assignment.python, source.python : source.python b : source.python -. : source.python -False : keyword.illegal.name.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +False : keyword.illegal.name.python, meta.member.access.python, source.python : source.python = : keyword.operator.assignment.python, source.python : source.python d : source.python -. : source.python -None : keyword.illegal.name.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +None : keyword.illegal.name.python, meta.member.access.python, source.python diff --git a/test/expressions/keywords.py b/test/expressions/keywords.py index 82ae6521..7c14d5c2 100644 --- a/test/expressions/keywords.py +++ b/test/expressions/keywords.py @@ -1,12 +1,14 @@ -as async await continue del assert break finally for +as async await continue del assert break finally from elif else if import except pass raise return try while with nonlocal global class def +for -as : keyword.control.flow.python, source.python + +as : keyword.control.import.python, source.python : source.python async : keyword.control.flow.python, source.python : source.python @@ -21,9 +23,7 @@ break : keyword.control.flow.python, source.python : source.python finally : keyword.control.flow.python, source.python - : source.python -for : keyword.control.flow.python, source.python -from : keyword.control.flow.python, source.python +from : keyword.control.import.python, source.python : source.python elif : keyword.control.flow.python, source.python : source.python @@ -52,3 +52,5 @@ : source.python class : source.python, storage.type.class.python def : source.python, storage.type.function.python + : source.python +for : keyword.control.flow.python, source.python diff --git a/test/expressions/special2.py b/test/expressions/special2.py new file mode 100644 index 00000000..727d664f --- /dev/null +++ b/test/expressions/special2.py @@ -0,0 +1,16 @@ +__post_init__ +def __class_getitem__(): pass +__mro_entries__ + + + +__post_init__ : source.python, support.variable.magic.python +def : meta.function.python, source.python, storage.type.function.python + : meta.function.python, source.python +__class_getitem__ : meta.function.python, source.python, support.function.magic.python +( : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.begin.python, source.python +) : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.end.python, source.python +: : meta.function.python, punctuation.section.function.begin.python, source.python + : source.python +pass : keyword.control.flow.python, source.python +__mro_entries__ : source.python, support.variable.magic.python diff --git a/test/fstrings/comment4.py b/test/fstrings/comment4.py index 7f298de9..66943523 100644 --- a/test/fstrings/comment4.py +++ b/test/fstrings/comment4.py @@ -4,36 +4,36 @@ self : source.python, variable.language.special.self.python -. : source.python -assertEqual : meta.function-call.generic.python, meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python -' : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python -{ : constant.character.format.placeholder.other.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -10 : constant.numeric.dec.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -: : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, support.other.format.python -# : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, support.other.format.python -{ : constant.character.format.placeholder.other.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -3 : constant.numeric.dec.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python - : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -!= : keyword.operator.comparison.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python - : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -{ : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.dict.begin.python, source.python -4 : constant.numeric.dec.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -: : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.dict.python, source.python -5 : constant.numeric.dec.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -} : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.dict.end.python, source.python - : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -and : keyword.operator.logical.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python - : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -width : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -} : constant.character.format.placeholder.other.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -x : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, support.other.format.python -} : constant.character.format.placeholder.other.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -' : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python -, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -' : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python - 0xa : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python -' : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +assertEqual : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +' : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +10 : constant.numeric.dec.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +: : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, storage.type.format.python +# : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, storage.type.format.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +3 : constant.numeric.dec.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python + : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +!= : keyword.operator.comparison.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python + : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +{ : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.dict.begin.python, source.python +4 : constant.numeric.dec.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +: : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.separator.dict.python, source.python +5 : constant.numeric.dec.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +} : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.dict.end.python, source.python + : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +and : keyword.operator.logical.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python + : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +width : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +x : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, storage.type.format.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +' : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +, : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +' : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python + 0xa : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.quoted.single.python +' : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/fstrings/empty2.py b/test/fstrings/empty2.py index 77dc9087..9a007e14 100644 --- a/test/fstrings/empty2.py +++ b/test/fstrings/empty2.py @@ -7,20 +7,22 @@ -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -{ : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.single.python - : source.python, string.interpolated.python, string.regexp.quoted.single.python -{ : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.single.python - : invalid.illegal.brace.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -{ : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -} : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -{ : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python - : invalid.illegal.brace.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -} : constant.character.format.placeholder.other.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : invalid.illegal.brace.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +""" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : invalid.illegal.brace.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +""" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python diff --git a/test/fstrings/fraw1.py b/test/fstrings/fraw1.py new file mode 100644 index 00000000..31e2196e --- /dev/null +++ b/test/fstrings/fraw1.py @@ -0,0 +1,40 @@ +a = fr'[a-z]' +a = Fr'[a-z]' +a = rf'[a-z]' +a = rF'[a-z]' + + + + +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +fr : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +[a-z] : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +Fr : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +[a-z] : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +[a-z] : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rF : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +[a-z] : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python diff --git a/test/fstrings/fraw2.py b/test/fstrings/fraw2.py new file mode 100644 index 00000000..df0a2fed --- /dev/null +++ b/test/fstrings/fraw2.py @@ -0,0 +1,36 @@ +rf'fo{{2}}' +rf"fo{{2}}" +rf'''fo{{2}}''' +rf"""fo{{2}}""" + + + + +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +}} : constant.character.escape.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +}} : constant.character.escape.python, meta.fstring.python, source.python +" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +''' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +}} : constant.character.escape.python, meta.fstring.python, source.python +''' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +""" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +}} : constant.character.escape.python, meta.fstring.python, source.python +""" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python diff --git a/test/fstrings/fraw3.py b/test/fstrings/fraw3.py new file mode 100644 index 00000000..da1a6aff --- /dev/null +++ b/test/fstrings/fraw3.py @@ -0,0 +1,36 @@ +rf'fo{2}' +rf"fo{2}" +rf'''fo{2}''' +rf"""fo{2}""" + + + + +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +''' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +''' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.multi.python +""" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.multi.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +""" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.multi.python diff --git a/test/fstrings/fraw4.py b/test/fstrings/fraw4.py new file mode 100644 index 00000000..2bf2d465 --- /dev/null +++ b/test/fstrings/fraw4.py @@ -0,0 +1,45 @@ +a = rf'fo{{{2}}}' +a = rf'fo{{{bar}}}' +a = rf'fo{{2}}' + + + + + +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +2 : constant.numeric.dec.python, meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +}} : constant.character.escape.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +bar : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +}} : constant.character.escape.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +fo : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{{ : constant.character.escape.python, meta.fstring.python, source.python +2 : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +}} : constant.character.escape.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python diff --git a/test/fstrings/nested1.py b/test/fstrings/nested1.py index d7d1c8bf..91f15907 100644 --- a/test/fstrings/nested1.py +++ b/test/fstrings/nested1.py @@ -26,7 +26,7 @@ : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.multi.python { : constant.character.format.placeholder.other.python, meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python -bar : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.item-access.python, source.python +bar : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.indexed-name.python, meta.item-access.python, source.python [ : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.item-access.python, punctuation.definition.arguments.begin.python, source.python " : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.item-access.arguments.python, meta.item-access.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python q : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, meta.item-access.arguments.python, meta.item-access.python, source.python, string.quoted.single.python diff --git a/test/fstrings/nested2.py b/test/fstrings/nested2.py index fe92f4e1..276501e2 100644 --- a/test/fstrings/nested2.py +++ b/test/fstrings/nested2.py @@ -8,7 +8,7 @@ result: : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python value : meta.fstring.python, source.python -: : meta.fstring.python, source.python, support.other.format.python +: : meta.fstring.python, source.python, storage.type.format.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python 60 : constant.numeric.dec.python, meta.fstring.python, source.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python @@ -16,7 +16,7 @@ { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python 16 : constant.numeric.dec.python, meta.fstring.python, source.python !s : meta.fstring.python, source.python, storage.type.format.python -:2 : meta.fstring.python, source.python, support.other.format.python +:2 : meta.fstring.python, source.python, storage.type.format.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python ' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python diff --git a/test/fstrings/nested3.py b/test/fstrings/nested3.py index 80d07e26..64589b18 100644 --- a/test/fstrings/nested3.py +++ b/test/fstrings/nested3.py @@ -1,6 +1,6 @@ f"result: {value:{60}.{16!s:2}{'qwerty' [2]}}" - +# comment @@ -9,7 +9,7 @@ result: : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python value : meta.fstring.python, source.python -: : meta.fstring.python, source.python, support.other.format.python +: : meta.fstring.python, source.python, storage.type.format.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python 60 : constant.numeric.dec.python, meta.fstring.python, source.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python @@ -17,7 +17,7 @@ { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python 16 : constant.numeric.dec.python, meta.fstring.python, source.python !s : meta.fstring.python, source.python, storage.type.format.python -:2 : meta.fstring.python, source.python, support.other.format.python +:2 : meta.fstring.python, source.python, storage.type.format.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python ' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python @@ -30,3 +30,5 @@ }} : source.python " : punctuation.definition.string.begin.python, source.python, string.quoted.single.python : invalid.illegal.newline.python, source.python, string.quoted.single.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + comment : comment.line.number-sign.python, source.python diff --git a/test/fstrings/nested4.py b/test/fstrings/nested4.py index bd5325c7..01b9cd16 100644 --- a/test/fstrings/nested4.py +++ b/test/fstrings/nested4.py @@ -10,7 +10,7 @@ def foo(): pass result: : meta.fstring.python, source.python, string.interpolated.python, string.quoted.multi.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python value : meta.fstring.python, source.python -: : meta.fstring.python, source.python, support.other.format.python +: : meta.fstring.python, source.python, storage.type.format.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python 60 : constant.numeric.dec.python, meta.fstring.python, source.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python @@ -18,7 +18,7 @@ def foo(): pass { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python 16 : constant.numeric.dec.python, meta.fstring.python, source.python !s : meta.fstring.python, source.python, storage.type.format.python -:2 : meta.fstring.python, source.python, support.other.format.python +:2 : meta.fstring.python, source.python, storage.type.format.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python ' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python diff --git a/test/fstrings/prefixes2.py b/test/fstrings/prefixes2.py index 861d2b66..78ee4246 100644 --- a/test/fstrings/prefixes2.py +++ b/test/fstrings/prefixes2.py @@ -22,16 +22,20 @@ obj : meta.fstring.python, source.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python ' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -some : source.python, string.interpolated.python, string.regexp.quoted.single.python -{obj} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rF : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -some : source.python, string.interpolated.python, string.regexp.quoted.single.python -{obj} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python +rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +obj : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +rF : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +obj : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python Rf : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python ' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python diff --git a/test/fstrings/prefixes3.py b/test/fstrings/prefixes3.py index 200fa896..46d12ffb 100644 --- a/test/fstrings/prefixes3.py +++ b/test/fstrings/prefixes3.py @@ -6,16 +6,20 @@ -fr : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -some : source.python, string.interpolated.python, string.regexp.quoted.single.python -{obj} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -Fr : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -some : source.python, string.interpolated.python, string.regexp.quoted.single.python -{obj} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python +fr : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +obj : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python +Fr : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python +some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +obj : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.raw.single.python fR : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.raw.single.python ' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python some : meta.fstring.python, source.python, string.interpolated.python, string.quoted.raw.single.python diff --git a/test/fstrings/simple1.py b/test/fstrings/simple1.py index 813e7270..d7281270 100644 --- a/test/fstrings/simple1.py +++ b/test/fstrings/simple1.py @@ -16,7 +16,7 @@ [ : meta.fstring.python, punctuation.definition.list.begin.python, source.python : meta.fstring.python, source.python ] : meta.fstring.python, punctuation.definition.list.end.python, source.python -:X>+10d : meta.fstring.python, source.python, support.other.format.python +:X>+10d : meta.fstring.python, source.python, storage.type.format.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python " : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python a : source.python @@ -31,7 +31,7 @@ : meta.fstring.python, source.python ] : meta.fstring.python, punctuation.definition.list.end.python, source.python !s : meta.fstring.python, source.python, storage.type.format.python -:X>+10d : meta.fstring.python, source.python, support.other.format.python +:X>+10d : meta.fstring.python, source.python, storage.type.format.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python " : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python a : source.python diff --git a/test/fstrings/simple10.py b/test/fstrings/simple10.py new file mode 100644 index 00000000..ae793021 --- /dev/null +++ b/test/fstrings/simple10.py @@ -0,0 +1,18 @@ +f'values: {a=} {b=!r}' + + + + +f : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +values: : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +a : meta.fstring.python, source.python += : meta.fstring.python, source.python, storage.type.format.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python + : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +b : meta.fstring.python, source.python +=!r : meta.fstring.python, source.python, storage.type.format.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python diff --git a/test/fstrings/simple2.py b/test/fstrings/simple2.py index 967bb1f3..e4e1ff37 100644 --- a/test/fstrings/simple2.py +++ b/test/fstrings/simple2.py @@ -2,6 +2,7 @@ + a : source.python : source.python = : keyword.operator.assignment.python, source.python @@ -22,8 +23,8 @@ normal : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python { : constant.character.format.placeholder.other.python, meta.fstring.python, source.python fo : meta.fstring.python, source.python -. : meta.fstring.python, source.python -__add__ : meta.fstring.python, source.python, support.function.magic.python +. : meta.fstring.python, meta.member.access.python, punctuation.separator.period.python, source.python +__add__ : meta.fstring.python, meta.member.access.python, source.python, support.function.magic.python !s : meta.fstring.python, source.python, storage.type.format.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python " : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python diff --git a/test/fstrings/simple3.py b/test/fstrings/simple3.py index a67ea19c..6ea20170 100644 --- a/test/fstrings/simple3.py +++ b/test/fstrings/simple3.py @@ -22,7 +22,7 @@ 23 : constant.numeric.dec.python, meta.fstring.python, source.python : meta.fstring.python, source.python !r : meta.fstring.python, source.python, storage.type.format.python -:f : meta.fstring.python, source.python, support.other.format.python +:f : meta.fstring.python, source.python, storage.type.format.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python times : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python ' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python diff --git a/test/fstrings/simple4.py b/test/fstrings/simple4.py index 73891eb0..f0630423 100644 --- a/test/fstrings/simple4.py +++ b/test/fstrings/simple4.py @@ -22,7 +22,7 @@ 23 : constant.numeric.dec.python, meta.fstring.python, source.python : meta.fstring.python, source.python !r : meta.fstring.python, source.python, storage.type.format.python -:f : meta.fstring.python, source.python, support.other.format.python +:f : meta.fstring.python, source.python, storage.type.format.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python times : meta.fstring.python, source.python, string.interpolated.python, string.quoted.multi.python ''' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.multi.python diff --git a/test/fstrings/simple5.py b/test/fstrings/simple5.py index 42d3f400..074af929 100644 --- a/test/fstrings/simple5.py +++ b/test/fstrings/simple5.py @@ -24,7 +24,7 @@ 23 : constant.numeric.dec.python, meta.fstring.python, source.python : meta.fstring.python, source.python !r : meta.fstring.python, source.python, storage.type.format.python -:f : meta.fstring.python, source.python, support.other.format.python +:f : meta.fstring.python, source.python, storage.type.format.python } : constant.character.format.placeholder.other.python, meta.fstring.python, source.python times : meta.fstring.python, source.python, string.interpolated.python, string.quoted.multi.python ''' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.multi.python diff --git a/test/functions/async1.py b/test/functions/async1.py index cb6fd70e..0218fafd 100644 --- a/test/functions/async1.py +++ b/test/functions/async1.py @@ -7,7 +7,7 @@ async def foo(): -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python foo : entity.name.function.decorator.python, meta.function.decorator.python, source.python async : meta.function.python, source.python, storage.type.function.async.python : meta.function.python, source.python @@ -36,7 +36,7 @@ def : meta.function.python, source.python, storage.type.function.pytho : source.python c : source.python : source.python -in : keyword.operator.logical.python, source.python +in : keyword.control.flow.python, source.python : source.python b : source.python : : punctuation.separator.colon.python, source.python diff --git a/test/functions/decl15.py b/test/functions/decl15.py new file mode 100644 index 00000000..7f5c4ecc --- /dev/null +++ b/test/functions/decl15.py @@ -0,0 +1,34 @@ +def showcase(a, b, /, c, d, *, e, f): + return + + + + +def : meta.function.python, source.python, storage.type.function.python + : meta.function.python, source.python +showcase : entity.name.function.python, meta.function.python, source.python +( : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.begin.python, source.python +a : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +b : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +/ : keyword.operator.positional.parameter.python, meta.function.parameters.python, meta.function.python, source.python +, : meta.function.parameters.python, meta.function.python, source.python +c : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +d : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +* : keyword.operator.unpacking.parameter.python, meta.function.parameters.python, meta.function.python, source.python +, : meta.function.parameters.python, meta.function.python, source.python +e : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +, : meta.function.parameters.python, meta.function.python, punctuation.separator.parameters.python, source.python + : meta.function.parameters.python, meta.function.python, source.python +f : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +) : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.end.python, source.python +: : meta.function.python, punctuation.section.function.begin.python, source.python + : source.python +return : keyword.control.flow.python, source.python diff --git a/test/functions/decl2.py b/test/functions/decl2.py index 7da74fcb..f858ea26 100644 --- a/test/functions/decl2.py +++ b/test/functions/decl2.py @@ -2,6 +2,8 @@ def result_annot(lambda, lambda=) -> qqq[None]: pass + + def : meta.function.python, source.python, storage.type.function.python : meta.function.python, source.python result_annot : entity.name.function.python, meta.function.python, source.python @@ -14,7 +16,7 @@ def : meta.function.python, source.python, storage.type.function.pytho : meta.function.python, source.python -> : meta.function.python, punctuation.separator.annotation.result.python, source.python : meta.function.python, source.python -qqq : meta.function.python, meta.item-access.python, source.python +qqq : meta.function.python, meta.indexed-name.python, meta.item-access.python, source.python [ : meta.function.python, meta.item-access.python, punctuation.definition.arguments.begin.python, source.python None : constant.language.python, meta.function.python, meta.item-access.arguments.python, meta.item-access.python, source.python ] : meta.function.python, meta.item-access.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/functions/decl4.py b/test/functions/decl4.py index f3b40fe5..5a452ea4 100644 --- a/test/functions/decl4.py +++ b/test/functions/decl4.py @@ -38,7 +38,7 @@ def : meta.function.python, source.python, storage.type.function.pytho : meta.function.parameters.python, meta.function.python, source.python val : meta.function.parameters.python, meta.function.python, source.python : meta.function.parameters.python, meta.function.python, source.python -in : keyword.operator.logical.python, meta.function.parameters.python, meta.function.python, source.python +in : keyword.control.flow.python, meta.function.parameters.python, meta.function.python, source.python : meta.function.parameters.python, meta.function.python, source.python ( : meta.function.parameters.python, meta.function.python, punctuation.parenthesis.begin.python, source.python x : meta.function.parameters.python, meta.function.python, source.python diff --git a/test/functions/decorators1.py b/test/functions/decorators1.py index 6fc61aea..bf03fad0 100644 --- a/test/functions/decorators1.py +++ b/test/functions/decorators1.py @@ -8,25 +8,25 @@ def decorated(a): pass -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python some_decorator : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python with comment : comment.line.number-sign.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python some : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python class : keyword.control.flow.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python decorator : entity.name.function.decorator.python, meta.function.decorator.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python some_decorator : entity.name.function.decorator.python, meta.function.decorator.python, source.python ( : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python 1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python some : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python decorator : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python ( : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python @@ -35,7 +35,7 @@ class : keyword.control.flow.python, meta.function.decorator.python, sou : meta.function-call.arguments.python, meta.function.decorator.python, source.python 3 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python some_decorator : entity.name.function.decorator.python, meta.function.decorator.python, source.python ( : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python a : meta.function-call.arguments.python, meta.function.decorator.python, source.python, variable.parameter.function-call.python @@ -58,7 +58,7 @@ class : keyword.control.flow.python, meta.function.decorator.python, sou ** : keyword.operator.unpacking.arguments.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python kwargs : meta.function-call.arguments.python, meta.function.decorator.python, source.python ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python classmethod : meta.function.decorator.python, source.python, support.type.python def : meta.function.python, source.python, storage.type.function.python : meta.function.python, source.python diff --git a/test/functions/decorators2.py b/test/functions/decorators2.py index af5d7170..0b129649 100644 --- a/test/functions/decorators2.py +++ b/test/functions/decorators2.py @@ -3,19 +3,19 @@ def foo(): pass -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python : meta.function.decorator.python, source.python f : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python : meta.function.decorator.python, source.python bar : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python : meta.function.decorator.python, source.python a : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python : meta.function.decorator.python, source.python b : entity.name.function.decorator.python, meta.function.decorator.python, source.python def : meta.function.python, source.python, storage.type.function.python diff --git a/test/functions/decorators3.py b/test/functions/decorators3.py index 0cfe7a4f..24e7a58f 100644 --- a/test/functions/decorators3.py +++ b/test/functions/decorators3.py @@ -3,11 +3,11 @@ def foo(): pass -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python : meta.function.decorator.python, source.python f : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python : meta.function.decorator.python, source.python bar : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python diff --git a/test/functions/decorators4.py b/test/functions/decorators4.py index f2aab448..ba7df7bd 100644 --- a/test/functions/decorators4.py +++ b/test/functions/decorators4.py @@ -3,15 +3,15 @@ def foo(): pass -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python a : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python b : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python c : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python None : keyword.illegal.name.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python z : entity.name.function.decorator.python, meta.function.decorator.python, source.python def : meta.function.python, source.python, storage.type.function.python : meta.function.python, source.python diff --git a/test/functions/decorators5.py b/test/functions/decorators5.py index 412747ae..436d1a4b 100644 --- a/test/functions/decorators5.py +++ b/test/functions/decorators5.py @@ -7,24 +7,24 @@ def foo(): pass -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python a : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python : meta.function.decorator.python, source.python \ : meta.function.decorator.python, punctuation.separator.continuation.line.python, source.python : meta.function.decorator.python, source.python : meta.function.decorator.python, source.python b : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python : meta.function.decorator.python, source.python \ : meta.function.decorator.python, punctuation.separator.continuation.line.python, source.python : meta.function.decorator.python, source.python : meta.function.decorator.python, source.python c : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python None : keyword.illegal.name.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python z : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python \ : meta.function.decorator.python, punctuation.separator.continuation.line.python, source.python @@ -36,9 +36,9 @@ def foo(): pass = : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python 1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function.decorator.python, source.python ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python foo : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python ok : entity.name.function.decorator.python, meta.function.decorator.python, source.python def : meta.function.python, source.python, storage.type.function.python : meta.function.python, source.python diff --git a/test/functions/decorators6.py b/test/functions/decorators6.py index 53264a2e..5c9b4289 100644 --- a/test/functions/decorators6.py +++ b/test/functions/decorators6.py @@ -6,21 +6,23 @@ def foo(): pass -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python + + +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python a : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python : meta.function.decorator.python, source.python b : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python : meta.function.decorator.python, source.python \ : meta.function.decorator.python, punctuation.separator.continuation.line.python, source.python : meta.function.decorator.python, source.python : meta.function.decorator.python, source.python c : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python None : keyword.illegal.name.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python z : entity.name.function.decorator.python, meta.function.decorator.python, source.python ( : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python foo : meta.function-call.arguments.python, meta.function.decorator.python, source.python, variable.parameter.function-call.python @@ -29,22 +31,22 @@ def foo(): pass ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python . \ : invalid.illegal.decorator.python, meta.function.decorator.python, source.python : source.python -baz : meta.item-access.python, source.python +baz : meta.indexed-name.python, meta.item-access.python, source.python [ : meta.item-access.python, punctuation.definition.arguments.begin.python, source.python 1 : constant.numeric.dec.python, meta.item-access.arguments.python, meta.item-access.python, source.python : : meta.item-access.arguments.python, meta.item-access.python, punctuation.separator.slice.python, source.python 2 : constant.numeric.dec.python, meta.item-access.arguments.python, meta.item-access.python, source.python ] : meta.item-access.python, punctuation.definition.arguments.end.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python foo : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python class : keyword.control.flow.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python bar : entity.name.function.decorator.python, meta.function.decorator.python, source.python [] : invalid.illegal.decorator.python, meta.function.decorator.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python foo : entity.name.function.decorator.python, meta.function.decorator.python, source.python -. : entity.name.function.decorator.python, meta.function.decorator.python, source.python +. : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.separator.period.python, source.python ok : entity.name.function.decorator.python, meta.function.decorator.python, source.python : invalid.illegal.decorator.python, meta.function.decorator.python, source.python ''' : invalid.illegal.decorator.python, meta.function.decorator.python, source.python diff --git a/test/functions/decorators7.py b/test/functions/decorators7.py index 19d3f3bd..c604f818 100644 --- a/test/functions/decorators7.py +++ b/test/functions/decorators7.py @@ -16,23 +16,23 @@ def baz(): pass this is testing trailing whitespace after the decorator : comment.line.number-sign.python, source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python DO NOT DELETE TRAILING WHITESTAPCE IN THIS FILE : comment.line.number-sign.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python foo : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python foo : entity.name.function.decorator.python, meta.function.decorator.python, source.python ( : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python : source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python bar : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.decorator.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python bar : entity.name.function.decorator.python, meta.function.decorator.python, source.python ( : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python : source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python bar : entity.name.function.decorator.python, meta.function.decorator.python, source.python ( : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python @@ -40,7 +40,7 @@ def baz(): pass : source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python legal : comment.line.number-sign.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python bar : entity.name.function.decorator.python, meta.function.decorator.python, source.python ( : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/functions/decorators8.py b/test/functions/decorators8.py index f3898789..aeb2454e 100644 --- a/test/functions/decorators8.py +++ b/test/functions/decorators8.py @@ -3,7 +3,7 @@ def foo(): pass -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python deco : entity.name.function.decorator.python, meta.function.decorator.python, source.python ( : meta.function.decorator.python, punctuation.definition.arguments.begin.python, source.python ) : meta.function.decorator.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/functions/lambda10.py b/test/functions/lambda10.py new file mode 100644 index 00000000..c033ac71 --- /dev/null +++ b/test/functions/lambda10.py @@ -0,0 +1,35 @@ +showcase = lambda a, /, b, *, c: (a + b + c) + + + + +showcase : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +lambda : meta.lambda-function.python, source.python, storage.type.function.lambda.python + : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +a : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python, variable.parameter.function.language.python +, : meta.function.lambda.parameters.python, meta.lambda-function.python, punctuation.separator.parameters.python, source.python + : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +/ : keyword.operator.positional.parameter.python, meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +, : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +b : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python, variable.parameter.function.language.python +, : meta.function.lambda.parameters.python, meta.lambda-function.python, punctuation.separator.parameters.python, source.python + : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +* : keyword.operator.unpacking.parameter.python, meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +, : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python +c : meta.function.lambda.parameters.python, meta.lambda-function.python, source.python, variable.parameter.function.language.python +: : meta.lambda-function.python, punctuation.section.function.lambda.begin.python, source.python + : source.python +( : punctuation.parenthesis.begin.python, source.python +a : source.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +b : source.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +c : source.python +) : punctuation.parenthesis.end.python, source.python diff --git a/test/illegals/backticks3.py b/test/illegals/backticks3.py index b4ae29c9..d99080e8 100644 --- a/test/illegals/backticks3.py +++ b/test/illegals/backticks3.py @@ -1,4 +1,5 @@ a = lambda `123` +# comment @@ -12,3 +13,5 @@ 123 : constant.numeric.dec.python, invalid.deprecated.backtick.python, meta.function.lambda.parameters.python, meta.lambda-function.python, source.python ` : invalid.deprecated.backtick.python, meta.function.lambda.parameters.python, meta.lambda-function.python, source.python : meta.lambda-function.python, source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + comment : comment.line.number-sign.python, source.python diff --git a/test/illegals/illegal1.py b/test/illegals/illegal1.py index 61c22dce..5ee9654d 100644 --- a/test/illegals/illegal1.py +++ b/test/illegals/illegal1.py @@ -9,6 +9,7 @@ def f(): pass + -> : invalid.illegal.annotation.python, source.python def : meta.function.python, source.python, storage.type.function.python : meta.function.python, source.python @@ -28,13 +29,13 @@ def : meta.function.python, source.python, storage.type.function.pytho .class : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python ) : punctuation.parenthesis.end.python, source.python -. : source.python -fuuuu : meta.function-call.generic.python, meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -baz : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +fuuuu : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +baz : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python we recover just fine : comment.line.number-sign.python, source.python b : source.python diff --git a/test/regexp/comments3.re b/test/regexp/comments3.re new file mode 100644 index 00000000..8f1cb962 --- /dev/null +++ b/test/regexp/comments3.re @@ -0,0 +1,10 @@ +foo(?#NOTE:comment)bar + + + +foo : source.regexp.python +(?# : comment.regexp, punctuation.comment.begin.regexp, source.regexp.python +NOTE : comment.regexp, keyword.codetag.notation.python, source.regexp.python +:comment : comment.regexp, source.regexp.python +) : comment.regexp, punctuation.comment.end.regexp, source.regexp.python +bar : source.regexp.python diff --git a/test/regexp/fregexp1.py b/test/regexp/fregexp1.py deleted file mode 100644 index 9e77d66b..00000000 --- a/test/regexp/fregexp1.py +++ /dev/null @@ -1,56 +0,0 @@ -a = fr'[a-z]' -a = Fr'[a-z]' -a = rf'[a-z]' -a = rF'[a-z]' - - - - -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -fr : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -[ : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.begin.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -- : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -z : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -] : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.end.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -Fr : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -[ : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.begin.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -- : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -z : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -] : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.end.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -[ : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.begin.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -- : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -z : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -] : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.end.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rF : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -[ : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.begin.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -- : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -z : constant.character.set.regexp, meta.character.set.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -] : constant.other.set.regexp, meta.character.set.regexp, punctuation.character.set.end.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python diff --git a/test/regexp/fregexp2.py b/test/regexp/fregexp2.py deleted file mode 100644 index 72eccc28..00000000 --- a/test/regexp/fregexp2.py +++ /dev/null @@ -1,28 +0,0 @@ -rf'fo{{2}}' -rf"fo{{2}}" -rf'''fo{{2}}''' -rf"""fo{{2}}""" - - - - -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -''' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -fo : source.python, string.interpolated.python, string.regexp.quoted.multi.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.multi.python -''' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -fo : source.python, string.interpolated.python, string.regexp.quoted.multi.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python diff --git a/test/regexp/fregexp3.py b/test/regexp/fregexp3.py deleted file mode 100644 index ede8a6c3..00000000 --- a/test/regexp/fregexp3.py +++ /dev/null @@ -1,28 +0,0 @@ -rf'fo{2}' -rf"fo{2}" -rf'''fo{2}''' -rf"""fo{2}""" - - - - -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.single.python -" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -''' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -fo : source.python, string.interpolated.python, string.regexp.quoted.multi.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.multi.python -''' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python -fo : source.python, string.interpolated.python, string.regexp.quoted.multi.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.multi.python -""" : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.multi.python diff --git a/test/regexp/fregexp4.py b/test/regexp/fregexp4.py deleted file mode 100644 index 5b707ad3..00000000 --- a/test/regexp/fregexp4.py +++ /dev/null @@ -1,36 +0,0 @@ -a = rf'fo{{2}}' -a = r'fo{{2}}' -a = r'fo{2}' - - - - - -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -r : source.python, storage.type.string.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.regexp.quoted.single.python -fo{ : source.python, string.regexp.quoted.single.python -{2} : keyword.operator.quantifier.regexp, source.python, string.regexp.quoted.single.python -} : source.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -r : source.python, storage.type.string.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.regexp.quoted.single.python -fo : source.python, string.regexp.quoted.single.python -{2} : keyword.operator.quantifier.regexp, source.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.regexp.quoted.single.python diff --git a/test/regexp/fregexp5.py b/test/regexp/fregexp5.py deleted file mode 100644 index 0d339e27..00000000 --- a/test/regexp/fregexp5.py +++ /dev/null @@ -1,26 +0,0 @@ -a = rf'{{foo}}' -a = r'\{foo\}' - - - - -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -{{ : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -foo : source.python, string.interpolated.python, string.regexp.quoted.single.python -}} : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -r : source.python, storage.type.string.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.regexp.quoted.single.python -\{ : constant.character.escape.regexp, source.python, string.regexp.quoted.single.python -foo : source.python, string.regexp.quoted.single.python -\} : constant.character.escape.regexp, source.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.regexp.quoted.single.python diff --git a/test/regexp/fregexp6.py b/test/regexp/fregexp6.py deleted file mode 100644 index a51311d0..00000000 --- a/test/regexp/fregexp6.py +++ /dev/null @@ -1,39 +0,0 @@ -a = rf'fo{{{2}}}' -a = rf'fo{{{bar}}}' -a = rf'fo{{2}}' - - - - - -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{ : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -{2} : source.python, string.interpolated.python, string.regexp.quoted.single.python -}} : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{ : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -{bar} : source.python, string.interpolated.python, string.regexp.quoted.single.python -}} : constant.character.escape.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -a : source.python - : source.python -= : keyword.operator.assignment.python, source.python - : source.python -rf : source.python, storage.type.string.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.regexp.quoted.single.python -fo : source.python, string.interpolated.python, string.regexp.quoted.single.python -{{2}} : keyword.operator.quantifier.regexp, source.python, string.interpolated.python, string.regexp.quoted.single.python -' : punctuation.definition.string.end.python, source.python, string.interpolated.python, string.regexp.quoted.single.python diff --git a/test/regexp/python2.py b/test/regexp/python2.py index bda2bb0b..1738c203 100644 --- a/test/regexp/python2.py +++ b/test/regexp/python2.py @@ -2,6 +2,7 @@ (?x) foo ' +# comment @@ -21,3 +22,5 @@ foo : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.docstring.single.python : invalid.illegal.newline.python, source.python, string.quoted.docstring.single.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + comment : comment.line.number-sign.python, source.python diff --git a/test/regexp/python8.py b/test/regexp/python8.py index b0080e77..ba72a7be 100644 --- a/test/regexp/python8.py +++ b/test/regexp/python8.py @@ -22,6 +22,7 @@ + # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python And now something fun! : comment.line.number-sign.python, source.python : source.python @@ -30,166 +31,166 @@ = : keyword.operator.assignment.python, source.python : source.python re : source.python -. : source.python -compile : meta.function-call.generic.python, meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -r : meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.regexp.quoted.multi.python -""" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -(?x) : meta.function-call.arguments.python, meta.function-call.python, source.python, storage.modifier.flag.regexp, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - This is a verbose pattern : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -\s : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Optional whitespace at start of cookie : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -( : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.parenthesis.named.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp -?P : entity.name.tag.named.group.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Start of group 'key' : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python -""" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -+ : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, source.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -_LegalKeyChars : meta.function-call.arguments.python, meta.function-call.python, source.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -+ : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, source.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -r : meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.regexp.quoted.multi.python -""" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.regexp.quoted.multi.python -] : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -+? : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Any word of at least one letter : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - ) : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - End of group 'key' : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -( : meta.function-call.arguments.python, meta.function-call.python, punctuation.parenthesis.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Optional group: there may not be a value. : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -\s : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -= : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -\s : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Equal Sign : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -( : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.parenthesis.named.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp -?P : entity.name.tag.named.group.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Start of group 'val' : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - " : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -(?: : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.parenthesis.non-capturing.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp -[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python -^ : keyword.operator.negation.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -\\ : constant.character.escape.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -" : constant.character.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -] : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.character.set.end.regexp, source.python, string.regexp.quoted.multi.python -| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -\\ : constant.character.escape.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -. : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.match.any.regexp -) : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.parenthesis.non-capturing.end.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp -* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -" : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Any doublequoted string : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - or : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -\w : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -{3} : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -, : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -\s : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python -\w : meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -\d : meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -\s : meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -- : constant.character.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -] : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.character.set.end.regexp, source.python, string.regexp.quoted.multi.python -{9,11} : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -\s : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python -\d : meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -: : constant.character.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -] : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.character.set.end.regexp, source.python, string.regexp.quoted.multi.python -{8} : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -\s : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -GMT : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Special case for "expires" attr : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - or : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python -[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python -""" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -+ : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, source.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -_LegalValueChars : meta.function-call.arguments.python, meta.function-call.python, source.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -+ : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, source.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -r : meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.regexp.quoted.multi.python -""" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.regexp.quoted.multi.python -] : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Any word or empty string : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - ) : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - End of group 'val' : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - ) : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -? : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - End of optional value group : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -\s : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Any number of spaces. : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -( : meta.function-call.arguments.python, meta.function-call.python, punctuation.parenthesis.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp -\s : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp -+ : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -; : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -$ : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python, support.other.match.end.regexp -) : meta.function-call.arguments.python, meta.function-call.python, punctuation.parenthesis.end.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python - Ending either at space, semicolon, or EOS. : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python - : meta.function-call.arguments.python, meta.function-call.python, source.python, string.regexp.quoted.multi.python -""" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.regexp.quoted.multi.python -, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python - : meta.function-call.arguments.python, meta.function-call.python, source.python -re : meta.function-call.arguments.python, meta.function-call.python, source.python -. : meta.function-call.arguments.python, meta.function-call.python, source.python -ASCII : constant.other.caps.python, meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +compile : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +r : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, storage.type.string.python, string.regexp.quoted.multi.python +""" : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.begin.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +(?x) : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, storage.modifier.flag.regexp, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + This is a verbose pattern : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +\s : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Optional whitespace at start of cookie : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +( : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.parenthesis.named.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp +?P : entity.name.tag.named.group.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Start of group 'key' : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python +""" : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.end.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python ++ : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +_LegalKeyChars : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python ++ : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +r : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, storage.type.string.python, string.regexp.quoted.multi.python +""" : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.begin.python, source.python, string.regexp.quoted.multi.python +] : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python ++? : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Any word of at least one letter : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + ) : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + End of group 'key' : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +( : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.parenthesis.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Optional group: there may not be a value. : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +\s : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python += : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +\s : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Equal Sign : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +( : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.parenthesis.named.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp +?P : entity.name.tag.named.group.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Start of group 'val' : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + " : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +(?: : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.parenthesis.non-capturing.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp +[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python +^ : keyword.operator.negation.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +\\ : constant.character.escape.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +" : constant.character.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +] : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.character.set.end.regexp, source.python, string.regexp.quoted.multi.python +| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +\\ : constant.character.escape.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +. : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.match.any.regexp +) : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.parenthesis.non-capturing.end.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp +* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +" : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Any doublequoted string : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + or : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +\w : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +{3} : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +, : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +\s : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python +\w : meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +\d : meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +\s : meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +- : constant.character.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +] : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.character.set.end.regexp, source.python, string.regexp.quoted.multi.python +{9,11} : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +\s : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python +\d : meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +: : constant.character.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +] : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.character.set.end.regexp, source.python, string.regexp.quoted.multi.python +{8} : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +\s : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +GMT : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Special case for "expires" attr : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + or : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, source.python, string.regexp.quoted.multi.python +[ : constant.other.set.regexp, meta.character.set.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, meta.named.regexp, punctuation.character.set.begin.regexp, source.python, string.regexp.quoted.multi.python +""" : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.end.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python ++ : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +_LegalValueChars : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python ++ : keyword.operator.arithmetic.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +r : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, storage.type.string.python, string.regexp.quoted.multi.python +""" : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.begin.python, source.python, string.regexp.quoted.multi.python +] : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Any word or empty string : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + ) : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + End of group 'val' : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + ) : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +? : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + End of optional value group : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +\s : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp +* : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Any number of spaces. : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +( : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.parenthesis.begin.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp +\s : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python, support.other.escape.special.regexp ++ : keyword.operator.quantifier.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +; : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +| : keyword.operator.disjunction.regexp, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +$ : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python, support.other.match.end.regexp +) : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.parenthesis.end.regexp, source.python, string.regexp.quoted.multi.python, support.other.parenthesis.regexp + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +# : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.comment.python, source.python, string.regexp.quoted.multi.python + Ending either at space, semicolon, or EOS. : comment.line.number-sign.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, string.regexp.quoted.multi.python +""" : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.definition.string.end.python, source.python, string.regexp.quoted.multi.python +, : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +re : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +. : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, punctuation.separator.period.python, source.python +ASCII : constant.other.caps.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python : source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python May be removed if safe. : comment.line.number-sign.python, source.python diff --git a/test/statements/annotation1.py b/test/statements/annotation1.py index a72bafc6..332cd26a 100644 --- a/test/statements/annotation1.py +++ b/test/statements/annotation1.py @@ -3,6 +3,7 @@ + some_number : source.python : : punctuation.separator.colon.python, source.python : source.python @@ -13,7 +14,7 @@ some_list : source.python : : punctuation.separator.colon.python, source.python : source.python -List : meta.item-access.python, source.python +List : meta.indexed-name.python, meta.item-access.python, source.python [ : meta.item-access.python, punctuation.definition.arguments.begin.python, source.python int : meta.item-access.arguments.python, meta.item-access.python, source.python, support.type.python ] : meta.item-access.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/statements/for1.py b/test/statements/for1.py index 3300f8cb..b379ef19 100644 --- a/test/statements/for1.py +++ b/test/statements/for1.py @@ -15,7 +15,7 @@ : source.python c : source.python : source.python -in : keyword.operator.logical.python, source.python +in : keyword.control.flow.python, source.python : source.python b : source.python : : punctuation.separator.colon.python, source.python diff --git a/test/statements/for2.py b/test/statements/for2.py new file mode 100644 index 00000000..8468e08e --- /dev/null +++ b/test/statements/for2.py @@ -0,0 +1,44 @@ +forvariable = None +for a, b, c, invariable in [2 in q, 2 in w]: + pass + + + + +forvariable : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +None : constant.language.python, source.python +for : keyword.control.flow.python, source.python + : source.python +a : source.python +, : punctuation.separator.element.python, source.python + : source.python +b : source.python +, : punctuation.separator.element.python, source.python + : source.python +c : source.python +, : punctuation.separator.element.python, source.python + : source.python +invariable : source.python + : source.python +in : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +2 : constant.numeric.dec.python, source.python + : source.python +in : keyword.operator.logical.python, source.python + : source.python +q : source.python +, : punctuation.separator.element.python, source.python + : source.python +2 : constant.numeric.dec.python, source.python + : source.python +in : keyword.operator.logical.python, source.python + : source.python +w : source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +pass : keyword.control.flow.python, source.python diff --git a/test/statements/for3.py b/test/statements/for3.py new file mode 100644 index 00000000..5ebfc592 --- /dev/null +++ b/test/statements/for3.py @@ -0,0 +1,38 @@ +for(a, b), c, invariable in[2 in q, 2 in w]: + pass + + + + +for : keyword.control.flow.python, source.python +( : punctuation.parenthesis.begin.python, source.python +a : source.python +, : punctuation.separator.element.python, source.python + : source.python +b : source.python +) : punctuation.parenthesis.end.python, source.python +, : punctuation.separator.element.python, source.python + : source.python +c : source.python +, : punctuation.separator.element.python, source.python + : source.python +invariable : source.python + : source.python +in : keyword.control.flow.python, source.python +[ : punctuation.definition.list.begin.python, source.python +2 : constant.numeric.dec.python, source.python + : source.python +in : keyword.operator.logical.python, source.python + : source.python +q : source.python +, : punctuation.separator.element.python, source.python + : source.python +2 : constant.numeric.dec.python, source.python + : source.python +in : keyword.operator.logical.python, source.python + : source.python +w : source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +pass : keyword.control.flow.python, source.python diff --git a/test/statements/from1.py b/test/statements/from1.py new file mode 100644 index 00000000..99d3a670 --- /dev/null +++ b/test/statements/from1.py @@ -0,0 +1,33 @@ +from ... import foo as bar +raise Exception('done') from exc +yield from foo + + + + +from : keyword.control.import.python, source.python + : source.python +... : punctuation.separator.period.python, source.python + : source.python +import : keyword.control.import.python, source.python + : source.python +foo : source.python + : source.python +as : keyword.control.import.python, source.python + : source.python +bar : source.python +raise : keyword.control.flow.python, source.python + : source.python +Exception : meta.function-call.python, source.python, support.type.exception.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +' : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +done : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +' : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +from : keyword.control.flow.python, source.python + : source.python +exc : source.python +yield from : keyword.control.flow.python, source.python + : source.python +foo : source.python diff --git a/test/statements/import1.py b/test/statements/import1.py index 1e322e8f..6faca8b3 100644 --- a/test/statements/import1.py +++ b/test/statements/import1.py @@ -4,14 +4,15 @@ from : keyword.control.import.python, source.python - ... : source.python + : source.python +... : punctuation.separator.period.python, source.python foo : source.python : source.python import : keyword.control.import.python, source.python : source.python bar : source.python : source.python -as : keyword.control.flow.python, source.python +as : keyword.control.import.python, source.python : source.python spam : source.python , : punctuation.separator.element.python, source.python @@ -21,7 +22,7 @@ : source.python time : source.python : source.python -as : keyword.control.flow.python, source.python +as : keyword.control.import.python, source.python : source.python ham : source.python , : punctuation.separator.element.python, source.python diff --git a/test/statements/import2.py b/test/statements/import2.py index 18a8049e..842f68d2 100644 --- a/test/statements/import2.py +++ b/test/statements/import2.py @@ -5,17 +5,23 @@ from : keyword.control.import.python, source.python - .... : source.python + : source.python +.... : punctuation.separator.period.python, source.python + : source.python import : keyword.control.import.python, source.python : source.python a : source.python from : keyword.control.import.python, source.python - ... : source.python + : source.python +... : punctuation.separator.period.python, source.python + : source.python import : keyword.control.import.python, source.python : source.python b : source.python from : keyword.control.import.python, source.python - .. : source.python + : source.python +.. : punctuation.separator.period.python, source.python + : source.python import : keyword.control.import.python, source.python : source.python c : source.python diff --git a/test/statements/import3.py b/test/statements/import3.py index 79f06128..1952bf59 100644 --- a/test/statements/import3.py +++ b/test/statements/import3.py @@ -6,22 +6,22 @@ from : keyword.control.import.python, source.python -.... : source.python +.... : punctuation.separator.period.python, source.python import : keyword.control.import.python, source.python : source.python a : source.python from : keyword.control.import.python, source.python -... : source.python +... : punctuation.separator.period.python, source.python import : keyword.control.import.python, source.python : source.python b : source.python from : keyword.control.import.python, source.python -.. : source.python +.. : punctuation.separator.period.python, source.python import : keyword.control.import.python, source.python : source.python c : source.python from : keyword.control.import.python, source.python -. : source.python +. : punctuation.separator.period.python, source.python import : keyword.control.import.python, source.python : source.python d : source.python diff --git a/test/statements/import4.py b/test/statements/import4.py index c45b328d..b2cee743 100644 --- a/test/statements/import4.py +++ b/test/statements/import4.py @@ -6,28 +6,28 @@ from : keyword.control.import.python, source.python -.... : source.python +.... : punctuation.separator.period.python, source.python foo : source.python : source.python import : keyword.control.import.python, source.python : source.python a : source.python from : keyword.control.import.python, source.python -... : source.python +... : punctuation.separator.period.python, source.python foo : source.python : source.python import : keyword.control.import.python, source.python : source.python b : source.python from : keyword.control.import.python, source.python -.. : source.python +.. : punctuation.separator.period.python, source.python foo : source.python : source.python import : keyword.control.import.python, source.python : source.python c : source.python from : keyword.control.import.python, source.python -. : source.python +. : punctuation.separator.period.python, source.python foo : source.python : source.python import : keyword.control.import.python, source.python diff --git a/test/statements/import5.py b/test/statements/import5.py index a805e4d3..6050a777 100644 --- a/test/statements/import5.py +++ b/test/statements/import5.py @@ -10,22 +10,23 @@ def bar(): + from : keyword.control.import.python, source.python -. : source.python +. : punctuation.separator.period.python, source.python foo : source.python : source.python import : keyword.control.import.python, source.python : source.python d : source.python from : keyword.control.import.python, source.python -. : source.python +. : punctuation.separator.period.python, source.python import : keyword.control.import.python, source.python : source.python a : source.python : source.python foo : source.python -. : source.python -import : keyword.control.flow.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +import : keyword.control.import.python, meta.member.access.python, source.python : source.python raise : keyword.control.flow.python, source.python : source.python diff --git a/test/statements/import6.py b/test/statements/import6.py index 18bacd0e..a39a790e 100644 --- a/test/statements/import6.py +++ b/test/statements/import6.py @@ -2,7 +2,7 @@ -from : keyword.control.flow.python, source.python +from : keyword.control.import.python, source.python : source.python __future__ : source.python, support.variable.magic.python : source.python diff --git a/test/statements/import7.py b/test/statements/import7.py new file mode 100644 index 00000000..7951802b --- /dev/null +++ b/test/statements/import7.py @@ -0,0 +1,20 @@ +from .importing import * +from importing import * + + + +from : keyword.control.import.python, source.python + : source.python +. : punctuation.separator.period.python, source.python +importing : source.python + : source.python +import : keyword.control.import.python, source.python + : source.python +* : keyword.operator.arithmetic.python, source.python +from : keyword.control.import.python, source.python + : source.python +importing : source.python + : source.python +import : keyword.control.import.python, source.python + : source.python +* : keyword.operator.arithmetic.python, source.python diff --git a/test/statements/import8.py b/test/statements/import8.py new file mode 100644 index 00000000..1d49bbd1 --- /dev/null +++ b/test/statements/import8.py @@ -0,0 +1,48 @@ +try: + import \ + time as ham, \ + datetime \ + # XXX: comment at the end of import +except Exception as exc: + pass + + + + +try : keyword.control.flow.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +import : keyword.control.import.python, source.python + : source.python +\ : punctuation.separator.continuation.line.python, source.python + : source.python + : source.python +time : source.python + : source.python +as : keyword.control.import.python, source.python + : source.python +ham : source.python +, : punctuation.separator.element.python, source.python + : source.python +\ : punctuation.separator.continuation.line.python, source.python + : source.python + : source.python +datetime : source.python + : source.python +\ : punctuation.separator.continuation.line.python, source.python + : source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + : comment.line.number-sign.python, source.python +XXX : comment.line.number-sign.python, keyword.codetag.notation.python, source.python +: comment at the end of import : comment.line.number-sign.python, source.python +except : keyword.control.flow.python, source.python + : source.python +Exception : source.python, support.type.exception.python + : source.python +as : keyword.control.flow.python, source.python + : source.python +exc : source.python +: : punctuation.separator.colon.python, source.python + : source.python +pass : keyword.control.flow.python, source.python diff --git a/test/statements/import9.py b/test/statements/import9.py new file mode 100644 index 00000000..2fa6bfac --- /dev/null +++ b/test/statements/import9.py @@ -0,0 +1,67 @@ +from . . . foo import \ + ( + # XXX: legal comment inside import + time as bar, + # another comment + baz, + datetime as ham + ) +raise Exception('!') from None + + + + +from : keyword.control.import.python, source.python + : source.python +. : punctuation.separator.period.python, source.python + : source.python +. : punctuation.separator.period.python, source.python + : source.python +. : punctuation.separator.period.python, source.python + : source.python +foo : source.python + : source.python +import : keyword.control.import.python, source.python + : source.python +\ : punctuation.separator.continuation.line.python, source.python + : source.python + : source.python +( : punctuation.parenthesis.begin.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + : comment.line.number-sign.python, source.python +XXX : comment.line.number-sign.python, keyword.codetag.notation.python, source.python +: legal comment inside import : comment.line.number-sign.python, source.python + : source.python +time : source.python + : source.python +as : keyword.control.import.python, source.python + : source.python +bar : source.python +, : punctuation.separator.element.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + another comment : comment.line.number-sign.python, source.python + : source.python +baz : source.python +, : punctuation.separator.element.python, source.python + : source.python +datetime : source.python + : source.python +as : keyword.control.import.python, source.python + : source.python +ham : source.python + : source.python +) : punctuation.parenthesis.end.python, source.python +raise : keyword.control.flow.python, source.python + : source.python +Exception : meta.function-call.python, source.python, support.type.exception.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +' : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +! : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +' : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +from : keyword.control.flow.python, source.python + : source.python +None : constant.language.python, source.python diff --git a/test/statements/match1.py b/test/statements/match1.py new file mode 100644 index 00000000..f393066a --- /dev/null +++ b/test/statements/match1.py @@ -0,0 +1,61 @@ +def foo(status): + match status: + case 404: + return "Not found" + case 401 | 403: + return "Not allowed" + case _: + return "Something's wrong with the internet" + + + + +def : meta.function.python, source.python, storage.type.function.python + : meta.function.python, source.python +foo : entity.name.function.python, meta.function.python, source.python +( : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.begin.python, source.python +status : meta.function.parameters.python, meta.function.python, source.python, variable.parameter.function.language.python +) : meta.function.parameters.python, meta.function.python, punctuation.definition.parameters.end.python, source.python +: : meta.function.python, punctuation.section.function.begin.python, source.python + : source.python +match : keyword.control.flow.python, source.python + : source.python +status : source.python +: : punctuation.separator.colon.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +404 : constant.numeric.dec.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +return : keyword.control.flow.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Not found : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +401 : constant.numeric.dec.python, source.python + : source.python +| : keyword.operator.bitwise.python, source.python + : source.python +403 : constant.numeric.dec.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +return : keyword.control.flow.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Not allowed : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +_ : source.python +: : punctuation.separator.colon.python, source.python + : source.python +return : keyword.control.flow.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Something's wrong with the internet : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python diff --git a/test/statements/match2.py b/test/statements/match2.py new file mode 100644 index 00000000..a88ba055 --- /dev/null +++ b/test/statements/match2.py @@ -0,0 +1,112 @@ +match point: + case Point(x=0, y=0): + print("Origin is the point's location.") + case Point(x=0, y=y): + print(f"The point is on the y-axis.") + case Point(x=x, y=0): + print(f"The point is on the x-axis.") + case Point(): + print("The point is located somewhere else on the plane.") + case _: + print("Not a point") + + + +match : keyword.control.flow.python, source.python + : source.python +point : source.python +: : punctuation.separator.colon.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Origin is the point's location. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +The point is on the y-axis. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +The point is on the x-axis. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +The point is located somewhere else on the plane. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +_ : source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Not a point : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/statements/match3.py b/test/statements/match3.py new file mode 100644 index 00000000..ac8e176c --- /dev/null +++ b/test/statements/match3.py @@ -0,0 +1,114 @@ +match points: + case []: + print("No points in the list.") + case [Point(0, 0)]: + print("The origin is the only point in the list.") + case [Point(x, y)]: + print(f"A single point is in the list.") + case [Point(0, y1), Point(0, y2)]: + print(f"Two points on the Y axis are in the list.") + case _: + print("Something else is found in the list.") + + + +match : keyword.control.flow.python, source.python + : source.python +points : source.python +: : punctuation.separator.colon.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +No points in the list. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +The origin is the only point in the list. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +A single point is in the list. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y1 : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +, : punctuation.separator.element.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +0 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y2 : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +Two points on the Y axis are in the list. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +_ : source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.quoted.single.python +Something else is found in the list. : meta.function-call.arguments.python, meta.function-call.python, source.python, string.quoted.single.python +" : meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/statements/match4.py b/test/statements/match4.py new file mode 100644 index 00000000..fc4857f5 --- /dev/null +++ b/test/statements/match4.py @@ -0,0 +1,59 @@ +match point: + case Point(x, y) if x == y: + print(f"The point is located on the diagonal Y=X.") + case Point(x, y): + print(f"Point is not on the diagonal.") + + + + +match : keyword.control.flow.python, source.python + : source.python +point : source.python +: : punctuation.separator.colon.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +if : keyword.control.flow.python, source.python + : source.python +x : source.python + : source.python +== : keyword.operator.comparison.python, source.python + : source.python +y : source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +The point is located on the diagonal Y=X. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +Point : meta.function-call.generic.python, meta.function-call.python, source.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +x : meta.function-call.arguments.python, meta.function-call.python, source.python +, : meta.function-call.arguments.python, meta.function-call.python, punctuation.separator.arguments.python, source.python + : meta.function-call.arguments.python, meta.function-call.python, source.python +y : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +print : meta.function-call.python, source.python, support.function.builtin.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +f : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +Point is not on the diagonal. : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, source.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, meta.function-call.arguments.python, meta.function-call.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/statements/match5.py b/test/statements/match5.py new file mode 100644 index 00000000..905ef8ba --- /dev/null +++ b/test/statements/match5.py @@ -0,0 +1,112 @@ +match command.split() if command else ['default']: + ... # Other cases + case ["north"] | ["go", "north"]: + ... # handle case + case ["get", obj] | ["pick", "up", *other] | ["pick", obj, "up"]: + ... # handle case + + + + +match : keyword.control.flow.python, source.python + : source.python +command : source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +split : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python + : source.python +if : keyword.control.flow.python, source.python + : source.python +command : source.python + : source.python +else : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +default : source.python, string.quoted.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.single.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + Other cases : comment.line.number-sign.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +north : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +] : punctuation.definition.list.end.python, source.python + : source.python +| : keyword.operator.bitwise.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +go : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +north : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + handle case : comment.line.number-sign.python, source.python + : source.python +case : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +get : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +obj : source.python +] : punctuation.definition.list.end.python, source.python + : source.python +| : keyword.operator.bitwise.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +pick : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +up : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +* : keyword.operator.arithmetic.python, source.python +other : source.python +] : punctuation.definition.list.end.python, source.python + : source.python +| : keyword.operator.bitwise.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +pick : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +, : punctuation.separator.element.python, source.python + : source.python +obj : source.python +, : punctuation.separator.element.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +up : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + handle case : comment.line.number-sign.python, source.python diff --git a/test/statements/match6.py b/test/statements/match6.py new file mode 100644 index 00000000..b74491f7 --- /dev/null +++ b/test/statements/match6.py @@ -0,0 +1,53 @@ +match (foo + bar): + ... # cases +match [foo, bar]: + ... # cases +match {foo, bar}: + ... # cases + + + + +match : keyword.control.flow.python, source.python + : source.python +( : punctuation.parenthesis.begin.python, source.python +foo : source.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +bar : source.python +) : punctuation.parenthesis.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +[ : punctuation.definition.list.begin.python, source.python +foo : source.python +, : punctuation.separator.element.python, source.python + : source.python +bar : source.python +] : punctuation.definition.list.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +{ : punctuation.definition.dict.begin.python, source.python +foo : source.python +, : punctuation.separator.element.python, source.python + : source.python +bar : source.python +} : punctuation.definition.dict.end.python, source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python \ No newline at end of file diff --git a/test/statements/match7.py b/test/statements/match7.py new file mode 100644 index 00000000..878fa813 --- /dev/null +++ b/test/statements/match7.py @@ -0,0 +1,97 @@ +match 'prefix' + foo: + ... # cases +match "prefix" + foo: + ... # cases +match f'prefix{foo}': + ... # cases +match f"prefix{foo}": + ... # cases +match -foo: + ... # cases +match not foo: + ... # cases + + + + +match : keyword.control.flow.python, source.python + : source.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +prefix : source.python, string.quoted.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.single.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +foo : source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +prefix : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python + : source.python ++ : keyword.operator.arithmetic.python, source.python + : source.python +foo : source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +f : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +' : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +prefix : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +foo : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +' : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +f : meta.fstring.python, source.python, storage.type.string.python, string.interpolated.python, string.quoted.single.python +" : meta.fstring.python, punctuation.definition.string.begin.python, source.python, string.interpolated.python, string.quoted.single.python +prefix : meta.fstring.python, source.python, string.interpolated.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +foo : meta.fstring.python, source.python +} : constant.character.format.placeholder.other.python, meta.fstring.python, source.python +" : meta.fstring.python, punctuation.definition.string.end.python, source.python, string.interpolated.python, string.quoted.single.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +- : keyword.operator.arithmetic.python, source.python +foo : source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python +match : keyword.control.flow.python, source.python + : source.python +not : keyword.operator.logical.python, source.python + : source.python +foo : source.python +: : punctuation.separator.colon.python, source.python + : source.python +... : constant.other.ellipsis.python, source.python + : source.python +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + cases : comment.line.number-sign.python, source.python \ No newline at end of file diff --git a/test/strings/bug1.py b/test/strings/bug1.py new file mode 100644 index 00000000..eeb0f950 --- /dev/null +++ b/test/strings/bug1.py @@ -0,0 +1,47 @@ +# issue 150 +record = { + "a": {k: str(v) for k, v in foo if ""} +} + + + + +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + issue 150 : comment.line.number-sign.python, source.python +record : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +{ : punctuation.definition.dict.begin.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +a : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +: : punctuation.separator.dict.python, source.python + : source.python +{ : punctuation.definition.dict.begin.python, source.python +k : source.python +: : punctuation.separator.dict.python, source.python + : source.python +str : meta.function-call.python, source.python, support.type.python +( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python +v : meta.function-call.arguments.python, meta.function-call.python, source.python +) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python + : source.python +for : keyword.control.flow.python, source.python + : source.python +k : source.python +, : punctuation.separator.element.python, source.python + : source.python +v : source.python + : source.python +in : keyword.control.flow.python, source.python + : source.python +foo : source.python + : source.python +if : keyword.control.flow.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +} : punctuation.definition.dict.end.python, source.python +} : punctuation.definition.dict.end.python, source.python diff --git a/test/strings/bug2.py b/test/strings/bug2.py new file mode 100644 index 00000000..3f4622e8 --- /dev/null +++ b/test/strings/bug2.py @@ -0,0 +1,35 @@ +# issue 150 +cmd = "git-clang-format --style=\"{{BasedOnStyle: Google, ColumnLimit: 100, IndentWidth: 2, " \ + "AlignConsecutiveAssignments: true}}\" {COMMIT_SHA} -- ./**/*.proto > {OUTPUT}".format( + + + + +# : comment.line.number-sign.python, punctuation.definition.comment.python, source.python + issue 150 : comment.line.number-sign.python, source.python +cmd : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +git-clang-format --style= : source.python, string.quoted.single.python +\" : constant.character.escape.python, source.python, string.quoted.single.python +{{ : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +BasedOnStyle: Google, ColumnLimit: 100, IndentWidth: 2, : source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python + : source.python +\ : punctuation.separator.continuation.line.python, source.python + : source.python + : source.python +" : punctuation.definition.string.begin.python, source.python, string.quoted.single.python +AlignConsecutiveAssignments: true : source.python, string.quoted.single.python +}} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +\" : constant.character.escape.python, source.python, string.quoted.single.python + : source.python, string.quoted.single.python +{COMMIT_SHA} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python + -- ./**/*.proto > : source.python, string.quoted.single.python +{OUTPUT} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +" : punctuation.definition.string.end.python, source.python, string.quoted.single.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +format : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python diff --git a/test/strings/bytes2.py b/test/strings/bytes2.py index 22df557e..7aefd973 100644 --- a/test/strings/bytes2.py +++ b/test/strings/bytes2.py @@ -8,7 +8,8 @@ -a : meta.item-access.python, source.python + +a : meta.indexed-name.python, meta.item-access.python, source.python [ : meta.item-access.python, punctuation.definition.arguments.begin.python, source.python 1 : constant.numeric.dec.python, meta.item-access.arguments.python, meta.item-access.python, source.python ] : meta.item-access.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/strings/format1.py b/test/strings/format1.py index beba8cff..55327cae 100644 --- a/test/strings/format1.py +++ b/test/strings/format1.py @@ -9,29 +9,28 @@ = : keyword.operator.assignment.python, source.python : source.python " : punctuation.definition.string.begin.python, source.python, string.quoted.single.python -{0[ ] : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -:X>+10d : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python, support.other.format.python -} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{0[ ] : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +:X>+10d : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python " : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python : source.python = : keyword.operator.assignment.python, source.python : source.python " : punctuation.definition.string.begin.python, source.python, string.quoted.single.python -{0[ ] : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -!s : constant.character.format.placeholder.other.python, source.python, storage.type.format.python, string.quoted.single.python -:X>+10d : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python, support.other.format.python -} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{0[ ] : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +!s : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +:X>+10d : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python " : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python : source.python = : keyword.operator.assignment.python, source.python : source.python " : punctuation.definition.string.begin.python, source.python, string.quoted.single.python -{0[ ] : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -: : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python, support.other.format.python -Xd>+10d : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{0[ ] : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +: : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +Xd>+10d} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python " : punctuation.definition.string.end.python, source.python, string.quoted.single.python : source.python # : comment.line.number-sign.python, punctuation.definition.comment.python, source.python diff --git a/test/strings/format10.py b/test/strings/format10.py index 01f848f6..3cd456f7 100644 --- a/test/strings/format10.py +++ b/test/strings/format10.py @@ -11,15 +11,15 @@ : source.python ''' : punctuation.definition.string.begin.python, source.python, string.quoted.multi.python blah {foo-bar : source.python, string.quoted.multi.python -%d : constant.character.format.placeholder.other.python, source.python, string.quoted.multi.python +%d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.multi.python blah : source.python, string.quoted.multi.python {foo-bar : source.python, string.quoted.multi.python -%d : constant.character.format.placeholder.other.python, source.python, string.quoted.multi.python +%d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.multi.python } : source.python, string.quoted.multi.python blah {foo-bar : source.python, string.quoted.multi.python -%d : constant.character.format.placeholder.other.python, source.python, string.quoted.multi.python +%d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.multi.python //insane {}} : source.python, string.quoted.multi.python {}blah {foo-bar : source.python, string.quoted.multi.python -%d : constant.character.format.placeholder.other.python, source.python, string.quoted.multi.python +%d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.multi.python //insane {}} : source.python, string.quoted.multi.python ''' : punctuation.definition.string.end.python, source.python, string.quoted.multi.python diff --git a/test/strings/format11.py b/test/strings/format11.py index c16d3fbf..358bb9f7 100644 --- a/test/strings/format11.py +++ b/test/strings/format11.py @@ -11,9 +11,9 @@ R : source.python, storage.type.string.python, string.quoted.raw.multi.python ''' : punctuation.definition.string.begin.python, source.python, string.quoted.raw.multi.python \fr : source.python, string.quoted.raw.multi.python -{still_ok} : constant.character.format.placeholder.other.python, source.python, string.quoted.raw.multi.python +{still_ok} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.raw.multi.python ac : source.python, string.quoted.raw.multi.python {m_{j \rightarrow i}(\mathrm{good})} : source.python, string.quoted.raw.multi.python {not_ok} : source.python, string.quoted.raw.multi.python -%d : constant.character.format.placeholder.other.python, source.python, string.quoted.raw.multi.python +%d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.raw.multi.python ''' : punctuation.definition.string.end.python, source.python, string.quoted.raw.multi.python diff --git a/test/strings/format12.py b/test/strings/format12.py index 6b9fd6fc..e370a39d 100644 --- a/test/strings/format12.py +++ b/test/strings/format12.py @@ -10,8 +10,8 @@ ' : punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python $\frac : source.python, string.quoted.raw.single.python {m_{j \ : source.python, string.quoted.raw.single.python -%s : constant.character.format.placeholder.other.python, source.python, string.quoted.raw.single.python +%s : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.raw.single.python rightarrow i}(\mathrm{ : source.python, string.quoted.raw.single.python -%s : constant.character.format.placeholder.other.python, source.python, string.quoted.raw.single.python +%s : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.raw.single.python good})}{\su%m{m_{j \rightarrow i}}}$ : source.python, string.quoted.raw.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.raw.single.python diff --git a/test/strings/format15.py b/test/strings/format15.py index b1f7ddfe..68743f4d 100644 --- a/test/strings/format15.py +++ b/test/strings/format15.py @@ -11,9 +11,9 @@ = : keyword.operator.assignment.python, source.python : source.python ''' : punctuation.definition.string.begin.python, source.python, string.quoted.multi.python -{{ : constant.character.format.placeholder.other.python, source.python, string.quoted.multi.python +{{ : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.multi.python before detection : source.python, string.quoted.multi.python -}} : constant.character.format.placeholder.other.python, source.python, string.quoted.multi.python +}} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.multi.python {# jinja comment #} : source.python, string.quoted.multi.python {{ after detection }} : source.python, string.quoted.multi.python ''' : punctuation.definition.string.end.python, source.python, string.quoted.multi.python diff --git a/test/strings/format16.py b/test/strings/format16.py new file mode 100644 index 00000000..9b81ed31 --- /dev/null +++ b/test/strings/format16.py @@ -0,0 +1,19 @@ +a = b'%b' % b'foo' + + + +a : source.python + : source.python += : keyword.operator.assignment.python, source.python + : source.python +b : source.python, storage.type.string.python, string.quoted.binary.single.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.binary.single.python +%b : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.binary.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.binary.single.python + : source.python +% : keyword.operator.arithmetic.python, source.python + : source.python +b : source.python, storage.type.string.python, string.quoted.binary.single.python +' : punctuation.definition.string.begin.python, source.python, string.quoted.binary.single.python +foo : source.python, string.quoted.binary.single.python +' : punctuation.definition.string.end.python, source.python, string.quoted.binary.single.python diff --git a/test/strings/format2.py b/test/strings/format2.py index e9b0c240..d6079535 100644 --- a/test/strings/format2.py +++ b/test/strings/format2.py @@ -2,28 +2,29 @@ + a : source.python : source.python = : keyword.operator.assignment.python, source.python : source.python " : punctuation.definition.string.begin.python, source.python, string.quoted.single.python normal : source.python, string.quoted.single.python -{{ : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{{ : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python normal : source.python, string.quoted.single.python -}} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +}} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python normal : source.python, string.quoted.single.python -{10 : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -!r : constant.character.format.placeholder.other.python, source.python, storage.type.format.python, string.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{10 : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +!r : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python normal : source.python, string.quoted.single.python -{fo.__add__ : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -!s : constant.character.format.placeholder.other.python, source.python, storage.type.format.python, string.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{fo.__add__ : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +!s : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python " : punctuation.definition.string.end.python, source.python, string.quoted.single.python -. : source.python -format : meta.function-call.generic.python, meta.function-call.python, source.python -( : meta.function-call.python, punctuation.definition.arguments.begin.python, source.python -fo : meta.function-call.arguments.python, meta.function-call.python, source.python, variable.parameter.function-call.python -= : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, source.python -1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, source.python -) : meta.function-call.python, punctuation.definition.arguments.end.python, source.python +. : meta.member.access.python, punctuation.separator.period.python, source.python +format : meta.function-call.generic.python, meta.function-call.python, meta.member.access.python, source.python +( : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.begin.python, source.python +fo : meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python, variable.parameter.function-call.python += : keyword.operator.assignment.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +1 : constant.numeric.dec.python, meta.function-call.arguments.python, meta.function-call.python, meta.member.access.python, source.python +) : meta.function-call.python, meta.member.access.python, punctuation.definition.arguments.end.python, source.python diff --git a/test/strings/format3.py b/test/strings/format3.py index b699bdff..2a63d31a 100644 --- a/test/strings/format3.py +++ b/test/strings/format3.py @@ -10,7 +10,7 @@ = : keyword.operator.assignment.python, source.python : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python -%i : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +%i : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python : source.python % : keyword.operator.arithmetic.python, source.python @@ -21,9 +21,9 @@ = : keyword.operator.assignment.python, source.python : source.python " : punctuation.definition.string.begin.python, source.python, string.quoted.single.python -%(language)s : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +%(language)s : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.single.python has : source.python, string.quoted.single.python -%(number)03d : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +%(number)03d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.single.python quote types. : source.python, string.quoted.single.python " : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python @@ -32,9 +32,9 @@ : source.python b : source.python, storage.type.string.python, string.quoted.binary.single.python " : punctuation.definition.string.begin.python, source.python, string.quoted.binary.single.python -%(language)s : constant.character.format.placeholder.other.python, source.python, string.quoted.binary.single.python +%(language)s : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.binary.single.python has : source.python, string.quoted.binary.single.python -%(number)03d : constant.character.format.placeholder.other.python, source.python, string.quoted.binary.single.python +%(number)03d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.binary.single.python quote types. : source.python, string.quoted.binary.single.python " : punctuation.definition.string.end.python, source.python, string.quoted.binary.single.python a : source.python @@ -43,8 +43,8 @@ : source.python R : source.python, storage.type.string.python, string.quoted.raw.single.python " : punctuation.definition.string.begin.python, source.python, string.quoted.raw.single.python -%(language)s : constant.character.format.placeholder.other.python, source.python, string.quoted.raw.single.python +%(language)s : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.raw.single.python has : source.python, string.quoted.raw.single.python -%(number)03d : constant.character.format.placeholder.other.python, source.python, string.quoted.raw.single.python +%(number)03d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.raw.single.python quote types. : source.python, string.quoted.raw.single.python " : punctuation.definition.string.end.python, source.python, string.quoted.raw.single.python diff --git a/test/strings/format4.py b/test/strings/format4.py index b33daf32..4f21cd9c 100644 --- a/test/strings/format4.py +++ b/test/strings/format4.py @@ -10,10 +10,9 @@ : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python qqq : source.python, string.quoted.single.python -{ : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -: : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python, support.other.format.python -%Y-%m-%d %H:%M:%S : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{ : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +: : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +%Y-%m-%d %H:%M:%S} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python www : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python @@ -22,11 +21,8 @@ : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python qqq : source.python, string.quoted.single.python -{0 : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -: : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python, support.other.format.python -{fill} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -{align} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -16 : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{0 : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +: : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +{fill}{align}16} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python www : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python diff --git a/test/strings/format5.py b/test/strings/format5.py index 5d7194df..230253a3 100644 --- a/test/strings/format5.py +++ b/test/strings/format5.py @@ -11,11 +11,9 @@ : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python qqq : source.python, string.quoted.single.python -{0 : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -: : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python, support.other.format.python -{width} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -{base} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{0 : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +: : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +{width}{base}} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python www : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python @@ -24,10 +22,9 @@ : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python qqq : source.python, string.quoted.single.python -{0 : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -: : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python, support.other.format.python -$20 : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python -} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{0 : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python +: : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, storage.type.format.python, string.quoted.single.python +$20} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python www : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python @@ -36,6 +33,6 @@ : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python qqq : source.python, string.quoted.single.python -{0} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{0} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python www : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python diff --git a/test/strings/format6.py b/test/strings/format6.py index cf0ea3c7..aff1a320 100644 --- a/test/strings/format6.py +++ b/test/strings/format6.py @@ -31,6 +31,6 @@ : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python h} : source.python, string.quoted.single.python -{base} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{base} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python }www : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python diff --git a/test/strings/format7.py b/test/strings/format7.py index e80e3d27..cafb5dcd 100644 --- a/test/strings/format7.py +++ b/test/strings/format7.py @@ -19,9 +19,9 @@ : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python 12 : source.python, string.quoted.single.python -% o : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +% o : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.single.python f : source.python, string.quoted.single.python -%s : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +%s : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python : source.python % : keyword.operator.arithmetic.python, source.python diff --git a/test/strings/format9.py b/test/strings/format9.py index 26e5d643..1b493484 100644 --- a/test/strings/format9.py +++ b/test/strings/format9.py @@ -11,7 +11,7 @@ : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python blah {foo-bar : source.python, string.quoted.single.python -%d : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +%d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python : source.python @@ -20,7 +20,7 @@ ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python blah : source.python, string.quoted.single.python {foo-bar : source.python, string.quoted.single.python -%d : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +%d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.single.python } : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python @@ -30,7 +30,7 @@ ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python blah : source.python, string.quoted.single.python {foo-bar : source.python, string.quoted.single.python -%d : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +%d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.single.python //insane {}} : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python a : source.python @@ -38,9 +38,9 @@ = : keyword.operator.assignment.python, source.python : source.python ' : punctuation.definition.string.begin.python, source.python, string.quoted.single.python -{} : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +{} : constant.character.format.placeholder.other.python, meta.format.brace.python, source.python, string.quoted.single.python blah : source.python, string.quoted.single.python {foo-bar : source.python, string.quoted.single.python -%d : constant.character.format.placeholder.other.python, source.python, string.quoted.single.python +%d : constant.character.format.placeholder.other.python, meta.format.percent.python, source.python, string.quoted.single.python //insane {}} : source.python, string.quoted.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.single.python diff --git a/test/unicode/unicode1.py b/test/unicode/unicode1.py index 9d853df6..795a2645 100644 --- a/test/unicode/unicode1.py +++ b/test/unicode/unicode1.py @@ -24,7 +24,7 @@ class : meta.class.python, source.python, storage.type.class.python Ń : source.python, string.quoted.docstring.single.python ' : punctuation.definition.string.end.python, source.python, string.quoted.docstring.single.python : meta.function.decorator.python, source.python -@ : entity.name.function.decorator.python, meta.function.decorator.python, source.python +@ : entity.name.function.decorator.python, meta.function.decorator.python, punctuation.definition.decorator.python, source.python æaœ : entity.name.function.decorator.python, meta.function.decorator.python, source.python : meta.function.python, source.python def : meta.function.python, source.python, storage.type.function.python