Skip to content

Add soft keywords #20370

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 26, 2020
Merged

Add soft keywords #20370

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3cbe7c3
Add soft keywords
gvanrossum May 25, 2020
8225723
Fix D() macro usage for soft keywords
gvanrossum May 25, 2020
9deae0e
Add some tests for soft keywords
gvanrossum May 26, 2020
702e7ae
Remove nonfunctional debug stuff from test
gvanrossum May 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add soft keywords 
These are like keywords but they only work in context;
they are not reserved except when there is an exact match.

This would enable things like match statements.
  • Loading branch information
@gvanrossum
gvanrossum committed May 25, 2020
commit 3cbe7c3651d64cb15277f5fae6d7976020846f04
24 changes: 24 additions & 0 deletions Parser/pegen/pegen.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -753,6 +753,30 @@ _PyPegen_expect_token(Parser *p, int type)
return t;
}

expr_ty
_PyPegen_expect_soft_keyword(Parser *p, const char *keyword)
{
if (p->mark == p->fill) {
if (_PyPegen_fill_token(p) < 0) {
p->error_indicator = 1;
return NULL;
}
}
Token *t = p->tokens[p->mark];
if (t->type != NAME) {
return NULL;
}
char* s = PyBytes_AsString(t->bytes);
if (!s) {
return NULL;
}
if (strcmp(s, keyword) != 0) {
return NULL;
}
expr_ty res = _PyPegen_name_token(p);
return res;
}

Token *
_PyPegen_get_last_nonnwhitespace_token(Parser *p)
{
Expand Down
1 change: 1 addition & 0 deletions Parser/pegen/pegen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int
int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *);

Token *_PyPegen_expect_token(Parser *p, int type);
expr_ty _PyPegen_expect_soft_keyword(Parser *p, const char *keyword);
Token *_PyPegen_get_last_nonnwhitespace_token(Parser *);
int _PyPegen_fill_token(Parser *p);
expr_ty _PyPegen_name_token(Parser *p);
Expand Down
15 changes: 14 additions & 1 deletion Tools/peg_generator/pegen/c_generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,16 @@ def keyword_helper(self, keyword: str) -> FunctionCall:
comment=f"token='{keyword}'",
)

def soft_keyword_helper(self, value: str) -> FunctionCall:
return FunctionCall(
assigned_variable="_keyword",
function="_PyPegen_expect_soft_keyword",
arguments=["p", value],
return_type="expr_ty",
nodetype=NodeTypes.NAME_TOKEN,
comment=f"soft_keyword='{value}'",
)

def visit_NameLeaf(self, node: NameLeaf) -> FunctionCall:
name = node.value
if name in self.non_exact_tokens:
Expand Down Expand Up @@ -154,7 +164,10 @@ def visit_NameLeaf(self, node: NameLeaf) -> FunctionCall:
def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall:
val = ast.literal_eval(node.value)
if re.match(r"[a-zA-Z_]\w*\Z", val): # This is a keyword
return self.keyword_helper(val)
if node.value.endswith("'"):
return self.keyword_helper(val)
else:
return self.soft_keyword_helper(node.value)
else:
assert val in self.exact_tokens, f"{node.value} is not a known literal"
type = self.exact_tokens[val]
Expand Down