Skip to content

Commit e64610d

Browse files
borsbors
committed
Auto merge of #17037 - davidsemakula:token-set-collisions, r=Veykril
internal: improve `TokenSet` implementation and add reserved keywords The current `TokenSet` type represents "A bit-set of `SyntaxKind`s" as a newtype `u128`. Internally, the flag for each `SyntaxKind` variant in the bit-set is set as the n-th LSB (least significant bit) via a bit-wise left shift operation, where n is the discriminant. Edit: This is problematic because there's currently ~121 token `SyntaxKind`s, so adding new token kinds for missing reserved keywords increases the number of token `SyntaxKind`s above 128, thus making this ["mask"](https://github.com/rust-lang/rust-analyzer/blob/7a8374c162c64c17e865b98aad282d16b16e96d6/crates/parser/src/token_set.rs#L31-L33) operation overflow. ~~This is problematic because there's currently 266 SyntaxKinds, so this ["mask"](https://github.com/rust-lang/rust-analyzer/blob/7a8374c162c64c17e865b98aad282d16b16e96d6/crates/parser/src/token_set.rs#L31-L33) operation silently overflows in release mode.~~ ~~This leads to a single flag/bit in the bit-set being shared by multiple `SyntaxKind`s~~. This PR: - Changes the wrapped type for `TokenSet` from `u128` to `[u64; 3]` ~~`[u*; N]` (currently `[u16; 17]`) where `u*` can be any desirable unsigned integer type and `N` is the minimum array length needed to represent all token `SyntaxKind`s without any collisions~~. - Edit: Add assertion that `TokenSet`s only include token `SyntaxKind`s - Edit: Add ~7 missing [reserved keywords](https://doc.rust-lang.org/stable/reference/keywords.html#reserved-keywords) - ~~Moves the definition of the `TokenSet` type to grammar codegen in xtask, so that `N` is adjusted automatically (depending on the chosen `u*` "base" type) when new `SyntaxKind`s are added~~. - ~~Updates the `token_set_works_for_tokens` unit test to include the `__LAST` `SyntaxKind` as a way of catching overflows in tests.~~ ~~Currently `u16` is arbitrarily chosen as the `u*` "base" type mostly because it strikes a good balance (IMO) between unused bits and readability of the generated `TokenSet` code (especially the [`union` method](https://github.com/rust-lang/rust-analyzer/blob/7a8374c162c64c17e865b98aad282d16b16e96d6/crates/parser/src/token_set.rs#L26-L28)), but I'm open to other suggestions or a better methodology for choosing `u*` type.~~ ~~I considered using a third-party crate for the bit-set, but a direct implementation seems simple enough without adding any new dependencies. I'm not strongly opposed to using a third-party crate though, if that's preferred.~~ ~~Finally, I haven't had the chance to review issues, to figure out if there are any parser issues caused by collisions due the current implementation that may be fixed by this PR - I just stumbled upon the issue while adding "new" keywords to solve #16858~~ Edit: fixes#16858
2 parents dcbb27a + 69fe457 commit e64610d

4 files changed

Lines changed: 101 additions & 20 deletions

File tree

‎crates/ide-assists/src/handlers/auto_import.rs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1620,4 +1620,50 @@ mod foo {
16201620
"#,
16211621
);
16221622
}
1623+
1624+
#[test]
1625+
fnpreserve_raw_identifiers_strict(){
1626+
check_assist(
1627+
auto_import,
1628+
r"
1629+
r#as$0
1630+
1631+
pub mod ffi_mod {
1632+
pub fn r#as() {};
1633+
}
1634+
",
1635+
r"
1636+
use ffi_mod::r#as;
1637+
1638+
r#as
1639+
1640+
pub mod ffi_mod {
1641+
pub fn r#as() {};
1642+
}
1643+
",
1644+
);
1645+
}
1646+
1647+
#[test]
1648+
fnpreserve_raw_identifiers_reserved(){
1649+
check_assist(
1650+
auto_import,
1651+
r"
1652+
r#abstract$0
1653+
1654+
pub mod ffi_mod {
1655+
pub fn r#abstract() {};
1656+
}
1657+
",
1658+
r"
1659+
use ffi_mod::r#abstract;
1660+
1661+
r#abstract
1662+
1663+
pub mod ffi_mod {
1664+
pub fn r#abstract() {};
1665+
}
1666+
",
1667+
);
1668+
}
16231669
}

‎crates/parser/src/syntax_kind/generated.rs‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,11 @@ pub enum SyntaxKind {
6161
SHR,
6262
SHLEQ,
6363
SHREQ,
64+
ABSTRACT_KW,
6465
AS_KW,
6566
ASYNC_KW,
6667
AWAIT_KW,
68+
BECOME_KW,
6769
BOX_KW,
6870
BREAK_KW,
6971
CONST_KW,
@@ -75,6 +77,7 @@ pub enum SyntaxKind {
7577
ENUM_KW,
7678
EXTERN_KW,
7779
FALSE_KW,
80+
FINAL_KW,
7881
FN_KW,
7982
FOR_KW,
8083
IF_KW,
@@ -87,10 +90,11 @@ pub enum SyntaxKind {
8790
MOD_KW,
8891
MOVE_KW,
8992
MUT_KW,
93+
OVERRIDE_KW,
94+
PRIV_KW,
9095
PUB_KW,
9196
REF_KW,
9297
RETURN_KW,
93-
BECOME_KW,
9498
SELF_KW,
9599
SELF_TYPE_KW,
96100
STATIC_KW,
@@ -100,8 +104,11 @@ pub enum SyntaxKind {
100104
TRUE_KW,
101105
TRY_KW,
102106
TYPE_KW,
107+
TYPEOF_KW,
103108
UNSAFE_KW,
109+
UNSIZED_KW,
104110
USE_KW,
111+
VIRTUAL_KW,
105112
WHERE_KW,
106113
WHILE_KW,
107114
YIELD_KW,
@@ -280,9 +287,11 @@ impl SyntaxKind {
280287
pubfnis_keyword(self) -> bool{
281288
matches!(
282289
self,
283-
AS_KW
290+
ABSTRACT_KW
291+
| AS_KW
284292
| ASYNC_KW
285293
| AWAIT_KW
294+
| BECOME_KW
286295
| BOX_KW
287296
| BREAK_KW
288297
| CONST_KW
@@ -294,6 +303,7 @@ impl SyntaxKind {
294303
| ENUM_KW
295304
| EXTERN_KW
296305
| FALSE_KW
306+
| FINAL_KW
297307
| FN_KW
298308
| FOR_KW
299309
| IF_KW
@@ -306,10 +316,11 @@ impl SyntaxKind {
306316
| MOD_KW
307317
| MOVE_KW
308318
| MUT_KW
319+
| OVERRIDE_KW
320+
| PRIV_KW
309321
| PUB_KW
310322
| REF_KW
311323
| RETURN_KW
312-
| BECOME_KW
313324
| SELF_KW
314325
| SELF_TYPE_KW
315326
| STATIC_KW
@@ -319,8 +330,11 @@ impl SyntaxKind {
319330
| TRUE_KW
320331
| TRY_KW
321332
| TYPE_KW
333+
| TYPEOF_KW
322334
| UNSAFE_KW
335+
| UNSIZED_KW
323336
| USE_KW
337+
| VIRTUAL_KW
324338
| WHERE_KW
325339
| WHILE_KW
326340
| YIELD_KW
@@ -399,9 +413,11 @@ impl SyntaxKind {
399413
}
400414
pubfnfrom_keyword(ident:&str) -> Option<SyntaxKind>{
401415
let kw = match ident {
416+
"abstract" => ABSTRACT_KW,
402417
"as" => AS_KW,
403418
"async" => ASYNC_KW,
404419
"await" => AWAIT_KW,
420+
"become" => BECOME_KW,
405421
"box" => BOX_KW,
406422
"break" => BREAK_KW,
407423
"const" => CONST_KW,
@@ -413,6 +429,7 @@ impl SyntaxKind {
413429
"enum" => ENUM_KW,
414430
"extern" => EXTERN_KW,
415431
"false" => FALSE_KW,
432+
"final" => FINAL_KW,
416433
"fn" => FN_KW,
417434
"for" => FOR_KW,
418435
"if" => IF_KW,
@@ -425,10 +442,11 @@ impl SyntaxKind {
425442
"mod" => MOD_KW,
426443
"move" => MOVE_KW,
427444
"mut" => MUT_KW,
445+
"override" => OVERRIDE_KW,
446+
"priv" => PRIV_KW,
428447
"pub" => PUB_KW,
429448
"ref" => REF_KW,
430449
"return" => RETURN_KW,
431-
"become" => BECOME_KW,
432450
"self" => SELF_KW,
433451
"Self" => SELF_TYPE_KW,
434452
"static" => STATIC_KW,
@@ -438,8 +456,11 @@ impl SyntaxKind {
438456
"true" => TRUE_KW,
439457
"try" => TRY_KW,
440458
"type" => TYPE_KW,
459+
"typeof" => TYPEOF_KW,
441460
"unsafe" => UNSAFE_KW,
461+
"unsized" => UNSIZED_KW,
442462
"use" => USE_KW,
463+
"virtual" => VIRTUAL_KW,
443464
"where" => WHERE_KW,
444465
"while" => WHILE_KW,
445466
"yield" => YIELD_KW,
@@ -500,4 +521,4 @@ impl SyntaxKind {
500521
}
501522
}
502523
#[macro_export]
503-
macro_rules!T{[;] => { $ crate::SyntaxKind::SEMICOLON};[,] => { $ crate::SyntaxKind::COMMA};['('] => { $ crate::SyntaxKind::L_PAREN};[')'] => { $ crate::SyntaxKind::R_PAREN};['{'] => { $ crate::SyntaxKind::L_CURLY};['}'] => { $ crate::SyntaxKind::R_CURLY};['['] => { $ crate::SyntaxKind::L_BRACK};[']'] => { $ crate::SyntaxKind::R_BRACK};[<] => { $ crate::SyntaxKind::L_ANGLE};[>] => { $ crate::SyntaxKind::R_ANGLE};[@] => { $ crate::SyntaxKind::AT};[#] => { $ crate::SyntaxKind::POUND};[~] => { $ crate::SyntaxKind::TILDE};[?] => { $ crate::SyntaxKind::QUESTION};[$] => { $ crate::SyntaxKind::DOLLAR};[&] => { $ crate::SyntaxKind::AMP};[|] => { $ crate::SyntaxKind::PIPE};[+] => { $ crate::SyntaxKind::PLUS};[*] => { $ crate::SyntaxKind::STAR};[/] => { $ crate::SyntaxKind::SLASH};[^] => { $ crate::SyntaxKind::CARET};[%] => { $ crate::SyntaxKind::PERCENT};[_] => { $ crate::SyntaxKind::UNDERSCORE};[.] => { $ crate::SyntaxKind::DOT};[..] => { $ crate::SyntaxKind::DOT2};[...] => { $ crate::SyntaxKind::DOT3};[..=] => { $ crate::SyntaxKind::DOT2EQ};[:] => { $ crate::SyntaxKind::COLON};[::] => { $ crate::SyntaxKind::COLON2};[=] => { $ crate::SyntaxKind::EQ};[==] => { $ crate::SyntaxKind::EQ2};[=>] => { $ crate::SyntaxKind::FAT_ARROW};[!] => { $ crate::SyntaxKind::BANG};[!=] => { $ crate::SyntaxKind::NEQ};[-] => { $ crate::SyntaxKind::MINUS};[->] => { $ crate::SyntaxKind::THIN_ARROW};[<=] => { $ crate::SyntaxKind::LTEQ};[>=] => { $ crate::SyntaxKind::GTEQ};[+=] => { $ crate::SyntaxKind::PLUSEQ};[-=] => { $ crate::SyntaxKind::MINUSEQ};[|=] => { $ crate::SyntaxKind::PIPEEQ};[&=] => { $ crate::SyntaxKind::AMPEQ};[^=] => { $ crate::SyntaxKind::CARETEQ};[/=] => { $ crate::SyntaxKind::SLASHEQ};[*=] => { $ crate::SyntaxKind::STAREQ};[%=] => { $ crate::SyntaxKind::PERCENTEQ};[&&] => { $ crate::SyntaxKind::AMP2};[||] => { $ crate::SyntaxKind::PIPE2};[<<] => { $ crate::SyntaxKind::SHL};[>>] => { $ crate::SyntaxKind::SHR};[<<=] => { $ crate::SyntaxKind::SHLEQ};[>>=] => { $ crate::SyntaxKind::SHREQ};[as] => { $ crate::SyntaxKind::AS_KW};[async] => { $ crate::SyntaxKind::ASYNC_KW};[await] => { $ crate::SyntaxKind::AWAIT_KW};[box] => { $ crate::SyntaxKind::BOX_KW};[break] => { $ crate::SyntaxKind::BREAK_KW};[const] => { $ crate::SyntaxKind::CONST_KW};[continue] => { $ crate::SyntaxKind::CONTINUE_KW};[crate] => { $ crate::SyntaxKind::CRATE_KW};[do] => { $ crate::SyntaxKind::DO_KW};[dyn] => { $ crate::SyntaxKind::DYN_KW};[else] => { $ crate::SyntaxKind::ELSE_KW};[enum] => { $ crate::SyntaxKind::ENUM_KW};[extern] => { $ crate::SyntaxKind::EXTERN_KW};[false] => { $ crate::SyntaxKind::FALSE_KW};[fn] => { $ crate::SyntaxKind::FN_KW};[for] => { $ crate::SyntaxKind::FOR_KW};[if] => { $ crate::SyntaxKind::IF_KW};[impl] => { $ crate::SyntaxKind::IMPL_KW};[in] => { $ crate::SyntaxKind::IN_KW};[let] => { $ crate::SyntaxKind::LET_KW};[loop] => { $ crate::SyntaxKind::LOOP_KW};[macro] => { $ crate::SyntaxKind::MACRO_KW};[match] => { $ crate::SyntaxKind::MATCH_KW};[mod] => { $ crate::SyntaxKind::MOD_KW};[move] => { $ crate::SyntaxKind::MOVE_KW};[mut] => { $ crate::SyntaxKind::MUT_KW};[pub] => { $ crate::SyntaxKind::PUB_KW};[ref] => { $ crate::SyntaxKind::REF_KW};[return] => { $ crate::SyntaxKind::RETURN_KW};[become] => { $ crate::SyntaxKind::BECOME_KW};[self] => { $ crate::SyntaxKind::SELF_KW};[Self] => { $ crate::SyntaxKind::SELF_TYPE_KW};[static] => { $ crate::SyntaxKind::STATIC_KW};[struct] => { $ crate::SyntaxKind::STRUCT_KW};[super] => { $ crate::SyntaxKind::SUPER_KW};[trait] => { $ crate::SyntaxKind::TRAIT_KW};[true] => { $ crate::SyntaxKind::TRUE_KW};[try] => { $ crate::SyntaxKind::TRY_KW};[type] => { $ crate::SyntaxKind::TYPE_KW};[unsafe] => { $ crate::SyntaxKind::UNSAFE_KW};[use] => { $ crate::SyntaxKind::USE_KW};[where] => { $ crate::SyntaxKind::WHERE_KW};[while] => { $ crate::SyntaxKind::WHILE_KW};[yield] => { $ crate::SyntaxKind::YIELD_KW};[auto] => { $ crate::SyntaxKind::AUTO_KW};[builtin] => { $ crate::SyntaxKind::BUILTIN_KW};[default] => { $ crate::SyntaxKind::DEFAULT_KW};[existential] => { $ crate::SyntaxKind::EXISTENTIAL_KW};[union] => { $ crate::SyntaxKind::UNION_KW};[raw] => { $ crate::SyntaxKind::RAW_KW};[macro_rules] => { $ crate::SyntaxKind::MACRO_RULES_KW};[yeet] => { $ crate::SyntaxKind::YEET_KW};[offset_of] => { $ crate::SyntaxKind::OFFSET_OF_KW};[asm] => { $ crate::SyntaxKind::ASM_KW};[format_args] => { $ crate::SyntaxKind::FORMAT_ARGS_KW};[lifetime_ident] => { $ crate::SyntaxKind::LIFETIME_IDENT};[ident] => { $ crate::SyntaxKind::IDENT};[shebang] => { $ crate::SyntaxKind::SHEBANG};}
524+
macro_rules!T{[;] => { $ crate::SyntaxKind::SEMICOLON};[,] => { $ crate::SyntaxKind::COMMA};['('] => { $ crate::SyntaxKind::L_PAREN};[')'] => { $ crate::SyntaxKind::R_PAREN};['{'] => { $ crate::SyntaxKind::L_CURLY};['}'] => { $ crate::SyntaxKind::R_CURLY};['['] => { $ crate::SyntaxKind::L_BRACK};[']'] => { $ crate::SyntaxKind::R_BRACK};[<] => { $ crate::SyntaxKind::L_ANGLE};[>] => { $ crate::SyntaxKind::R_ANGLE};[@] => { $ crate::SyntaxKind::AT};[#] => { $ crate::SyntaxKind::POUND};[~] => { $ crate::SyntaxKind::TILDE};[?] => { $ crate::SyntaxKind::QUESTION};[$] => { $ crate::SyntaxKind::DOLLAR};[&] => { $ crate::SyntaxKind::AMP};[|] => { $ crate::SyntaxKind::PIPE};[+] => { $ crate::SyntaxKind::PLUS};[*] => { $ crate::SyntaxKind::STAR};[/] => { $ crate::SyntaxKind::SLASH};[^] => { $ crate::SyntaxKind::CARET};[%] => { $ crate::SyntaxKind::PERCENT};[_] => { $ crate::SyntaxKind::UNDERSCORE};[.] => { $ crate::SyntaxKind::DOT};[..] => { $ crate::SyntaxKind::DOT2};[...] => { $ crate::SyntaxKind::DOT3};[..=] => { $ crate::SyntaxKind::DOT2EQ};[:] => { $ crate::SyntaxKind::COLON};[::] => { $ crate::SyntaxKind::COLON2};[=] => { $ crate::SyntaxKind::EQ};[==] => { $ crate::SyntaxKind::EQ2};[=>] => { $ crate::SyntaxKind::FAT_ARROW};[!] => { $ crate::SyntaxKind::BANG};[!=] => { $ crate::SyntaxKind::NEQ};[-] => { $ crate::SyntaxKind::MINUS};[->] => { $ crate::SyntaxKind::THIN_ARROW};[<=] => { $ crate::SyntaxKind::LTEQ};[>=] => { $ crate::SyntaxKind::GTEQ};[+=] => { $ crate::SyntaxKind::PLUSEQ};[-=] => { $ crate::SyntaxKind::MINUSEQ};[|=] => { $ crate::SyntaxKind::PIPEEQ};[&=] => { $ crate::SyntaxKind::AMPEQ};[^=] => { $ crate::SyntaxKind::CARETEQ};[/=] => { $ crate::SyntaxKind::SLASHEQ};[*=] => { $ crate::SyntaxKind::STAREQ};[%=] => { $ crate::SyntaxKind::PERCENTEQ};[&&] => { $ crate::SyntaxKind::AMP2};[||] => { $ crate::SyntaxKind::PIPE2};[<<] => { $ crate::SyntaxKind::SHL};[>>] => { $ crate::SyntaxKind::SHR};[<<=] => { $ crate::SyntaxKind::SHLEQ};[>>=] => { $ crate::SyntaxKind::SHREQ};[abstract] => { $ crate::SyntaxKind::ABSTRACT_KW};[as] => { $ crate::SyntaxKind::AS_KW};[async] => { $ crate::SyntaxKind::ASYNC_KW};[await] => { $ crate::SyntaxKind::AWAIT_KW};[become] => { $ crate::SyntaxKind::BECOME_KW};[box] => { $ crate::SyntaxKind::BOX_KW};[break] => { $ crate::SyntaxKind::BREAK_KW};[const] => { $ crate::SyntaxKind::CONST_KW};[continue] => { $ crate::SyntaxKind::CONTINUE_KW};[crate] => { $ crate::SyntaxKind::CRATE_KW};[do] => { $ crate::SyntaxKind::DO_KW};[dyn] => { $ crate::SyntaxKind::DYN_KW};[else] => { $ crate::SyntaxKind::ELSE_KW};[enum] => { $ crate::SyntaxKind::ENUM_KW};[extern] => { $ crate::SyntaxKind::EXTERN_KW};[false] => { $ crate::SyntaxKind::FALSE_KW};[final] => { $ crate::SyntaxKind::FINAL_KW};[fn] => { $ crate::SyntaxKind::FN_KW};[for] => { $ crate::SyntaxKind::FOR_KW};[if] => { $ crate::SyntaxKind::IF_KW};[impl] => { $ crate::SyntaxKind::IMPL_KW};[in] => { $ crate::SyntaxKind::IN_KW};[let] => { $ crate::SyntaxKind::LET_KW};[loop] => { $ crate::SyntaxKind::LOOP_KW};[macro] => { $ crate::SyntaxKind::MACRO_KW};[match] => { $ crate::SyntaxKind::MATCH_KW};[mod] => { $ crate::SyntaxKind::MOD_KW};[move] => { $ crate::SyntaxKind::MOVE_KW};[mut] => { $ crate::SyntaxKind::MUT_KW};[override] => { $ crate::SyntaxKind::OVERRIDE_KW};[priv] => { $ crate::SyntaxKind::PRIV_KW};[pub] => { $ crate::SyntaxKind::PUB_KW};[ref] => { $ crate::SyntaxKind::REF_KW};[return] => { $ crate::SyntaxKind::RETURN_KW};[self] => { $ crate::SyntaxKind::SELF_KW};[Self] => { $ crate::SyntaxKind::SELF_TYPE_KW};[static] => { $ crate::SyntaxKind::STATIC_KW};[struct] => { $ crate::SyntaxKind::STRUCT_KW};[super] => { $ crate::SyntaxKind::SUPER_KW};[trait] => { $ crate::SyntaxKind::TRAIT_KW};[true] => { $ crate::SyntaxKind::TRUE_KW};[try] => { $ crate::SyntaxKind::TRY_KW};[type] => { $ crate::SyntaxKind::TYPE_KW};[typeof] => { $ crate::SyntaxKind::TYPEOF_KW};[unsafe] => { $ crate::SyntaxKind::UNSAFE_KW};[unsized] => { $ crate::SyntaxKind::UNSIZED_KW};[use] => { $ crate::SyntaxKind::USE_KW};[virtual] => { $ crate::SyntaxKind::VIRTUAL_KW};[where] => { $ crate::SyntaxKind::WHERE_KW};[while] => { $ crate::SyntaxKind::WHILE_KW};[yield] => { $ crate::SyntaxKind::YIELD_KW};[auto] => { $ crate::SyntaxKind::AUTO_KW};[builtin] => { $ crate::SyntaxKind::BUILTIN_KW};[default] => { $ crate::SyntaxKind::DEFAULT_KW};[existential] => { $ crate::SyntaxKind::EXISTENTIAL_KW};[union] => { $ crate::SyntaxKind::UNION_KW};[raw] => { $ crate::SyntaxKind::RAW_KW};[macro_rules] => { $ crate::SyntaxKind::MACRO_RULES_KW};[yeet] => { $ crate::SyntaxKind::YEET_KW};[offset_of] => { $ crate::SyntaxKind::OFFSET_OF_KW};[asm] => { $ crate::SyntaxKind::ASM_KW};[format_args] => { $ crate::SyntaxKind::FORMAT_ARGS_KW};[lifetime_ident] => { $ crate::SyntaxKind::LIFETIME_IDENT};[ident] => { $ crate::SyntaxKind::IDENT};[shebang] => { $ crate::SyntaxKind::SHEBANG};}

‎crates/parser/src/token_set.rs‎

Lines changed: 24 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,34 +4,48 @@ use crate::SyntaxKind;
44

55
/// A bit-set of `SyntaxKind`s
66
#[derive(Clone,Copy)]
7-
pub(crate)structTokenSet(u128);
7+
pub(crate)structTokenSet([u64;3]);
8+
9+
/// `TokenSet`s should only include token `SyntaxKind`s, so the discriminant of any passed/included
10+
/// `SyntaxKind` must *not* be greater than that of the last token `SyntaxKind`.
11+
/// See #17037.
12+
constLAST_TOKEN_KIND_DISCRIMINANT:usize = SyntaxKind::SHEBANGasusize;
813

914
implTokenSet{
10-
pub(crate)constEMPTY:TokenSet = TokenSet(0);
15+
pub(crate)constEMPTY:TokenSet = TokenSet([0;3]);
1116

1217
pub(crate)constfnnew(kinds:&[SyntaxKind]) -> TokenSet{
13-
letmut res = 0u128;
18+
letmut res = [0;3];
1419
letmut i = 0;
1520
while i < kinds.len(){
16-
res |= mask(kinds[i]);
21+
let discriminant = kinds[i]asusize;
22+
debug_assert!(
23+
discriminant <= LAST_TOKEN_KIND_DISCRIMINANT,
24+
"Expected a token `SyntaxKind`"
25+
);
26+
let idx = discriminant / 64;
27+
res[idx] |= 1 << (discriminant % 64);
1728
i += 1;
1829
}
1930
TokenSet(res)
2031
}
2132

2233
pub(crate)constfnunion(self,other:TokenSet) -> TokenSet{
23-
TokenSet(self.0 | other.0)
34+
TokenSet([self.0[0] | other.0[0],self.0[1] | other.0[1],self.0[2] | other.0[2]])
2435
}
2536

2637
pub(crate)constfncontains(&self,kind:SyntaxKind) -> bool{
27-
self.0&mask(kind) != 0
38+
let discriminant = kind asusize;
39+
debug_assert!(
40+
discriminant <= LAST_TOKEN_KIND_DISCRIMINANT,
41+
"Expected a token `SyntaxKind`"
42+
);
43+
let idx = discriminant / 64;
44+
let mask = 1 << (discriminant % 64);
45+
self.0[idx]& mask != 0
2846
}
2947
}
3048

31-
constfnmask(kind:SyntaxKind) -> u128{
32-
1u128 << (kind asusize)
33-
}
34-
3549
#[test]
3650
fntoken_set_works_for_tokens(){
3751
usecrate::SyntaxKind::*;

‎xtask/src/codegen/grammar/ast_src.rs‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,11 @@ pub(crate) const KINDS_SRC: KindsSrc<'_> = KindsSrc {
6565
(">>=","SHREQ"),
6666
],
6767
keywords:&[
68-
"as","async","await","box","break","const","continue","crate","do","dyn","else",
69-
"enum","extern","false","fn","for","if","impl","in","let","loop","macro",
70-
"match","mod","move","mut","pub","ref","return","become","self","Self","static",
71-
"struct","super","trait","true","try","type","unsafe","use","where","while",
72-
"yield",
68+
"abstract","as","async","await","become","box","break","const","continue","crate",
69+
"do","dyn","else","enum","extern","false","final","fn","for","if","impl","in",
70+
"let","loop","macro","match","mod","move","mut","override","priv","pub","ref",
71+
"return","self","Self","static","struct","super","trait","true","try","type",
72+
"typeof","unsafe","unsized","use","virtual","where","while","yield",
7373
],
7474
contextual_keywords:&[
7575
"auto",

0 commit comments

Comments
 (0)