internal: improve TokenSet implementation

Author: davidsemakula

crates/parser/src/token_set.rs

@@ -4,32 +4,46 @@ use crate::SyntaxKind;
 
 /// A bit-set of `SyntaxKind`s
 #[derive(Clone,Copy)]
-pub(crate)structTokenSet(u128);
+pub(crate)structTokenSet([u64;3]);
+
+constLAST_TOKEN_KIND_DISCRIMINANT:usize = SyntaxKind::SHEBANGasusize;
 
 implTokenSet{
-pub(crate)constEMPTY:TokenSet = TokenSet(0);
+pub(crate)constEMPTY:TokenSet = TokenSet([0;3]);
 
 pub(crate)constfnnew(kinds:&[SyntaxKind]) -> TokenSet{
-letmut res = 0u128;
+letmut res = [0;3];
 letmut i = 0;
 while i < kinds.len(){
- res |= mask(kinds[i]);
+let kind = kinds[i];
+debug_assert!(
+ kind asusize <= LAST_TOKEN_KIND_DISCRIMINANT,
+"Expected a token `SyntaxKind`"
+);
+let idx = kind asusize / 64;
+ res[idx] |= mask(kind);
  i += 1;
 }
 TokenSet(res)
 }
 
 pub(crate)constfnunion(self,other:TokenSet) -> TokenSet{
-TokenSet(self.0 | other.0)
+TokenSet([self.0[0] | other.0[0],self.0[1] | other.0[1],self.0[2] | other.0[2]])
 }
 
 pub(crate)constfncontains(&self,kind:SyntaxKind) -> bool{
-self.0&mask(kind) != 0
+debug_assert!(
+ kind asusize <= LAST_TOKEN_KIND_DISCRIMINANT,
+"Expected a token `SyntaxKind`"
+);
+let idx = kind asusize / 64;
+self.0[idx]&mask(kind) != 0
 }
 }
 
-constfnmask(kind:SyntaxKind) -> u128{
-1u128 << (kind asusize)
+constfnmask(kind:SyntaxKind) -> u64{
+debug_assert!(kind asusize <= LAST_TOKEN_KIND_DISCRIMINANT,"Expected a token `SyntaxKind`");
+1 << (kind asusize % 64)
 }
 
 #[test]