rustc_lexer_zig

Zig port of rustc_lexer

Rust 🦀	Zig ⚡
char	u21
&str	std.unicode.Utf8View
std::str::Chars	std.unicode.Utf8Iterator

Highlights

comptime + @unionInit

Rust 🦀	match first_char {// ...'b' => self.c_or_byte_string( |terminated| ByteStr{ terminated }, |n_hashes| RawByteStr{ n_hashes },Some(|terminated| Byte{ terminated }),),'c' => self.c_or_byte_string( |terminated| CStr{ terminated }, |n_hashes| RawCStr{ n_hashes },None,),// ...}
Zig ⚡	switch (c) { // ...'b'=>self.cOrByteString("byte_str", "raw_byte_str", "byte"), 'c'=>self.cOrByteString("c_str", "raw_c_str", null), // ... }

Rust 🦀	fn c_or_byte_string(&mutself,mk_kind:implFnOnce(bool) -> LiteralKind,mk_kind_raw:implFnOnce(Option<u8>) -> LiteralKind,single_quoted:Option<fn(bool) -> LiteralKind>,) -> TokenKind{
Zig ⚡	fncOrByteString( self: *Self, comptimekind_name: []constu8, comptimeraw_kind_name: []constu8, comptimesingle_quoted: ?[]constu8, ) TokenKind {

Rust 🦀	let kind = mk_kind(terminated);
Zig ⚡	constkind=@unionInit(LiteralKind, kind_name, .{ .terminated=terminated });

Multiline String Literals

Rust 🦀	#[test]fncomment_flavors(){check_lexing(r"// line//// line as well/// outer doc line//! inner doc line/* block *//**//*** also block *//** outer doc block *//*! inner doc block */",// ...)}
Zig ⚡	test"comment flavors" { trycheckLexing( \\// line\\//// line as well\\/// outer doc line\\//! inner doc line\\/* block */\\/**/\\/*** also block */\\/** outer doc block */\\/*! inner doc block */ , // ... ); }

Anonymous Struct Literals

Rust 🦀	#[test]fnsmoke_test(){check_lexing("/* my source file */ fn main() { println!(\"zebra\"); }\n",expect![[r#" Token { kind: BlockComment { doc_style: None, terminated: true }, len: 20 } Token { kind: Whitespace, len: 1 } ... "#]],)}
Zig ⚡	test"smoke test" { trycheckLexing( "/* my source file */ fn main() { println!(\"zebra\"); }\n", &.{ .{ .kind= .{ .block_comment= .{ .doc_style=null, .terminated=true } }, .len=20 }, .{ .kind=.whitespace, .len=1 }, // ... }, ); }