Add a lower bound check to unicode-table-generator output

This adds a dedicated check for the lower bound
(if it is outside of ASCII range) to the output of the `unicode-table-generator` tool.
This generalized the ASCII-only fast-path, but only for the `Grapheme_Extend` property for now,
as that is the only one with a lower bound outside of ASCII.

Author: Swatinem

library/core/src/char/methods.rs

@@ -927,7 +927,7 @@ impl char {
 #[must_use]
 #[inline]
 pub(crate)fnis_grapheme_extended(self) -> bool{
-self > '\x7f' && unicode::Grapheme_Extend(self)
+ unicode::Grapheme_Extend(self)
 }
 
 /// Returns `true` if this `char` has one of the general categories for numbers.

library/core/src/unicode/unicode_data.rs

@@ -315,7 +315,11 @@ pub mod grapheme_extend {
 15,0,7,1,17,2,7,1,2,1,5,100,1,160,7,0,1,61,4,0,4,0,7,109,7,0,96,
 128,240,0,
 ];
+#[inline]
 pubfnlookup(c:char) -> bool{
+(c asu32) >= 0x300 && lookup_slow(c)
+}
+fnlookup_slow(c:char) -> bool{
 super::skip_search(
  c asu32,
 &SHORT_OFFSET_RUNS,

src/tools/unicode-table-generator/src/raw_emitter.rs

@@ -23,6 +23,7 @@ impl RawEmitter {
 }
 
 fnemit_bitset(&mutself,ranges:&[Range<u32>]) -> Result<(),String>{
+let first_code_point = ranges.first().unwrap().start;
 let last_code_point = ranges.last().unwrap().end;
 // bitset for every bit in the codepoint range
 //
@@ -101,7 +102,10 @@ impl RawEmitter {
 )
 .unwrap();
 writeln!(&mutself.file,"pub const fn lookup(c: char) -> bool {{").unwrap();
-writeln!(&mutself.file," super::bitset_search(",).unwrap();
+if first_code_point > 0x7f{
+writeln!(&mutself.file," (c as u32) >= {first_code_point:#04x} &&").unwrap();
+}
+writeln!(&mutself.file," super::bitset_search(").unwrap();
 writeln!(&mutself.file," c as u32,").unwrap();
 writeln!(&mutself.file," &BITSET_CHUNKS_MAP,").unwrap();
 writeln!(&mutself.file," &BITSET_INDEX_CHUNKS,").unwrap();

src/tools/unicode-table-generator/src/skiplist.rs

@@ -25,8 +25,9 @@ impl ShortOffsetRunHeader {
 
 implRawEmitter{
 pubfnemit_skiplist(&mutself,ranges:&[Range<u32>]){
+let first_code_point = ranges.first().unwrap().start;
 letmut offsets = Vec::<u32>::new();
-let points = ranges.iter().flat_map(|r| vec![r.start, r.end]).collect::<Vec<u32>>();
+let points = ranges.iter().flat_map(|r| [r.start, r.end]).collect::<Vec<u32>>();
 letmut offset = 0;
 for pt in points {
 let delta = pt - offset;
@@ -86,7 +87,26 @@ impl RawEmitter {
 .unwrap();
 self.bytes_used += coded_offsets.len();
 
-writeln!(&mutself.file,"pub fn lookup(c: char) -> bool {{").unwrap();
+// The inlining in this code works like the following:
+//
+// The `skip_search` function is always inlined into the parent `lookup` fn,
+// thus the compiler can generate optimal code based on the referenced `static`s.
+//
+// In the case of ASCII optimization, the lower-bounds check is inlined into
+// the caller, and slower-path `skip_search` is outlined into a separate `lookup_slow` fn.
+//
+// Thus, in both cases, the `skip_search` function is specialized for the `static`s,
+// and outlined into the prebuilt `std`.
+if first_code_point > 0x7f{
+writeln!(&mutself.file,"#[inline]").unwrap();
+writeln!(&mutself.file,"pub fn lookup(c: char) -> bool {{").unwrap();
+writeln!(&mutself.file," (c as u32) >= {first_code_point:#04x} && lookup_slow(c)")
+.unwrap();
+writeln!(&mutself.file,"}}").unwrap();
+writeln!(&mutself.file,"fn lookup_slow(c: char) -> bool {{").unwrap();
+}else{
+writeln!(&mutself.file,"pub fn lookup(c: char) -> bool {{").unwrap();
+}
 writeln!(&mutself.file," super::skip_search(",).unwrap();
 writeln!(&mutself.file," c as u32,").unwrap();
 writeln!(&mutself.file," &SHORT_OFFSET_RUNS,").unwrap();