From 1aa24222f724d3f33bcd94d605a484af129f4e09 Mon Sep 17 00:00:00 2001 From: Gunnlaugur Thor Briem Date: Wed, 26 Aug 2026 10:12:14 +0000 Subject: [PATCH 1/2] fix: lex .5 and 5. as numbers, and not 5E and 5e+ Fix three misreadings of number forms, inconsistent with Excel: - A numeric literal with no integer part (`.95`, `-.5`) was lexed as `unknown` followed by an integer number. The `.` can now start a number. A lone `.` without a digit on at least one side stays `unknown`. - A numeric literal with a dot but no fractional part (`1.`, `1.*2`, `1.E5`) was lexed as a number only at the end of the input (accidentally, because of a NaN gotcha, see below) and `unknown` elsewhere. Now a number everywhere. - An unfinished exponent at the end of the input (`1E`, `1.5e+`) was lexed as a number, because of a NaN gotcha in `advDigits`. The NaN gotcha in `advDigits`: it used a do-while loop, so didn't check string length before the first iteration. So if `pos` was after the end of the string, `str.charCodeAt` returned NaN. The result of `c < 48 || c > 57` (false for NaN) was then misunderstood as meaning that `c` was in the 0-9 range. --- lib/lexers/lexNumber.ts | 17 +++++---- lib/tokenize.spec.ts | 81 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 8 deletions(-) diff --git a/lib/lexers/lexNumber.ts b/lib/lexers/lexNumber.ts index 2b04dae..b4490a6 100644 --- a/lib/lexers/lexNumber.ts +++ b/lib/lexers/lexNumber.ts @@ -6,34 +6,35 @@ const COLON = 58; // : function advDigits (str: string, pos: number): number { const start = pos; - do { + while (pos < str.length) { const c = str.charCodeAt(pos); if (c < 48 || c > 57) { // 0-9 break; } pos++; } - while (pos < str.length); return pos - start; } -// \d+(\.\d+)?(?:[eE][+-]?\d+)? +// (?:\d+(\.\d*)?|\.\d+)(?:[eE][+-]?\d+)? export function lexNumber (str: string, pos: number): Token | undefined { const start = pos; - // integer + // integer part, optional when there is a fraction part (.5) const lead = advDigits(str, pos); - if (!lead) { return; } pos += lead; - // optional fraction part + // fraction part, optional when there is an integer part (5.) const c0 = str.charCodeAt(pos); + let frac = 0; if (c0 === 46) { // . pos++; - const frac = advDigits(str, pos); - if (!frac) { return; } + frac = advDigits(str, pos); pos += frac; } + if (!frac && !lead) { + return; + } // optional exponent part const c1 = str.charCodeAt(pos); if (c1 === 69 || c1 === 101) { // E e diff --git a/lib/tokenize.spec.ts b/lib/tokenize.spec.ts index 7b24adb..98857d2 100644 --- a/lib/tokenize.spec.ts +++ b/lib/tokenize.spec.ts @@ -511,6 +511,87 @@ describe('lexer', () => { ]); }); + test('decimals with no integer part', () => { + isTokens('=.5', [ + { type: FX_PREFIX, value: '=' }, + { type: NUMBER, value: '.5' } + ]); + isTokens('=A1*.95', [ + { type: FX_PREFIX, value: '=' }, + { type: REF_RANGE, value: 'A1' }, + { type: OPERATOR, value: '*' }, + { type: NUMBER, value: '.95' } + ]); + isTokens('=.5E-1', [ + { type: FX_PREFIX, value: '=' }, + { type: NUMBER, value: '.5E-1' } + ]); + isTokensNeg('=-.5', [ + { type: FX_PREFIX, value: '=' }, + { type: NUMBER, value: '-.5' } + ]); + // unary minus after a binary operator + isTokensNeg('=A1*-.95', [ + { type: FX_PREFIX, value: '=' }, + { type: REF_RANGE, value: 'A1' }, + { type: OPERATOR, value: '*' }, + { type: NUMBER, value: '-.95' } + ]); + isTokens('=A1*-.95', [ + { type: FX_PREFIX, value: '=' }, + { type: REF_RANGE, value: 'A1' }, + { type: OPERATOR, value: '*' }, + { type: OPERATOR, value: '-' }, + { type: NUMBER, value: '.95' } + ]); + // a lone period is not a number + isTokens('=.', [ + { type: FX_PREFIX, value: '=' }, + { type: UNKNOWN, value: '.' } + ]); + }); + + test('decimals with no fraction part', () => { + isTokens('=1.*2', [ + { type: FX_PREFIX, value: '=' }, + { type: NUMBER, value: '1.' }, + { type: OPERATOR, value: '*' }, + { type: NUMBER, value: '2' } + ]); + isTokens('=(1.)', [ + { type: FX_PREFIX, value: '=' }, + { type: OPERATOR, value: '(' }, + { type: NUMBER, value: '1.' }, + { type: OPERATOR, value: ')' } + ]); + isTokens('=1.E5', [ + { type: FX_PREFIX, value: '=' }, + { type: NUMBER, value: '1.E5' } + ]); + isTokens('=1.e-2', [ + { type: FX_PREFIX, value: '=' }, + { type: NUMBER, value: '1.e-2' } + ]); + // Excel reads `=1.` as 1, at the end of the input like anywhere else + isTokens('=1.', [ + { type: FX_PREFIX, value: '=' }, + { type: NUMBER, value: '1.' } + ]); + }); + + test('dangling exponents are not numbers', () => { + // Excel refuses `=1E` and `=1.5e+` at entry + isTokens('=1E', [ + { type: FX_PREFIX, value: '=' }, + { type: UNKNOWN, value: '1E' } + ]); + isTokens('=1.5e+', [ + { type: FX_PREFIX, value: '=' }, + { type: UNKNOWN, value: '1.5e' }, + { type: OPERATOR, value: '+' } + ]); + }); + test('scientific notation', () => { isTokens('=1E-1', [ { type: FX_PREFIX, value: '=' }, From 40d183093a0e69ac0c9ca230ac2bf55c5d5b3cb2 Mon Sep 17 00:00:00 2001 From: Gunnlaugur Thor Briem Date: Wed, 26 Aug 2026 12:47:45 +0000 Subject: [PATCH 2/2] Pin that a leading-dot decimal yields to a following : or ! The tail guard that keeps a number from swallowing a sheet or range operator applies to the new form; pin it, per review. --- lib/tokenize.spec.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/tokenize.spec.ts b/lib/tokenize.spec.ts index 98857d2..bd825c3 100644 --- a/lib/tokenize.spec.ts +++ b/lib/tokenize.spec.ts @@ -551,6 +551,13 @@ describe('lexer', () => { ]); }); + test('a decimal with no integer part still yields to a following : or !', () => { + // the tail guard applies to the new form as to any number + for (const expr of [ '=.5:A1', '=.5!A1' ]) { + expect(tokenize(expr).filter(t => t.type === NUMBER)).toEqual([]); + } + }); + test('decimals with no fraction part', () => { isTokens('=1.*2', [ { type: FX_PREFIX, value: '=' },