Skip to content
This repository was archived by the owner on Mar 6, 2026. It is now read-only.

refactor: simplify OrderedDict arguments in lexer#598

Merged
plamut merged 1 commit into from
Apr 13, 2021
Merged

refactor: simplify OrderedDict arguments in lexer #598

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 37 additions & 82 deletions google/cloud/bigquery/magics/line_arg_parser/lexer.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -49,90 +49,45 @@
# the value of an option other than "--params", we do not really care about its
# structure, and thus do not want to use any of the "Python tokens" for pattern matching.
#
# Since token definition order is important, an OrderedDict is needed with tightly
# controlled member definitions (i.e. passed as a sequence, and *not* via kwargs).
# Token definition order is important, thus an OrderedDict is used. In addition, PEP 468
# guarantees us that the order of kwargs is preserved in Python 3.6+.
Comment on lines +52 to +53

@plamutplamutApr 8, 2021

Copy link
Copy Markdown
ContributorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note - we still need to use OrderedDict instad of a regular dict, because the regular dict's key insertion order should be considered an implementation detail of CPython 3.6 (that behavior has only been promoted to a language spec in Python 3.7).

@tswasttswastApr 13, 2021

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah! I didn't realize kwargs became ordered a version earlier :-)

token_types = OrderedDict(
[
(
"state_parse_pos_args",
OrderedDict(
[
(
"GOTO_PARSE_NON_PARAMS_OPTIONS",
r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--))", # double dash - starting the options list
),
(
"DEST_VAR",
r"(?P<DEST_VAR>[^\d\W]\w*)", # essentially a Python ID
),
]
),
),
(
"state_parse_non_params_options",
OrderedDict(
[
(
"GOTO_PARSE_PARAMS_OPTION",
r"(?P<GOTO_PARSE_PARAMS_OPTION>(?=--params(?:\s|=|--|$)))", # the --params option
),
("OPTION_SPEC", r"(?P<OPTION_SPEC>--\w+)"),
("OPTION_EQ", r"(?P<OPTION_EQ>=)"),
("OPT_VAL", r"(?P<OPT_VAL>\S+?(?=\s|--|$))"),
]
),
),
(
"state_parse_params_option",
OrderedDict(
[
(
"PY_STRING",
r"(?P<PY_STRING>(?:{})|(?:{}))".format(
r"'(?:[^'\\]|\.)*'",
r'"(?:[^"\\]|\.)*"', # single and double quoted strings
),
),
("PARAMS_OPT_SPEC", r"(?P<PARAMS_OPT_SPEC>--params(?=\s|=|--|$))"),
("PARAMS_OPT_EQ", r"(?P<PARAMS_OPT_EQ>=)"),
(
"GOTO_PARSE_NON_PARAMS_OPTIONS",
r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--\w+))", # found another option spec
),
("PY_BOOL", r"(?P<PY_BOOL>True|False)"),
("DOLLAR_PY_ID", r"(?P<DOLLAR_PY_ID>\$[^\d\W]\w*)"),
(
"PY_NUMBER",
r"(?P<PY_NUMBER>-?[1-9]\d*(?:\.\d+)?(:?[e|E][+-]?\d+)?)",
),
("SQUOTE", r"(?P<SQUOTE>')"),
("DQUOTE", r'(?P<DQUOTE>")'),
("COLON", r"(?P<COLON>:)"),
("COMMA", r"(?P<COMMA>,)"),
("LCURL", r"(?P<LCURL>\{)"),
("RCURL", r"(?P<RCURL>})"),
("LSQUARE", r"(?P<LSQUARE>\[)"),
("RSQUARE", r"(?P<RSQUARE>])"),
("LPAREN", r"(?P<LPAREN>\()"),
("RPAREN", r"(?P<RPAREN>\))"),
]
),
),
(
"common",
OrderedDict(
[
("WS", r"(?P<WS>\s+)"),
("EOL", r"(?P<EOL>$)"),
(
# anything not a whitespace or matched by something else
"UNKNOWN",
r"(?P<UNKNOWN>\S+)",
),
]
),
state_parse_pos_args=OrderedDict(
GOTO_PARSE_NON_PARAMS_OPTIONS=r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--))", # double dash - starting the options list
DEST_VAR=r"(?P<DEST_VAR>[^\d\W]\w*)", # essentially a Python ID
),
state_parse_non_params_options=OrderedDict(
GOTO_PARSE_PARAMS_OPTION=r"(?P<GOTO_PARSE_PARAMS_OPTION>(?=--params(?:\s|=|--|$)))", # the --params option
OPTION_SPEC=r"(?P<OPTION_SPEC>--\w+)",
OPTION_EQ=r"(?P<OPTION_EQ>=)",
OPT_VAL=r"(?P<OPT_VAL>\S+?(?=\s|--|$))",
),
state_parse_params_option=OrderedDict(
PY_STRING=r"(?P<PY_STRING>(?:{})|(?:{}))".format( # single and double quoted strings
r"'(?:[^'\\]|\.)*'", r'"(?:[^"\\]|\.)*"'
),
]
PARAMS_OPT_SPEC=r"(?P<PARAMS_OPT_SPEC>--params(?=\s|=|--|$))",
PARAMS_OPT_EQ=r"(?P<PARAMS_OPT_EQ>=)",
GOTO_PARSE_NON_PARAMS_OPTIONS=r"(?P<GOTO_PARSE_NON_PARAMS_OPTIONS>(?=--\w+))", # found another option spec
PY_BOOL=r"(?P<PY_BOOL>True|False)",
DOLLAR_PY_ID=r"(?P<DOLLAR_PY_ID>\$[^\d\W]\w*)",
PY_NUMBER=r"(?P<PY_NUMBER>-?[1-9]\d*(?:\.\d+)?(:?[e|E][+-]?\d+)?)",
SQUOTE=r"(?P<SQUOTE>')",
DQUOTE=r'(?P<DQUOTE>")',
COLON=r"(?P<COLON>:)",
COMMA=r"(?P<COMMA>,)",
LCURL=r"(?P<LCURL>\{)",
RCURL=r"(?P<RCURL>})",
LSQUARE=r"(?P<LSQUARE>\[)",
RSQUARE=r"(?P<RSQUARE>])",
LPAREN=r"(?P<LPAREN>\()",
RPAREN=r"(?P<RPAREN>\))",
),
common=OrderedDict(
WS=r"(?P<WS>\s+)",
EOL=r"(?P<EOL>$)",
UNKNOWN=r"(?P<UNKNOWN>\S+)", # anything not a whitespace or matched by something else
),
)


Expand Down