gh-131507: Add support for syntax highlighting in PyREPL

Doc/whatsnew/3.14.rst

@@ -556,6 +556,23 @@ For further information on how to build Python, see
 (Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this
 in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.)
 
+Syntax highlighting in PyREPL
+-----------------------------
+
+The default :term:`interactive` shell now highlights Python syntax as you
+type. The feature is enabled by default unless the
+:envvar:`PYTHON_BASIC_REPL` environment is set or any color-disabling
+environment variables are used. See :ref:`using-on-controlling-color` for
+details.
+
+The default color theme for syntax highlighting strives for good contrast
+and uses exclusively the 4-bit VGA standard ANSI color codes for maximum
+compatibility. The theme can be customized using an experimental API
+``_colorize.set_theme()``. This can be called interactively, as well as
+in the :envvar:`PYTHONSTARTUP` script.
+
+(Contributed by Łukasz Langa in :gh:`131507`.)
+
 
 Other language changes
 ======================

Lib/_colorize.py

@@ -7,7 +7,22 @@
 
 # types
 if False:
- from typing import IO
+ from typing import IO, Literal
+
+ type ColorTag = Literal[
+ "PROMPT",
+ "KEYWORD",
+ "BUILTIN",
+ "COMMENT",
+ "STRING",
+ "NUMBER",
+ "OP",
+ "DEFINITION",
+ "SOFT_KEYWORD",
+ "RESET",
+ ]
+
+ theme: dict[ColorTag, str]
 
 
 class ANSIColors:
@@ -22,6 +37,7 @@ class ANSIColors:
  WHITE = "\x1b[37m" # more like LIGHT GRAY
  YELLOW = "\x1b[33m"
 
+ BOLD = "\x1b[1m"
  BOLD_BLACK = "\x1b[1;30m" # DARK GRAY
  BOLD_BLUE = "\x1b[1;34m"
  BOLD_CYAN = "\x1b[1;36m"
@@ -110,3 +126,28 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool:
  return os.isatty(file.fileno())
  except io.UnsupportedOperation:
  return hasattr(file, "isatty") and file.isatty()
+
+
+def set_theme(t: dict[ColorTag, str] | None = None) -> None:
+ global theme
+
+ if t:
+ theme = t
+ return
+
+ colors = get_colors()
+ theme = {
+ "PROMPT": colors.BOLD_MAGENTA,
+ "KEYWORD": colors.BOLD_BLUE,
+ "BUILTIN": colors.CYAN,
+ "COMMENT": colors.RED,
+ "STRING": colors.GREEN,
+ "NUMBER": colors.YELLOW,
+ "OP": colors.RESET,
+ "DEFINITION": colors.BOLD,
+ "SOFT_KEYWORD": colors.BOLD_BLUE,
+ "RESET": colors.RESET,
+ }
+
+
+set_theme()

Lib/_pyrepl/_module_completer.py

@@ -2,6 +2,7 @@
 
 import pkgutil
 import sys
+import token
 import tokenize
 from io import StringIO
 from contextlib import contextmanager
@@ -180,8 +181,8 @@ class ImportParser:
  when parsing multiple statements.
  """
  _ignored_tokens = {
-tokenize.INDENT, tokenize.DEDENT, tokenize.COMMENT,
-tokenize.NL, tokenize.NEWLINE, tokenize.ENDMARKER
+token.INDENT, token.DEDENT, token.COMMENT,
+token.NL, token.NEWLINE, token.ENDMARKER
  }
  _keywords = {'import', 'from', 'as'}
 
@@ -350,11 +351,11 @@ def peek(self) -> TokenInfo | None:
  def peek_name(self) -> bool:
  if not (tok := self.peek()):
  return False
- return tok.type == tokenize.NAME
+ return tok.type == token.NAME
 
  def pop_name(self) -> str:
  tok = self.pop()
- if tok.type != tokenize.NAME:
+ if tok.type != token.NAME:
  raise ParseError('pop_name')
  return tok.string
 

Lib/_pyrepl/commands.py

@@ -21,6 +21,7 @@
 
 from __future__ import annotations
 import os
+import time
 
 # Categories of actions:
 # killing
@@ -31,6 +32,7 @@
 # finishing
 # [completion]
 
+from .trace import trace
 
 # types
 if False:
@@ -471,19 +473,24 @@ def do(self) -> None:
 
 
 class paste_mode(Command):
-
  def do(self) -> None:
  self.reader.paste_mode = not self.reader.paste_mode
  self.reader.dirty = True
 
 
-class enable_bracketed_paste(Command):
- def do(self) -> None:
- self.reader.paste_mode = True
- self.reader.in_bracketed_paste = True
-
-class disable_bracketed_paste(Command):
- def do(self) -> None:
- self.reader.paste_mode = False
- self.reader.in_bracketed_paste = False
- self.reader.dirty = True
+class perform_bracketed_paste(Command):
+ def do(self) -> None:
+ done = "\x1b[201~"
+ data = ""
+ start = time.time()
+ while done not in data:
+ self.reader.console.wait(100)
+ ev = self.reader.console.getpending()
+ data += ev.data
+ trace(
+ "bracketed pasting of {l} chars done in {s:.2f}s",
+ l=len(data),
+ s=time.time() - start,
+ )
+ self.reader.insert(data.replace(done, ""))
+ self.reader.last_refresh_cache.invalidated = True

Lib/_pyrepl/mypy.ini

@@ -23,7 +23,3 @@ check_untyped_defs = False
 # Various internal modules that typeshed deliberately doesn't have stubs for:
 [mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*]
 ignore_missing_imports = True
-
-# Other untyped parts of the stdlib
-[mypy-idlelib.*]
-ignore_missing_imports = True

Lib/_pyrepl/reader.py

@@ -22,14 +22,13 @@
 from __future__ import annotations
 
 import sys
+import _colorize
 
 from contextlib import contextmanager
 from dataclasses import dataclass, field, fields
-from _colorize import can_colorize, ANSIColors
-
 
 from . import commands, console, input
-from .utils import wlen, unbracket, disp_str
+from .utils import wlen, unbracket, disp_str, gen_colors
 from .trace import trace
 
 
@@ -38,8 +37,7 @@
 from .types import Callback, SimpleContextManager, KeySpec, CommandName
 
 
-# syntax classes:
-
+# syntax classes
 SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3)
 
 
@@ -105,8 +103,7 @@ def make_default_commands() -> dict[CommandName, type[Command]]:
  (r"\M-9", "digit-arg"),
  (r"\M-\n", "accept"),
  ("\\\\", "self-insert"),
- (r"\x1b[200~", "enable_bracketed_paste"),
- (r"\x1b[201~", "disable_bracketed_paste"),
+ (r"\x1b[200~", "perform-bracketed-paste"),
  (r"\x03", "ctrl-c"),
  ]
  + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"]
@@ -144,16 +141,17 @@ class Reader:
  Instance variables of note include:
 
  * buffer:
- A *list* (*not* a string at the moment :-) containing all the
-characters that have been entered.
+ A per-character list containing all the characters that have been
+entered. Does not include color information.
  * console:
  Hopefully encapsulates the OS dependent stuff.
  * pos:
  A 0-based index into 'buffer' for where the insertion point
  is.
  * screeninfo:
- Ahem. This list contains some info needed to move the
- insertion point around reasonably efficiently.
+ A list of screen position tuples. Each list element is a tuple
+ representing information on visible line length for a given line.
+ Allows for efficient skipping of color escape sequences.
  * cxy, lxy:
  the position of the insertion point in screen ...
  * syntax_table:
@@ -203,7 +201,6 @@ class Reader:
  dirty: bool = False
  finished: bool = False
  paste_mode: bool = False
- in_bracketed_paste: bool = False
  commands: dict[str, type[Command]] = field(default_factory=make_default_commands)
  last_command: type[Command] | None = None
  syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table)
@@ -221,7 +218,6 @@ class Reader:
  ## cached metadata to speed up screen refreshes
  @dataclass
  class RefreshCache:
- in_bracketed_paste: bool = False
  screen: list[str] = field(default_factory=list)
  screeninfo: list[tuple[int, list[int]]] = field(init=False)
  line_end_offsets: list[int] = field(default_factory=list)
@@ -235,7 +231,6 @@ def update_cache(self,
  screen: list[str],
  screeninfo: list[tuple[int, list[int]]],
  ) -> None:
- self.in_bracketed_paste = reader.in_bracketed_paste
  self.screen = screen.copy()
  self.screeninfo = screeninfo.copy()
  self.pos = reader.pos
@@ -248,8 +243,7 @@ def valid(self, reader: Reader) -> bool:
  return False
  dimensions = reader.console.width, reader.console.height
  dimensions_changed = dimensions != self.dimensions
- paste_changed = reader.in_bracketed_paste != self.in_bracketed_paste
- return not (dimensions_changed or paste_changed)
+ return not dimensions_changed
 
  def get_cached_location(self, reader: Reader) -> tuple[int, int]:
  if self.invalidated:
@@ -279,7 +273,7 @@ def __post_init__(self) -> None:
  self.screeninfo = [(0, [])]
  self.cxy = self.pos2xy()
  self.lxy = (self.pos, 0)
- self.can_colorize = can_colorize()
+ self.can_colorize = _colorize.can_colorize()
 
  self.last_refresh_cache.screeninfo = self.screeninfo
  self.last_refresh_cache.pos = self.pos
@@ -316,6 +310,12 @@ def calc_screen(self) -> list[str]:
  pos -= offset
 
  prompt_from_cache = (offset and self.buffer[offset - 1] != "\n")
+
+ if self.can_colorize:
+ colors = list(gen_colors(self.get_unicode()))
+ else:
+ colors = None
+ trace("colors = {colors}", colors=colors)
  lines = "".join(self.buffer[offset:]).split("\n")
  cursor_found = False
  lines_beyond_cursor = 0
@@ -343,9 +343,8 @@ def calc_screen(self) -> list[str]:
  screeninfo.append((0, []))
  pos -= line_len + 1
  prompt, prompt_len = self.process_prompt(prompt)
- chars, char_widths = disp_str(line)
+ chars, char_widths = disp_str(line, colors, offset)
  wrapcount = (sum(char_widths) + prompt_len) // self.console.width
- trace("wrapcount = {wrapcount}", wrapcount=wrapcount)
  if wrapcount == 0 or not char_widths:
  offset += line_len + 1 # Takes all of the line plus the newline
  last_refresh_line_end_offsets.append(offset)
@@ -479,7 +478,7 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
  'lineno'."""
  if self.arg is not None and cursor_on_line:
  prompt = f"(arg: {self.arg}) "
- elif self.paste_mode and not self.in_bracketed_paste:
+ elif self.paste_mode:
  prompt = "(paste) "
  elif "\n" in self.buffer:
  if lineno == 0:
@@ -492,7 +491,11 @@ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
  prompt = self.ps1
 
  if self.can_colorize:
- prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}"
+ prompt = (
+ f"{_colorize.theme["PROMPT"]}"
+ f"{prompt}"
+ f"{_colorize.theme["RESET"]}"
+ )
  return prompt
 
  def push_input_trans(self, itrans: input.KeymapTranslator) -> None:
@@ -567,6 +570,7 @@ def insert(self, text: str | list[str]) -> None:
  def update_cursor(self) -> None:
  """Move the cursor to reflect changes in self.pos"""
  self.cxy = self.pos2xy()
+ trace("update_cursor({pos}) = {cxy}", pos=self.pos, cxy=self.cxy)
  self.console.move_cursor(*self.cxy)
 
  def after_command(self, cmd: Command) -> None:
@@ -633,9 +637,6 @@ def update_screen(self) -> None:
 
  def refresh(self) -> None:
  """Recalculate and refresh the screen."""
- if self.in_bracketed_paste and self.buffer and not self.buffer[-1] == "\n":
- return
-
  # this call sets up self.cxy, so call it first.
  self.screen = self.calc_screen()
  self.console.refresh(self.screen, self.cxy)

Lib/_pyrepl/readline.py

@@ -276,10 +276,6 @@ def do(self) -> None:
  r = self.reader # type: ignore[assignment]
  r.dirty = True # this is needed to hide the completion menu, if visible
 
- if self.reader.in_bracketed_paste:
- r.insert("\n")
- return
-
  # if there are already several lines and the cursor
  # is not on the last one, always insert a new \n.
  text = r.get_unicode()

Lib/_pyrepl/simple_interact.py

@@ -157,7 +157,6 @@ def maybe_run_command(statement: str) -> bool:
  r.pos = len(r.get_unicode())
  r.dirty = True
  r.refresh()
- r.in_bracketed_paste = False
  console.write("\nKeyboardInterrupt\n")
  console.resetbuffer()
  except MemoryError: