[3.15] gh-109638: Avoid pathological backtracking in csv.Sniffer (GH-153694)

Lib/csv.py

@@ -332,20 +332,59 @@ def _guess_quote_and_delimiter(self, data, delimiters):
  delim = ''
  skipinitialspace = 0
 
- # if we see an extra quote between delimiters, we've got a
- # double quoted format
- dq_regexp = re.compile(
- r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \
- {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE)
+ doublequote = self._detect_doublequote(data, delim, quotechar)
 
+ return (quotechar, doublequote, delim, skipinitialspace)
 
 
- if dq_regexp.search(data):
- doublequote = True
- else:
- doublequote = False
+ def _detect_doublequote(self, data, delimiter, quotechar):
+ """
+ Return whether a doubled quote occurs inside a quoted field.
 
- return (quotechar, doublequote, delim, skipinitialspace)
+ The first regexp is a fast, linear pre-filter. Since it is not
+ anchored, a match can start at a delimiter inside another quoted
+ field. The second regexp rules out well-formed input without doubled
+ quotes. Both regexps use possessive repetition to avoid backtracking.
+ """
+ import re
+
+ escaped_delimiter = re.escape(delimiter)
+ escaped_quote = re.escape(quotechar)
+ values = {'delim': escaped_delimiter, 'quote': escaped_quote}
+ if delimiter:
+ # Spaces after a space delimiter are delimiters, not padding.
+ values['space'] = '' if delimiter == ' ' else ' *+'
+ candidate = re.compile(
+ r"(?:%(delim)s|\r|^)%(space)s%(quote)s"
+ r"[^%(quote)s]*+%(quote)s%(quote)s"
+ r"(?:%(quote)s%(quote)s|[^%(quote)s]++)*+"
+ r"%(quote)s(?:%(delim)s|(?=\r)|$)"
+ % values, re.MULTILINE)
+ separator = rf"(?:{escaped_delimiter}|\r\n|\r|\n)"
+ plain = (
+ rf"(?! *+{escaped_quote})"
+ rf"[^{escaped_delimiter}\r\n]*+")
+ else:
+ # An empty delimiter must not create a zero-width alternative
+ # which makes re.search() retry the pattern at every position.
+ candidate = re.compile(
+ r"(?:[\r\n]|\A) *+%(quote)s"
+ r"[^%(quote)s]*+%(quote)s%(quote)s"
+ r"(?:%(quote)s%(quote)s|[^%(quote)s]++)*+"
+ r"%(quote)s(?=[\r\n]|\Z)"
+ % values, re.MULTILINE)
+ separator = r"(?:\r\n|\r|\n)"
+ plain = rf"(?! *+{escaped_quote})[^\r\n]*+"
+
+ if candidate.search(data) is None:
+ return False
+
+ quoted = (
+ rf" *+{escaped_quote}[^{escaped_quote}]*+{escaped_quote}")
+ field = rf"(?:{quoted}|{plain})"
+ without_doublequote = re.compile(
+ rf"\A{field}(?:{separator}{field})*+\Z")
+ return without_doublequote.match(data) is None
 
 
  def _guess_delimiter(self, data, delimiters):

Lib/test/test_csv.py

@@ -1501,6 +1501,25 @@ def test_delimiters(self):
  self.assertEqual(dialect.delimiter, ',')
  self.assertEqual(dialect.quotechar, '"')
 
+ def test_sniff_regex_backtracking(self):
+ # gh-109638: this artificial sample used to take minutes.
+ sniffer = csv.Sniffer()
+ sample = '"",' * 100 + '"' * 100 + '0' + '"' * 100 + '0'
+ self.assertEqual(sniffer.sniff(sample).delimiter, ',')
+
+ def test_sniff_space_delimiter(self):
+ # This sample used to be quadratic.
+ sniffer = csv.Sniffer()
+ sample = '"a" "b"\n' + ' ' * 100_000
+ dialect = sniffer.sniff(sample)
+ self.assertEqual(dialect.delimiter, ' ')
+ self.assertIs(dialect.doublequote, False)
+
+ # A quoted field can still start after multiple space delimiters.
+ dialect = sniffer.sniff('"a" "b""c"')
+ self.assertEqual(dialect.delimiter, ' ')
+ self.assertIs(dialect.doublequote, True)
+
  def test_doublequote(self):
  sniffer = csv.Sniffer()
  dialect = sniffer.sniff(self.header1)
@@ -1514,6 +1533,86 @@ def test_doublequote(self):
  dialect = sniffer.sniff(self.sample9)
  self.assertTrue(dialect.doublequote)
 
+ def test_doublequote_without_delimiter(self):
+ sniffer = csv.Sniffer()
+ for quotechar in ('"', "'"):
+ with self.subTest(quotechar=quotechar):
+ self.assertEqual(
+ sniffer._guess_quote_and_delimiter(
+ f'{quotechar}a{quotechar}{quotechar}b{quotechar}',
+ None,
+ ),
+ (quotechar, True, '', 0),
+ )
+ self.assertEqual(
+ sniffer._guess_quote_and_delimiter(
+ f'{quotechar}ab{quotechar}',
+ None,
+ ),
+ (quotechar, False, '', 0),
+ )
+ self.assertEqual(
+ sniffer._guess_quote_and_delimiter(
+ f'{quotechar}a{quotechar}\n'
+ f'x{quotechar}{quotechar}{quotechar}{quotechar}x',
+ None,
+ ),
+ (quotechar, False, '', 0),
+ )
+
+ def test_doublequote_with_carriage_return(self):
+ sniffer = csv.Sniffer()
+ for record_delimiter in ('\r\n', '\r'):
+ for quotechar in ('"', "'"):
+ with self.subTest(
+ record_delimiter=record_delimiter,
+ quotechar=quotechar,
+ ):
+ data = (
+ f'x,{quotechar}plain{quotechar}{record_delimiter}'
+ f'{quotechar}a{quotechar}{quotechar}b{quotechar},y'
+ f'{record_delimiter}'
+ )
+ self.assertEqual(
+ sniffer._guess_quote_and_delimiter(data, None),
+ (quotechar, True, ',', 0),
+ )
+ self.assertIs(sniffer.sniff(data).doublequote, True)
+
+ def test_doublequote_across_quoted_fields(self):
+ sniffer = csv.Sniffer()
+ for delimiter in (',', ' ', ''):
+ for quotechar in ('"', "'"):
+ with self.subTest(
+ delimiter=delimiter,
+ quotechar=quotechar,
+ ):
+ separator = delimiter or '\n'
+ data = separator.join((
+ f'{quotechar}{separator}{quotechar}',
+ quotechar * 2,
+ f'{quotechar}{separator}{quotechar}',
+ ))
+ result = sniffer._guess_quote_and_delimiter(data, None)
+ self.assertEqual(result[0], quotechar)
+ self.assertIs(result[1], False)
+ self.assertEqual(result[2], delimiter)
+ if delimiter:
+ self.assertIs(
+ sniffer.sniff(data).doublequote,
+ False,
+ )
+ for literal_quote_count in range(1, 5):
+ with self.subTest(
+ literal_quote_count=literal_quote_count,
+ ):
+ self.assertFalse(sniffer._detect_doublequote(
+ f'a{quotechar * literal_quote_count}b'
+ f'{separator}{data}',
+ delimiter,
+ quotechar,
+ ))
+
  def test_guess_delimiter_crlf_not_chosen(self):
  # Ensure that we pick the real delimiter ("|") over "\r" in a tie.
  sniffer = csv.Sniffer()

Misc/NEWS.d/next/Library/2026-07-29-13-00-00.gh-issue-109638.SnifF1.rst

@@ -0,0 +1,3 @@
+Prevent pathological regular expression backtracking in :class:`csv.Sniffer`
+when detecting doubled quote characters. Improve detection for quoted fields,
+avoid matching across quoted fields, and handle CRLF or CR record delimiters.