Test failure on Python 3.14

[gotmax23]

One of the tests is failing on Python 3.14 due to changes in the tarfile module. See https://peps.python.org/pep-0706/. Starting in Python 3.14, tarfile.extractall defaults to data_filter() when unpacking archives which does not allow special files.

Downstream bug: https://bugzilla.redhat.com/show_bug.cgi?id=2325197

____________________________ TypeTest.test_get_type ____________________________
self = <test_filetype.TypeTest testMethod=test_get_type>
 def test_get_type(self):
> test_dir = self.extract_test_tar("filetype/types.tar", verbatim=True)
tests/test_filetype.py:37: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
src/commoncode/testcase.py:220: in extract_test_tar
 return self.__extract(test_path, extract_tar, verbatim)
src/commoncode/testcase.py:210: in __extract
 extract_func(original_archive, target_dir, verbatim=verbatim)
src/commoncode/archive.py:61: in extract_tar
 tar.extractall(target_dir, members=to_extract)
/usr/lib64/python3.14/tarfile.py:2389: in extractall
 tarinfo = self._get_extract_tarinfo(member, filter_function, path)
/usr/lib64/python3.14/tarfile.py:2443: in _get_extract_tarinfo
 self._handle_fatal_error(e)
/usr/lib64/python3.14/tarfile.py:2441: in _get_extract_tarinfo
 tarinfo = filter_function(tarinfo, path)
/usr/lib64/python3.14/tarfile.py:842: in data_filter
 new_attrs = _get_filtered_attrs(member, dest_path, True)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
member = <TarInfo '6-FIFOTYPE' at 0x7fff8df6b400>
dest_path = '/tmp/scancode-tk-tests -_mtr2thg/2dqbpc1l/types.tar'
for_data = True
 def _get_filtered_attrs(member, dest_path, for_data=True):
 new_attrs = {}
 name = member.name
 dest_path = os.path.realpath(dest_path)
 # Strip leading / (tar's directory separator) from filenames.
 # Include os.sep (target OS directory separator) as well.
 if name.startswith(('/', os.sep)):
 name = new_attrs['name'] = member.path.lstrip('/' + os.sep)
 if os.path.isabs(name):
 # Path is absolute even after stripping.
 # For example, 'C:/foo' on Windows.
 raise AbsolutePathError(member)
 # Ensure we stay in the destination
 target_path = os.path.realpath(os.path.join(dest_path, name))
 if os.path.commonpath([target_path, dest_path]) != dest_path:
 raise OutsideDestinationError(member, target_path)
 # Limit permissions (no high bits, and go-w)
 mode = member.mode
 if mode is not None:
 # Strip high bits & group/other write bits
 mode = mode & 0o755
 if for_data:
 # For data, handle permissions & file types
 if member.isreg() or member.islnk():
 if not mode & 0o100:
 # Clear executable bits if not executable by user
 mode &= ~0o111
 # Ensure owner can read & write
 mode |= 0o600
 elif member.isdir() or member.issym():
 # Ignore mode for directories & symlinks
 mode = None
 else:
 # Reject special files
> raise SpecialFileError(member)
E tarfile.SpecialFileError: '6-FIFOTYPE' is a special file
/usr/lib64/python3.14/tarfile.py:803: SpecialFileError

[gotmax23]

See also #75 where the associated tarfile DeprecationWarning was originally reported.

[AyanSinhaMahapatra]

Thanks for pointing this out @gotmax23, this should be fixed before we have a stable python3.14 release as it's a blocker otherwise.

[AyanSinhaMahapatra]

I've modified the test archive to remove the special '6-FIFOTYPE' file and also modified the test expectations for now. It does not seem like we are reducing any test coverage with this, but we can always reopen this later if so.

Thanks for the report @gotmax23, we also have a commoncode release with python 3.14 support, now that we have a stable python3.14 release out.

[AyanSinhaMahapatra]

It is actually better to set the filter='fully_trusted' in extractall() functions as this is used in a very specific context of extracting archives meant as testfiles (which we create ourselves), so these are trusted and safe.

This was also needed for aboutcode-org/scancode-toolkit#4595 as we were failing some tests because of symlinks present in test archives for alpine package detection tests.

[stefan6419846]

Strangely enough, this works on Python 3.14 now, but fails on the current Python 3.15 alpha:

____________________________ TypeTest.test_get_type ____________________________
[gw2] linux -- Python 3.15.0 /opt/hostedtoolcache/Python/3.15.0-alpha.5/x64/bin/python

self = <test_filetype.TypeTest testMethod=test_get_type>

 def test_get_type(self):
> test_dir = self.extract_test_tar(
 "filetype/types.tar", verbatim=True, filter="fully_trusted"
 )

tests/test_filetype.py:37: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/opt/hostedtoolcache/Python/3.15.0-alpha.5/x64/lib/python3.15/site-packages/commoncode/testcase.py:219: in extract_test_tar
 return self.__extract(test_path, extract_tar, verbatim, filter)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
/opt/hostedtoolcache/Python/3.15.0-alpha.5/x64/lib/python3.15/site-packages/commoncode/testcase.py:209: in __extract
 extract_func(original_archive, target_dir, verbatim=verbatim, filter=filter)
/opt/hostedtoolcache/Python/3.15.0-alpha.5/x64/lib/python3.15/site-packages/commoncode/archive.py:65: in extract_tar
 tar.extractall(target_dir, members=to_extract)
/opt/hostedtoolcache/Python/3.15.0-alpha.5/x64/lib/python3.15/tarfile.py:2408: in extractall
 tarinfo, unfiltered = self._get_extract_tarinfo(
/opt/hostedtoolcache/Python/3.15.0-alpha.5/x64/lib/python3.15/tarfile.py:2497: in _get_extract_tarinfo
 self._handle_fatal_error(e)
/opt/hostedtoolcache/Python/3.15.0-alpha.5/x64/lib/python3.15/tarfile.py:2495: in _get_extract_tarinfo
 filtered = filter_function(unfiltered, path)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
/opt/hostedtoolcache/Python/3.15.0-alpha.5/x64/lib/python3.15/tarfile.py:858: in data_filter
 new_attrs = _get_filtered_attrs(member, dest_path, True)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

member = <TarInfo '6-FIFOTYPE' at 0x7f36b78dc7c0>
dest_path = '/tmp/scancode-tk-tests -d7q2jp9v/cew9uh0b/types.tar'
for_data = True

 def _get_filtered_attrs(member, dest_path, for_data=True):
 new_attrs = {}
 name = member.name
 dest_path = os.path.realpath(dest_path, strict=os.path.ALLOW_MISSING)
 # Strip leading / (tar's directory separator) from filenames.
 # Include os.sep (target OS directory separator) as well.
 if name.startswith(('/', os.sep)):
 name = new_attrs['name'] = member.path.lstrip('/' + os.sep)
 if os.path.isabs(name):
 # Path is absolute even after stripping.
 # For example, 'C:/foo' on Windows.
 raise AbsolutePathError(member)
 # Ensure we stay in the destination
 target_path = os.path.realpath(os.path.join(dest_path, name),
 strict=os.path.ALLOW_MISSING)
 if os.path.commonpath([target_path, dest_path]) != dest_path:
 raise OutsideDestinationError(member, target_path)
 # Limit permissions (no high bits, and go-w)
 mode = member.mode
 if mode is not None:
 # Strip high bits & group/other write bits
 mode = mode & 0o755
 if for_data:
 # For data, handle permissions & file types
 if member.isreg() or member.islnk():
 if not mode & 0o100:
 # Clear executable bits if not executable by user
 mode &= ~0o111
 # Ensure owner can read & write
 mode |= 0o600
 elif member.isdir() or member.issym():
 # Ignore mode for directories & symlinks
 mode = None
 else:
 # Reject special files
> raise SpecialFileError(member)
E tarfile.SpecialFileError: '6-FIFOTYPE' is a special file

/opt/hostedtoolcache/Python/3.15.0-alpha.5/x64/lib/python3.15/tarfile.py:815: SpecialFileError