From 18a9bf0caf805afb99adbc209dd06a43d67219d9 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:08:17 +0200 Subject: [PATCH 1/4] Update `configparser.py` from 3.13.5 (#6062) --- Lib/configparser.py | 370 +++++++++++++++----------- Lib/test/{ => configdata}/cfgparser.1 | 0 Lib/test/{ => configdata}/cfgparser.2 | 0 Lib/test/{ => configdata}/cfgparser.3 | 0 Lib/test/test_configparser.py | 201 ++++++++++---- 5 files changed, 360 insertions(+), 211 deletions(-) rename Lib/test/{ => configdata}/cfgparser.1 (100%) rename Lib/test/{ => configdata}/cfgparser.2 (100%) rename Lib/test/{ => configdata}/cfgparser.3 (100%) diff --git a/Lib/configparser.py b/Lib/configparser.py index e8aae21794..05b86acb91 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -18,8 +18,8 @@ delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', - interpolation=, converters=): - + interpolation=, converters=, + allow_unnamed_section=False): Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. @@ -68,6 +68,10 @@ converter gets its corresponding get*() method on the parser object and section proxies. + When `allow_unnamed_section` is True (default: False), options + without section are accepted: the section for these is + ``configparser.UNNAMED_SECTION``. + sections() Return all the configuration section names, sans DEFAULT. @@ -139,24 +143,28 @@ between keys and values are surrounded by spaces. """ -from collections.abc import MutableMapping +# Do not import dataclasses; overhead is unacceptable (gh-117703) + +from collections.abc import Iterable, MutableMapping from collections import ChainMap as _ChainMap +import contextlib import functools import io import itertools import os import re import sys -import warnings +import types __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", + "MultilineContinuationError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", - "LegacyInterpolation", "SectionProxy", "ConverterMapping", - "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH") + "SectionProxy", "ConverterMapping", + "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") _default_dict = dict DEFAULTSECT = "DEFAULT" @@ -298,15 +306,33 @@ def __init__(self, option, section, rawval): class ParsingError(Error): """Raised when a configuration file does not follow legal syntax.""" - def __init__(self, source): + def __init__(self, source, *args): super().__init__(f'Source contains parsing errors: {source!r}') self.source = source self.errors = [] self.args = (source, ) + if args: + self.append(*args) def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, line) + self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + + def combine(self, others): + for other in others: + for error in other.errors: + self.append(*error) + return self + + @staticmethod + def _raise_all(exceptions: Iterable['ParsingError']): + """ + Combine any number of ParsingErrors into one and raise it. + """ + exceptions = iter(exceptions) + with contextlib.suppress(StopIteration): + raise next(exceptions).combine(exceptions) + class MissingSectionHeaderError(ParsingError): @@ -323,6 +349,28 @@ def __init__(self, filename, lineno, line): self.args = (filename, lineno, line) +class MultilineContinuationError(ParsingError): + """Raised when a key without value is followed by continuation line""" + def __init__(self, filename, lineno, line): + Error.__init__( + self, + "Key without value continued with an indented line.\n" + "file: %r, line: %d\n%r" + %(filename, lineno, line)) + self.source = filename + self.lineno = lineno + self.line = line + self.args = (filename, lineno, line) + +class _UnnamedSection: + + def __repr__(self): + return "" + + +UNNAMED_SECTION = _UnnamedSection() + + # Used in parser getters to indicate the default behaviour when a specific # option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. @@ -478,6 +526,8 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, except (KeyError, NoSectionError, NoOptionError): raise InterpolationMissingOptionError( option, section, rawval, ":".join(path)) from None + if v is None: + continue if "$" in v: self._interpolate_some(parser, opt, accum, v, sect, dict(parser.items(sect, raw=True)), @@ -491,51 +541,50 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, "found: %r" % (rest,)) -class LegacyInterpolation(Interpolation): - """Deprecated interpolation used in old versions of ConfigParser. - Use BasicInterpolation or ExtendedInterpolation instead.""" +class _ReadState: + elements_added : set[str] + cursect : dict[str, str] | None = None + sectname : str | None = None + optname : str | None = None + lineno : int = 0 + indent_level : int = 0 + errors : list[ParsingError] - _KEYCRE = re.compile(r"%\(([^)]*)\)s|.") + def __init__(self): + self.elements_added = set() + self.errors = list() - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - warnings.warn( - "LegacyInterpolation has been deprecated since Python 3.2 " - "and will be removed from the configparser module in Python 3.13. " - "Use BasicInterpolation or ExtendedInterpolation instead.", - DeprecationWarning, stacklevel=2 - ) - def before_get(self, parser, section, option, value, vars): - rawval = value - depth = MAX_INTERPOLATION_DEPTH - while depth: # Loop through this until it's done - depth -= 1 - if value and "%(" in value: - replace = functools.partial(self._interpolation_replace, - parser=parser) - value = self._KEYCRE.sub(replace, value) - try: - value = value % vars - except KeyError as e: - raise InterpolationMissingOptionError( - option, section, rawval, e.args[0]) from None - else: - break - if value and "%(" in value: - raise InterpolationDepthError(option, section, rawval) - return value +class _Line(str): - def before_set(self, parser, section, option, value): - return value + def __new__(cls, val, *args, **kwargs): + return super().__new__(cls, val) - @staticmethod - def _interpolation_replace(match, parser): - s = match.group(1) - if s is None: - return match.group() - else: - return "%%(%s)s" % parser.optionxform(s) + def __init__(self, val, prefixes): + self.prefixes = prefixes + + @functools.cached_property + def clean(self): + return self._strip_full() and self._strip_inline() + + @property + def has_comments(self): + return self.strip() != self.clean + + def _strip_inline(self): + """ + Search for the earliest prefix at the beginning of the line or following a space. + """ + matcher = re.compile( + '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) + # match nothing if no prefixes + or '(?!)' + ) + match = matcher.search(self) + return self[:match.start() if match else None].strip() + + def _strip_full(self): + return '' if any(map(self.strip().startswith, self.prefixes.full)) else True class RawConfigParser(MutableMapping): @@ -584,7 +633,8 @@ def __init__(self, defaults=None, dict_type=_default_dict, comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, - interpolation=_UNSET, converters=_UNSET): + interpolation=_UNSET, converters=_UNSET, + allow_unnamed_section=False,): self._dict = dict_type self._sections = self._dict() @@ -603,8 +653,10 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._comment_prefixes = tuple(comment_prefixes or ()) - self._inline_comment_prefixes = tuple(inline_comment_prefixes or ()) + self._prefixes = types.SimpleNamespace( + full=tuple(comment_prefixes or ()), + inline=tuple(inline_comment_prefixes or ()), + ) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values @@ -623,6 +675,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, self._converters.update(converters) if defaults: self._read_defaults(defaults) + self._allow_unnamed_section = allow_unnamed_section def defaults(self): return self._defaults @@ -896,13 +949,19 @@ def write(self, fp, space_around_delimiters=True): if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) + if UNNAMED_SECTION in self._sections: + self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True) + for section in self._sections: + if section is UNNAMED_SECTION: + continue self._write_section(fp, section, self._sections[section].items(), d) - def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp`.""" - fp.write("[{}]\n".format(section_name)) + def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): + """Write a single section to the specified `fp'.""" + if not unnamed: + fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, value) @@ -988,110 +1047,113 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - elements_added = set() - cursect = None # None, or a dictionary - sectname = None - optname = None - lineno = 0 - indent_level = 0 - e = None # None, or an exception - for lineno, line in enumerate(fp, start=1): - comment_start = sys.maxsize - # strip inline comments - inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} - while comment_start == sys.maxsize and inline_prefixes: - next_prefixes = {} - for prefix, index in inline_prefixes.items(): - index = line.find(prefix, index+1) - if index == -1: - continue - next_prefixes[prefix] = index - if index == 0 or (index > 0 and line[index-1].isspace()): - comment_start = min(comment_start, index) - inline_prefixes = next_prefixes - # strip full line comments - for prefix in self._comment_prefixes: - if line.strip().startswith(prefix): - comment_start = 0 - break - if comment_start == sys.maxsize: - comment_start = None - value = line[:comment_start].strip() - if not value: + + try: + ParsingError._raise_all(self._read_inner(fp, fpname)) + finally: + self._join_multiline_values() + + def _read_inner(self, fp, fpname): + st = _ReadState() + + Line = functools.partial(_Line, prefixes=self._prefixes) + for st.lineno, line in enumerate(map(Line, fp), start=1): + if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no # comment on the line - if (comment_start is None and - cursect is not None and - optname and - cursect[optname] is not None): - cursect[optname].append('') # newlines added at join + if (not line.has_comments and + st.cursect is not None and + st.optname and + st.cursect[st.optname] is not None): + st.cursect[st.optname].append('') # newlines added at join else: # empty line marks end of value - indent_level = sys.maxsize + st.indent_level = sys.maxsize continue - # continuation line? + first_nonspace = self.NONSPACECRE.search(line) - cur_indent_level = first_nonspace.start() if first_nonspace else 0 - if (cursect is not None and optname and - cur_indent_level > indent_level): - cursect[optname].append(value) - # a section header or option header? - else: - indent_level = cur_indent_level - # is it a section header? - mo = self.SECTCRE.match(value) - if mo: - sectname = mo.group('header') - if sectname in self._sections: - if self._strict and sectname in elements_added: - raise DuplicateSectionError(sectname, fpname, - lineno) - cursect = self._sections[sectname] - elements_added.add(sectname) - elif sectname == self.default_section: - cursect = self._defaults - else: - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - # So sections can't start with a continuation line - optname = None - # no section header in the file? - elif cursect is None: - raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? - else: - mo = self._optcre.match(value) - if mo: - optname, vi, optval = mo.group('option', 'vi', 'value') - if not optname: - e = self._handle_error(e, fpname, lineno, line) - optname = self.optionxform(optname.rstrip()) - if (self._strict and - (sectname, optname) in elements_added): - raise DuplicateOptionError(sectname, optname, - fpname, lineno) - elements_added.add((sectname, optname)) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - optval = optval.strip() - cursect[optname] = [optval] - else: - # valueless option handling - cursect[optname] = None - else: - # a non-fatal parsing error occurred. set up the - # exception but keep going. the exception will be - # raised at the end of the file and will contain a - # list of all bogus lines - e = self._handle_error(e, fpname, lineno, line) - self._join_multiline_values() - # if any parsing errors occurred, raise an exception - if e: - raise e + st.cur_indent_level = first_nonspace.start() if first_nonspace else 0 + + if self._handle_continuation_line(st, line, fpname): + continue + + self._handle_rest(st, line, fpname) + + return st.errors + + def _handle_continuation_line(self, st, line, fpname): + # continuation line? + is_continue = (st.cursect is not None and st.optname and + st.cur_indent_level > st.indent_level) + if is_continue: + if st.cursect[st.optname] is None: + raise MultilineContinuationError(fpname, st.lineno, line) + st.cursect[st.optname].append(line.clean) + return is_continue + + def _handle_rest(self, st, line, fpname): + # a section header or option header? + if self._allow_unnamed_section and st.cursect is None: + self._handle_header(st, UNNAMED_SECTION, fpname) + + st.indent_level = st.cur_indent_level + # is it a section header? + mo = self.SECTCRE.match(line.clean) + + if not mo and st.cursect is None: + raise MissingSectionHeaderError(fpname, st.lineno, line) + + self._handle_header(st, mo.group('header'), fpname) if mo else self._handle_option(st, line, fpname) + + def _handle_header(self, st, sectname, fpname): + st.sectname = sectname + if st.sectname in self._sections: + if self._strict and st.sectname in st.elements_added: + raise DuplicateSectionError(st.sectname, fpname, + st.lineno) + st.cursect = self._sections[st.sectname] + st.elements_added.add(st.sectname) + elif st.sectname == self.default_section: + st.cursect = self._defaults + else: + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + # So sections can't start with a continuation line + st.optname = None + + def _handle_option(self, st, line, fpname): + # an option line? + st.indent_level = st.cur_indent_level + + mo = self._optcre.match(line.clean) + if not mo: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + st.errors.append(ParsingError(fpname, st.lineno, line)) + return + + st.optname, vi, optval = mo.group('option', 'vi', 'value') + if not st.optname: + st.errors.append(ParsingError(fpname, st.lineno, line)) + st.optname = self.optionxform(st.optname.rstrip()) + if (self._strict and + (st.sectname, st.optname) in st.elements_added): + raise DuplicateOptionError(st.sectname, st.optname, + fpname, st.lineno) + st.elements_added.add((st.sectname, st.optname)) + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + optval = optval.strip() + st.cursect[st.optname] = [optval] + else: + # valueless option handling + st.cursect[st.optname] = None def _join_multiline_values(self): defaults = self.default_section, self._defaults @@ -1111,12 +1173,6 @@ def _read_defaults(self, defaults): for key, value in defaults.items(): self._defaults[self.optionxform(key)] = value - def _handle_error(self, exc, fpname, lineno, line): - if not exc: - exc = ParsingError(fpname) - exc.append(lineno, repr(line)) - return exc - def _unify_values(self, section, vars): """Create a sequence of lookups with 'vars' taking priority over the 'section' which takes priority over the DEFAULTSECT. diff --git a/Lib/test/cfgparser.1 b/Lib/test/configdata/cfgparser.1 similarity index 100% rename from Lib/test/cfgparser.1 rename to Lib/test/configdata/cfgparser.1 diff --git a/Lib/test/cfgparser.2 b/Lib/test/configdata/cfgparser.2 similarity index 100% rename from Lib/test/cfgparser.2 rename to Lib/test/configdata/cfgparser.2 diff --git a/Lib/test/cfgparser.3 b/Lib/test/configdata/cfgparser.3 similarity index 100% rename from Lib/test/cfgparser.3 rename to Lib/test/configdata/cfgparser.3 diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 01e8e6c675..d793cc5890 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,10 +2,8 @@ import configparser import io import os -import pathlib import textwrap import unittest -import warnings from test import support from test.support import os_helper @@ -545,7 +543,7 @@ def test_parse_errors(self): "[Foo]\n wrong-indent\n") self.assertEqual(e.args, ('',)) # read_file on a real file - tricky = support.findfile("cfgparser.3") + tricky = support.findfile("cfgparser.3", subdir="configdata") if self.delimiters[0] == '=': error = configparser.ParsingError expected = (tricky,) @@ -648,6 +646,21 @@ def test_weird_errors(self): "'opt' in section 'Bar' already exists") self.assertEqual(e.args, ("Bar", "opt", "", None)) + def test_get_after_duplicate_option_error(self): + cf = self.newconfig() + ini = textwrap.dedent("""\ + [Foo] + x{equals}1 + y{equals}2 + y{equals}3 + """.format(equals=self.delimiters[0])) + if self.strict: + with self.assertRaises(configparser.DuplicateOptionError): + cf.read_string(ini) + else: + cf.read_string(ini) + self.assertEqual(cf.get('Foo', 'x'), '1') + def test_write(self): config_string = ( "[Long Line]\n" @@ -719,7 +732,7 @@ class mystr(str): def test_read_returns_file_list(self): if self.delimiters[0] != '=': self.skipTest('incompatible format') - file1 = support.findfile("cfgparser.1") + file1 = support.findfile("cfgparser.1", subdir="configdata") # check when we pass a mix of readable and non-readable files: cf = self.newconfig() parsed_files = cf.read([file1, "nonexistent-file"], encoding="utf-8") @@ -732,12 +745,12 @@ def test_read_returns_file_list(self): self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a Path object: cf = self.newconfig() - parsed_files = cf.read(pathlib.Path(file1), encoding="utf-8") + parsed_files = cf.read(os_helper.FakePath(file1), encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we passed both a filename and a Path object: cf = self.newconfig() - parsed_files = cf.read([pathlib.Path(file1), file1], encoding="utf-8") + parsed_files = cf.read([os_helper.FakePath(file1), file1], encoding="utf-8") self.assertEqual(parsed_files, [file1, file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: @@ -753,7 +766,7 @@ def test_read_returns_file_list(self): def test_read_returns_file_list_with_bytestring_path(self): if self.delimiters[0] != '=': self.skipTest('incompatible format') - file1_bytestring = support.findfile("cfgparser.1").encode() + file1_bytestring = support.findfile("cfgparser.1", subdir="configdata").encode() # check when passing an existing bytestring path cf = self.newconfig() parsed_files = cf.read(file1_bytestring, encoding="utf-8") @@ -909,9 +922,6 @@ def test_interpolation(self): if self.interpolation == configparser._UNSET: self.assertEqual(e.args, ("bar11", "Foo", "something %(with11)s lots of interpolation (11 steps)")) - elif isinstance(self.interpolation, configparser.LegacyInterpolation): - self.assertEqual(e.args, ("bar11", "Foo", - "something %(with11)s lots of interpolation (11 steps)")) def test_interpolation_missing_value(self): cf = self.get_interpolation_config() @@ -923,9 +933,6 @@ def test_interpolation_missing_value(self): if self.interpolation == configparser._UNSET: self.assertEqual(e.args, ('name', 'Interpolation Error', '%(reference)s', 'reference')) - elif isinstance(self.interpolation, configparser.LegacyInterpolation): - self.assertEqual(e.args, ('name', 'Interpolation Error', - '%(reference)s', 'reference')) def test_items(self): self.check_items_config([('default', ''), @@ -944,9 +951,6 @@ def test_safe_interpolation(self): self.assertEqual(cf.get("section", "ok"), "xxx/%s") if self.interpolation == configparser._UNSET: self.assertEqual(cf.get("section", "not_ok"), "xxx/xxx/%s") - elif isinstance(self.interpolation, configparser.LegacyInterpolation): - with self.assertRaises(TypeError): - cf.get("section", "not_ok") def test_set_malformatted_interpolation(self): cf = self.fromstring("[sect]\n" @@ -1027,31 +1031,6 @@ class CustomConfigParser(configparser.ConfigParser): cf.read_string(self.ini) self.assertMatchesIni(cf) - -class ConfigParserTestCaseLegacyInterpolation(ConfigParserTestCase): - config_class = configparser.ConfigParser - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - interpolation = configparser.LegacyInterpolation() - - def test_set_malformatted_interpolation(self): - cf = self.fromstring("[sect]\n" - "option1{eq}foo\n".format(eq=self.delimiters[0])) - - self.assertEqual(cf.get('sect', "option1"), "foo") - - cf.set("sect", "option1", "%foo") - self.assertEqual(cf.get('sect', "option1"), "%foo") - cf.set("sect", "option1", "foo%") - self.assertEqual(cf.get('sect', "option1"), "foo%") - cf.set("sect", "option1", "f%oo") - self.assertEqual(cf.get('sect', "option1"), "f%oo") - - # bug #5741: double percents are *not* malformed - cf.set("sect", "option2", "foo%%bar") - self.assertEqual(cf.get("sect", "option2"), "foo%%bar") - - class ConfigParserTestCaseInvalidInterpolationType(unittest.TestCase): def test_error_on_wrong_type_for_interpolation(self): for value in [configparser.ExtendedInterpolation, 42, "a string"]: @@ -1163,7 +1142,7 @@ class RawConfigParserTestSambaConf(CfgParserTestCaseClass, unittest.TestCase): empty_lines_in_values = False def test_reading(self): - smbconf = support.findfile("cfgparser.2") + smbconf = support.findfile("cfgparser.2", subdir="configdata") # check when we pass a mix of readable and non-readable files: cf = self.newconfig() parsed_files = cf.read([smbconf, "nonexistent-file"], encoding='utf-8') @@ -1351,6 +1330,47 @@ class ConfigParserTestCaseNoValue(ConfigParserTestCase): allow_no_value = True +class NoValueAndExtendedInterpolation(CfgParserTestCaseClass): + interpolation = configparser.ExtendedInterpolation() + allow_no_value = True + + def test_interpolation_with_allow_no_value(self): + config = textwrap.dedent(""" + [dummy] + a + b = ${a} + """) + cf = self.fromstring(config) + + self.assertIs(cf["dummy"]["a"], None) + self.assertEqual(cf["dummy"]["b"], "") + + def test_explicit_none(self): + config = textwrap.dedent(""" + [dummy] + a = None + b = ${a} + """) + cf = self.fromstring(config) + + self.assertEqual(cf["dummy"]["a"], "None") + self.assertEqual(cf["dummy"]["b"], "None") + + +class ConfigParserNoValueAndExtendedInterpolationTest( + NoValueAndExtendedInterpolation, + unittest.TestCase, +): + config_class = configparser.ConfigParser + + +class RawConfigParserNoValueAndExtendedInterpolationTest( + NoValueAndExtendedInterpolation, + unittest.TestCase, +): + config_class = configparser.RawConfigParser + + class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass, unittest.TestCase): config_class = configparser.ConfigParser delimiters = {'='} @@ -1358,7 +1378,7 @@ class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass, unittest.TestCase): allow_no_value = True def test_cfgparser_dot_3(self): - tricky = support.findfile("cfgparser.3") + tricky = support.findfile("cfgparser.3", subdir="configdata") cf = self.newconfig() self.assertEqual(len(cf.read(tricky, encoding='utf-8')), 1) self.assertEqual(cf.sections(), ['strange', @@ -1390,7 +1410,7 @@ def test_cfgparser_dot_3(self): self.assertEqual(cf.get('more interpolation', 'lets'), 'go shopping') def test_unicode_failure(self): - tricky = support.findfile("cfgparser.3") + tricky = support.findfile("cfgparser.3", subdir="configdata") cf = self.newconfig() with self.assertRaises(UnicodeDecodeError): cf.read(tricky, encoding='ascii') @@ -1491,7 +1511,7 @@ def fromstring(self, string, defaults=None): class FakeFile: def __init__(self): - file_path = support.findfile("cfgparser.1") + file_path = support.findfile("cfgparser.1", subdir="configdata") with open(file_path, encoding="utf-8") as f: self.lines = f.readlines() self.lines.reverse() @@ -1512,7 +1532,7 @@ def readline_generator(f): class ReadFileTestCase(unittest.TestCase): def test_file(self): - file_paths = [support.findfile("cfgparser.1")] + file_paths = [support.findfile("cfgparser.1", subdir="configdata")] try: file_paths.append(file_paths[0].encode('utf8')) except UnicodeEncodeError: @@ -1592,6 +1612,30 @@ def test_source_as_bytes(self): "'[badbad'" ) + def test_keys_without_value_with_extra_whitespace(self): + lines = [ + '[SECT]\n', + 'KEY1\n', + ' KEY2 = VAL2\n', # note the Space before the key! + ] + parser = configparser.ConfigParser( + comment_prefixes="", + allow_no_value=True, + strict=False, + delimiters=('=',), + interpolation=None, + ) + with self.assertRaises(configparser.MultilineContinuationError) as dse: + parser.read_file(lines) + self.assertEqual( + str(dse.exception), + "Key without value continued with an indented line.\n" + "file: '', line: 3\n" + "' KEY2 = VAL2\\n'" + ) + + + class CoverageOneHundredTestCase(unittest.TestCase): """Covers edge cases in the codebase.""" @@ -1638,14 +1682,6 @@ def test_interpolation_validation(self): self.assertEqual(str(cm.exception), "bad interpolation variable " "reference '%(()'") - def test_legacyinterpolation_deprecation(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always", DeprecationWarning) - configparser.LegacyInterpolation() - self.assertGreaterEqual(len(w), 1) - for warning in w: - self.assertIs(warning.category, DeprecationWarning) - def test_sectionproxy_repr(self): parser = configparser.ConfigParser() parser.read_string(""" @@ -2121,6 +2157,63 @@ def test_instance_assignment(self): self.assertEqual(cfg['two'].getlen('one'), 5) +class SectionlessTestCase(unittest.TestCase): + + def fromstring(self, string): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.read_string(string) + return cfg + + def test_no_first_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + [sect1] + c = 3 + """) + + self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg1.sections())) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg1['sect1']['c']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + #self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg2.sections())) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg2['sect1']['c']) + + def test_no_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + """) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg1.sections()) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg2.sections()) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + + def test_multiple_configs(self): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.read_string('a = 1') + cfg.read_string('b = 2') + + self.assertEqual([configparser.UNNAMED_SECTION], cfg.sections()) + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg[configparser.UNNAMED_SECTION]['b']) + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, configparser, not_exported={"Error"}) From 566d9aabae6f942c1781e6702124d4075aaf30eb Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:08:35 +0200 Subject: [PATCH 2/4] Update `gettext.py` from 3.13.5 (#6063) --- Lib/gettext.py | 25 ++- Lib/test/test_gettext.py | 445 ++++++++++++++++++++++++++++++++++----- 2 files changed, 419 insertions(+), 51 deletions(-) diff --git a/Lib/gettext.py b/Lib/gettext.py index b72b15f82d..62cff81b7b 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -46,6 +46,7 @@ # find this format documented anywhere. +import operator import os import re import sys @@ -166,14 +167,28 @@ def _parse(tokens, priority=-1): def _as_int(n): try: - i = round(n) + round(n) except TypeError: raise TypeError('Plural value must be an integer, got %s' % (n.__class__.__name__,)) from None + return _as_int2(n) + +def _as_int2(n): + try: + return operator.index(n) + except TypeError: + pass + import warnings + frame = sys._getframe(1) + stacklevel = 2 + while frame.f_back is not None and frame.f_globals.get('__name__') == __name__: + stacklevel += 1 + frame = frame.f_back warnings.warn('Plural value must be an integer, got %s' % (n.__class__.__name__,), - DeprecationWarning, 4) + DeprecationWarning, + stacklevel) return n @@ -200,7 +215,7 @@ def c2py(plural): elif c == ')': depth -= 1 - ns = {'_as_int': _as_int} + ns = {'_as_int': _as_int, '__name__': __name__} exec('''if True: def func(n): if not isinstance(n, int): @@ -280,6 +295,7 @@ def gettext(self, message): def ngettext(self, msgid1, msgid2, n): if self._fallback: return self._fallback.ngettext(msgid1, msgid2, n) + n = _as_int2(n) if n == 1: return msgid1 else: @@ -293,6 +309,7 @@ def pgettext(self, context, message): def npgettext(self, context, msgid1, msgid2, n): if self._fallback: return self._fallback.npgettext(context, msgid1, msgid2, n) + n = _as_int2(n) if n == 1: return msgid1 else: @@ -579,6 +596,7 @@ def dngettext(domain, msgid1, msgid2, n): try: t = translation(domain, _localedirs.get(domain, None)) except OSError: + n = _as_int2(n) if n == 1: return msgid1 else: @@ -598,6 +616,7 @@ def dnpgettext(domain, context, msgid1, msgid2, n): try: t = translation(domain, _localedirs.get(domain, None)) except OSError: + n = _as_int2(n) if n == 1: return msgid1 else: diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 8430fc234d..0653bb762a 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -2,6 +2,8 @@ import base64 import gettext import unittest +import unittest.mock +from functools import partial from test import support from test.support import os_helper @@ -37,6 +39,9 @@ bmsgd2luayAoaW4gIm15IG90aGVyIGNvbnRleHQiKQB3aW5rIHdpbmsA ''' +# .mo file with an invalid magic number +GNU_MO_DATA_BAD_MAGIC_NUMBER = base64.b64encode(b'ABCD') + # This data contains an invalid major version number (5) # An unexpected major version number should be treated as an error when # parsing a .mo file @@ -85,6 +90,49 @@ ciBUQUgKdHJnZ3JrZyB6cmZmbnRyIHBuZ255YnQgeXZvZW5lbC4AYmFjb24Ad2luayB3aW5rAA== ''' +# Corrupt .mo file +# Generated from +# +# msgid "foo" +# msgstr "bar" +# +# with msgfmt --no-hash +# +# The translation offset is changed to 0xFFFFFFFF, +# making it larger than the file size, which should +# raise an error when parsing. +GNU_MO_DATA_CORRUPT = base64.b64encode(bytes([ + 0xDE, 0x12, 0x04, 0x95, # Magic + 0x00, 0x00, 0x00, 0x00, # Version + 0x01, 0x00, 0x00, 0x00, # Message count + 0x1C, 0x00, 0x00, 0x00, # Message offset + 0x24, 0x00, 0x00, 0x00, # Translation offset + 0x00, 0x00, 0x00, 0x00, # Hash table size + 0x2C, 0x00, 0x00, 0x00, # Hash table offset + 0x03, 0x00, 0x00, 0x00, # 1st message length + 0x2C, 0x00, 0x00, 0x00, # 1st message offset + 0x03, 0x00, 0x00, 0x00, # 1st trans length + 0xFF, 0xFF, 0xFF, 0xFF, # 1st trans offset (Modified to make it invalid) + 0x66, 0x6F, 0x6F, 0x00, # Message data + 0x62, 0x61, 0x72, 0x00, # Message data +])) + + +GNU_MO_DATA_BIG_ENDIAN = base64.b64encode(bytes([ + 0x95, 0x04, 0x12, 0xDE, # Magic + 0x00, 0x00, 0x00, 0x00, # Version + 0x00, 0x00, 0x00, 0x01, # Message count + 0x00, 0x00, 0x00, 0x1C, # Message offset + 0x00, 0x00, 0x00, 0x24, # Translation offset + 0x00, 0x00, 0x00, 0x00, # Hash table size + 0x00, 0x00, 0x00, 0x2C, # Hash table offset + 0x00, 0x00, 0x00, 0x03, # 1st message length + 0x00, 0x00, 0x00, 0x2C, # 1st message offset + 0x00, 0x00, 0x00, 0x03, # 1st trans length + 0x00, 0x00, 0x00, 0x30, # 1st trans offset + 0x66, 0x6F, 0x6F, 0x00, # Message data + 0x62, 0x61, 0x72, 0x00, # Message data +])) UMO_DATA = b'''\ 3hIElQAAAAADAAAAHAAAADQAAAAAAAAAAAAAAAAAAABMAAAABAAAAE0AAAAQAAAAUgAAAA8BAABj @@ -109,30 +157,49 @@ LOCALEDIR = os.path.join('xx', 'LC_MESSAGES') MOFILE = os.path.join(LOCALEDIR, 'gettext.mo') +MOFILE_BAD_MAGIC_NUMBER = os.path.join(LOCALEDIR, 'gettext_bad_magic_number.mo') MOFILE_BAD_MAJOR_VERSION = os.path.join(LOCALEDIR, 'gettext_bad_major_version.mo') MOFILE_BAD_MINOR_VERSION = os.path.join(LOCALEDIR, 'gettext_bad_minor_version.mo') +MOFILE_CORRUPT = os.path.join(LOCALEDIR, 'gettext_corrupt.mo') +MOFILE_BIG_ENDIAN = os.path.join(LOCALEDIR, 'gettext_big_endian.mo') UMOFILE = os.path.join(LOCALEDIR, 'ugettext.mo') MMOFILE = os.path.join(LOCALEDIR, 'metadata.mo') +def reset_gettext(): + gettext._localedirs.clear() + gettext._current_domain = 'messages' + gettext._translations.clear() + + class GettextBaseTest(unittest.TestCase): - def setUp(self): - self.addCleanup(os_helper.rmtree, os.path.split(LOCALEDIR)[0]) + @classmethod + def setUpClass(cls): + cls.addClassCleanup(os_helper.rmtree, os.path.split(LOCALEDIR)[0]) if not os.path.isdir(LOCALEDIR): os.makedirs(LOCALEDIR) with open(MOFILE, 'wb') as fp: fp.write(base64.decodebytes(GNU_MO_DATA)) + with open(MOFILE_BAD_MAGIC_NUMBER, 'wb') as fp: + fp.write(base64.decodebytes(GNU_MO_DATA_BAD_MAGIC_NUMBER)) with open(MOFILE_BAD_MAJOR_VERSION, 'wb') as fp: fp.write(base64.decodebytes(GNU_MO_DATA_BAD_MAJOR_VERSION)) with open(MOFILE_BAD_MINOR_VERSION, 'wb') as fp: fp.write(base64.decodebytes(GNU_MO_DATA_BAD_MINOR_VERSION)) + with open(MOFILE_CORRUPT, 'wb') as fp: + fp.write(base64.decodebytes(GNU_MO_DATA_CORRUPT)) + with open(MOFILE_BIG_ENDIAN, 'wb') as fp: + fp.write(base64.decodebytes(GNU_MO_DATA_BIG_ENDIAN)) with open(UMOFILE, 'wb') as fp: fp.write(base64.decodebytes(UMO_DATA)) with open(MMOFILE, 'wb') as fp: fp.write(base64.decodebytes(MMO_DATA)) + + def setUp(self): self.env = self.enterContext(os_helper.EnvironmentVarGuard()) self.env['LANGUAGE'] = 'xx' - gettext._translations.clear() + reset_gettext() + self.addCleanup(reset_gettext) GNU_MO_DATA_ISSUE_17898 = b'''\ @@ -237,6 +304,16 @@ def test_bindtextdomain(self): def test_textdomain(self): self.assertEqual(gettext.textdomain(), 'gettext') + def test_bad_magic_number(self): + with open(MOFILE_BAD_MAGIC_NUMBER, 'rb') as fp: + with self.assertRaises(OSError) as cm: + gettext.GNUTranslations(fp) + + exception = cm.exception + self.assertEqual(exception.errno, 0) + self.assertEqual(exception.strerror, "Bad magic number") + self.assertEqual(exception.filename, MOFILE_BAD_MAGIC_NUMBER) + def test_bad_major_version(self): with open(MOFILE_BAD_MAJOR_VERSION, 'rb') as fp: with self.assertRaises(OSError) as cm: @@ -252,6 +329,22 @@ def test_bad_minor_version(self): # Check that no error is thrown with a bad minor version number gettext.GNUTranslations(fp) + def test_corrupt_file(self): + with open(MOFILE_CORRUPT, 'rb') as fp: + with self.assertRaises(OSError) as cm: + gettext.GNUTranslations(fp) + + exception = cm.exception + self.assertEqual(exception.errno, 0) + self.assertEqual(exception.strerror, "File is corrupt") + self.assertEqual(exception.filename, MOFILE_CORRUPT) + + def test_big_endian_file(self): + with open(MOFILE_BIG_ENDIAN, 'rb') as fp: + t = gettext.GNUTranslations(fp) + + self.assertEqual(t.gettext('foo'), 'bar') + def test_some_translations(self): eq = self.assertEqual # test some translations @@ -309,55 +402,153 @@ def test_multiline_strings(self): trggrkg zrffntr pngnybt yvoenel.''') -class PluralFormsTestCase(GettextBaseTest): +class PluralFormsTests: + + def _test_plural_forms(self, ngettext, gettext, + singular, plural, tsingular, tplural, + numbers_only=True): + x = ngettext(singular, plural, 1) + self.assertEqual(x, tsingular) + x = ngettext(singular, plural, 2) + self.assertEqual(x, tplural) + x = gettext(singular) + self.assertEqual(x, tsingular) + + lineno = self._test_plural_forms.__code__.co_firstlineno + 12 + with self.assertWarns(DeprecationWarning) as cm: + x = ngettext(singular, plural, 1.0) + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno) + self.assertEqual(x, tsingular) + with self.assertWarns(DeprecationWarning) as cm: + x = ngettext(singular, plural, 1.1) + self.assertEqual(cm.filename, __file__) + self.assertEqual(cm.lineno, lineno + 5) + self.assertEqual(x, tplural) + + if numbers_only: + with self.assertRaises(TypeError): + ngettext(singular, plural, None) + else: + with self.assertWarns(DeprecationWarning) as cm: + x = ngettext(singular, plural, None) + self.assertEqual(x, tplural) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_plural_forms(self): + self._test_plural_forms( + self.ngettext, self.gettext, + 'There is %s file', 'There are %s files', + 'Hay %s fichero', 'Hay %s ficheros') + self._test_plural_forms( + self.ngettext, self.gettext, + '%d file deleted', '%d files deleted', + '%d file deleted', '%d files deleted') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_plural_context_forms(self): + ngettext = partial(self.npgettext, 'With context') + gettext = partial(self.pgettext, 'With context') + self._test_plural_forms( + ngettext, gettext, + 'There is %s file', 'There are %s files', + 'Hay %s fichero (context)', 'Hay %s ficheros (context)') + self._test_plural_forms( + ngettext, gettext, + '%d file deleted', '%d files deleted', + '%d file deleted', '%d files deleted') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_plural_wrong_context_forms(self): + self._test_plural_forms( + partial(self.npgettext, 'Unknown context'), + partial(self.pgettext, 'Unknown context'), + 'There is %s file', 'There are %s files', + 'There is %s file', 'There are %s files') + + +class GNUTranslationsPluralFormsTestCase(PluralFormsTests, GettextBaseTest): def setUp(self): GettextBaseTest.setUp(self) - self.mofile = MOFILE + # Set up the bindings + gettext.bindtextdomain('gettext', os.curdir) + gettext.textdomain('gettext') - def test_plural_forms1(self): - eq = self.assertEqual - x = gettext.ngettext('There is %s file', 'There are %s files', 1) - eq(x, 'Hay %s fichero') - x = gettext.ngettext('There is %s file', 'There are %s files', 2) - eq(x, 'Hay %s ficheros') - x = gettext.gettext('There is %s file') - eq(x, 'Hay %s fichero') - - def test_plural_context_forms1(self): - eq = self.assertEqual - x = gettext.npgettext('With context', - 'There is %s file', 'There are %s files', 1) - eq(x, 'Hay %s fichero (context)') - x = gettext.npgettext('With context', - 'There is %s file', 'There are %s files', 2) - eq(x, 'Hay %s ficheros (context)') - x = gettext.pgettext('With context', 'There is %s file') - eq(x, 'Hay %s fichero (context)') - - def test_plural_forms2(self): - eq = self.assertEqual - with open(self.mofile, 'rb') as fp: - t = gettext.GNUTranslations(fp) - x = t.ngettext('There is %s file', 'There are %s files', 1) - eq(x, 'Hay %s fichero') - x = t.ngettext('There is %s file', 'There are %s files', 2) - eq(x, 'Hay %s ficheros') - x = t.gettext('There is %s file') - eq(x, 'Hay %s fichero') - - def test_plural_context_forms2(self): - eq = self.assertEqual - with open(self.mofile, 'rb') as fp: + self.gettext = gettext.gettext + self.ngettext = gettext.ngettext + self.pgettext = gettext.pgettext + self.npgettext = gettext.npgettext + + +class GNUTranslationsWithDomainPluralFormsTestCase(PluralFormsTests, GettextBaseTest): + def setUp(self): + GettextBaseTest.setUp(self) + # Set up the bindings + gettext.bindtextdomain('gettext', os.curdir) + + self.gettext = partial(gettext.dgettext, 'gettext') + self.ngettext = partial(gettext.dngettext, 'gettext') + self.pgettext = partial(gettext.dpgettext, 'gettext') + self.npgettext = partial(gettext.dnpgettext, 'gettext') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_plural_forms_wrong_domain(self): + self._test_plural_forms( + partial(gettext.dngettext, 'unknown'), + partial(gettext.dgettext, 'unknown'), + 'There is %s file', 'There are %s files', + 'There is %s file', 'There are %s files', + numbers_only=False) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_plural_context_forms_wrong_domain(self): + self._test_plural_forms( + partial(gettext.dnpgettext, 'unknown', 'With context'), + partial(gettext.dpgettext, 'unknown', 'With context'), + 'There is %s file', 'There are %s files', + 'There is %s file', 'There are %s files', + numbers_only=False) + + +class GNUTranslationsClassPluralFormsTestCase(PluralFormsTests, GettextBaseTest): + def setUp(self): + GettextBaseTest.setUp(self) + with open(MOFILE, 'rb') as fp: t = gettext.GNUTranslations(fp) - x = t.npgettext('With context', - 'There is %s file', 'There are %s files', 1) - eq(x, 'Hay %s fichero (context)') - x = t.npgettext('With context', - 'There is %s file', 'There are %s files', 2) - eq(x, 'Hay %s ficheros (context)') - x = gettext.pgettext('With context', 'There is %s file') - eq(x, 'Hay %s fichero (context)') + self.gettext = t.gettext + self.ngettext = t.ngettext + self.pgettext = t.pgettext + self.npgettext = t.npgettext + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_plural_forms_null_translations(self): + t = gettext.NullTranslations() + self._test_plural_forms( + t.ngettext, t.gettext, + 'There is %s file', 'There are %s files', + 'There is %s file', 'There are %s files', + numbers_only=False) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_plural_context_forms_null_translations(self): + t = gettext.NullTranslations() + self._test_plural_forms( + partial(t.npgettext, 'With context'), + partial(t.pgettext, 'With context'), + 'There is %s file', 'There are %s files', + 'There is %s file', 'There are %s files', + numbers_only=False) + + +class PluralFormsInternalTestCase(unittest.TestCase): # Examples from http://www.gnu.org/software/gettext/manual/gettext.html def test_ja(self): @@ -472,12 +663,18 @@ def test_decimal_number(self): def test_invalid_syntax(self): invalid_expressions = [ 'x>1', '(n>1', 'n>1)', '42**42**42', '0xa', '1.0', '1e2', - 'n>0x1', '+n', '-n', 'n()', 'n(1)', '1+', 'nn', 'n n', + 'n>0x1', '+n', '-n', 'n()', 'n(1)', '1+', 'nn', 'n n', 'n ? 1 2' ] for expr in invalid_expressions: with self.assertRaises(ValueError): gettext.c2py(expr) + def test_negation(self): + f = gettext.c2py('!!!n') + self.assertEqual(f(0), 1) + self.assertEqual(f(1), 0) + self.assertEqual(f(2), 0) + def test_nested_condition_operator(self): self.assertEqual(gettext.c2py('n?1?2:3:4')(0), 4) self.assertEqual(gettext.c2py('n?1?2:3:4')(1), 2) @@ -640,6 +837,158 @@ def test_cache(self): self.assertEqual(t.__class__, DummyGNUTranslations) +class FallbackTranslations(gettext.NullTranslations): + def gettext(self, message): + return f'gettext: {message}' + + def ngettext(self, msgid1, msgid2, n): + return f'ngettext: {msgid1}, {msgid2}, {n}' + + def pgettext(self, context, message): + return f'pgettext: {context}, {message}' + + def npgettext(self, context, msgid1, msgid2, n): + return f'npgettext: {context}, {msgid1}, {msgid2}, {n}' + + +class FallbackTestCase(GettextBaseTest): + def test_null_translations_fallback(self): + t = gettext.NullTranslations() + t.add_fallback(FallbackTranslations()) + self.assertEqual(t.gettext('foo'), 'gettext: foo') + self.assertEqual(t.ngettext('foo', 'foos', 1), + 'ngettext: foo, foos, 1') + self.assertEqual(t.pgettext('context', 'foo'), + 'pgettext: context, foo') + self.assertEqual(t.npgettext('context', 'foo', 'foos', 1), + 'npgettext: context, foo, foos, 1') + + def test_gnu_translations_fallback(self): + with open(MOFILE, 'rb') as fp: + t = gettext.GNUTranslations(fp) + t.add_fallback(FallbackTranslations()) + self.assertEqual(t.gettext('foo'), 'gettext: foo') + self.assertEqual(t.ngettext('foo', 'foos', 1), + 'ngettext: foo, foos, 1') + self.assertEqual(t.pgettext('context', 'foo'), + 'pgettext: context, foo') + self.assertEqual(t.npgettext('context', 'foo', 'foos', 1), + 'npgettext: context, foo, foos, 1') + + def test_nested_fallbacks(self): + class NestedFallback(gettext.NullTranslations): + def gettext(self, message): + if message == 'foo': + return 'fallback' + return super().gettext(message) + + fallback1 = NestedFallback() + fallback2 = FallbackTranslations() + t = gettext.NullTranslations() + t.add_fallback(fallback1) + t.add_fallback(fallback2) + + self.assertEqual(fallback1.gettext('bar'), 'gettext: bar') + self.assertEqual(t.gettext('foo'), 'fallback') + self.assertEqual(t.gettext('bar'), 'gettext: bar') + + +class ExpandLangTestCase(unittest.TestCase): + def test_expand_lang(self): + # Test all combinations of territory, charset and + # modifier (locale extension) + locales = { + 'cs': ['cs'], + 'cs_CZ': ['cs_CZ', 'cs'], + 'cs.ISO8859-2': ['cs.ISO8859-2', 'cs'], + 'cs@euro': ['cs@euro', 'cs'], + 'cs_CZ.ISO8859-2': ['cs_CZ.ISO8859-2', 'cs_CZ', 'cs.ISO8859-2', + 'cs'], + 'cs_CZ@euro': ['cs_CZ@euro', 'cs@euro', 'cs_CZ', 'cs'], + 'cs.ISO8859-2@euro': ['cs.ISO8859-2@euro', 'cs@euro', + 'cs.ISO8859-2', 'cs'], + 'cs_CZ.ISO8859-2@euro': ['cs_CZ.ISO8859-2@euro', 'cs_CZ@euro', + 'cs.ISO8859-2@euro', 'cs@euro', + 'cs_CZ.ISO8859-2', 'cs_CZ', + 'cs.ISO8859-2', 'cs'], + } + for locale, expanded in locales.items(): + with self.subTest(locale=locale): + with unittest.mock.patch("locale.normalize", + return_value=locale): + self.assertEqual(gettext._expand_lang(locale), expanded) + + +class FindTestCase(unittest.TestCase): + + def setUp(self): + self.env = self.enterContext(os_helper.EnvironmentVarGuard()) + self.tempdir = self.enterContext(os_helper.temp_cwd()) + + for key in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'): + self.env.unset(key) + + def create_mo_file(self, lang): + locale_dir = os.path.join(self.tempdir, "locale") + mofile_dir = os.path.join(locale_dir, lang, "LC_MESSAGES") + os.makedirs(mofile_dir) + mo_file = os.path.join(mofile_dir, "mofile.mo") + with open(mo_file, "wb") as f: + f.write(GNU_MO_DATA) + return mo_file + + def test_find_with_env_vars(self): + # test that find correctly finds the environment variables + # when languages are not supplied + mo_file = self.create_mo_file("ga_IE") + for var in ('LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG'): + self.env.set(var, 'ga_IE') + result = gettext.find("mofile", + localedir=os.path.join(self.tempdir, "locale")) + self.assertEqual(result, mo_file) + self.env.unset(var) + + def test_find_with_languages(self): + # test that passed languages are used + self.env.set('LANGUAGE', 'pt_BR') + mo_file = self.create_mo_file("ga_IE") + + result = gettext.find("mofile", + localedir=os.path.join(self.tempdir, "locale"), + languages=['ga_IE']) + self.assertEqual(result, mo_file) + + @unittest.mock.patch('gettext._expand_lang') + def test_find_with_no_lang(self, patch_expand_lang): + # no language can be found + gettext.find('foo') + patch_expand_lang.assert_called_with('C') + + @unittest.mock.patch('gettext._expand_lang') + def test_find_with_c(self, patch_expand_lang): + # 'C' is already in languages + self.env.set('LANGUAGE', 'C') + gettext.find('foo') + patch_expand_lang.assert_called_with('C') + + def test_find_all(self): + # test that all are returned when all is set + paths = [] + for lang in ["ga_IE", "es_ES"]: + paths.append(self.create_mo_file(lang)) + result = gettext.find('mofile', + localedir=os.path.join(self.tempdir, "locale"), + languages=["ga_IE", "es_ES"], all=True) + self.assertEqual(sorted(result), sorted(paths)) + + def test_find_deduplication(self): + # test that find removes duplicate languages + mo_file = [self.create_mo_file('ga_IE')] + result = gettext.find("mofile", localedir=os.path.join(self.tempdir, "locale"), + languages=['ga_IE', 'ga_IE'], all=True) + self.assertEqual(result, mo_file) + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, gettext, From 72fc3c0ba4daf4a0f4eb5a27994a6ea0b891e209 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:09:36 +0200 Subject: [PATCH 3/4] Update `pickle{tools,}.py` from 3.13.5 (#6064) --- Lib/pickle.py | 148 +++++++++++++++++++++++------------ Lib/pickletools.py | 11 ++- Lib/test/test_pickle.py | 56 ------------- Lib/test/test_pickletools.py | 18 ----- 4 files changed, 106 insertions(+), 127 deletions(-) diff --git a/Lib/pickle.py b/Lib/pickle.py index 6e3c61fd0b..550f8675f2 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -314,16 +314,17 @@ def load_frame(self, frame_size): # Tools used for pickling. def _getattribute(obj, name): + top = obj for subpath in name.split('.'): if subpath == '': raise AttributeError("Can't get local attribute {!r} on {!r}" - .format(name, obj)) + .format(name, top)) try: parent = obj obj = getattr(obj, subpath) except AttributeError: raise AttributeError("Can't get attribute {!r} on {!r}" - .format(name, obj)) from None + .format(name, top)) from None return obj, parent def whichmodule(obj, name): @@ -396,6 +397,8 @@ def decode_long(data): return int.from_bytes(data, byteorder='little', signed=True) +_NoValue = object() + # Pickling machinery class _Pickler: @@ -530,10 +533,11 @@ def save(self, obj, save_persistent_id=True): self.framer.commit_frame() # Check for persistent id (defined by a subclass) - pid = self.persistent_id(obj) - if pid is not None and save_persistent_id: - self.save_pers(pid) - return + if save_persistent_id: + pid = self.persistent_id(obj) + if pid is not None: + self.save_pers(pid) + return # Check the memo x = self.memo.get(id(obj)) @@ -542,8 +546,8 @@ def save(self, obj, save_persistent_id=True): return rv = NotImplemented - reduce = getattr(self, "reducer_override", None) - if reduce is not None: + reduce = getattr(self, "reducer_override", _NoValue) + if reduce is not _NoValue: rv = reduce(obj) if rv is NotImplemented: @@ -556,8 +560,8 @@ def save(self, obj, save_persistent_id=True): # Check private dispatch table if any, or else # copyreg.dispatch_table - reduce = getattr(self, 'dispatch_table', dispatch_table).get(t) - if reduce is not None: + reduce = getattr(self, 'dispatch_table', dispatch_table).get(t, _NoValue) + if reduce is not _NoValue: rv = reduce(obj) else: # Check for a class with a custom metaclass; treat as regular @@ -567,12 +571,12 @@ def save(self, obj, save_persistent_id=True): return # Check for a __reduce_ex__ method, fall back to __reduce__ - reduce = getattr(obj, "__reduce_ex__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce_ex__", _NoValue) + if reduce is not _NoValue: rv = reduce(self.proto) else: - reduce = getattr(obj, "__reduce__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce__", _NoValue) + if reduce is not _NoValue: rv = reduce() else: raise PicklingError("Can't pickle %r object: %r" % @@ -780,14 +784,10 @@ def save_float(self, obj): self.write(FLOAT + repr(obj).encode("ascii") + b'\n') dispatch[float] = save_float - def save_bytes(self, obj): - if self.proto < 3: - if not obj: # bytes object is empty - self.save_reduce(bytes, (), obj=obj) - else: - self.save_reduce(codecs.encode, - (str(obj, 'latin1'), 'latin1'), obj=obj) - return + def _save_bytes_no_memo(self, obj): + # helper for writing bytes objects for protocol >= 3 + # without memoizing them + assert self.proto >= 3 n = len(obj) if n <= 0xff: self.write(SHORT_BINBYTES + pack("= 5 + # without memoizing them + assert self.proto >= 5 + n = len(obj) + if n >= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BYTEARRAY8 + pack("= self.framer._FRAME_SIZE_TARGET: - self._write_large_bytes(BYTEARRAY8 + pack("= 5") with obj.raw() as m: if not m.contiguous: @@ -830,10 +846,18 @@ def save_picklebuffer(self, obj): if in_band: # Write data in-band # XXX The C implementation avoids a copy here + buf = m.tobytes() + in_memo = id(buf) in self.memo if m.readonly: - self.save_bytes(m.tobytes()) + if in_memo: + self._save_bytes_no_memo(buf) + else: + self.save_bytes(buf) else: - self.save_bytearray(m.tobytes()) + if in_memo: + self._save_bytearray_no_memo(buf) + else: + self.save_bytearray(buf) else: # Write data out-of-band self.write(NEXT_BUFFER) @@ -1070,11 +1094,16 @@ def save_global(self, obj, name=None): (obj, module_name, name)) if self.proto >= 2: - code = _extension_registry.get((module_name, name)) - if code: - assert code > 0 + code = _extension_registry.get((module_name, name), _NoValue) + if code is not _NoValue: if code <= 0xff: - write(EXT1 + pack("= 3: - write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + - bytes(name, "utf-8") + b'\n') + elif '.' in name: + # In protocol < 4, objects with multi-part __qualname__ + # are represented as + # getattr(getattr(..., attrname1), attrname2). + dotted_path = name.split('.') + name = dotted_path.pop(0) + save = self.save + for attrname in dotted_path: + save(getattr) + if self.proto < 2: + write(MARK) + self._save_toplevel_by_name(module_name, name) + for attrname in dotted_path: + save(attrname) + if self.proto < 2: + write(TUPLE) + else: + write(TUPLE2) + write(REDUCE) + else: + self._save_toplevel_by_name(module_name, name) + + self.memoize(obj) + + def _save_toplevel_by_name(self, module_name, name): + if self.proto >= 3: + # Non-ASCII identifiers are supported only with protocols >= 3. + self.write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING @@ -1102,14 +1155,12 @@ def save_global(self, obj, name=None): elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: - write(GLOBAL + bytes(module_name, "ascii") + b'\n' + - bytes(name, "ascii") + b'\n') + self.write(GLOBAL + bytes(module_name, "ascii") + b'\n' + + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " - "pickle protocol %i" % (module, name, self.proto)) from None - - self.memoize(obj) + "pickle protocol %i" % (module_name, name, self.proto)) from None def save_type(self, obj): if obj is type(None): @@ -1546,9 +1597,8 @@ def load_ext4(self): dispatch[EXT4[0]] = load_ext4 def get_extension(self, code): - nil = [] - obj = _extension_cache.get(code, nil) - if obj is not nil: + obj = _extension_cache.get(code, _NoValue) + if obj is not _NoValue: self.append(obj) return key = _inverted_registry.get(code) @@ -1705,8 +1755,8 @@ def load_build(self): stack = self.stack state = stack.pop() inst = stack[-1] - setstate = getattr(inst, "__setstate__", None) - if setstate is not None: + setstate = getattr(inst, "__setstate__", _NoValue) + if setstate is not _NoValue: setstate(state) return slotstate = None diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 51ee4a7a26..33a51492ea 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -312,7 +312,7 @@ def read_uint8(f): doc="Eight-byte unsigned integer, little-endian.") -def read_stringnl(f, decode=True, stripquotes=True): +def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'): r""" >>> import io >>> read_stringnl(io.BytesIO(b"'abcd'\nefg\n")) @@ -356,7 +356,7 @@ def read_stringnl(f, decode=True, stripquotes=True): raise ValueError("no string quotes around %r" % data) if decode: - data = codecs.escape_decode(data)[0].decode("ascii") + data = codecs.escape_decode(data)[0].decode(encoding) return data stringnl = ArgumentDescriptor( @@ -370,7 +370,7 @@ def read_stringnl(f, decode=True, stripquotes=True): """) def read_stringnl_noescape(f): - return read_stringnl(f, stripquotes=False) + return read_stringnl(f, stripquotes=False, encoding='utf-8') stringnl_noescape = ArgumentDescriptor( name='stringnl_noescape', @@ -2513,7 +2513,10 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): # make a mild effort to align arguments line += ' ' * (10 - len(opcode.name)) if arg is not None: - line += ' ' + repr(arg) + if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"): + line += ' ' + ascii(arg) + else: + line += ' ' + repr(arg) if markmsg: line += ' ' + markmsg if annotate: diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index a9177ada39..070e277c2a 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -97,11 +97,6 @@ def dumps(self, arg, proto=None, **kwargs): def test_picklebuffer_error(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_picklebuffer_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reduce_ex_None(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_reduce_ex_None() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_bad_getattr(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -190,16 +185,6 @@ def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this def test_optional_frames(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_optional_frames() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickle_setstate_None(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_pickle_setstate_None() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_recursive_nested_names2(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_recursive_nested_names2() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -240,16 +225,6 @@ def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this def test_optional_frames(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_optional_frames() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickle_setstate_None(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_pickle_setstate_None() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_recursive_nested_names2(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_recursive_nested_names2() - class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, BigmemPickleTests, unittest.TestCase): @@ -299,11 +274,6 @@ def test_load_python2_str_as_bytes(self): # TODO(RUSTPYTHON): Remove this test w def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_py_methods() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_recursive_nested_names2(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_recursive_nested_names2() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -344,11 +314,6 @@ def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes def test_optional_frames(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_optional_frames() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickle_setstate_None(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_pickle_setstate_None() - class PersistentPicklerUnpicklerMixin(object): def dumps(self, arg, proto=None): @@ -465,8 +430,6 @@ def persistent_load(pid): return pid check(PersUnpickler) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickler_super(self): class PersPickler(self.pickler): def persistent_id(subself, obj): @@ -496,8 +459,6 @@ def persistent_load(subself, pid): self.assertEqual(unpickler.load(), 'abc') self.assertEqual(called, ['abc']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickler_instance_attribute(self): def persistent_id(obj): called.append(obj) @@ -532,8 +493,6 @@ def persistent_load(pid): del unpickler.persistent_load self.assertEqual(unpickler.persistent_load, old_persistent_load) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickler_super_instance_attribute(self): class PersPickler(self.pickler): def persistent_id(subself, obj): @@ -582,11 +541,6 @@ class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittes pickler_class = pickle._Pickler unpickler_class = pickle._Unpickler - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickle_invalid_reducer_override(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_pickle_invalid_reducer_override() - class PyDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): @@ -595,11 +549,6 @@ class PyDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): def get_dispatch_table(self): return pickle.dispatch_table.copy() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dispatch_table_None_item(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_dispatch_table_None_item() - class PyChainDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): @@ -608,11 +557,6 @@ class PyChainDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): def get_dispatch_table(self): return collections.ChainMap({}, pickle.dispatch_table) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dispatch_table_None_item(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_dispatch_table_None_item() - class PyPicklerHookTests(AbstractHookTests, unittest.TestCase): class CustomPyPicklerClass(pickle._Pickler, diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 2a19976ce2..6c38bef3d3 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -102,16 +102,6 @@ def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this def test_optional_frames(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_optional_frames() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_pickle_setstate_None(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_pickle_setstate_None() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_recursive_nested_names2(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_recursive_nested_names2() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -436,8 +426,6 @@ def test_annotate(self): highest protocol among opcodes = 0 ''', annotate=20) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_string(self): self.check_dis(b"S'abc'\n.", '''\ 0: S STRING 'abc' @@ -469,8 +457,6 @@ def test_string_without_quotes(self): self.check_dis_error(b"S\"abc'\n.", '', r"""strinq quote b'"' not found at both ends of b'"abc\\''""") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_binstring(self): self.check_dis(b"T\x03\x00\x00\x00abc.", '''\ 0: T BINSTRING 'abc' @@ -483,8 +469,6 @@ def test_binstring(self): highest protocol among opcodes = 1 ''') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_short_binstring(self): self.check_dis(b"U\x03abc.", '''\ 0: U SHORT_BINSTRING 'abc' @@ -497,8 +481,6 @@ def test_short_binstring(self): highest protocol among opcodes = 1 ''') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_global(self): self.check_dis(b"cmodule\nname\n.", '''\ 0: c GLOBAL 'module name' From c4a805107f5bd9ad608a3582912c1c703350f475 Mon Sep 17 00:00:00 2001 From: Shahar Naveh <50263213+ShaharNaveh@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:10:02 +0200 Subject: [PATCH 4/4] Update `genericpath.py` from 3.13.5 (#6065) --- Lib/genericpath.py | 37 ++++++++++++++++++++++++-- Lib/test/test_genericpath.py | 50 ++++++++++++++++++++++++------------ 2 files changed, 69 insertions(+), 18 deletions(-) diff --git a/Lib/genericpath.py b/Lib/genericpath.py index 1bd5b3897c..9363f564aa 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -7,8 +7,8 @@ import stat __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', - 'getsize', 'isdir', 'isfile', 'islink', 'samefile', 'sameopenfile', - 'samestat'] + 'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink', + 'lexists', 'samefile', 'sameopenfile', 'samestat', 'ALLOW_MISSING'] # Does a path exist? @@ -22,6 +22,15 @@ def exists(path): return True +# Being true for dangling symbolic links is also useful. +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + os.lstat(path) + except (OSError, ValueError): + return False + return True + # This follows symbolic links, so both islink() and isdir() can be true # for the same path on systems that support symlinks def isfile(path): @@ -57,6 +66,21 @@ def islink(path): return stat.S_ISLNK(st.st_mode) +# Is a path a junction? +def isjunction(path): + """Test whether a path is a junction + Junctions are not supported on the current platform""" + os.fspath(path) + return False + + +def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive. + Dev Drives are not supported on the current platform""" + os.fspath(path) + return False + + def getsize(filename): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size @@ -165,3 +189,12 @@ def _check_arg_types(funcname, *args): f'os.PathLike object, not {s.__class__.__name__!r}') from None if hasstr and hasbytes: raise TypeError("Can't mix strings and bytes in path components") from None + +# A singleton with a true boolean value. +@object.__new__ +class ALLOW_MISSING: + """Special value for use in realpath().""" + def __repr__(self): + return 'os.path.ALLOW_MISSING' + def __reduce__(self): + return self.__class__.__name__ diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 4f311c2d49..2e28d3cfb7 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -7,9 +7,9 @@ import sys import unittest import warnings -from test.support import is_emscripten -from test.support import os_helper -from test.support import warnings_helper +from test.support import ( + is_apple, is_emscripten, os_helper, warnings_helper +) from test.support.script_helper import assert_python_ok from test.support.os_helper import FakePath @@ -135,6 +135,9 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename), False) self.assertIs(self.pathmodule.exists(bfilename), False) + self.assertIs(self.pathmodule.lexists(filename), False) + self.assertIs(self.pathmodule.lexists(bfilename), False) + create_file(filename) self.assertIs(self.pathmodule.exists(filename), True) @@ -145,14 +148,17 @@ def test_exists(self): self.assertIs(self.pathmodule.exists(filename + '\x00'), False) self.assertIs(self.pathmodule.exists(bfilename + b'\x00'), False) - if self.pathmodule is not genericpath: - self.assertIs(self.pathmodule.lexists(filename), True) - self.assertIs(self.pathmodule.lexists(bfilename), True) + self.assertIs(self.pathmodule.lexists(filename), True) + self.assertIs(self.pathmodule.lexists(bfilename), True) + + self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) + self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) + self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) - self.assertIs(self.pathmodule.lexists(filename + '\udfff'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\xff'), False) - self.assertIs(self.pathmodule.lexists(filename + '\x00'), False) - self.assertIs(self.pathmodule.lexists(bfilename + b'\x00'), False) + # Keyword arguments are accepted + self.assertIs(self.pathmodule.exists(path=filename), True) + self.assertIs(self.pathmodule.lexists(path=filename), True) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") @unittest.skipIf(is_emscripten, "Emscripten pipe fds have no stat") @@ -165,6 +171,14 @@ def test_exists_fd(self): os.close(w) self.assertFalse(self.pathmodule.exists(r)) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exists_bool(self): + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor'): + self.pathmodule.exists(fd) + def test_isdir(self): filename = os_helper.TESTFN bfilename = os.fsencode(filename) @@ -483,12 +497,16 @@ def test_abspath_issue3426(self): self.assertIsInstance(abspath(path), str) def test_nonascii_abspath(self): - if (os_helper.TESTFN_UNDECODABLE - # macOS and Emscripten deny the creation of a directory with an - # invalid UTF-8 name. Windows allows creating a directory with an - # arbitrary bytes name, but fails to enter this directory - # (when the bytes name is used). - and sys.platform not in ('win32', 'darwin', 'emscripten', 'wasi')): + if ( + os_helper.TESTFN_UNDECODABLE + # Apple platforms and Emscripten/WASI deny the creation of a + # directory with an invalid UTF-8 name. Windows allows creating a + # directory with an arbitrary bytes name, but fails to enter this + # directory (when the bytes name is used). + and sys.platform not in { + "win32", "emscripten", "wasi" + } and not is_apple + ): name = os_helper.TESTFN_UNDECODABLE elif os_helper.TESTFN_NONASCII: name = os_helper.TESTFN_NONASCII