Add some support for Python 3.11

- add Python 3.11 to CI tests
- adapt the pathlib tests to work with Python 3.11
- adapt handling of pathlib in unfaked modules:
  need to ensure that the original os module is used,
  as pathlib has removed the accessor layer and now
  directly accesses os
- add target_is_directory argument to symlink (ignored)
- 'U' open mode is no longer allowed in Python 3.11
- closes #677

6 files changed, 159 insertions, 80 deletions

diff --git a/pyfakefs/fake_filesystem.py b/pyfakefs/fake_filesystem.py
index d51cb00..0bfa463 100644
--- a/pyfakefs/fake_filesystem.py
+++ b/pyfakefs/fake_filesystem.py
@@ -94,7 +94,9 @@ True
 True
 """
 import errno
+import functools
 import heapq
+import inspect
 import io
 import locale
 import os
@@ -103,6 +105,7 @@ import sys
 import traceback
 import uuid
 from collections import namedtuple, OrderedDict
+from contextlib import contextmanager
 from doctest import TestResults
 from enum import Enum
 from stat import (
@@ -3851,6 +3854,8 @@ class FakeOsModule:
     my_os_module = fake_filesystem.FakeOsModule(filesystem)
     """
 
+    use_original = False
+
     @staticmethod
     def dir() -> List[str]:
         """Return the list of patched function names. Used for patching
@@ -3879,7 +3884,7 @@ class FakeOsModule:
             filesystem: FakeFilesystem used to provide file system information
         """
         self.filesystem = filesystem
-        self._os_module: Any = os
+        self.os_module: Any = os
         self.path = FakePathModule(self.filesystem, self)
 
     @property
@@ -4790,13 +4795,15 @@ class FakeOsModule:
             tail, mode & ~self.filesystem.umask,
             filesystem=self.filesystem))
 
-    def symlink(self, src: AnyStr, dst: AnyStr, *,
+    def symlink(self, src: AnyStr, dst: AnyStr,
+                target_is_directory: bool = False, *,
                 dir_fd: Optional[int] = None) -> None:
         """Creates the specified symlink, pointed at the specified link target.
 
         Args:
             src: The target of the symlink.
             dst: Path to the symlink to create.
+            target_is_directory: Currently ignored.
             dir_fd: If not `None`, the file descriptor of a directory,
                 with `src` being relative to this directory.
 
@@ -4915,7 +4922,38 @@ class FakeOsModule:
 
     def __getattr__(self, name: str) -> Any:
         """Forwards any unfaked calls to the standard os module."""
-        return getattr(self._os_module, name)
+        return getattr(self.os_module, name)
+
+
+if sys.version_info > (3, 10):
+    def handle_original_call(f: Callable) -> Callable:
+        """Decorator used for real pathlib Path methods to ensure that
+        real os functions instead of faked ones are used.
+        Applied to all non-private methods of `FakeOsModule`."""
+        @functools.wraps(f)
+        def wrapped(*args, **kwargs):
+            if not f.__name__.startswith('_') and FakeOsModule.use_original:
+                # remove the `self` argument for FakeOsModule methods
+                if args and isinstance(args[0], FakeOsModule):
+                    args = args[1:]
+                return getattr(os, f.__name__)(*args, **kwargs)
+            return f(*args, **kwargs)
+        return wrapped
+
+    for name, fn in inspect.getmembers(FakeOsModule, inspect.isfunction):
+        setattr(FakeOsModule, name, handle_original_call(fn))
+
+
+@contextmanager
+def use_original_os():
+    """Temporarily use original os functions instead of faked ones.
+    Used to ensure that skipped modules do not use faked calls.
+    """
+    try:
+        FakeOsModule.use_original = True
+        yield
+    finally:
+        FakeOsModule.use_original = False
 
 
 class FakeIoModule:
@@ -5847,7 +5885,8 @@ class FakeFileOpen:
                                                        _OpenModes]:
         orig_modes = mode  # Save original modes for error messages.
         # Normalize modes. Handle 't' and 'U'.
-        if 'b' in mode and 't' in mode:
+        if (('b' in mode and 't' in mode) or
+                (sys.version_info > (3, 10) and 'U' in mode)):
             raise ValueError('Invalid mode: ' + mode)
         mode = mode.replace('t', '').replace('b', '')
         mode = mode.replace('rU', 'r').replace('U', 'r')
diff --git a/pyfakefs/fake_pathlib.py b/pyfakefs/fake_pathlib.py
index 375843f..5935c61 100644
--- a/pyfakefs/fake_pathlib.py
+++ b/pyfakefs/fake_pathlib.py
@@ -31,16 +31,20 @@ get the properties of the underlying fake filesystem.
 import errno
 import fnmatch
 import functools
+import inspect
 import os
 import pathlib
-from pathlib import PurePath
 import re
 import sys
+from pathlib import PurePath
+from typing import Callable
 from urllib.parse import quote_from_bytes as urlquote_from_bytes
 
 from pyfakefs import fake_scandir
 from pyfakefs.extra_packages import use_scandir
-from pyfakefs.fake_filesystem import FakeFileOpen, FakeFilesystem
+from pyfakefs.fake_filesystem import (
+    FakeFileOpen, FakeFilesystem, use_original_os
+)
 
 
 def init_module(filesystem):
@@ -98,27 +102,26 @@ class _FakeAccessor(accessor):  # type: ignore [valid-type, misc]
     if use_scandir:
         scandir = _wrap_strfunc(fake_scandir.scandir)
 
-    chmod = _wrap_strfunc(FakeFilesystem.chmod)
-
     if hasattr(os, "lchmod"):
         lchmod = _wrap_strfunc(lambda fs, path, mode: FakeFilesystem.chmod(
             fs, path, mode, follow_symlinks=False))
     else:
-        def lchmod(self, pathobj,  *args, **kwargs):
+        def lchmod(self, pathobj, *args, **kwargs):
             """Raises not implemented for Windows systems."""
             raise NotImplementedError("lchmod() not available on this system")
 
-        def chmod(self, pathobj, *args, **kwargs):
-            if "follow_symlinks" in kwargs:
-                if sys.version_info < (3, 10):
-                    raise TypeError("chmod() got an unexpected keyword "
-                                    "argument 'follow_synlinks'")
-                if (not kwargs["follow_symlinks"] and
-                        os.chmod not in os.supports_follow_symlinks):
-                    raise NotImplementedError(
-                        "`follow_symlinks` for chmod() is not available "
-                        "on this system")
-            return pathobj.filesystem.chmod(str(pathobj), *args, **kwargs)
+    def chmod(self, pathobj, *args, **kwargs):
+        if "follow_symlinks" in kwargs:
+            if sys.version_info < (3, 10):
+                raise TypeError("chmod() got an unexpected keyword "
+                                "argument 'follow_symlinks'")
+
+            if (not kwargs["follow_symlinks"] and
+                    os.os_module.chmod not in os.supports_follow_symlinks):
+                raise NotImplementedError(
+                    "`follow_symlinks` for chmod() is not available "
+                    "on this system")
+        return pathobj.filesystem.chmod(str(pathobj), *args, **kwargs)
 
     mkdir = _wrap_strfunc(FakeFilesystem.makedir)
 
@@ -793,6 +796,8 @@ class RealPath(pathlib.Path):
     Needed because `Path` in `pathlib` is always faked, even if `pathlib`
     itself is not.
     """
+    _flavour = (pathlib._WindowsFlavour() if os.name == 'nt'  # type:ignore
+                else pathlib._PosixFlavour())  # type:ignore
 
     def __new__(cls, *args, **kwargs):
         """Creates the correct subclass based on OS."""
@@ -803,6 +808,41 @@ class RealPath(pathlib.Path):
         return self
 
 
+if sys.version_info > (3, 10):
+    def with_original_os(f: Callable) -> Callable:
+        """Decorator used for real pathlib Path methods to ensure that
+        real os functions instead of faked ones are used."""
+        @functools.wraps(f)
+        def wrapped(*args, **kwargs):
+            with use_original_os():
+                return f(*args, **kwargs)
+        return wrapped
+
+    for name, fn in inspect.getmembers(RealPath, inspect.isfunction):
+        setattr(RealPath, name, with_original_os(fn))
+
+
+class RealPathlibPathModule:
+    """Patches `pathlib.Path` by passing all calls to RealPathlibModule."""
+    real_pathlib = None
+
+    @classmethod
+    def __instancecheck__(cls, instance):
+        # as we cannot derive from pathlib.Path, we fake
+        # the inheritance to pass isinstance checks - see #666
+        return isinstance(instance, PurePath)
+
+    def __init__(self):
+        if self.real_pathlib is None:
+            self.__class__.real_pathlib = RealPathlibModule()
+
+    def __call__(self, *args, **kwargs):
+        return RealPath(*args, **kwargs)
+
+    def __getattr__(self, name):
+        return getattr(self.real_pathlib.Path, name)
+
+
 class RealPathlibModule:
     """Used to replace `pathlib` for skipped modules.
     As the original `pathlib` is always patched to use the fake path,
@@ -810,8 +850,6 @@ class RealPathlibModule:
     """
 
     def __init__(self):
-        RealPathlibModule.PureWindowsPath._flavour = pathlib._WindowsFlavour()
-        RealPathlibModule.PurePosixPath._flavour = pathlib._PosixFlavour()
         self._pathlib_module = pathlib
 
     class PurePosixPath(PurePath):
@@ -840,24 +878,3 @@ class RealPathlibModule:
     def __getattr__(self, name):
         """Forwards any unfaked calls to the standard pathlib module."""
         return getattr(self._pathlib_module, name)
-
-
-class RealPathlibPathModule:
-    """Patches `pathlib.Path` by passing all calls to RealPathlibModule."""
-    real_pathlib = None
-
-    @classmethod
-    def __instancecheck__(cls, instance):
-        # as we cannot derive from pathlib.Path, we fake
-        # the inheritance to pass isinstance checks - see #666
-        return isinstance(instance, PurePath)
-
-    def __init__(self):
-        if self.real_pathlib is None:
-            self.__class__.real_pathlib = RealPathlibModule()
-
-    def __call__(self, *args, **kwargs):
-        return self.real_pathlib.Path(*args, **kwargs)
-
-    def __getattr__(self, name):
-        return getattr(self.real_pathlib.Path, name)
diff --git a/pyfakefs/helpers.py b/pyfakefs/helpers.py
index ec8c230..3e1ae69 100644
--- a/pyfakefs/helpers.py
+++ b/pyfakefs/helpers.py
@@ -221,13 +221,13 @@ class FakeStatResult:
         mode = 0
         st_mode = self.st_mode
         if st_mode & stat.S_IFDIR:
-            mode |= stat.FILE_ATTRIBUTE_DIRECTORY
+            mode |= stat.FILE_ATTRIBUTE_DIRECTORY  # type:ignore[attr-defined]
         if st_mode & stat.S_IFREG:
-            mode |= stat.FILE_ATTRIBUTE_NORMAL
+            mode |= stat.FILE_ATTRIBUTE_NORMAL  # type:ignore[attr-defined]
         if st_mode & (stat.S_IFCHR | stat.S_IFBLK):
-            mode |= stat.FILE_ATTRIBUTE_DEVICE
+            mode |= stat.FILE_ATTRIBUTE_DEVICE  # type:ignore[attr-defined]
         if st_mode & stat.S_IFLNK:
-            mode |= stat.FILE_ATTRIBUTE_REPARSE_POINT
+            mode |= stat.FILE_ATTRIBUTE_REPARSE_POINT  # type:ignore
         return mode
 
     @property
diff --git a/pyfakefs/tests/fake_open_test.py b/pyfakefs/tests/fake_open_test.py
index 9e942f9..399660c 100644
--- a/pyfakefs/tests/fake_open_test.py
+++ b/pyfakefs/tests/fake_open_test.py
@@ -1486,6 +1486,7 @@ class FakeFileOpenLineEndingTest(FakeFileOpenTestBase):
         with self.open(file_path, mode='r', newline='\r\n') as f:
             self.assertEqual(['1\r\n', '2\n3\r4'], f.readlines())
 
+    @unittest.skipIf(sys.version_info >= (3, 10), "U flag no longer supported")
     def test_read_with_ignored_universal_newlines_flag(self):
         file_path = self.make_path('some_file')
         file_contents = b'1\r\n2\n3\r4'
@@ -1497,6 +1498,14 @@ class FakeFileOpenLineEndingTest(FakeFileOpenTestBase):
         with self.open(file_path, mode='U', newline='\r') as f:
             self.assertEqual('1\r\n2\n3\r4', f.read())
 
+    @unittest.skipIf(sys.version_info < (3, 11), "U flag still supported")
+    def test_universal_newlines_flag_not_supported(self):
+        file_path = self.make_path('some_file')
+        file_contents = b'1\r\n2\n3\r4'
+        self.create_file(file_path, contents=file_contents)
+        with self.assertRaises(ValueError):
+            self.open(file_path, mode='U', newline='\r')
+
     def test_write_with_newline_arg(self):
         file_path = self.make_path('some_file')
         with self.open(file_path, 'w', newline='') as f:
diff --git a/pyfakefs/tests/fake_pathlib_test.py b/pyfakefs/tests/fake_pathlib_test.py
index 1a3cdfc..0ffe082 100644
--- a/pyfakefs/tests/fake_pathlib_test.py
+++ b/pyfakefs/tests/fake_pathlib_test.py
@@ -27,26 +27,36 @@ import stat
 import sys
 import unittest
 
+from pyfakefs import fake_pathlib, fake_filesystem, fake_filesystem_unittest
 from pyfakefs.fake_filesystem import is_root
-
-from pyfakefs import fake_pathlib, fake_filesystem
 from pyfakefs.helpers import IS_PYPY
-from pyfakefs.tests.test_utils import RealFsTestCase
+from pyfakefs.tests.test_utils import RealFsTestMixin
 
 is_windows = sys.platform == 'win32'
 
 
-class RealPathlibTestCase(RealFsTestCase):
+class RealPathlibTestCase(fake_filesystem_unittest.TestCase, RealFsTestMixin):
+    is_windows = sys.platform == 'win32'
+
     def __init__(self, methodName='runTest'):
-        super(RealPathlibTestCase, self).__init__(methodName)
-        self.pathlib = pathlib
-        self.path = None
+        fake_filesystem_unittest.TestCase.__init__(self, methodName)
+        RealFsTestMixin.__init__(self)
 
     def setUp(self):
-        super().setUp()
+        RealFsTestMixin.setUp(self)
+        self.filesystem = None
+        self.real_os = os
         if not self.use_real_fs():
-            self.pathlib = fake_pathlib.FakePathlibModule(self.filesystem)
-        self.path = self.pathlib.Path
+            self.setUpPyfakefs()
+            self.filesystem = self.fs
+            self.create_basepath()
+        self.pathlib = pathlib
+        self.path = pathlib.Path
+        self.os = os
+        self.open = open
+
+    def use_real_fs(self):
+        return False
 
 
 class FakePathlibInitializationTest(RealPathlibTestCase):
@@ -178,7 +188,8 @@ class FakePathlibInitializationWithDriveTest(RealPathlibTestCase):
 
 
 class RealPathlibInitializationWithDriveTest(
-        FakePathlibInitializationWithDriveTest):
+    FakePathlibInitializationWithDriveTest
+):
     def use_real_fs(self):
         return True
 
@@ -383,7 +394,7 @@ class FakePathlibFileObjectPropertyTest(RealPathlibTestCase):
         self.skip_if_symlink_not_supported()
         file_stat = self.os.stat(self.file_path)
         link_stat = self.os.lstat(self.file_link_path)
-        if not hasattr(os, "lchmod"):
+        if not hasattr(self.real_os, "lchmod"):
             with self.assertRaises(NotImplementedError):
                 self.path(self.file_link_path).lchmod(0o444)
         else:
@@ -393,13 +404,16 @@ class FakePathlibFileObjectPropertyTest(RealPathlibTestCase):
             self.assertEqual(link_stat.st_mode & 0o777700,
                              stat.S_IFLNK | 0o700)
 
-    @unittest.skipIf(sys.version_info < (3, 10),
-                     "follow_symlinks argument new in Python 3.10")
+    @unittest.skipIf(
+        sys.version_info < (3, 10),
+        "follow_symlinks argument new in Python 3.10"
+    )
     def test_chmod_no_followsymlinks(self):
         self.skip_if_symlink_not_supported()
         file_stat = self.os.stat(self.file_path)
         link_stat = self.os.lstat(self.file_link_path)
-        if os.chmod not in os.supports_follow_symlinks or IS_PYPY:
+        if (self.real_os.chmod not in os.supports_follow_symlinks
+                or IS_PYPY):
             with self.assertRaises(NotImplementedError):
                 self.path(self.file_link_path).chmod(0o444,
                                                      follow_symlinks=False)
@@ -443,11 +457,11 @@ class FakePathlibFileObjectPropertyTest(RealPathlibTestCase):
         file_path = self.os.path.join(dir_path, 'some_file')
         self.create_file(file_path)
         self.os.chmod(dir_path, 0o000)
-        iter = self.path(dir_path).iterdir()
+        it = self.path(dir_path).iterdir()
         if not is_root():
-            self.assert_raises_os_error(errno.EACCES, list, iter)
+            self.assert_raises_os_error(errno.EACCES, list, it)
         else:
-            path = str(list(iter)[0])
+            path = str(list(it)[0])
             self.assertTrue(path.endswith('some_file'))
 
     def test_resolve_nonexisting_file(self):
diff --git a/pyfakefs/tests/test_utils.py b/pyfakefs/tests/test_utils.py
index eea81c4..b8cf79a 100644
--- a/pyfakefs/tests/test_utils.py
+++ b/pyfakefs/tests/test_utils.py
@@ -81,17 +81,6 @@ class TestCase(unittest.TestCase):
             else:
                 self.assertEqual(subtype, exc.errno)
 
-    def assert_raises_os_error(self, subtype, expression, *args, **kwargs):
-        """Asserts that a specific subtype of OSError is raised."""
-        try:
-            expression(*args, **kwargs)
-            self.fail('No exception was raised, OSError expected')
-        except OSError as exc:
-            if isinstance(subtype, list):
-                self.assertIn(exc.errno, subtype)
-            else:
-                self.assertEqual(subtype, exc.errno)
-
 
 class RealFsTestMixin:
     """Test mixin to allow tests to run both in the fake filesystem and in the
@@ -149,6 +138,9 @@ class RealFsTestMixin:
         """Return True if the real file system shall be tested."""
         return False
 
+    def setUpFileSystem(self):
+        pass
+
     def path_separator(self):
         """Can be overwritten to use a specific separator in the
         fake filesystem."""
@@ -362,7 +354,7 @@ class RealFsTestMixin:
         if self.filesystem is not None:
             old_base_path = self.base_path
             self.base_path = self.filesystem.path_separator + 'basepath'
-            if self.is_windows_fs:
+            if self.filesystem.is_windows_fs:
                 self.base_path = 'C:' + self.base_path
             if old_base_path != self.base_path:
                 if old_base_path is not None:
@@ -402,6 +394,17 @@ class RealFsTestMixin:
                               DummyTime(start, step))
         return DummyMock()
 
+    def assert_raises_os_error(self, subtype, expression, *args, **kwargs):
+        """Asserts that a specific subtype of OSError is raised."""
+        try:
+            expression(*args, **kwargs)
+            self.fail('No exception was raised, OSError expected')
+        except OSError as exc:
+            if isinstance(subtype, list):
+                self.assertIn(exc.errno, subtype)
+            else:
+                self.assertEqual(subtype, exc.errno)
+
 
 class RealFsTestCase(TestCase, RealFsTestMixin):
     """Can be used as base class for tests also running in the real
@@ -426,9 +429,6 @@ class RealFsTestCase(TestCase, RealFsTestMixin):
     def tearDown(self):
         RealFsTestMixin.tearDown(self)
 
-    def setUpFileSystem(self):
-        pass
-
     @property
     def is_windows_fs(self):
         if self.use_real_fs():