diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:01:02 +0000 |
---|---|---|
committer | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-07-07 05:01:02 +0000 |
commit | 9c96ddeb7a05036707b39f2103110321ee6f29d6 (patch) | |
tree | 94e3f97552dde847854b0a8dc558efb385a2d210 | |
parent | 6fc9dc112a9b29b44804d25ab28768fecbdbbba1 (diff) | |
parent | 8cfa3cf18049bb756eb0013ba512dc1aac7ba8d1 (diff) | |
download | tests-9c96ddeb7a05036707b39f2103110321ee6f29d6.tar.gz |
Snap for 10453563 from 8cfa3cf18049bb756eb0013ba512dc1aac7ba8d1 to mainline-media-swcodec-releaseaml_swc_341312300aml_swc_341312020aml_swc_341111000aml_swc_341011020aml_swc_340922010android14-mainline-media-swcodec-release
Change-Id: I6731fb2bb9c177180113ea4c9e856e171b75536e
64 files changed, 1800 insertions, 1491 deletions
diff --git a/Android.bp b/Android.bp deleted file mode 100644 index e6b4444..0000000 --- a/Android.bp +++ /dev/null @@ -1,17 +0,0 @@ -package { - // See: http://go/android-license-faq - default_applicable_licenses: ["Android-Apache-2.0"], -} - -python_defaults { - name: "kernel_tests_defaults", - version: { - py2: { - embedded_launcher: true, - enabled: true, - }, - py3: { - enabled: false, - }, - }, -} diff --git a/devicetree/early_mount/Android.bp b/devicetree/early_mount/Android.bp deleted file mode 100644 index 01d149e..0000000 --- a/devicetree/early_mount/Android.bp +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (C) 2016 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package { - // See: http://go/android-license-faq - default_applicable_licenses: ["Android-Apache-2.0"], -} - -python_test { - name: "dt_early_mount_test", - srcs: [ - "**/*.py", - ], - version: { - py2: { - embedded_launcher: true, - enabled: true, - }, - py3: { - enabled: false, - }, - }, -} diff --git a/devicetree/early_mount/dt_early_mount_test.py b/devicetree/early_mount/dt_early_mount_test.py deleted file mode 100755 index 6cabde4..0000000 --- a/devicetree/early_mount/dt_early_mount_test.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Test cases for device tree overlays for early mounting partitions.""" - -import os -import unittest - - -# Early mount fstab entry must have following properties defined. -REQUIRED_FSTAB_PROPERTIES = [ - 'dev', - 'type', - 'mnt_flags', - 'fsmgr_flags', -] - - -def ReadFile(file_path): - with open(file_path, 'r') as f: - # Strip all trailing spaces, newline and null characters. - return f.read().rstrip(' \n\x00') - - -def GetAndroidDtDir(): - """Returns location of android device tree directory.""" - with open('/proc/cmdline', 'r') as f: - cmdline_list = f.read().split() - - # Find android device tree directory path passed through kernel cmdline. - for option in cmdline_list: - if option.startswith('androidboot.android_dt_dir'): - return option.split('=')[1] - - # If no custom path found, return the default location. - return '/proc/device-tree/firmware/android' - - -class DtEarlyMountTest(unittest.TestCase): - """Test device tree overlays for early mounting.""" - - def setUp(self): - self._android_dt_dir = GetAndroidDtDir() - self._fstab_dt_dir = os.path.join(self._android_dt_dir, 'fstab') - - def GetEarlyMountedPartitions(self): - """Returns a list of partitions specified in fstab for early mount.""" - # Device tree nodes are represented as directories in the filesystem. - return [x for x in os.listdir(self._fstab_dt_dir) if os.path.isdir(x)] - - def VerifyFstabEntry(self, partition): - partition_dt_dir = os.path.join(self._fstab_dt_dir, partition) - properties = [x for x in os.listdir(partition_dt_dir)] - - self.assertTrue( - set(REQUIRED_FSTAB_PROPERTIES).issubset(properties), - 'fstab entry for /%s is missing required properties' % partition) - - def testFstabCompatible(self): - """Verify fstab compatible string.""" - compatible = ReadFile(os.path.join(self._fstab_dt_dir, 'compatible')) - self.assertEqual('android,fstab', compatible) - - def testFstabEntries(self): - """Verify properties of early mount fstab entries.""" - for partition in self.GetEarlyMountedPartitions(): - self.VerifyFstabEntry(partition) - -if __name__ == '__main__': - unittest.main() - diff --git a/net/test/Android.bp b/net/test/Android.bp index 2d789a2..b16b81f 100644 --- a/net/test/Android.bp +++ b/net/test/Android.bp @@ -3,32 +3,22 @@ package { default_applicable_licenses: ["Android-Apache-2.0"], } -python_defaults { - name: "kernel_net_tests_defaults", +// Main target used for VTS tests. +python_test { + name: "vts_kernel_net_tests", + stem: "kernel_net_tests_bin", srcs: [ "*.py", ], libs: [ "scapy", ], - defaults: ["kernel_tests_defaults",], -} - -// Currently, we keep it for vts10. This could be useful to produce a binary -// that can be run manually on the device. -// TODO(b/146651404): Remove all vts10 only test modules after vts11 -// is released. -python_test { - name: "kernel_net_tests", - main: "all_tests.py", - defaults: ["kernel_net_tests_defaults",], -} - -python_test { - name: "vts_kernel_net_tests", - stem: "kernel_net_tests_bin", main: "all_tests.py", - defaults: ["kernel_net_tests_defaults",], - test_suites: ["vts", "general-tests"], + version: { + py3: { + embedded_launcher: true, + }, + }, test_config: "vts_kernel_net_tests.xml", + test_suites: ["vts", "general-tests"], } diff --git a/net/test/TEST_MAPPING b/net/test/TEST_MAPPING new file mode 100644 index 0000000..bc27d17 --- /dev/null +++ b/net/test/TEST_MAPPING @@ -0,0 +1,12 @@ +{ + "presubmit": [ + { + "name": "vts_kernel_net_tests" + } + ], + "kernel-presubmit": [ + { + "name": "vts_kernel_net_tests" + } + ] +} diff --git a/net/test/all_tests.py b/net/test/all_tests.py index 2305354..4fd20dd 100755 --- a/net/test/all_tests.py +++ b/net/test/all_tests.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2018 The Android Open Source Project # @@ -26,33 +26,35 @@ test_modules = [ 'bpf_test', 'csocket_test', 'cstruct_test', - 'forwarding_test', 'leak_test', 'multinetwork_test', 'neighbour_test', + 'netlink_test', 'nf_test', + 'parameterization_test', 'pf_key_test', 'ping6_test', 'policy_crash_test', - 'qtaguid_test', 'removed_feature_test', 'resilient_rs_test', 'sock_diag_test', 'srcaddr_selection_test', + 'sysctls_test', 'tcp_fastopen_test', 'tcp_nuke_addr_test', 'tcp_repair_test', - 'tcp_test', 'xfrm_algorithm_test', 'xfrm_test', 'xfrm_tunnel_test', ] if __name__ == '__main__': - # Check whether ADB over TCP is occupying TCP port 5555, - # or if we're on a real Android device - if os.path.isdir('/system') or namespace.HasEstablishedTcpSessionOnPort(5555): - namespace.IfPossibleEnterNewNetworkNamespace() + namespace.EnterNewNetworkNamespace() + + # If one or more tests were passed in on the command line, only run those. + if len(sys.argv) > 1: + test_modules = sys.argv[1:] + # First, run InjectTests on all modules, to ensure that any parameterized # tests in those modules are injected. for name in test_modules: @@ -60,11 +62,7 @@ if __name__ == '__main__': if hasattr(sys.modules[name], 'InjectTests'): sys.modules[name].InjectTests() - loader = unittest.defaultTestLoader - if len(sys.argv) > 1: - test_suite = loader.loadTestsFromNames(sys.argv[1:]) - else: - test_suite = loader.loadTestsFromNames(test_modules) + test_suite = unittest.defaultTestLoader.loadTestsFromNames(test_modules) assert test_suite.countTestCases() > 0, ( 'Inconceivable: no tests found! Command line: %s' % ' '.join(sys.argv)) diff --git a/net/test/all_tests.sh b/net/test/all_tests.sh index 63576b0..aa63cdd 100755 --- a/net/test/all_tests.sh +++ b/net/test/all_tests.sh @@ -18,6 +18,10 @@ readonly PREFIX="#####" readonly RETRIES=2 test_prefix= +# The tests currently have hundreds of ResourceWarnings that make it hard +# to see errors/failures. Disable this warning for now. +export PYTHONWARNINGS="ignore::ResourceWarning" + function checkArgOrExit() { if [[ $# -lt 2 ]]; then echo "Missing argument for option $1" >&2 diff --git a/net/test/anycast_test.py b/net/test/anycast_test.py index 6222580..a63f661 100644..100755 --- a/net/test/anycast_test.py +++ b/net/test/anycast_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2014 The Android Open Source Project # @@ -35,7 +35,8 @@ _CLOSE_HUNG = False def CauseOops(): - open("/proc/sysrq-trigger", "w").write("c") + with open("/proc/sysrq-trigger", "w") as trigger: + trigger.write("c") class CloseFileDescriptorThread(threading.Thread): @@ -111,6 +112,7 @@ class AnycastTest(multinetwork_base.MultiNetworkBaseTest): # This doesn't seem to help, but still. self.AnycastSetsockopt(s, False, netid, addr) self.assertTrue(thread.finished) + s.close() if __name__ == "__main__": diff --git a/net/test/bpf.py b/net/test/bpf.py index 6d22423..b96c82a 100755 --- a/net/test/bpf.py +++ b/net/test/bpf.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2016 The Android Open Source Project # @@ -18,9 +18,9 @@ import ctypes import os -import platform import resource import socket +import sys import csocket import cstruct @@ -32,10 +32,6 @@ import net_test # around this problem and pick the right syscall nr, we can additionally check # the bitness of the python interpreter. Assume that the 64-bit architectures # are not running with COMPAT_UTS_MACHINE and must be 64-bit at all times. -# -# Is there a better way of doing this? -# Is it correct to use os.uname()[4] instead of platform.machine() ? -# Should we use 'sys.maxsize > 2**32' instead of platform.architecture()[0] ? __NR_bpf = { # pylint: disable=invalid-name "aarch64-32bit": 386, "aarch64-64bit": 280, @@ -46,7 +42,18 @@ __NR_bpf = { # pylint: disable=invalid-name "i686-64bit": 321, "x86_64-32bit": 357, "x86_64-64bit": 321, -}[os.uname()[4] + "-" + platform.architecture()[0]] + "riscv64-64bit": 280, +}[os.uname()[4] + "-" + ("64" if sys.maxsize > 0x7FFFFFFF else "32") + "bit"] + +# After ACK merge of 5.10.168 is when support for this was backported from +# upstream Linux 5.14 and was merged into ACK android{12,13}-5.10 branches. +# ACK android12-5.10 was >= 5.10.168 without this support only for ~4.5 hours +# ACK android13-4.10 was >= 5.10.168 without this support only for ~25 hours +# as such we can >= 5.10.168 instead of > 5.10.168 +HAVE_SO_NETNS_COOKIE = net_test.LINUX_VERSION >= (5, 10, 168) + +# Note: This is *not* correct for parisc & sparc architectures +SO_NETNS_COOKIE = 71 LOG_LEVEL = 1 LOG_SIZE = 65536 @@ -191,9 +198,6 @@ BpfInsn = cstruct.Struct("bpf_insn", "=BBhi", "code dst_src_reg off imm") # pylint: enable=invalid-name libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) -HAVE_EBPF_SUPPORT = net_test.LINUX_VERSION >= (4, 4, 0) -HAVE_EBPF_4_9 = net_test.LINUX_VERSION >= (4, 9, 0) -HAVE_EBPF_4_14 = net_test.LINUX_VERSION >= (4, 14, 0) HAVE_EBPF_4_19 = net_test.LINUX_VERSION >= (4, 19, 0) HAVE_EBPF_5_4 = net_test.LINUX_VERSION >= (5, 4, 0) @@ -257,11 +261,11 @@ def DeleteMap(map_fd, key): def BpfProgLoad(prog_type, instructions, prog_license=b"GPL"): - bpf_prog = "".join(instructions) + bpf_prog = b"".join(instructions) insn_buff = ctypes.create_string_buffer(bpf_prog) gpl_license = ctypes.create_string_buffer(prog_license) log_buf = ctypes.create_string_buffer(b"", LOG_SIZE) - attr = BpfAttrProgLoad((prog_type, len(insn_buff) / len(BpfInsn), + attr = BpfAttrProgLoad((prog_type, len(insn_buff) // len(BpfInsn), ctypes.addressof(insn_buff), ctypes.addressof(gpl_license), LOG_LEVEL, LOG_SIZE, ctypes.addressof(log_buf), 0)) diff --git a/net/test/bpf_test.py b/net/test/bpf_test.py index a014918..343ca97 100755 --- a/net/test/bpf_test.py +++ b/net/test/bpf_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2016 The Android Open Source Project # @@ -18,7 +18,6 @@ import ctypes import errno import os import socket -import subprocess import tempfile import unittest @@ -40,6 +39,7 @@ from bpf import BPF_FUNC_map_lookup_elem from bpf import BPF_FUNC_map_update_elem from bpf import BPF_FUNC_skb_change_head from bpf import BPF_JNE +from bpf import BPF_MAP_TYPE_ARRAY from bpf import BPF_MAP_TYPE_HASH from bpf import BPF_PROG_TYPE_CGROUP_SKB from bpf import BPF_PROG_TYPE_CGROUP_SOCK @@ -79,27 +79,11 @@ from bpf import LookupMap from bpf import UpdateMap import csocket import net_test -from net_test import LINUX_VERSION import sock_diag libc = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True) -HAVE_EBPF_ACCOUNTING = bpf.HAVE_EBPF_4_9 -HAVE_EBPF_SOCKET = bpf.HAVE_EBPF_4_14 - -# bpf_ktime_get_ns() was made non-GPL requiring in 5.8 and at the same time -# bpf_ktime_get_boot_ns() was added, both of these changes were backported to -# Android Common Kernel in 4.14.221, 4.19.175, 5.4.97. -# As such we require 4.14.222+ 4.19.176+ 5.4.98+ 5.8.0+, -# but since we only really care about LTS releases: -HAVE_EBPF_KTIME_GET_NS_APACHE2 = ( - ((LINUX_VERSION > (4, 14, 221)) and (LINUX_VERSION < (4, 19, 0))) or - ((LINUX_VERSION > (4, 19, 175)) and (LINUX_VERSION < (5, 4, 0))) or - (LINUX_VERSION > (5, 4, 97)) -) -HAVE_EBPF_KTIME_GET_BOOT_NS = HAVE_EBPF_KTIME_GET_NS_APACHE2 - -KEY_SIZE = 8 +KEY_SIZE = 4 VALUE_SIZE = 4 TOTAL_ENTRIES = 20 TEST_UID = 54321 @@ -129,19 +113,23 @@ def PrintMapInfo(map_fd): def SocketUDPLoopBack(packet_count, version, prog_fd): family = {4: socket.AF_INET, 6: socket.AF_INET6}[version] sock = socket.socket(family, socket.SOCK_DGRAM, 0) - if prog_fd is not None: - BpfProgAttachSocket(sock.fileno(), prog_fd) - net_test.SetNonBlocking(sock) - addr = {4: "127.0.0.1", 6: "::1"}[version] - sock.bind((addr, 0)) - addr = sock.getsockname() - sockaddr = csocket.Sockaddr(addr) - for _ in range(packet_count): - sock.sendto("foo", addr) - data, retaddr = csocket.Recvfrom(sock, 4096, 0) - assert "foo" == data - assert sockaddr == retaddr - return sock + try: + if prog_fd is not None: + BpfProgAttachSocket(sock.fileno(), prog_fd) + net_test.SetNonBlocking(sock) + addr = {4: "127.0.0.1", 6: "::1"}[version] + sock.bind((addr, 0)) + addr = sock.getsockname() + sockaddr = csocket.Sockaddr(addr) + for _ in range(packet_count): + sock.sendto(b"foo", addr) + data, retaddr = csocket.Recvfrom(sock, 4096, 0) + assert b"foo" == data + assert sockaddr == retaddr + return sock + except Exception as e: + sock.close() + raise e # The main code block for eBPF packet counting program. It takes a preloaded @@ -217,8 +205,6 @@ INS_BPF_PARAM_STORE = [ ] -@unittest.skipUnless(HAVE_EBPF_ACCOUNTING, - "BPF helper function is not fully supported") class BpfTest(net_test.NetworkTest): def setUp(self): @@ -230,10 +216,13 @@ class BpfTest(net_test.NetworkTest): def tearDown(self): if self.prog_fd >= 0: os.close(self.prog_fd) + self.prog_fd = -1 if self.map_fd >= 0: os.close(self.map_fd) + self.map_fd = -1 if self.sock: self.sock.close() + self.sock = None super(BpfTest, self).tearDown() def testCreateMap(self): @@ -281,6 +270,13 @@ class BpfTest(net_test.NetworkTest): key = first_key.value self.CheckAllMapEntry(key, TOTAL_ENTRIES - 1, value) + def testArrayNonZeroOffset(self): + self.map_fd = CreateMap(BPF_MAP_TYPE_ARRAY, KEY_SIZE, VALUE_SIZE, 2) + key = 1 + value = 123 + UpdateMap(self.map_fd, key, value) + self.assertEqual(value, LookupMap(self.map_fd, key).value) + def testRdOnlyMap(self): self.map_fd = CreateMap(BPF_MAP_TYPE_HASH, KEY_SIZE, VALUE_SIZE, TOTAL_ENTRIES, map_flags=BPF_F_RDONLY) @@ -304,8 +300,8 @@ class BpfTest(net_test.NetworkTest): ] instructions += INS_SK_FILTER_ACCEPT self.prog_fd = BpfProgLoad(BPF_PROG_TYPE_SOCKET_FILTER, instructions) - SocketUDPLoopBack(1, 4, self.prog_fd) - SocketUDPLoopBack(1, 6, self.prog_fd) + SocketUDPLoopBack(1, 4, self.prog_fd).close() + SocketUDPLoopBack(1, 6, self.prog_fd).close() def testPacketBlock(self): self.prog_fd = BpfProgLoad(BPF_PROG_TYPE_SOCKET_FILTER, INS_BPF_EXIT_BLOCK) @@ -327,8 +323,8 @@ class BpfTest(net_test.NetworkTest): + INS_SK_FILTER_ACCEPT) self.prog_fd = BpfProgLoad(BPF_PROG_TYPE_SOCKET_FILTER, instructions) packet_count = 10 - SocketUDPLoopBack(packet_count, 4, self.prog_fd) - SocketUDPLoopBack(packet_count, 6, self.prog_fd) + SocketUDPLoopBack(packet_count, 4, self.prog_fd).close() + SocketUDPLoopBack(packet_count, 6, self.prog_fd).close() self.assertEqual(packet_count * 2, LookupMap(self.map_fd, key).value) ############################################################################## @@ -350,8 +346,6 @@ class BpfTest(net_test.NetworkTest): # net: bpf: Allow TC programs to call BPF_FUNC_skb_change_head # commit 6f3f65d80dac8f2bafce2213005821fccdce194c # - @unittest.skipUnless(bpf.HAVE_EBPF_4_14, - "no bpf_skb_change_head() support for pre-4.14 kernels") def testSkbChangeHead(self): # long bpf_skb_change_head(struct sk_buff *skb, u32 len, u64 flags) instructions = [ @@ -383,8 +377,6 @@ class BpfTest(net_test.NetworkTest): # 5.4: https://android-review.googlesource.com/c/kernel/common/+/1355422 # commit 45217b91eaaa3a563247c4f470f4cb785de6b1c6 # - @unittest.skipUnless(HAVE_EBPF_KTIME_GET_NS_APACHE2, - "no bpf_ktime_get_ns() support for non-GPL programs") def testKtimeGetNsApache2(self): instructions = [BpfFuncCall(BPF_FUNC_ktime_get_ns)] + INS_BPF_EXIT_BLOCK self.prog_fd = BpfProgLoad(BPF_PROG_TYPE_SCHED_CLS, instructions, @@ -406,8 +398,6 @@ class BpfTest(net_test.NetworkTest): # 5.4: https://android-review.googlesource.com/c/kernel/common/+/1585252 # commit 57b3f4830fb66a6038c4c1c66ca2e138fe8be231 # - @unittest.skipUnless(HAVE_EBPF_KTIME_GET_BOOT_NS, - "no bpf_ktime_get_boot_ns() support") def testKtimeGetBootNs(self): instructions = [ BpfFuncCall(BPF_FUNC_ktime_get_boot_ns), @@ -416,6 +406,43 @@ class BpfTest(net_test.NetworkTest): b"Apache 2.0") # No exceptions? Good. + ############################################################################## + # + # Test for presence of upstream 5.14 kernel patches: + # + # Android12-5.10: + # UPSTREAM: net: initialize net->net_cookie at netns setup + # https://android-review.git.corp.google.com/c/kernel/common/+/2503195 + # + # UPSTREAM: net: retrieve netns cookie via getsocketopt + # https://android-review.git.corp.google.com/c/kernel/common/+/2503056 + # + # (and potentially if you care about kernel ABI) + # + # ANDROID: fix ABI by undoing atomic64_t -> u64 type conversion + # https://android-review.git.corp.google.com/c/kernel/common/+/2504335 + # + # Android13-5.10: + # UPSTREAM: net: initialize net->net_cookie at netns setup + # https://android-review.git.corp.google.com/c/kernel/common/+/2503795 + # + # UPSTREAM: net: retrieve netns cookie via getsocketopt + # https://android-review.git.corp.google.com/c/kernel/common/+/2503796 + # + # (and potentially if you care about kernel ABI) + # + # ANDROID: fix ABI by undoing atomic64_t -> u64 type conversion + # https://android-review.git.corp.google.com/c/kernel/common/+/2506895 + # + @unittest.skipUnless(bpf.HAVE_SO_NETNS_COOKIE, "no SO_NETNS_COOKIE support") + def testGetNetNsCookie(self): + sk = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM, 0) + cookie = sk.getsockopt(socket.SOL_SOCKET, bpf.SO_NETNS_COOKIE, 8) # sizeof(u64) == 8 + sk.close() + self.assertEqual(len(cookie), 8) + cookie = int.from_bytes(cookie, "little") + self.assertGreaterEqual(cookie, 0) + def testGetSocketCookie(self): self.map_fd = CreateMap(BPF_MAP_TYPE_HASH, KEY_SIZE, VALUE_SIZE, TOTAL_ENTRIES) @@ -455,36 +482,23 @@ class BpfTest(net_test.NetworkTest): uid = TEST_UID with net_test.RunAsUid(uid): self.assertRaisesErrno(errno.ENOENT, LookupMap, self.map_fd, uid) - SocketUDPLoopBack(packet_count, 4, self.prog_fd) + SocketUDPLoopBack(packet_count, 4, self.prog_fd).close() self.assertEqual(packet_count, LookupMap(self.map_fd, uid).value) DeleteMap(self.map_fd, uid) - SocketUDPLoopBack(packet_count, 6, self.prog_fd) + SocketUDPLoopBack(packet_count, 6, self.prog_fd).close() self.assertEqual(packet_count, LookupMap(self.map_fd, uid).value) -@unittest.skipUnless(HAVE_EBPF_ACCOUNTING, - "Cgroup BPF is not fully supported") class BpfCgroupTest(net_test.NetworkTest): @classmethod def setUpClass(cls): super(BpfCgroupTest, cls).setUpClass() - cls._cg_dir = tempfile.mkdtemp(prefix="cg_bpf-") - cmd = "mount -t cgroup2 cg_bpf %s" % cls._cg_dir - try: - subprocess.check_call(cmd.split()) - except subprocess.CalledProcessError: - # If an exception is thrown in setUpClass, the test fails and - # tearDownClass is not called. - os.rmdir(cls._cg_dir) - raise - cls._cg_fd = os.open(cls._cg_dir, os.O_DIRECTORY | os.O_RDONLY) + cls._cg_fd = os.open("/sys/fs/cgroup", os.O_DIRECTORY | os.O_RDONLY) @classmethod def tearDownClass(cls): os.close(cls._cg_fd) - subprocess.call(("umount %s" % cls._cg_dir).split()) - os.rmdir(cls._cg_dir) super(BpfCgroupTest, cls).tearDownClass() def setUp(self): @@ -522,8 +536,8 @@ class BpfCgroupTest(net_test.NetworkTest): self.assertRaisesErrno(errno.EAGAIN, SocketUDPLoopBack, 1, 4, None) self.assertRaisesErrno(errno.EAGAIN, SocketUDPLoopBack, 1, 6, None) BpfProgDetach(self._cg_fd, BPF_CGROUP_INET_INGRESS) - SocketUDPLoopBack(1, 4, None) - SocketUDPLoopBack(1, 6, None) + SocketUDPLoopBack(1, 4, None).close() + SocketUDPLoopBack(1, 6, None).close() def testCgroupEgress(self): self.prog_fd = BpfProgLoad(BPF_PROG_TYPE_CGROUP_SKB, INS_BPF_EXIT_BLOCK) @@ -531,8 +545,8 @@ class BpfCgroupTest(net_test.NetworkTest): self.assertRaisesErrno(errno.EPERM, SocketUDPLoopBack, 1, 4, None) self.assertRaisesErrno(errno.EPERM, SocketUDPLoopBack, 1, 6, None) BpfProgDetach(self._cg_fd, BPF_CGROUP_INET_EGRESS) - SocketUDPLoopBack(1, 4, None) - SocketUDPLoopBack(1, 6, None) + SocketUDPLoopBack(1, 4, None).close() + SocketUDPLoopBack(1, 6, None).close() def testCgroupBpfUid(self): self.map_fd = CreateMap(BPF_MAP_TYPE_HASH, KEY_SIZE, VALUE_SIZE, @@ -551,10 +565,10 @@ class BpfCgroupTest(net_test.NetworkTest): uid = TEST_UID with net_test.RunAsUid(uid): self.assertRaisesErrno(errno.ENOENT, LookupMap, self.map_fd, uid) - SocketUDPLoopBack(packet_count, 4, None) + SocketUDPLoopBack(packet_count, 4, None).close() self.assertEqual(packet_count, LookupMap(self.map_fd, uid).value) DeleteMap(self.map_fd, uid) - SocketUDPLoopBack(packet_count, 6, None) + SocketUDPLoopBack(packet_count, 6, None).close() self.assertEqual(packet_count, LookupMap(self.map_fd, uid).value) BpfProgDetach(self._cg_fd, BPF_CGROUP_INET_INGRESS) @@ -576,8 +590,6 @@ class BpfCgroupTest(net_test.NetworkTest): for socktype in [socket.SOCK_DGRAM, socket.SOCK_STREAM]: self.checkSocketCreate(family, socktype, success) - @unittest.skipUnless(HAVE_EBPF_SOCKET, - "Cgroup BPF socket is not supported") def testCgroupSocketCreateBlock(self): instructions = [ BpfFuncCall(BPF_FUNC_get_current_uid_gid), diff --git a/net/test/build_rootfs.sh b/net/test/build_rootfs.sh index e631fe8..ee79c86 100755 --- a/net/test/build_rootfs.sh +++ b/net/test/build_rootfs.sh @@ -21,24 +21,27 @@ set -u SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) usage() { - echo -n "usage: $0 [-h] [-s bullseye|bullseye-cuttlefish|bullseye-rockpi] " + echo -n "usage: $0 [-h] [-s bullseye|bullseye-cuttlefish|bullseye-rockpi|bullseye-server] " echo -n "[-a i386|amd64|armhf|arm64] -k /path/to/kernel " echo -n "-i /path/to/initramfs.gz [-d /path/to/dtb:subdir] " - echo "[-m http://mirror/debian] [-n rootfs] [-r initrd] [-e]" + echo "[-m http://mirror/debian] [-n rootfs|disk] [-r initrd] [-e] [-g]" exit 1 } mirror=http://ftp.debian.org/debian +embed_kernel_initrd_dtb=0 +install_grub=0 suite=bullseye arch=amd64 -embed_kernel_initrd_dtb= dtb_subdir= +initramfs= +kernel= ramdisk= -rootfs= +disk= dtb= -while getopts ":hs:a:m:n:r:k:i:d:e" opt; do +while getopts ":hs:a:m:n:r:k:i:d:eg" opt; do case "${opt}" in h) usage @@ -57,7 +60,7 @@ while getopts ":hs:a:m:n:r:k:i:d:e" opt; do mirror="${OPTARG}" ;; n) - rootfs="${OPTARG}" + disk="${OPTARG}" ;; r) ramdisk="${OPTARG}" @@ -77,6 +80,9 @@ while getopts ":hs:a:m:n:r:k:i:d:e" opt; do e) embed_kernel_initrd_dtb=1 ;; + g) + install_grub=1 + ;; \?) echo "Invalid option: ${OPTARG}" >&2 usage @@ -89,36 +95,37 @@ while getopts ":hs:a:m:n:r:k:i:d:e" opt; do done # Disable Debian's "persistent" network device renaming -cmdline="net.ifnames=0 rw 8250.nr_uarts=2 PATH=/usr/sbin:/usr/bin" - -# Pass down embedding option, if specified -if [ -n "${embed_kernel_initrd_dtb}" ]; then - cmdline="${cmdline} embed_kernel_initrd_dtb=${embed_kernel_initrd_dtb}" -fi +cmdline="net.ifnames=0 rw 8250.nr_uarts=2 PATH=/usr/sbin:/bin:/usr/bin" +cmdline="${cmdline} embed_kernel_initrd_dtb=${embed_kernel_initrd_dtb}" +cmdline="${cmdline} install_grub=${install_grub}" case "${arch}" in i386) cmdline="${cmdline} console=ttyS0 exitcode=/dev/ttyS1" machine="pc-i440fx-2.8,accel=kvm" qemu="qemu-system-i386" + partguid="8303" cpu="max" ;; amd64) cmdline="${cmdline} console=ttyS0 exitcode=/dev/ttyS1" machine="pc-i440fx-2.8,accel=kvm" qemu="qemu-system-x86_64" + partguid="8304" cpu="max" ;; armhf) cmdline="${cmdline} console=ttyAMA0 exitcode=/dev/ttyS0" machine="virt,gic-version=2" qemu="qemu-system-arm" + partguid="8307" cpu="cortex-a15" ;; arm64) cmdline="${cmdline} console=ttyAMA0 exitcode=/dev/ttyS0" machine="virt,gic-version=2" qemu="qemu-system-aarch64" + partguid="8305" cpu="cortex-a53" # "max" is too slow ;; *) @@ -127,10 +134,15 @@ case "${arch}" in ;; esac -if [[ -z "${rootfs}" ]]; then - rootfs="rootfs.${arch}.${suite}.$(date +%Y%m%d)" +if [[ -z "${disk}" ]]; then + if [[ "${install_grub}" = "1" ]]; then + base_image_name=disk + else + base_image_name=rootfs + fi + disk="${base_image_name}.${arch}.${suite}.$(date +%Y%m%d)" fi -rootfs=$(realpath "${rootfs}") +disk=$(realpath "${disk}") if [[ -z "${ramdisk}" ]]; then ramdisk="initrd.${arch}.${suite}.$(date +%Y%m%d)" @@ -156,7 +168,7 @@ fi # Sometimes it isn't obvious when the script fails failure() { echo "Filesystem generation process failed." >&2 - rm -f "${rootfs}" "${ramdisk}" + rm -f "${disk}" "${ramdisk}" } trap failure ERR @@ -178,8 +190,17 @@ sudo chown root:root "${workdir}" # Run the debootstrap first cd "${workdir}" -sudo debootstrap --arch="${arch}" --variant=minbase --include="${packages}" \ - --foreign "${suite%-*}" . "${mirror}" + +retries=5 +while ! sudo debootstrap --arch="${arch}" --variant=minbase --include="${packages}" \ + --foreign "${suite%-*}" . "${mirror}"; do + retries=$((${retries} - 1)) + if [ ${retries} -le 0 ]; then + failure + exit 1 + fi + echo "debootstrap failed - trying again - ${retries} retries left" +done # Copy some bootstrapping scripts into the rootfs sudo cp -a "${SCRIPT_DIR}"/rootfs/*.sh root/ @@ -192,6 +213,15 @@ lz4 -lcd "${initramfs}" | sudo cpio -idum lib/modules/* # Create /host, for the pivot_root and 9p mount use cases sudo mkdir host +# debootstrap workaround: Run debootstrap in docker sometimes causes the +# /proc being a symlink in first stage. We need to fix the symlink to an empty +# directory. +if [ -L "${workdir}/proc" ]; then + echo "/proc in debootstrap 1st stage is a symlink. Fixed!" + sudo rm -f "${workdir}/proc" + sudo mkdir "${workdir}/proc" +fi + # Leave the workdir, to build the filesystem cd - @@ -212,10 +242,10 @@ initrd_remove() { } trap initrd_remove EXIT truncate -s 512M "${initrd}" -mke2fs -F -t ext3 -L ROOT "${initrd}" +/sbin/mke2fs -F -t ext4 -L ROOT "${initrd}" # Mount the new filesystem locally -sudo mount -o loop -t ext3 "${initrd}" "${mount}" +sudo mount -o loop -t ext4 "${initrd}" "${mount}" image_unmount() { sudo umount "${mount}" initrd_remove @@ -230,11 +260,78 @@ sudo rm -rf "${workdir}" sudo umount "${mount}" trap initrd_remove EXIT -# Copy the initial ramdisk to the final rootfs name and extend it -sudo cp -a "${initrd}" "${rootfs}" -truncate -s 2G "${rootfs}" -e2fsck -p -f "${rootfs}" || true -resize2fs "${rootfs}" +if [[ "${install_grub}" = 1 ]]; then + part_num=0 + # $1 partition size + # $2 gpt partition type + # $3 partition name + # $4 bypass alignment checks (use on <1MB partitions only) + # $5 partition attribute bit to set + sgdisk() { + part_num=$((part_num+1)) + [[ -n "${4:-}" ]] && prefix="-a1" || prefix= + [[ -n "${5:-}" ]] && suffix="-A:${part_num}:set:$5" || suffix= + /sbin/sgdisk ${prefix} \ + "-n:${part_num}:$1" "-t:${part_num}:$2" "-c:${part_num}:$3" \ + ${suffix} "${disk}" >/dev/null 2>&1 + } + # If there's a bootloader, we need to make space for the GPT header, GPT + # footer and EFI system partition (legacy boot is not supported) + # Keep this simple - modern gdisk reserves 1MB for the GPT header and + # assumes all partitions are 1MB aligned + truncate -s "$((1 + 128 + 10 * 1024 + 1))M" "${disk}" + /sbin/sgdisk --zap-all "${disk}" >/dev/null 2>&1 + # On RockPi devices, steal a bit of space at the start of the disk for + # some special bootloader partitions. Some of these have to start/end + # at specific offsets as well + if [[ "${suite#*-}" = "rockpi" ]]; then + # See https://opensource.rock-chips.com/wiki_Boot_option + # Keep in sync with rootfs/*-rockpi.sh + sgdisk "64:8127" "8301" "idbloader" "true" + sgdisk "8128:+64" "8301" "uboot_env" "true" + sgdisk "8M:+4M" "8301" "uboot" + sgdisk "12M:+4M" "8301" "trust" + sgdisk "16M:+1M" "8301" "misc" + sgdisk "17M:+128M" "ef00" "esp" "" "0" + sgdisk "145M:0" "8305" "rootfs" "" "2" + system_partition="6" + rootfs_partition="7" + else + sgdisk "0:+128M" "ef00" "esp" "" "0" + sgdisk "0:0" "${partguid}" "rootfs" "" "2" + system_partition="1" + rootfs_partition="2" + fi + + # Create an empty EFI system partition; it will be initialized later + system_partition_start=$(partx -g -o START -s -n "${system_partition}" "${disk}" | xargs) + system_partition_end=$(partx -g -o END -s -n "${system_partition}" "${disk}" | xargs) + system_partition_num_sectors=$((${system_partition_end} - ${system_partition_start} + 1)) + system_partition_num_vfat_blocks=$((${system_partition_num_sectors} / 2)) + /sbin/mkfs.vfat -n SYSTEM -F 16 --offset=${system_partition_start} "${disk}" ${system_partition_num_vfat_blocks} >/dev/null + # Copy the rootfs to just after the EFI system partition + rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1)) + rootfs_partition_offset=$((${rootfs_partition_start} * 512)) + rootfs_partition_size=$((${rootfs_partition_num_sectors} * 512)) + dd if="${initrd}" of="${disk}" bs=512 seek="${rootfs_partition_start}" conv=fsync,notrunc 2>/dev/null + /sbin/e2fsck -p -f "${disk}"?offset=${rootfs_partition_offset} || true + disksize=$(stat -c %s "${disk}") + /sbin/resize2fs "${disk}"?offset=${rootfs_partition_offset} ${rootfs_partition_num_sectors}s + truncate -s "${disksize}" "${disk}" + /sbin/sgdisk -e "${disk}" + /sbin/e2fsck -p -f "${disk}"?offset=${rootfs_partition_offset} || true + /sbin/e2fsck -fy "${disk}"?offset=${rootfs_partition_offset} || true +else + # If there's no bootloader, the initrd is the disk image + cp -a "${initrd}" "${disk}" + truncate -s 10G "${disk}" + /sbin/e2fsck -p -f "${disk}" || true + /sbin/resize2fs "${disk}" + system_partition= + rootfs_partition="raw" +fi # Create another fake block device for initrd.img writeout raw_initrd=$(mktemp) @@ -245,14 +342,20 @@ raw_initrd_remove() { trap raw_initrd_remove EXIT truncate -s 64M "${raw_initrd}" +# Get number of cores for qemu. Restrict the maximum value to 8. +qemucpucores=$(nproc) +if [[ ${qemucpucores} -gt 8 ]]; then + qemucpucores=8 +fi + # Complete the bootstrap process using QEMU and the specified kernel ${qemu} -machine "${machine}" -cpu "${cpu}" -m 2048 >&2 \ -kernel "${kernel}" -initrd "${initrd}" -no-user-config -nodefaults \ -no-reboot -display none -nographic -serial stdio -parallel none \ - -smp 8,sockets=8,cores=1,threads=1 \ + -smp "${qemucpucores}",sockets="${qemucpucores}",cores=1,threads=1 \ -object rng-random,id=objrng0,filename=/dev/urandom \ -device virtio-rng-pci-non-transitional,rng=objrng0,id=rng0,max-bytes=1024,period=2000 \ - -drive file="${rootfs}",format=raw,if=none,aio=threads,id=drive-virtio-disk0 \ + -drive file="${disk}",format=raw,if=none,aio=threads,id=drive-virtio-disk0 \ -device virtio-blk-pci-non-transitional,scsi=off,drive=drive-virtio-disk0 \ -drive file="${raw_initrd}",format=raw,if=none,aio=threads,id=drive-virtio-disk1 \ -device virtio-blk-pci-non-transitional,scsi=off,drive=drive-virtio-disk1 \ @@ -267,7 +370,32 @@ if [ "${exitcode}" != "0" ]; then fi # Fix up any issues from the unclean shutdown -e2fsck -p -f "${rootfs}" || true +if [[ ${rootfs_partition} = "raw" ]]; then + sudo e2fsck -p -f "${disk}" || true +else + rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1)) + rootfs_partition_offset=$((${rootfs_partition_start} * 512)) + rootfs_partition_tempfile2=$(mktemp) + dd if="${disk}" of="${rootfs_partition_tempfile2}" bs=512 skip=${rootfs_partition_start} count=${rootfs_partition_num_sectors} + e2fsck -p -f "${rootfs_partition_tempfile2}" || true + dd if="${rootfs_partition_tempfile2}" of="${disk}" bs=512 seek=${rootfs_partition_start} count=${rootfs_partition_num_sectors} conv=fsync,notrunc + rm -f "${rootfs_partition_tempfile2}" + e2fsck -fy "${disk}"?offset=${rootfs_partition_offset} || true +fi +if [[ -n "${system_partition}" ]]; then + system_partition_start=$(partx -g -o START -s -n "${system_partition}" "${disk}" | xargs) + system_partition_end=$(partx -g -o END -s -n "${system_partition}" "${disk}" | xargs) + system_partition_num_sectors=$((${system_partition_end} - ${system_partition_start} + 1)) + system_partition_offset=$((${system_partition_start} * 512)) + system_partition_size=$((${system_partition_num_sectors} * 512)) + system_partition_tempfile=$(mktemp) + dd if="${disk}" of="${system_partition_tempfile}" bs=512 skip=${system_partition_start} count=${system_partition_num_sectors} + /sbin/fsck.vfat -a "${system_partition_tempfile}" || true + dd if="${system_partition_tempfile}" of="${disk}" bs=512 seek=${system_partition_start} count=${system_partition_num_sectors} conv=fsync,notrunc + rm -f "${system_partition_tempfile}" +fi # New workdir for the initrd extraction workdir="${tmpdir}/initrd" @@ -280,6 +408,7 @@ cd "${workdir}" # Process the initrd to remove kernel-specific metadata kernel_version=$(basename $(lz4 -lcd "${raw_initrd}" | sudo cpio -idumv 2>&1 | grep usr/lib/modules/ - | head -n1)) +lz4 -lcd "${raw_initrd}" | sudo cpio -idumv sudo rm -rf usr/lib/modules sudo mkdir -p usr/lib/modules @@ -299,37 +428,66 @@ cat "${ramdisk}" "${initramfs}" >"${initrd}" # Leave workdir to boot-test combined initrd cd - +rootfs_partition_tempfile=$(mktemp) # Mount the new filesystem locally -sudo mount -o loop -t ext3 "${rootfs}" "${mount}" +if [[ ${rootfs_partition} = "raw" ]]; then + sudo mount -o loop -t ext4 "${disk}" "${mount}" +else + rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_offset=$((${rootfs_partition_start} * 512)) + rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1)) + dd if="${disk}" of="${rootfs_partition_tempfile}" bs=512 skip=${rootfs_partition_start} count=${rootfs_partition_num_sectors} +fi image_unmount2() { sudo umount "${mount}" raw_initrd_remove } -trap image_unmount2 EXIT +if [[ ${rootfs_partition} = "raw" ]]; then + trap image_unmount2 EXIT +fi # Embed the kernel and dtb images now, if requested -if [ -n "${embed_kernel_initrd_dtb}" ]; then - if [ -n "${dtb}" ]; then - sudo mkdir -p "${mount}/boot/dtb/${dtb_subdir}" - sudo cp -a "${dtb}" "${mount}/boot/dtb/${dtb_subdir}" - sudo chown -R root:root "${mount}/boot/dtb/${dtb_subdir}" - fi - sudo cp -a "${kernel}" "${mount}/boot/vmlinuz-${kernel_version}" - sudo chown root:root "${mount}/boot/vmlinuz-${kernel_version}" +if [[ ${rootfs_partition} = "raw" ]]; then + if [[ "${embed_kernel_initrd_dtb}" = "1" ]]; then + if [ -n "${dtb}" ]; then + sudo mkdir -p "${mount}/boot/dtb/${dtb_subdir}" + sudo cp -a "${dtb}" "${mount}/boot/dtb/${dtb_subdir}" + sudo chown -R root:root "${mount}/boot/dtb/${dtb_subdir}" + fi + sudo cp -a "${kernel}" "${mount}/boot/vmlinuz-${kernel_version}" + sudo chown root:root "${mount}/boot/vmlinuz-${kernel_version}" + fi +else + if [[ "${embed_kernel_initrd_dtb}" = "1" ]]; then + if [ -n "${dtb}" ]; then + e2mkdir -G 0 -O 0 "${rootfs_partition_tempfile}":"/boot/dtb/${dtb_subdir}" + e2cp -G 0 -O 0 "${dtb}" "${rootfs_partition_tempfile}":"/boot/dtb/${dtb_subdir}" + fi + e2cp -G 0 -O 0 "${kernel}" "${rootfs_partition_tempfile}":"/boot/vmlinuz-${kernel_version}" + fi fi # Unmount the initial ramdisk -sudo umount "${mount}" +if [[ ${rootfs_partition} = "raw" ]]; then + sudo umount "${mount}" +else + rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1)) + dd if="${rootfs_partition_tempfile}" of="${disk}" bs=512 seek=${rootfs_partition_start} count=${rootfs_partition_num_sectors} conv=fsync,notrunc +fi +rm -f "${rootfs_partition_tempfile}" trap raw_initrd_remove EXIT # Boot test the new system and run stage 3 ${qemu} -machine "${machine}" -cpu "${cpu}" -m 2048 >&2 \ -kernel "${kernel}" -initrd "${initrd}" -no-user-config -nodefaults \ -no-reboot -display none -nographic -serial stdio -parallel none \ - -smp 8,sockets=8,cores=1,threads=1 \ + -smp "${qemucpucores}",sockets="${qemucpucores}",cores=1,threads=1 \ -object rng-random,id=objrng0,filename=/dev/urandom \ -device virtio-rng-pci-non-transitional,rng=objrng0,id=rng0,max-bytes=1024,period=2000 \ - -drive file="${rootfs}",format=raw,if=none,aio=threads,id=drive-virtio-disk0 \ + -drive file="${disk}",format=raw,if=none,aio=threads,id=drive-virtio-disk0 \ -device virtio-blk-pci-non-transitional,scsi=off,drive=drive-virtio-disk0 \ -chardev file,id=exitcode,path=exitcode \ -device pci-serial,chardev=exitcode \ @@ -344,10 +502,41 @@ if [ "${exitcode}" != "0" ]; then fi # Fix up any issues from the unclean shutdown -e2fsck -p -f "${rootfs}" || true +if [[ ${rootfs_partition} = "raw" ]]; then + sudo e2fsck -p -f "${disk}" || true +else + rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_end=$(partx -g -o END -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_num_sectors=$((${rootfs_partition_end} - ${rootfs_partition_start} + 1)) + rootfs_partition_offset=$((${rootfs_partition_start} * 512)) + rootfs_partition_tempfile2=$(mktemp) + dd if="${disk}" of="${rootfs_partition_tempfile2}" bs=512 skip=${rootfs_partition_start} count=${rootfs_partition_num_sectors} + e2fsck -p -f "${rootfs_partition_tempfile2}" || true + dd if="${rootfs_partition_tempfile2}" of="${disk}" bs=512 seek=${rootfs_partition_start} count=${rootfs_partition_num_sectors} conv=fsync,notrunc + rm -f "${rootfs_partition_tempfile2}" + e2fsck -fy "${disk}"?offset=${rootfs_partition_offset} || true +fi +if [[ -n "${system_partition}" ]]; then + system_partition_start=$(partx -g -o START -s -n "${system_partition}" "${disk}" | xargs) + system_partition_end=$(partx -g -o END -s -n "${system_partition}" "${disk}" | xargs) + system_partition_num_sectors=$((${system_partition_end} - ${system_partition_start} + 1)) + system_partition_offset=$((${system_partition_start} * 512)) + system_partition_size=$((${system_partition_num_sectors} * 512)) + system_partition_tempfile=$(mktemp) + dd if="${disk}" of="${system_partition_tempfile}" bs=512 skip=${system_partition_start} count=${system_partition_num_sectors} + /sbin/fsck.vfat -a "${system_partition_tempfile}" || true + dd if="${system_partition_tempfile}" of="${disk}" bs=512 seek=${system_partition_start} count=${system_partition_num_sectors} conv=fsync,notrunc + rm -f "${system_partition_tempfile}" +fi -# Mount the final rootfs locally -sudo mount -o loop -t ext3 "${rootfs}" "${mount}" +# Mount the final disk image locally +if [[ ${rootfs_partition} = "raw" ]]; then + sudo mount -o loop -t ext4 "${disk}" "${mount}" +else + rootfs_partition_start=$(partx -g -o START -s -n "${rootfs_partition}" "${disk}" | xargs) + rootfs_partition_offset=$((${rootfs_partition_start} * 512)) + sudo mount -o loop,offset=${rootfs_partition_offset} -t ext4 "${disk}" "${mount}" +fi image_unmount3() { sudo umount "${mount}" raw_initrd_remove @@ -358,5 +547,5 @@ trap image_unmount3 EXIT sudo dd if=/dev/zero of="${mount}/sparse" bs=1M 2>/dev/null || true sudo rm -f "${mount}/sparse" -echo "Debian ${suite} for ${arch} filesystem generated at '${rootfs}'." +echo "Debian ${suite} for ${arch} filesystem generated at '${disk}'." echo "Initial ramdisk generated at '${ramdisk}'." diff --git a/net/test/csocket.py b/net/test/csocket.py index ccabf4a..fac988b 100644 --- a/net/test/csocket.py +++ b/net/test/csocket.py @@ -93,7 +93,7 @@ def AddressVersion(addr): def SetSocketTimeout(sock, ms): - s = ms / 1000 + s = ms // 1000 us = (ms % 1000) * 1000 sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, struct.pack("LL", s, us)) @@ -147,7 +147,7 @@ def _MakeMsgControl(optlist): Raises: TypeError: Option data is neither an integer nor a string. """ - msg_control = "" + msg_control = b"" for i, opt in enumerate(optlist): msg_level, msg_type, data = opt @@ -155,13 +155,13 @@ def _MakeMsgControl(optlist): data = struct.pack("=I", data) elif isinstance(data, ctypes.c_uint32): data = struct.pack("=I", data.value) - elif not isinstance(data, str): + elif not isinstance(data, bytes): raise TypeError("unknown data type for opt (%d, %d): %s" % ( msg_level, msg_type, type(data))) datalen = len(data) msg_len = len(CMsgHdr) + datalen - padding = "\x00" * util.GetPadLength(CMSG_ALIGNTO, datalen) + padding = b"\x00" * util.GetPadLength(CMSG_ALIGNTO, datalen) msg_control += CMsgHdr((msg_len, msg_level, msg_type)).Pack() msg_control += data + padding @@ -326,7 +326,7 @@ def Recvmsg(s, buflen, controllen, flags, addrlen=len(SockaddrStorage)): MaybeRaiseSocketError(ret) data = buf.raw[:ret] - msghdr = MsgHdr(str(msghdr._buffer.raw)) + msghdr = MsgHdr(msghdr._buffer.raw) addr = _ToSocketAddress(addr, msghdr.namelen) control = control.raw[:msghdr.msg_controllen] msglist = _ParseMsgControl(control) diff --git a/net/test/csocket_test.py b/net/test/csocket_test.py index 19760fe..57afaa9 100755 --- a/net/test/csocket_test.py +++ b/net/test/csocket_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2016 The Android Open Source Project # @@ -45,9 +45,9 @@ class CsocketTest(unittest.TestCase): s = self._BuildSocket(family, addr) addr = s.getsockname() sockaddr = csocket.Sockaddr(addr) - s.sendto("foo", addr) + s.sendto(b"foo", addr) data, addr = csocket.Recvfrom(s, 4096, 0) - self.assertEqual("foo", data) + self.assertEqual(b"foo", data) self.assertEqual(sockaddr, addr) s.close() @@ -77,9 +77,9 @@ class CsocketTest(unittest.TestCase): addr = s.getsockname() sockaddr = csocket.Sockaddr(addr) - s.sendto("foo", addr) + s.sendto(b"foo", addr) data, addr, cmsg = csocket.Recvmsg(s, 4096, 1024, 0) - self.assertEqual("foo", data) + self.assertEqual(b"foo", data) self.assertEqual(sockaddr, addr) self.assertEqual([pktinfo, ttl], cmsg) diff --git a/net/test/cstruct.py b/net/test/cstruct.py index c675c9e..c10667a 100644 --- a/net/test/cstruct.py +++ b/net/test/cstruct.py @@ -67,44 +67,56 @@ NLMsgHdr(length=44, type=33, flags=0, seq=0, pid=0) >>> """ +import binascii import ctypes import string import struct import re -def CalcSize(fmt): +def _PythonFormat(fmt): if "A" in fmt: fmt = fmt.replace("A", "s") - # Remove the last digital since it will cause error in python3. - fmt = (re.split('\d+$', fmt)[0]) - return struct.calcsize(fmt) + return re.split('\d+$', fmt)[0] + +def CalcSize(fmt): + return struct.calcsize(_PythonFormat(fmt)) def CalcNumElements(fmt): + fmt = _PythonFormat(fmt) prevlen = len(fmt) fmt = fmt.replace("S", "") numstructs = prevlen - len(fmt) - size = CalcSize(fmt) + size = struct.calcsize(fmt) elements = struct.unpack(fmt, b"\x00" * size) return len(elements) + numstructs -def Struct(name, fmt, fieldnames, substructs={}): - """Function that returns struct classes.""" +class StructMetaclass(type): - class Meta(type): + def __len__(cls): + return cls._length - def __len__(cls): - return cls._length + def __init__(cls, unused_name, unused_bases, namespace): + # Make the class object have the name that's passed in. + type.__init__(cls, namespace["_name"], unused_bases, namespace) - def __init__(cls, unused_name, unused_bases, namespace): - # Make the class object have the name that's passed in. - type.__init__(cls, namespace["_name"], unused_bases, namespace) - class CStruct(object): - """Class representing a C-like structure.""" +def Struct(name, fmt, fieldnames, substructs={}): + """Function that returns struct classes.""" - __metaclass__ = Meta + # Hack to make struct classes use the StructMetaclass class on both python2 and + # python3. This is needed because in python2 the metaclass is assigned in the + # class definition, but in python3 it's passed into the constructor via + # keyword argument. Works by making all structs subclass CStructSuperclass, + # whose __new__ method uses StructMetaclass as its metaclass. + # + # A better option would be to use six.with_metaclass, but the existing python2 + # VM image doesn't have the six module. + CStructSuperclass = type.__new__(StructMetaclass, 'unused', (), {}) + + class CStruct(CStructSuperclass): + """Class representing a C-like structure.""" # Name of the struct. _name = name @@ -129,8 +141,11 @@ def Struct(name, fmt, fieldnames, substructs={}): laststructindex += 1 _format += "%ds" % len(_nested[index]) elif fmt[i] == "A": - # Null-terminated ASCII string. - index = CalcNumElements(fmt[:i]) + # Null-terminated ASCII string. Remove digits before the A, so we don't + # call CalcNumElements on an (invalid) format that ends with a digit. + start = i + while start > 0 and fmt[start - 1].isdigit(): start -= 1 + index = CalcNumElements(fmt[:start]) _asciiz.add(index) _format += "s" else: @@ -165,7 +180,7 @@ def Struct(name, fmt, fieldnames, substructs={}): data = data[:self._length] values = list(struct.unpack(self._format, data)) for index, value in enumerate(values): - if isinstance(value, str) and index in self._nested: + if isinstance(value, bytes) and index in self._nested: values[index] = self._nested[index](value) self._SetValues(values) @@ -175,7 +190,7 @@ def Struct(name, fmt, fieldnames, substructs={}): 1. With no args, the whole struct is zero-initialized. 2. With keyword args, the matching fields are populated; rest are zeroed. 3. With one tuple as the arg, the fields are assigned based on position. - 4. With one string arg, the Struct is parsed from bytes. + 4. With one bytes arg, the Struct is parsed from bytes. """ if tuple_or_bytes and kwargs: raise TypeError( @@ -183,22 +198,23 @@ def Struct(name, fmt, fieldnames, substructs={}): if tuple_or_bytes is None: # Default construct from null bytes. - self._Parse("\x00" * len(self)) + self._Parse(b"\x00" * len(self)) # If any keywords were supplied, set those fields. for k, v in kwargs.items(): setattr(self, k, v) - elif isinstance(tuple_or_bytes, str): - # Initializing from a string. + elif isinstance(tuple_or_bytes, bytes): + # Initializing from bytes. if len(tuple_or_bytes) < self._length: - raise TypeError("%s requires string of length %d, got %d" % + raise TypeError("%s requires a bytes object of length %d, got %d" % (self._name, self._length, len(tuple_or_bytes))) self._Parse(tuple_or_bytes) else: # Initializing from a tuple. if len(tuple_or_bytes) != len(self._fieldnames): - raise TypeError("%s has exactly %d fieldnames (%d given)" % + raise TypeError("%s has exactly %d fieldnames: (%s), %d given: (%s)" % (self._name, len(self._fieldnames), - len(tuple_or_bytes))) + ", ".join(self._fieldnames), len(tuple_or_bytes), + ", ".join(str(x) for x in tuple_or_bytes))) self._SetValues(tuple_or_bytes) def _FieldIndex(self, attr): @@ -236,7 +252,7 @@ def Struct(name, fmt, fieldnames, substructs={}): @staticmethod def _MaybePackStruct(value): - if hasattr(value, "__metaclass__"):# and value.__metaclass__ == Meta: + if isinstance(type(value), StructMetaclass): return value.Pack() else: return value @@ -246,13 +262,22 @@ def Struct(name, fmt, fieldnames, substructs={}): return struct.pack(self._format, *values) def __str__(self): + + def HasNonPrintableChar(s): + for c in s: + # Iterating over bytes yields chars in python2 but ints in python3. + if isinstance(c, int): c = chr(c) + if c not in string.printable: return True + return False + def FieldDesc(index, name, value): - if isinstance(value, str): + if isinstance(value, bytes) or isinstance(value, str): if index in self._asciiz: - value = value.rstrip("\x00") - elif any(c not in string.printable for c in value): - value = value.encode("hex") - return "%s=%s" % (name, value) + # TODO: use "backslashreplace" when python 2 is no longer supported. + value = value.rstrip(b"\x00").decode(errors="ignore") + elif HasNonPrintableChar(value): + value = binascii.hexlify(value).decode() + return "%s=%s" % (name, str(value)) descriptions = [ FieldDesc(i, n, v) for i, (n, v) in diff --git a/net/test/cstruct_test.py b/net/test/cstruct_test.py index af1fc42..b96e8bc 100755 --- a/net/test/cstruct_test.py +++ b/net/test/cstruct_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2016 The Android Open Source Project # @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import binascii import unittest import cstruct @@ -87,9 +88,9 @@ class CstructTest(unittest.TestCase): " nest2=Nested(word1=33214, nest2=TestStructA(byte1=3, int2=4)," " nest3=TestStructB(byte1=7, int2=33627591), int4=-55), byte3=252)") self.assertEqual(expected, str(d)) - expected = ("01" "02000000" - "81be" "03" "04000000" - "07" "c71d0102" "ffffffc9" "fc").decode("hex") + expected = binascii.unhexlify("01" "02000000" + "81be" "03" "04000000" + "07" "c71d0102" "ffffffc9" "fc") self.assertEqual(expected, d.Pack()) unpacked = DoubleNested(expected) self.CheckEquals(unpacked, d) @@ -97,50 +98,56 @@ class CstructTest(unittest.TestCase): def testNullTerminatedStrings(self): TestStruct = cstruct.Struct("TestStruct", "B16si16AH", "byte1 string2 int3 ascii4 word5") - nullstr = "hello" + (16 - len("hello")) * "\x00" + nullstr = b"hello" + (16 - len("hello")) * b"\x00" t = TestStruct((2, nullstr, 12345, nullstr, 33210)) expected = ("TestStruct(byte1=2, string2=68656c6c6f0000000000000000000000," " int3=12345, ascii4=hello, word5=33210)") self.assertEqual(expected, str(t)) - embeddednull = "hello\x00visible123" + embeddednull = b"hello\x00visible123" t = TestStruct((2, embeddednull, 12345, embeddednull, 33210)) expected = ("TestStruct(byte1=2, string2=68656c6c6f0076697369626c65313233," " int3=12345, ascii4=hello\x00visible123, word5=33210)") self.assertEqual(expected, str(t)) + embedded_non_ascii = b"hello\xc0visible123" + t = TestStruct((2, embedded_non_ascii, 12345, embeddednull, 33210)) + expected = ("TestStruct(byte1=2, string2=68656c6c6fc076697369626c65313233," + " int3=12345, ascii4=hello\x00visible123, word5=33210)") + self.assertEqual(expected, str(t)) + def testZeroInitialization(self): TestStruct = cstruct.Struct("TestStruct", "B16si16AH", "byte1 string2 int3 ascii4 word5") t = TestStruct() self.assertEqual(0, t.byte1) - self.assertEqual("\x00" * 16, t.string2) + self.assertEqual(b"\x00" * 16, t.string2) self.assertEqual(0, t.int3) - self.assertEqual("\x00" * 16, t.ascii4) + self.assertEqual(b"\x00" * 16, t.ascii4) self.assertEqual(0, t.word5) - self.assertEqual("\x00" * len(TestStruct), t.Pack()) + self.assertEqual(b"\x00" * len(TestStruct), t.Pack()) def testKeywordInitialization(self): TestStruct = cstruct.Struct("TestStruct", "=B16sIH", "byte1 string2 int3 word4") - text = "hello world! ^_^" - text_bytes = text.encode("hex") + bytes = b"hello world! ^_^" + hex_bytes = binascii.hexlify(bytes) # Populate all fields - t1 = TestStruct(byte1=1, string2=text, int3=0xFEDCBA98, word4=0x1234) - expected = ("01" + text_bytes + "98BADCFE" "3412").decode("hex") + t1 = TestStruct(byte1=1, string2=bytes, int3=0xFEDCBA98, word4=0x1234) + expected = binascii.unhexlify(b"01" + hex_bytes + b"98BADCFE" b"3412") self.assertEqual(expected, t1.Pack()) # Partially populated - t1 = TestStruct(string2=text, word4=0x1234) - expected = ("00" + text_bytes + "00000000" "3412").decode("hex") + t1 = TestStruct(string2=bytes, word4=0x1234) + expected = binascii.unhexlify(b"00" + hex_bytes + b"00000000" b"3412") self.assertEqual(expected, t1.Pack()) def testCstructOffset(self): TestStruct = cstruct.Struct("TestStruct", "B16si16AH", "byte1 string2 int3 ascii4 word5") - nullstr = "hello" + (16 - len("hello")) * "\x00" + nullstr = b"hello" + (16 - len("hello")) * b"\x00" t = TestStruct((2, nullstr, 12345, nullstr, 33210)) self.assertEqual(0, t.offset("byte1")) self.assertEqual(1, t.offset("string2")) # sizeof(byte) diff --git a/net/test/forwarding_test.py b/net/test/forwarding_test.py deleted file mode 100755 index b35e19f..0000000 --- a/net/test/forwarding_test.py +++ /dev/null @@ -1,172 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2015 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import itertools -import random -import unittest - -from socket import * - -import multinetwork_base -import net_test -import packets - -class ForwardingTest(multinetwork_base.MultiNetworkBaseTest): - TCP_TIME_WAIT = 6 - - def ForwardBetweenInterfaces(self, enabled, iface1, iface2): - for iif, oif in itertools.permutations([iface1, iface2]): - self.iproute.IifRule(6, enabled, self.GetInterfaceName(iif), - self._TableForNetid(oif), self.PRIORITY_IIF) - - def setUp(self): - self.SetSysctl("/proc/sys/net/ipv6/conf/all/forwarding", 1) - - def tearDown(self): - self.SetSysctl("/proc/sys/net/ipv6/conf/all/forwarding", 0) - - """Checks that IPv6 forwarding works for UDP packets and is not broken by early demux. - - Relevant kernel commits: - upstream: - 5425077d73e0c8e net: ipv6: Add early demux handler for UDP unicast - 0bd84065b19bca1 net: ipv6: Fix UDP early demux lookup with udp_l3mdev_accept=0 - Ifa9c2ddfaa5b51 net: ipv6: reset daddr and dport in sk if connect() fails - """ - def CheckForwardingUdp(self, netid, iface1, iface2): - # TODO: Make a test for IPv4 - # 1. Make version as an argument. Pick address to bind from array based - # on version. - # 2. The prefix length of the address is hardcoded to /64. Use the subnet - # mask there instead. - # 3. We recreate the address with SendRA, which obviously only works for - # IPv6. Use AddAddress for IPv4. - - # Create a UDP socket and bind to it - version = 6 - s = net_test.UDPSocket(AF_INET6) - self.SetSocketMark(s, netid) - s.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1) - s.bind(("::", 53)) - - remoteaddr = self.GetRemoteAddress(version) - myaddr = self.MyAddress(version, netid) - - try: - # Delete address and check if packet is forwarded - # (and not dropped because an incorrect socket match happened) - self.iproute.DelAddress(myaddr, 64, self.ifindices[netid]) - hoplimit = 39 - desc, udp_pkt = packets.UDPWithOptions(version, myaddr, remoteaddr, 53) - # Decrements the hoplimit of a packet to simulate forwarding. - desc_fwded, udp_fwd = packets.UDPWithOptions(version, myaddr, remoteaddr, - 53, hoplimit - 1) - msg = "Sent %s, expected %s" % (desc, desc_fwded) - self.ReceivePacketOn(iface1, udp_pkt) - self.ExpectPacketOn(iface2, msg, udp_fwd) - finally: - # Recreate the address. - self.SendRA(netid) - s.close() - - """Checks that IPv6 forwarding doesn't crash the system. - - Relevant kernel commits: - upstream net-next: - e7eadb4 ipv6: inet6_sk() should use sk_fullsock() - android-3.10: - feee3c1 ipv6: inet6_sk() should use sk_fullsock() - cdab04e net: add sk_fullsock() helper - android-3.18: - 8246f18 ipv6: inet6_sk() should use sk_fullsock() - bea19db net: add sk_fullsock() helper - """ - def CheckForwardingCrashTcp(self, netid, iface1, iface2): - version = 6 - listensocket = net_test.IPv6TCPSocket() - self.SetSocketMark(listensocket, netid) - listenport = net_test.BindRandomPort(version, listensocket) - - remoteaddr = self.GetRemoteAddress(version) - myaddr = self.MyAddress(version, netid) - - desc, syn = packets.SYN(listenport, version, remoteaddr, myaddr) - synack_desc, synack = packets.SYNACK(version, myaddr, remoteaddr, syn) - msg = "Sent %s, expected %s" % (desc, synack_desc) - reply = self._ReceiveAndExpectResponse(netid, syn, synack, msg) - - establishing_ack = packets.ACK(version, remoteaddr, myaddr, reply)[1] - self.ReceivePacketOn(netid, establishing_ack) - accepted, peer = listensocket.accept() - remoteport = accepted.getpeername()[1] - - accepted.close() - desc, fin = packets.FIN(version, myaddr, remoteaddr, establishing_ack) - self.ExpectPacketOn(netid, msg + ": expecting %s after close" % desc, fin) - - desc, finack = packets.FIN(version, remoteaddr, myaddr, fin) - self.ReceivePacketOn(netid, finack) - - # Check our socket is now in TIME_WAIT. - sockets = self.ReadProcNetSocket("tcp6") - mysrc = "%s:%04X" % (net_test.FormatSockStatAddress(myaddr), listenport) - mydst = "%s:%04X" % (net_test.FormatSockStatAddress(remoteaddr), remoteport) - state = None - sockets = [s for s in sockets if s[0] == mysrc and s[1] == mydst] - self.assertEqual(1, len(sockets)) - self.assertEqual("%02X" % self.TCP_TIME_WAIT, sockets[0][2]) - - # Remove our IP address. - try: - self.iproute.DelAddress(myaddr, 64, self.ifindices[netid]) - - self.ReceivePacketOn(iface1, finack) - self.ReceivePacketOn(iface1, establishing_ack) - self.ReceivePacketOn(iface1, establishing_ack) - # No crashes? Good. - - finally: - # Put back our IP address. - self.SendRA(netid) - listensocket.close() - - def CheckForwardingHandlerByProto(self, protocol, netid, iif, oif): - if protocol == IPPROTO_UDP: - self.CheckForwardingUdp(netid, iif, oif) - elif protocol == IPPROTO_TCP: - self.CheckForwardingCrashTcp(netid, iif, oif) - else: - raise NotImplementedError - - def CheckForwardingByProto(self, proto): - # Run the test a few times as it doesn't crash/hang the first time. - for netids in itertools.permutations(self.tuns): - # Pick an interface to send traffic on and two to forward traffic between. - netid, iface1, iface2 = random.sample(netids, 3) - self.ForwardBetweenInterfaces(True, iface1, iface2) - try: - self.CheckForwardingHandlerByProto(proto, netid, iface1, iface2) - finally: - self.ForwardBetweenInterfaces(False, iface1, iface2) - - def testForwardingUdp(self): - self.CheckForwardingByProto(IPPROTO_UDP) - - def testForwardingCrashTcp(self): - self.CheckForwardingByProto(IPPROTO_TCP) - -if __name__ == "__main__": - unittest.main() diff --git a/net/test/genetlink.py b/net/test/genetlink.py index 6928f07..d250ce1 100755 --- a/net/test/genetlink.py +++ b/net/test/genetlink.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -63,7 +63,7 @@ class GenericNetlink(netlink.NetlinkSocket): def _Dump(self, family, command, version): msg = Genlmsghdr((command, version)) - return super(GenericNetlink, self)._Dump(family, msg, Genlmsghdr, "") + return super(GenericNetlink, self)._Dump(family, msg, Genlmsghdr) class GenericNetlinkControl(GenericNetlink): @@ -75,6 +75,7 @@ class GenericNetlinkControl(GenericNetlink): def _DecodeOps(self, data): ops = [] Op = collections.namedtuple("Op", ["id", "flags"]) + # TODO: call _ParseAttributes on the nested data instead of manual parsing. while data: # Skip the nest marker. datalen, index, data = data[:2], data[2:4], data[4:] @@ -92,7 +93,7 @@ class GenericNetlinkControl(GenericNetlink): ops.append(Op(op_id, op_flags)) return ops - def _Decode(self, command, msg, nla_type, nla_data): + def _Decode(self, command, msg, nla_type, nla_data, nested): """Decodes generic netlink control attributes to human-readable format.""" name = self._GetConstantName(__name__, nla_type, "CTRL_ATTR_") @@ -100,7 +101,7 @@ class GenericNetlinkControl(GenericNetlink): if name == "CTRL_ATTR_FAMILY_ID": data = struct.unpack("=H", nla_data)[0] elif name == "CTRL_ATTR_FAMILY_NAME": - data = nla_data.strip("\x00") + data = nla_data.strip(b"\x00") elif name in ["CTRL_ATTR_VERSION", "CTRL_ATTR_HDRSIZE", "CTRL_ATTR_MAXATTR"]: data = struct.unpack("=I", nla_data)[0] elif name == "CTRL_ATTR_OPS": diff --git a/net/test/iproute.py b/net/test/iproute.py index 9036246..d61698c 100644 --- a/net/test/iproute.py +++ b/net/test/iproute.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2014 The Android Open Source Project # @@ -21,11 +21,13 @@ from socket import AF_INET from socket import AF_INET6 +import binascii import errno import os import socket import struct +import net_test import csocket import cstruct import netlink @@ -232,12 +234,18 @@ IFLA_VTI_LOCAL = 4 IFLA_VTI_REMOTE = 5 +CONSTANT_PREFIXES = netlink.MakeConstantPrefixes( + ["RTM_", "RTN_", "RTPROT_", "RT_SCOPE_", "RT_TABLE_", "RTA_", "RTMGRP_", + "RTNLGRP_", "RTAX_", "IFA_", "IFA_F_", "NDA_", "FRA_", "IFLA_", + "IFLA_INFO_", "IFLA_XFRM_", "IFLA_VTI_"]) + + def CommandVerb(command): return ["NEW", "DEL", "GET", "SET"][command % 4] def CommandSubject(command): - return ["LINK", "ADDR", "ROUTE", "NEIGH", "RULE"][(command - 16) / 4] + return ["LINK", "ADDR", "ROUTE", "NEIGH", "RULE"][(command - 16) // 4] def CommandName(command): @@ -251,12 +259,12 @@ class IPRoute(netlink.NetlinkSocket): """Provides a tiny subset of iproute functionality.""" def _NlAttrInterfaceName(self, nla_type, interface): - return self._NlAttr(nla_type, interface + "\x00") + return self._NlAttr(nla_type, interface.encode() + b"\x00") def _GetConstantName(self, value, prefix): return super(IPRoute, self)._GetConstantName(__name__, value, prefix) - def _Decode(self, command, msg, nla_type, nla_data, nested=0): + def _Decode(self, command, msg, nla_type, nla_data, nested): """Decodes netlink attributes to Python types. Values for which the code knows the type (e.g., the fwmark ID in a @@ -270,13 +278,11 @@ class IPRoute(netlink.NetlinkSocket): RTM_NEWROUTE command, attribute type 3 is the incoming interface and is an integer, but for a RTM_NEWRULE command, attribute type 3 is the incoming interface name and is a string. - - If negative, one of the following (negative) values: - - RTA_METRICS: Interpret as nested route metrics. - - IFLA_LINKINFO: Nested interface information. family: The address family. Used to convert IP addresses into strings. nla_type: An integer, then netlink attribute type. nla_data: A byte string, the netlink attribute data. - nested: An integer, how deep we're currently nested. + nested: A list, outermost first, of each of the attributes the NLAttrs are + nested inside. Empty for non-nested attributes. Returns: A tuple (name, data): @@ -287,11 +293,12 @@ class IPRoute(netlink.NetlinkSocket): (e.g., RTACacheinfo), etc. If we didn't understand the attribute, it will be the raw byte string. """ - if command == -RTA_METRICS: + lastnested = nested[-1] if nested else None + if lastnested == "RTA_METRICS": name = self._GetConstantName(nla_type, "RTAX_") - elif command == -IFLA_LINKINFO: + elif lastnested == "IFLA_LINKINFO": name = self._GetConstantName(nla_type, "IFLA_INFO_") - elif command == -IFLA_INFO_DATA: + elif lastnested == "IFLA_INFO_DATA": name = self._GetConstantName(nla_type, "IFLA_VTI_") elif CommandSubject(command) == "ADDR": name = self._GetConstantName(nla_type, "IFA_") @@ -326,13 +333,9 @@ class IPRoute(netlink.NetlinkSocket): data = socket.inet_ntop(msg.family, nla_data) elif name in ["FRA_IIFNAME", "FRA_OIFNAME", "IFLA_IFNAME", "IFLA_QDISC", "IFA_LABEL", "IFLA_INFO_KIND"]: - data = nla_data.strip("\x00") - elif name == "RTA_METRICS": - data = self._ParseAttributes(-RTA_METRICS, None, nla_data, nested + 1) - elif name == "IFLA_LINKINFO": - data = self._ParseAttributes(-IFLA_LINKINFO, None, nla_data, nested + 1) - elif name == "IFLA_INFO_DATA": - data = self._ParseAttributes(-IFLA_INFO_DATA, None, nla_data) + data = nla_data.strip(b"\x00") + elif name in ["RTA_METRICS", "IFLA_LINKINFO", "IFLA_INFO_DATA"]: + data = self._ParseAttributes(command, None, nla_data, nested + [name]) elif name == "RTA_CACHEINFO": data = RTACacheinfo(nla_data) elif name == "IFA_CACHEINFO": @@ -340,7 +343,7 @@ class IPRoute(netlink.NetlinkSocket): elif name == "NDA_CACHEINFO": data = NDACacheinfo(nla_data) elif name in ["NDA_LLADDR", "IFLA_ADDRESS", "IFLA_BROADCAST"]: - data = ":".join(x.encode("hex") for x in nla_data) + data = ":".join(net_test.ByteToHex(x) for x in nla_data) elif name == "FRA_UID_RANGE": data = FibRuleUidRange(nla_data) elif name == "IFLA_STATS": @@ -474,16 +477,16 @@ class IPRoute(netlink.NetlinkSocket): # Create a struct rtmsg specifying the table and the given match attributes. family = self._AddressFamily(version) rtmsg = RTMsg((family, 0, 0, 0, 0, 0, 0, 0, 0)) - return self._Dump(RTM_GETRULE, rtmsg, RTMsg, "") + return self._Dump(RTM_GETRULE, rtmsg, RTMsg) def DumpLinks(self): ifinfomsg = IfinfoMsg((0, 0, 0, 0, 0, 0)) - return self._Dump(RTM_GETLINK, ifinfomsg, IfinfoMsg, "") + return self._Dump(RTM_GETLINK, ifinfomsg, IfinfoMsg) def DumpAddresses(self, version): family = self._AddressFamily(version) ifaddrmsg = IfAddrMsg((family, 0, 0, 0, 0)) - return self._Dump(RTM_GETADDR, ifaddrmsg, IfAddrMsg, "") + return self._Dump(RTM_GETADDR, ifaddrmsg, IfAddrMsg) def _Address(self, version, command, addr, prefixlen, flags, scope, ifindex): """Adds or deletes an IP address.""" @@ -612,7 +615,7 @@ class IPRoute(netlink.NetlinkSocket): def DumpRoutes(self, version, ifindex): rtmsg = RTMsg(family=self._AddressFamily(version)) - return [(m, r) for (m, r) in self._Dump(RTM_GETROUTE, rtmsg, RTMsg, "") + return [(m, r) for (m, r) in self._Dump(RTM_GETROUTE, rtmsg, RTMsg) if r['RTA_TABLE'] == ifindex] def _Neighbour(self, version, is_add, addr, lladdr, dev, state, flags=0): @@ -622,9 +625,9 @@ class IPRoute(netlink.NetlinkSocket): # Convert the link-layer address to a raw byte string. if is_add and lladdr: lladdr = lladdr.split(":") - if len(lladdr) != 6: + if len(lladdr) != 6 or any (len(b) not in range(1, 3) for b in lladdr): raise ValueError("Invalid lladdr %s" % ":".join(lladdr)) - lladdr = "".join(chr(int(hexbyte, 16)) for hexbyte in lladdr) + lladdr = binascii.unhexlify("".join(lladdr)) ndmsg = NdMsg((family, dev, state, 0, RTN_UNICAST)).Pack() ndmsg += self._NlAttrIPAddress(NDA_DST, family, addr) @@ -645,7 +648,7 @@ class IPRoute(netlink.NetlinkSocket): def DumpNeighbours(self, version, ifindex): ndmsg = NdMsg((self._AddressFamily(version), 0, 0, 0, 0)) - attrs = self._NlAttrU32(NDA_IFINDEX, ifindex) if ifindex else "" + attrs = self._NlAttrU32(NDA_IFINDEX, ifindex) if ifindex else b"" return self._Dump(RTM_GETNEIGH, ndmsg, NdMsg, attrs) def ParseNeighbourMessage(self, msg): @@ -673,8 +676,8 @@ class IPRoute(netlink.NetlinkSocket): if hdr.type == RTM_NEWLINK: return cstruct.Read(data, IfinfoMsg) elif hdr.type == netlink.NLMSG_ERROR: - error = netlink.NLMsgErr(data).error - raise IOError(error, os.strerror(-error)) + error = -netlink.NLMsgErr(data).error + raise IOError(error, os.strerror(error)) else: raise ValueError("Unknown Netlink Message Type %d" % hdr.type) @@ -686,13 +689,13 @@ class IPRoute(netlink.NetlinkSocket): def GetIfaceStats(self, dev_name): """Returns an RtnlLinkStats64 stats object for the specified interface.""" _, attrs = self.GetIfinfo(dev_name) - attrs = self._ParseAttributes(RTM_NEWLINK, IfinfoMsg, attrs) + attrs = self._ParseAttributes(RTM_NEWLINK, IfinfoMsg, attrs, []) return attrs["IFLA_STATS64"] def GetIfinfoData(self, dev_name): """Returns an IFLA_INFO_DATA dict object for the specified interface.""" _, attrs = self.GetIfinfo(dev_name) - attrs = self._ParseAttributes(RTM_NEWLINK, IfinfoMsg, attrs) + attrs = self._ParseAttributes(RTM_NEWLINK, IfinfoMsg, attrs, []) return attrs["IFLA_LINKINFO"]["IFLA_INFO_DATA"] def GetRxTxPackets(self, dev_name): diff --git a/net/test/leak_test.py b/net/test/leak_test.py index a245817..54bbe73 100755 --- a/net/test/leak_test.py +++ b/net/test/leak_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2016 The Android Open Source Project # @@ -43,7 +43,7 @@ class LeakTest(net_test.NetworkTest): # testing for a bug where the kernel returns garbage, it's probably safer # to call the syscall directly. data, addr = csocket.Recvfrom(s, 4096) - self.assertEqual("", data) + self.assertEqual(b"", data) self.assertEqual(None, addr) @@ -72,6 +72,7 @@ class ForceSocketBufferOptionTest(net_test.NetworkTest): bogusval = 2 ** 31 - val s.setsockopt(SOL_SOCKET, force_option, bogusval) self.assertLessEqual(minbuf, s.getsockopt(SOL_SOCKET, option)) + s.close() def testRcvBufForce(self): self.CheckForceSocketBufferOption(SO_RCVBUF, self.SO_RCVBUFFORCE) diff --git a/net/test/multinetwork_base.py b/net/test/multinetwork_base.py index 6b79d4f..940be49 100644 --- a/net/test/multinetwork_base.py +++ b/net/test/multinetwork_base.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2014 The Android Open Source Project # @@ -59,6 +59,10 @@ IPV6_MARK_REFLECT_SYSCTL = "/proc/sys/net/ipv6/fwmark_reflect" HAVE_AUTOCONF_TABLE = os.path.isfile(AUTOCONF_TABLE_SYSCTL) +class ConfigurationError(AssertionError): + pass + + class UnexpectedPacketError(AssertionError): pass @@ -72,7 +76,7 @@ def MakePktInfo(version, addr, ifindex): if version == 6: return csocket.In6Pktinfo((addr, ifindex)).Pack() else: - return csocket.InPktinfo((ifindex, addr, "\x00" * 4)).Pack() + return csocket.InPktinfo((ifindex, addr, b"\x00" * 4)).Pack() class MultiNetworkBaseTest(net_test.NetworkTest): @@ -211,11 +215,11 @@ class MultiNetworkBaseTest(net_test.NetworkTest): def CreateTunInterface(cls, netid): iface = cls.GetInterfaceName(netid) try: - f = open("/dev/net/tun", "r+b") + f = open("/dev/net/tun", "r+b", buffering=0) except IOError: - f = open("/dev/tun", "r+b") - ifr = struct.pack("16sH", iface, IFF_TAP | IFF_NO_PI) - ifr += "\x00" * (40 - len(ifr)) + f = open("/dev/tun", "r+b", buffering=0) + ifr = struct.pack("16sH", iface.encode(), IFF_TAP | IFF_NO_PI) + ifr += b"\x00" * (40 - len(ifr)) fcntl.ioctl(f, TUNSETIFF, ifr) # Give ourselves a predictable MAC address. net_test.SetInterfaceHWAddr(iface, cls.MyMacAddress(netid)) @@ -256,7 +260,7 @@ class MultiNetworkBaseTest(net_test.NetworkTest): preferredlifetime=validity)) for option in options: ra /= option - posix.write(cls.tuns[netid].fileno(), str(ra)) + posix.write(cls.tuns[netid].fileno(), bytes(ra)) @classmethod def _RunSetupCommands(cls, netid, is_add): @@ -346,7 +350,8 @@ class MultiNetworkBaseTest(net_test.NetworkTest): @classmethod def GetSysctl(cls, sysctl): - return open(sysctl, "r").read() + with open(sysctl, "r") as sysctl_file: + return sysctl_file.read() @classmethod def SetSysctl(cls, sysctl, value): @@ -355,7 +360,8 @@ class MultiNetworkBaseTest(net_test.NetworkTest): # correctly at the end. if sysctl not in cls.saved_sysctls: cls.saved_sysctls[sysctl] = cls.GetSysctl(sysctl) - open(sysctl, "w").write(str(value) + "\n") + with open(sysctl, "w") as sysctl_file: + sysctl_file.write(str(value) + "\n") @classmethod def SetIPv6SysctlOnAllIfaces(cls, sysctl, value): @@ -368,7 +374,8 @@ class MultiNetworkBaseTest(net_test.NetworkTest): def _RestoreSysctls(cls): for sysctl, value in cls.saved_sysctls.items(): try: - open(sysctl, "w").write(value) + with open(sysctl, "w") as sysctl_file: + sysctl_file.write(value) except IOError: pass @@ -436,6 +443,7 @@ class MultiNetworkBaseTest(net_test.NetworkTest): cls._RunSetupCommands(netid, False) cls.tuns[netid].close() + cls.iproute.close() cls._RestoreSysctls() cls.SetConsoleLogLevel(cls.loglevel) @@ -456,7 +464,7 @@ class MultiNetworkBaseTest(net_test.NetworkTest): def BindToDevice(self, s, iface): if not iface: iface = "" - s.setsockopt(SOL_SOCKET, SO_BINDTODEVICE, iface) + s.setsockopt(SOL_SOCKET, SO_BINDTODEVICE, iface.encode()) def SetUnicastInterface(self, s, ifindex): # Otherwise, Python thinks it's a 1-byte option. @@ -529,7 +537,7 @@ class MultiNetworkBaseTest(net_test.NetworkTest): csocket.Sendmsg(s, (dstaddr, dstport), payload, cmsgs, csocket.MSG_CONFIRM) def ReceiveEtherPacketOn(self, netid, packet): - posix.write(self.tuns[netid].fileno(), str(packet)) + posix.write(self.tuns[netid].fileno(), bytes(packet)) def ReceivePacketOn(self, netid, ip_packet): routermac = self.RouterMacAddress(netid) @@ -560,7 +568,7 @@ class MultiNetworkBaseTest(net_test.NetworkTest): packets.append(ether.payload) except OSError as e: # EAGAIN means there are no more packets waiting. - if re.match(e.message, os.strerror(errno.EAGAIN)): + if e.errno == errno.EAGAIN: # If we didn't see any packets, try again for good luck. if not packets and retries < max_retries: time.sleep(0.01) @@ -664,8 +672,8 @@ class MultiNetworkBaseTest(net_test.NetworkTest): # Serialize the packet so that expected packet fields that are only set when # a packet is serialized e.g., the checksum) are filled in. - expected_real = expected.__class__(str(expected)) - actual_real = actual.__class__(str(actual)) + expected_real = expected.__class__(bytes(expected)) + actual_real = actual.__class__(bytes(actual)) # repr() can be expensive. Call it only if the test is going to fail and we # want to see the error. if expected_real != actual_real: @@ -712,7 +720,7 @@ class MultiNetworkBaseTest(net_test.NetworkTest): self.assertPacketMatches(expected, packets[-1]) except Exception as e: raise UnexpectedPacketError( - "%s: diff with last packet:\n%s" % (msg, e.message)) + "%s: diff with last packet:\n%s" % (msg, str(e))) def Combinations(self, version): """Produces a list of combinations to test.""" diff --git a/net/test/multinetwork_test.py b/net/test/multinetwork_test.py index 092736b..051b7d1 100755 --- a/net/test/multinetwork_test.py +++ b/net/test/multinetwork_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2014 The Android Open Source Project # @@ -41,15 +41,6 @@ IPV6_FLOWINFO = 11 SYNCOOKIES_SYSCTL = "/proc/sys/net/ipv4/tcp_syncookies" TCP_MARK_ACCEPT_SYSCTL = "/proc/sys/net/ipv4/tcp_fwmark_accept" -# The IP[V6]UNICAST_IF socket option was added between 3.1 and 3.4. -HAVE_UNICAST_IF = net_test.LINUX_VERSION >= (3, 4, 0) - -# RTPROT_RA is working properly with 4.14 -HAVE_RTPROT_RA = net_test.LINUX_VERSION >= (4, 14, 0) - -class ConfigurationError(AssertionError): - pass - class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): @@ -74,6 +65,7 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): s.sendto(packet + packets.PING_PAYLOAD, (dstsockaddr, 19321)) self.ExpectPacketOn(netid, msg, expected) + s.close() def CheckTCPSYNPacket(self, version, netid, routing_mode): s = self.BuildSocket(version, net_test.TCPSocket, netid, routing_mode) @@ -111,7 +103,8 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): s.connect((dstsockaddr, 53)) s.send(UDP_PAYLOAD) self.ExpectPacketOn(netid, msg % "connect/send", expected) - s.close() + + s.close() def CheckRawGrePacket(self, version, netid, routing_mode): s = self.BuildSocket(version, net_test.RawGRESocket, netid, routing_mode) @@ -119,7 +112,7 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): inner_version = {4: 6, 6: 4}[version] inner_src = self.MyAddress(inner_version, netid) inner_dst = self.GetRemoteAddress(inner_version) - inner = str(packets.UDP(inner_version, inner_src, inner_dst, sport=None)[1]) + inner = bytes(packets.UDP(inner_version, inner_src, inner_dst, sport=None)[1]) ethertype = {4: net_test.ETH_P_IP, 6: net_test.ETH_P_IPV6}[inner_version] # A GRE header can be as simple as two zero bytes and the ethertype. @@ -132,6 +125,7 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): msg = "Raw IPv%d GRE with inner IPv%d UDP: expected %s on %s" % ( version, inner_version, desc, self.GetInterfaceName(netid)) self.ExpectPacketOn(netid, msg, expected) + s.close() def CheckOutgoingPackets(self, routing_mode): for _ in range(self.ITERATIONS): @@ -171,7 +165,6 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): """Checks that oif routing selects the right outgoing interface.""" self.CheckOutgoingPackets("oif") - @unittest.skipUnless(HAVE_UNICAST_IF, "no support for UNICAST_IF") def testUcastOifRouting(self): """Checks that ucast oif routing selects the right outgoing interface.""" self.CheckOutgoingPackets("ucast_oif") @@ -179,7 +172,7 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): def CheckRemarking(self, version, use_connect): modes = ["mark", "oif", "uid"] # Setting UNICAST_IF on connected sockets does not work. - if not use_connect and HAVE_UNICAST_IF: + if not use_connect: modes += ["ucast_oif"] for mode in modes: @@ -259,6 +252,8 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): self.SelectInterface(s, None, mode) prevnetid = netid + s.close() + def testIPv4Remarking(self): """Checks that updating the mark on an IPv4 socket changes routing.""" self.CheckRemarking(4, False) @@ -279,11 +274,11 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): s.setsockopt(net_test.SOL_IPV6, net_test.IPV6_FLOWINFO_SEND, 1) # Set some destination options. - nonce = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c" - dstopts = "".join([ - "\x11\x02", # Next header=UDP, 24 bytes of options. - "\x01\x06", "\x00" * 6, # PadN, 6 bytes of padding. - "\x8b\x0c", # ILNP nonce, 12 bytes. + nonce = b"\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c" + dstopts = b"".join([ + b"\x11\x02", # Next header=UDP, 24 bytes of options. + b"\x01\x06", b"\x00" * 6, # PadN, 6 bytes of padding. + b"\x8b\x0c", # ILNP nonce, 12 bytes. nonce ]) s.setsockopt(net_test.SOL_IPV6, IPV6_DSTOPTS, dstopts) @@ -310,6 +305,7 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): msg = "IPv6 UDP using sticky pktinfo: expected UDP packet on %s" % ( self.GetInterfaceName(netid)) self.ExpectPacketOn(netid, msg, expected) + s.close() def CheckPktinfoRouting(self, version): for _ in range(self.ITERATIONS): @@ -350,6 +346,8 @@ class OutgoingTest(multinetwork_base.MultiNetworkBaseTest): version, desc, self.GetInterfaceName(netid)) self.ExpectPacketOn(netid, msg, expected) + s.close() + def testIPv4PktinfoRouting(self): self.CheckPktinfoRouting(4) @@ -660,13 +658,7 @@ class RIOTest(multinetwork_base.MultiNetworkBaseTest): table = self._TableForNetid(netid) router = self._RouterAddress(netid, version) ifindex = self.ifindices[netid] - # We actually want to specify RTPROT_RA, however an upstream - # kernel bug causes RAs to be installed with RTPROT_BOOT. - if HAVE_RTPROT_RA: - rtprot = iproute.RTPROT_RA - else: - rtprot = iproute.RTPROT_BOOT - self.iproute._Route(version, rtprot, iproute.RTM_DELROUTE, + self.iproute._Route(version, iproute.RTPROT_RA, iproute.RTM_DELROUTE, table, prefix, plen, router, ifindex, None, None) def testSetAcceptRaRtInfoMinPlen(self): @@ -817,6 +809,7 @@ class RATest(multinetwork_base.MultiNetworkBaseTest): else: self.assertRaisesErrno(errno.ENETUNREACH, s.sendto, UDP_PAYLOAD, (net_test.IPV6_ADDR, 1234)) + s.close() try: CheckIPv6Connectivity(True) @@ -866,12 +859,15 @@ class RATest(multinetwork_base.MultiNetworkBaseTest): msg = "After NA response, expecting %s" % desc self.ExpectPacketOn(netid, msg, expected) + s.close() + # This test documents a known issue: routing tables are never deleted. @unittest.skipUnless(multinetwork_base.HAVE_AUTOCONF_TABLE, "no support for per-table autoconf") def testLeftoverRoutes(self): def GetNumRoutes(): - return len(open("/proc/net/ipv6_route").readlines()) + with open("/proc/net/ipv6_route") as ipv6_route: + return len(ipv6_route.readlines()) num_routes = GetNumRoutes() for i in range(10, 20): @@ -893,7 +889,6 @@ class RATest(multinetwork_base.MultiNetworkBaseTest): lft_plc = (lifetime & 0xfff8) | 0 # 96-bit prefix length return self.Pref64Option((self.ND_OPT_PREF64, 2, lft_plc, prefix)) - @unittest.skipUnless(net_test.LINUX_VERSION >= (4, 9, 0), "not backported") def testPref64UserOption(self): # Open a netlink socket to receive RTM_NEWNDUSEROPT messages. s = netlink.NetlinkSocket(netlink.NETLINK_ROUTE, iproute.RTMGRP_ND_USEROPT) @@ -912,6 +907,7 @@ class RATest(multinetwork_base.MultiNetworkBaseTest): self.fail("Should have received an RTM_NEWNDUSEROPT message. " "Please ensure the kernel supports receiving the " "PREF64 RA option. Error: %s" % e) + s.close() # Check that the message is received correctly. nlmsghdr, data = cstruct.Read(data, netlink.NLMsgHdr) @@ -981,7 +977,7 @@ class PMTUTest(multinetwork_base.InboundMarkingTest): if use_connect: s.connect((dstaddr, 1234)) - payload = self.PAYLOAD_SIZE * "a" + payload = self.PAYLOAD_SIZE * b"a" # Send a packet and receive a packet too big. SendBigPacket(version, s, dstaddr, netid, payload) @@ -999,7 +995,7 @@ class PMTUTest(multinetwork_base.InboundMarkingTest): # If this is a connected socket, make sure the socket MTU was set. # Note that in IPv4 this only started working in Linux 3.6! - if use_connect and (version == 6 or net_test.LINUX_VERSION >= (3, 6)): + if use_connect: self.assertEqual(packets.PTB_MTU, self.GetSocketMTU(version, s)) s.close() @@ -1021,6 +1017,8 @@ class PMTUTest(multinetwork_base.InboundMarkingTest): metrics = attributes["RTA_METRICS"] self.assertEqual(packets.PTB_MTU, metrics["RTAX_MTU"]) + s2.close() + def testIPv4BasicPMTU(self): """Tests IPv4 path MTU discovery. @@ -1178,29 +1176,27 @@ class UidRoutingTest(multinetwork_base.MultiNetworkBaseTest): finally: self.iproute.FwmarkRule(version, False, 300, fwmask, 301, priority + 1) - # Test that EEXIST worksfor UID range rules too. This behaviour was only - # added in 4.8. - if net_test.LINUX_VERSION >= (4, 8, 0): - ranges = [(100, 101), (100, 102), (99, 101), (1234, 5678)] - dup = ranges[0] - try: - # Check that otherwise identical rules with different UID ranges can be - # created without EEXIST. - for start, end in ranges: - self.iproute.UidRangeRule(version, True, start, end, table, priority) - # ... but EEXIST is returned if the UID range is identical. - self.assertRaisesErrno( - errno.EEXIST, - self.iproute.UidRangeRule, version, True, dup[0], dup[1], table, - priority) - finally: - # Clean up. - for start, end in ranges + [dup]: - try: - self.iproute.UidRangeRule(version, False, start, end, table, - priority) - except IOError: - pass + # Test that EEXIST worksfor UID range rules too. + ranges = [(100, 101), (100, 102), (99, 101), (1234, 5678)] + dup = ranges[0] + try: + # Check that otherwise identical rules with different UID ranges can be + # created without EEXIST. + for start, end in ranges: + self.iproute.UidRangeRule(version, True, start, end, table, priority) + # ... but EEXIST is returned if the UID range is identical. + self.assertRaisesErrno( + errno.EEXIST, + self.iproute.UidRangeRule, version, True, dup[0], dup[1], table, + priority) + finally: + # Clean up. + for start, end in ranges + [dup]: + try: + self.iproute.UidRangeRule(version, False, start, end, table, + priority) + except IOError: + pass def testIPv4GetAndSetRules(self): self.CheckGetAndSetRules(4) @@ -1208,7 +1204,6 @@ class UidRoutingTest(multinetwork_base.MultiNetworkBaseTest): def testIPv6GetAndSetRules(self): self.CheckGetAndSetRules(6) - @unittest.skipUnless(net_test.LINUX_VERSION >= (4, 9, 0), "not backported") def testDeleteErrno(self): for version in [4, 6]: table = self._Random() @@ -1255,9 +1250,9 @@ class UidRoutingTest(multinetwork_base.MultiNetworkBaseTest): def CheckSendFails(): self.assertRaisesErrno(errno.ENETUNREACH, - s.sendto, "foo", (remoteaddr, 53)) + s.sendto, b"foo", (remoteaddr, 53)) def CheckSendSucceeds(): - self.assertEqual(len("foo"), s.sendto("foo", (remoteaddr, 53))) + self.assertEqual(len(b"foo"), s.sendto(b"foo", (remoteaddr, 53))) CheckSendFails() self.iproute.UidRangeRule(6, True, uid, uid, table, self.PRIORITY_UID) @@ -1275,6 +1270,7 @@ class UidRoutingTest(multinetwork_base.MultiNetworkBaseTest): CheckSendFails() finally: self.iproute.UidRangeRule(6, False, uid, uid, table, self.PRIORITY_UID) + s.close() class RulesTest(net_test.NetworkTest): diff --git a/net/test/namespace.py b/net/test/namespace.py index 3c0a0c1..fdea1e6 100644 --- a/net/test/namespace.py +++ b/net/test/namespace.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2020 The Android Open Source Project # @@ -72,13 +72,14 @@ libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) # https://docs.python.org/3/library/ctypes.html#fundamental-data-types libc.mount.argtypes = (ctypes.c_char_p, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_ulong, ctypes.c_void_p) -libc.sethostname.argtype = (ctypes.c_char_p, ctypes.c_size_t) +libc.sethostname.argtypes = (ctypes.c_char_p, ctypes.c_size_t) libc.umount2.argtypes = (ctypes.c_char_p, ctypes.c_int) libc.unshare.argtypes = (ctypes.c_int,) def Mount(src, tgt, fs, flags=MS_NODEV|MS_NOEXEC|MS_NOSUID|MS_RELATIME): - ret = libc.mount(src, tgt, fs, flags, None) + ret = libc.mount(src.encode(), tgt.encode(), fs.encode() if fs else None, + flags, None) if ret < 0: errno = ctypes.get_errno() raise OSError(errno, '%s mounting %s on %s (fs=%s flags=0x%x)' @@ -86,21 +87,27 @@ def Mount(src, tgt, fs, flags=MS_NODEV|MS_NOEXEC|MS_NOSUID|MS_RELATIME): def ReMountProc(): - libc.umount2('/proc', MNT_DETACH) # Ignore failure: might not be mounted + libc.umount2(b'/proc', MNT_DETACH) # Ignore failure: might not be mounted Mount('proc', '/proc', 'proc') def ReMountSys(): - libc.umount2('/sys', MNT_DETACH) # Ignore failure: might not be mounted + libc.umount2(b'/sys/fs/cgroup', MNT_DETACH) # Ignore failure: might not be mounted + libc.umount2(b'/sys/fs/bpf', MNT_DETACH) # Ignore failure: might not be mounted + libc.umount2(b'/sys', MNT_DETACH) # Ignore failure: might not be mounted Mount('sysfs', '/sys', 'sysfs') + Mount('bpf', '/sys/fs/bpf', 'bpf') + Mount('cgroup2', '/sys/fs/cgroup', 'cgroup2') def SetFileContents(f, s): - open(f, 'w').write(s) + with open(f, 'w') as set_file: + set_file.write(s) def SetHostname(s): - ret = libc.sethostname(s, len(s)) + hostname = s.encode() + ret = libc.sethostname(hostname, len(hostname)) if ret < 0: errno = ctypes.get_errno() raise OSError(errno, '%s while sethostname(%s)' % (os.strerror(errno), s)) @@ -116,7 +123,8 @@ def UnShare(flags): def DumpMounts(hdr): print('') print(hdr) - sys.stdout.write(open('/proc/mounts', 'r').read()) + with open('/proc/mounts', 'r') as mounts: + sys.stdout.write(mounts.read()) print('---') @@ -124,8 +132,8 @@ def DumpMounts(hdr): # CONFIG_NAMESPACES=y # CONFIG_NET_NS=y # CONFIG_UTS_NS=y -def IfPossibleEnterNewNetworkNamespace(): - """Instantiate and transition into a fresh new network namespace if possible.""" +def EnterNewNetworkNamespace(): + """Instantiate and transition into a fresh new network namespace.""" sys.stdout.write('Creating clean namespace... ') @@ -139,7 +147,7 @@ def IfPossibleEnterNewNetworkNamespace(): UnShare(CLONE_NEWNS | CLONE_NEWUTS | CLONE_NEWNET) except OSError as err: print('failed: %s (likely: no privs or lack of kernel support).' % err) - return False + raise try: # DumpMounts('Before:') @@ -176,7 +184,6 @@ def IfPossibleEnterNewNetworkNamespace(): init_rwnd_sysctl.write("60"); print('succeeded.') - return True def HasEstablishedTcpSessionOnPort(port): @@ -187,7 +194,7 @@ def HasEstablishedTcpSessionOnPort(port): states = 1 << tcp_test.TCP_ESTABLISHED - matches = sd.DumpAllInetSockets(socket.IPPROTO_TCP, "", + matches = sd.DumpAllInetSockets(socket.IPPROTO_TCP, b"", sock_id=sock_id, states=states) return len(matches) > 0 diff --git a/net/test/neighbour_test.py b/net/test/neighbour_test.py index 8cea6da..74b1156 100755 --- a/net/test/neighbour_test.py +++ b/net/test/neighbour_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2015 The Android Open Source Project # @@ -69,7 +69,7 @@ class NeighbourTest(multinetwork_base.MultiNetworkBaseTest): for proto in ["ipv4", "ipv6"]: cls.SetSysctl( "/proc/sys/net/%s/neigh/%s/delay_first_probe_time" % (proto, iface), - cls.DELAY_TIME_MS / 1000) + cls.DELAY_TIME_MS // 1000) cls.SetSysctl( "/proc/sys/net/%s/neigh/%s/retrans_time_ms" % (proto, iface), cls.RETRANS_TIME_MS) @@ -98,14 +98,6 @@ class NeighbourTest(multinetwork_base.MultiNetworkBaseTest): # MultinetworkBaseTest always uses NUD_PERMANENT for router ARP entries. # Temporarily change those entries to NUD_STALE so we can test them. - if net_test.LINUX_VERSION < (4, 9, 0): - # Cannot change state from NUD_PERMANENT to NUD_STALE directly, - # so delete it to make it NUD_FAILED then change it to NUD_STALE. - router = self._RouterAddress(self.netid, 4) - macaddr = self.RouterMacAddress(self.netid) - self.iproute.DelNeighbour(4, router, macaddr, self.ifindex) - self.ExpectNeighbourNotification(router, NUD_FAILED) - self.assertNeighbourState(NUD_FAILED, router) self.ChangeRouterNudState(4, NUD_STALE) def SetUnicastSolicit(self, proto, iface, value): @@ -124,6 +116,9 @@ class NeighbourTest(multinetwork_base.MultiNetworkBaseTest): # so as not to affect other tests. self.ChangeRouterNudState(4, NUD_PERMANENT) + self.sock.close() + self.sock = None + def ChangeRouterNudState(self, version, state): router = self._RouterAddress(self.netid, version) macaddr = self.RouterMacAddress(self.netid) @@ -167,8 +162,8 @@ class NeighbourTest(multinetwork_base.MultiNetworkBaseTest): dst = addr else: solicited = inet_pton(AF_INET6, addr) - last3bytes = tuple([ord(b) for b in solicited[-3:]]) - dst = "ff02::1:ff%02x:%02x%02x" % last3bytes + last3bytes = tuple([net_test.ByteToHex(b) for b in solicited[-3:]]) + dst = "ff02::1:ff%s:%s%s" % last3bytes src = self.MyAddress(6, self.netid) expected = ( scapy.IPv6(src=src, dst=dst) / @@ -267,11 +262,7 @@ class NeighbourTest(multinetwork_base.MultiNetworkBaseTest): # Respond to the NS and verify we're in REACHABLE again. self.ReceiveUnicastAdvertisement(router6, self.RouterMacAddress(self.netid)) self.assertNeighbourState(NUD_REACHABLE, router6) - if net_test.LINUX_VERSION >= (3, 13, 0): - # commit 53385d2 (v3.13) "neigh: Netlink notification for administrative - # NUD state change" produces notifications for NUD_REACHABLE, but these - # are not generated on earlier kernels. - self.ExpectNeighbourNotification(router6, NUD_REACHABLE) + self.ExpectNeighbourNotification(router6, NUD_REACHABLE) # Wait until the reachable time has passed, and verify we're in STALE. self.SleepMs(self.MAX_REACHABLE_TIME_MS * 1.2) @@ -280,6 +271,7 @@ class NeighbourTest(multinetwork_base.MultiNetworkBaseTest): # Send a packet, and verify we go into DELAY and then to PROBE. s.send(net_test.UDP_PAYLOAD) + s.close() self.assertNeighbourState(NUD_DELAY, router6) self.SleepMs(self.DELAY_TIME_MS * 1.1) self.assertNeighbourState(NUD_PROBE, router6) @@ -335,7 +327,7 @@ class NeighbourTest(multinetwork_base.MultiNetworkBaseTest): time.sleep(1) # Send another packet and expect a multicast NS. - self.SendDnsRequest(net_test.IPV6_ADDR) + self.SendDnsRequest(net_test.IPV6_ADDR).close() self.ExpectMulticastNS(router6) # Receive a unicast NA with the R flag set to 0. @@ -363,7 +355,7 @@ class NeighbourTest(multinetwork_base.MultiNetworkBaseTest): self.SetUnicastSolicit(proto, iface, self.UCAST_SOLICIT_LARGE) # Send a packet and check that we go into DELAY. - self.SendDnsRequest(ip_addr) + self.SendDnsRequest(ip_addr).close() self.assertNeighbourState(NUD_DELAY, router) # Probing 4 times but no reponse diff --git a/net/test/net_test.py b/net/test/net_test.py index c762cd8..bbff4e7 100755..100644 --- a/net/test/net_test.py +++ b/net/test/net_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2014 The Android Open Source Project # @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import contextlib import fcntl import os import random @@ -25,6 +26,7 @@ import unittest from scapy import all as scapy +import binascii import csocket # TODO: Move these to csocket.py. @@ -63,8 +65,8 @@ IPV6_FL_S_ANY = 255 IFNAMSIZ = 16 -IPV4_PING = "\x08\x00\x00\x00\x0a\xce\x00\x03" -IPV6_PING = "\x80\x00\x00\x00\x0a\xce\x00\x03" +IPV4_PING = b"\x08\x00\x00\x00\x0a\xce\x00\x03" +IPV6_PING = b"\x80\x00\x00\x00\x0a\xce\x00\x03" IPV4_ADDR = "8.8.8.8" IPV4_ADDR2 = "8.8.4.4" @@ -80,10 +82,10 @@ IPV6_SEQ_DGRAM_HEADER = (" sl " UDP_HDR_LEN = 8 # Arbitrary packet payload. -UDP_PAYLOAD = str(scapy.DNS(rd=1, - id=random.randint(0, 65535), - qd=scapy.DNSQR(qname="wWW.GoOGle.CoM", - qtype="AAAA"))) +UDP_PAYLOAD = bytes(scapy.DNS(rd=1, + id=random.randint(0, 65535), + qd=scapy.DNSQR(qname="wWW.GoOGle.CoM", + qtype="AAAA"))) # Unix group to use if we want to open sockets as non-root. AID_INET = 3003 @@ -94,6 +96,35 @@ KERN_INFO = 6 LINUX_VERSION = csocket.LinuxVersion() LINUX_ANY_VERSION = (0, 0) +def KernelAtLeast(versions): + """Checks the kernel version matches the specified versions. + + Args: + versions: a list of versions expressed as tuples, + e.g., [(5, 10, 108), (5, 15, 31)]. The kernel version matches if it's + between each specified version and the next minor version with last digit + set to 0. In this example, the kernel version must match either: + >= 5.10.108 and < 5.15.0 + >= 5.15.31 + While this is less flexible than matching exact tuples, it allows the caller + to pass in fewer arguments, because Android only supports certain minor + versions (4.19, 5.4, 5.10, ...) + + Returns: + True if the kernel version matches, False otherwise + """ + maxversion = (1000, 255, 65535) + for version in sorted(versions, reverse=True): + if version[:2] == maxversion[:2]: + raise ValueError("Duplicate minor version: %s %s", (version, maxversion)) + if LINUX_VERSION >= version and LINUX_VERSION < maxversion: + return True + maxversion = (version[0], version[1], 0) + return False + +def ByteToHex(b): + return "%02x" % (ord(b) if isinstance(b, str) else b) + def GetWildcardAddress(version): return {4: "0.0.0.0", 6: "::"}[version] @@ -208,33 +239,35 @@ def CreateSocketPair(family, socktype, addr): def GetInterfaceIndex(ifname): - s = UDPSocket(AF_INET) - ifr = struct.pack("%dsi" % IFNAMSIZ, ifname, 0) - ifr = fcntl.ioctl(s, scapy.SIOCGIFINDEX, ifr) - return struct.unpack("%dsi" % IFNAMSIZ, ifr)[1] + with UDPSocket(AF_INET) as s: + ifr = struct.pack("%dsi" % IFNAMSIZ, ifname.encode(), 0) + ifr = fcntl.ioctl(s, scapy.SIOCGIFINDEX, ifr) + return struct.unpack("%dsi" % IFNAMSIZ, ifr)[1] def SetInterfaceHWAddr(ifname, hwaddr): - s = UDPSocket(AF_INET) - hwaddr = hwaddr.replace(":", "") - hwaddr = hwaddr.decode("hex") - if len(hwaddr) != 6: - raise ValueError("Unknown hardware address length %d" % len(hwaddr)) - ifr = struct.pack("%dsH6s" % IFNAMSIZ, ifname, scapy.ARPHDR_ETHER, hwaddr) - fcntl.ioctl(s, SIOCSIFHWADDR, ifr) + with UDPSocket(AF_INET) as s: + hwaddr = hwaddr.replace(":", "") + hwaddr = binascii.unhexlify(hwaddr) + if len(hwaddr) != 6: + raise ValueError("Unknown hardware address length %d" % len(hwaddr)) + ifr = struct.pack("%dsH6s" % IFNAMSIZ, ifname.encode(), scapy.ARPHDR_ETHER, + hwaddr) + fcntl.ioctl(s, SIOCSIFHWADDR, ifr) def SetInterfaceState(ifname, up): - s = UDPSocket(AF_INET) - ifr = struct.pack("%dsH" % IFNAMSIZ, ifname, 0) - ifr = fcntl.ioctl(s, scapy.SIOCGIFFLAGS, ifr) - _, flags = struct.unpack("%dsH" % IFNAMSIZ, ifr) - if up: - flags |= scapy.IFF_UP - else: - flags &= ~scapy.IFF_UP - ifr = struct.pack("%dsH" % IFNAMSIZ, ifname, flags) - ifr = fcntl.ioctl(s, scapy.SIOCSIFFLAGS, ifr) + ifname_bytes = ifname.encode() + with UDPSocket(AF_INET) as s: + ifr = struct.pack("%dsH" % IFNAMSIZ, ifname_bytes, 0) + ifr = fcntl.ioctl(s, scapy.SIOCGIFFLAGS, ifr) + _, flags = struct.unpack("%dsH" % IFNAMSIZ, ifr) + if up: + flags |= scapy.IFF_UP + else: + flags &= ~scapy.IFF_UP + ifr = struct.pack("%dsH" % IFNAMSIZ, ifname_bytes, flags) + ifr = fcntl.ioctl(s, scapy.SIOCSIFFLAGS, ifr) def SetInterfaceUp(ifname): @@ -272,7 +305,8 @@ def FormatSockStatAddress(address): def GetLinkAddress(ifname, linklocal): - addresses = open("/proc/net/if_inet6").readlines() + with open("/proc/net/if_inet6") as if_inet6: + addresses = if_inet6.readlines() for address in addresses: address = [s for s in address.strip().split(" ") if s] if address[5] == ifname: @@ -285,7 +319,8 @@ def GetLinkAddress(ifname, linklocal): def GetDefaultRoute(version=6): if version == 6: - routes = open("/proc/net/ipv6_route").readlines() + with open("/proc/net/ipv6_route") as ipv6_route: + routes = ipv6_route.readlines() for route in routes: route = [s for s in route.strip().split(" ") if s] if (route[0] == "00000000000000000000000000000000" and route[1] == "00" @@ -294,12 +329,13 @@ def GetDefaultRoute(version=6): return FormatProcAddress(route[4]), route[9] raise ValueError("No IPv6 default route found") elif version == 4: - routes = open("/proc/net/route").readlines() + with open("/proc/net/route") as ipv4_route: + routes = ipv4_route.readlines() for route in routes: route = [s for s in route.strip().split("\t") if s] if route[1] == "00000000" and route[7] == "00000000": gw, iface = route[2], route[0] - gw = inet_ntop(AF_INET, gw.decode("hex")[::-1]) + gw = inet_ntop(AF_INET, binascii.unhexlify(gw)[::-1]) return gw, iface raise ValueError("No IPv4 default route found") else: @@ -331,7 +367,7 @@ def MakeFlowLabelOption(addr, label): action = IPV6_FL_A_GET share = IPV6_FL_S_ANY flags = IPV6_FL_F_CREATE - pad = "\x00" * 4 + pad = b"\x00" * 4 return struct.pack(fmt, addr, label, action, share, flags, 0, 0, pad) @@ -341,11 +377,31 @@ def SetFlowLabel(s, addr, label): # Caller also needs to do s.setsockopt(SOL_IPV6, IPV6_FLOWINFO_SEND, 1). +def GetIptablesBinaryPath(version): + if version == 4: + paths = ( + "/sbin/iptables-legacy", + "/sbin/iptables", + "/system/bin/iptables-legacy", + "/system/bin/iptables", + ) + elif version == 6: + paths = ( + "/sbin/ip6tables-legacy", + "/sbin/ip6tables", + "/system/bin/ip6tables-legacy", + "/system/bin/ip6tables", + ) + for iptables_path in paths: + if os.access(iptables_path, os.X_OK): + return iptables_path + raise FileNotFoundError( + "iptables binary for IPv{} not found".format(version) + + ", checked: {}".format(", ".join(paths))) + + def RunIptablesCommand(version, args): - iptables = {4: "iptables", 6: "ip6tables"}[version] - iptables_path = "/sbin/" + iptables - if not os.access(iptables_path, os.X_OK): - iptables_path = "/system/bin/" + iptables + iptables_path = GetIptablesBinaryPath(version) return os.spawnvp(os.P_WAIT, iptables_path, [iptables_path] + args.split(" ")) # Determine network configuration. @@ -393,11 +449,11 @@ class RunAsUid(RunAsUidGid): class NetworkTest(unittest.TestCase): - def assertRaisesRegex(self, *args, **kwargs): - if sys.version_info.major < 3: - return self.assertRaisesRegexp(*args, **kwargs) - else: - return super().assertRaisesRegex(*args, **kwargs) + @contextlib.contextmanager + def _errnoCheck(self, err_num): + with self.assertRaises(EnvironmentError) as context: + yield context + self.assertEqual(context.exception.errno, err_num) def assertRaisesErrno(self, err_num, f=None, *args): """Test that the system returns an errno error. @@ -415,16 +471,17 @@ class NetworkTest(unittest.TestCase): f: (optional) A callable that should result in error *args: arguments passed to f """ - msg = os.strerror(err_num) if f is None: - return self.assertRaisesRegex(EnvironmentError, msg) + return self._errnoCheck(err_num) else: - self.assertRaisesRegex(EnvironmentError, msg, f, *args) + with self._errnoCheck(err_num): + f(*args) def ReadProcNetSocket(self, protocol): # Read file. filename = "/proc/net/%s" % protocol - lines = open(filename).readlines() + with open(filename) as f: + lines = f.readlines() # Possibly check, and strip, header. if protocol in ["icmp6", "raw6", "udp6"]: @@ -474,11 +531,13 @@ class NetworkTest(unittest.TestCase): @staticmethod def GetConsoleLogLevel(): - return int(open("/proc/sys/kernel/printk").readline().split()[0]) + with open("/proc/sys/kernel/printk") as printk: + return int(printk.readline().split()[0]) @staticmethod def SetConsoleLogLevel(level): - return open("/proc/sys/kernel/printk", "w").write("%s\n" % level) + with open("/proc/sys/kernel/printk", "w") as printk: + return printk.write("%s\n" % level) if __name__ == "__main__": diff --git a/net/test/net_test.sh b/net/test/net_test.sh index e62120c..7185fd5 100755 --- a/net/test/net_test.sh +++ b/net/test/net_test.sh @@ -120,9 +120,9 @@ if [[ -n "${entropy}" ]]; then # In kernel/include/uapi/linux/random.h RNDADDENTROPY is defined as # _IOW('R', 0x03, int[2]) =(R is 0x52)= 0x40085203 = 1074287107 - /usr/bin/python 3>/dev/random <<EOF -import fcntl, struct -rnd = '${entropy}'.decode('base64') + /usr/bin/python3 3>/dev/random <<EOF +import base64, fcntl, struct +rnd = base64.b64decode('${entropy}') fcntl.ioctl(3, 0x40085203, struct.pack('ii', len(rnd) * 8, len(rnd)) + rnd) EOF @@ -139,6 +139,18 @@ sleep 1.1 # Reset it back to boot time default echo 60 > /proc/sys/kernel/random/urandom_min_reseed_secs +# Make sure /sys is mounted +[[ -d /sys/fs ]] || mount -t sysfs sysfs -o nosuid,nodev,noexec /sys + +if ! [[ "$(uname -r)" =~ ^([0-3]|4[.][0-8])[.] ]]; then + # Mount the bpf filesystem on Linux version 4.9+ + mount -t bpf bpf -o nosuid,nodev,noexec /sys/fs/bpf +fi + +if ! [[ "$(uname -r)" =~ ^([0-3]|4[.][0-9]|4[.]1[0-3])[.] ]]; then + # Mount the Cgroup v2 filesystem on Linux version 4.14+ + mount -t cgroup2 cgroup2 -o nosuid,nodev,noexec /sys/fs/cgroup +fi # In case IPv6 is compiled as a module. [ -f /proc/net/if_inet6 ] || insmod $DIR/kernel/net-next/net/ipv6/ipv6.ko @@ -146,11 +158,18 @@ echo 60 > /proc/sys/kernel/random/urandom_min_reseed_secs # Minimal network setup. ip link set lo up ip link set lo mtu 16436 -ip link set eth0 up +if [[ -d /sys/class/net/eth0 ]]; then + ip link set eth0 up +fi # Allow people to run ping. echo '0 2147483647' > /proc/sys/net/ipv4/ping_group_range +# Allow unprivileged use of eBPF (matches Android OS) +if [[ "$(< /proc/sys/kernel/unprivileged_bpf_disabled)" != '0' ]]; then + echo 0 > /proc/sys/kernel/unprivileged_bpf_disabled +fi + # Read environment variables passed to the kernel to determine if script is # running on builder and to find which test to run. diff --git a/net/test/netlink.py b/net/test/netlink.py index 2c9c757..b5efe11 100644 --- a/net/test/netlink.py +++ b/net/test/netlink.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2014 The Android Open Source Project # @@ -57,6 +57,11 @@ NLA_ALIGNTO = 4 # These can appear more than once but don't seem to contain any data. DUP_ATTRS_OK = ["INET_DIAG_NONE", "IFLA_PAD"] + +def MakeConstantPrefixes(prefixes): + return sorted(prefixes, key=len, reverse=True) + + class NetlinkSocket(object): """A basic netlink socket object.""" @@ -70,9 +75,10 @@ class NetlinkSocket(object): print(s) def _NlAttr(self, nla_type, data): + assert isinstance(data, bytes) datalen = len(data) # Pad the data if it's not a multiple of NLA_ALIGNTO bytes long. - padding = "\x00" * util.GetPadLength(NLA_ALIGNTO, datalen) + padding = b"\x00" * util.GetPadLength(NLA_ALIGNTO, datalen) nla_len = datalen + len(NLAttr) return NLAttr((nla_len, nla_type)).Pack() + data + padding @@ -86,18 +92,33 @@ class NetlinkSocket(object): def _NlAttrU32(self, nla_type, value): return self._NlAttr(nla_type, struct.pack("=I", value)) - def _GetConstantName(self, module, value, prefix): + @staticmethod + def _GetConstantName(module, value, prefix): + + def FirstMatching(name, prefixlist): + for prefix in prefixlist: + if name.startswith(prefix): + return prefix + return None + thismodule = sys.modules[module] + constant_prefixes = getattr(thismodule, "CONSTANT_PREFIXES", []) for name in dir(thismodule): - if name.startswith("INET_DIAG_BC"): + if value != getattr(thismodule, name) or not name.isupper(): + continue + # If the module explicitly specifies prefixes, only return this name if + # the passed-in prefix is the longest prefix that matches the name. + # This ensures, for example, that passing in a prefix of "IFA_" and a + # value of 1 returns "IFA_ADDRESS" instead of "IFA_F_SECONDARY". + # The longest matching prefix is always the first matching prefix because + # CONSTANT_PREFIXES must be sorted longest first. + if constant_prefixes and prefix != FirstMatching(name, constant_prefixes): continue - if (name.startswith(prefix) and - not name.startswith(prefix + "F_") and - name.isupper() and getattr(thismodule, name) == value): - return name + if name.startswith(prefix): + return name return value - def _Decode(self, command, msg, nla_type, nla_data): + def _Decode(self, command, msg, nla_type, nla_data, nested): """No-op, nonspecific version of decode.""" return nla_type, nla_data @@ -112,7 +133,7 @@ class NetlinkSocket(object): return nla, nla_data, data - def _ParseAttributes(self, command, msg, data, nested=0): + def _ParseAttributes(self, command, msg, data, nested): """Parses and decodes netlink attributes. Takes a block of NLAttr data structures, decodes them using Decode, and @@ -122,7 +143,8 @@ class NetlinkSocket(object): command: An integer, the rtnetlink command being carried out. msg: A Struct, the type of the data after the netlink header. data: A byte string containing a sequence of NLAttr data structures. - nested: An integer, how deep we're currently nested. + nested: A list, outermost first, of each of the attributes the NLAttrs are + nested inside. Empty for non-nested attributes. Returns: A dictionary mapping attribute types (integers) to decoded values. @@ -135,7 +157,7 @@ class NetlinkSocket(object): nla, nla_data, data = self._ReadNlAttr(data) # If it's an attribute we know about, try to decode it. - nla_name, nla_data = self._Decode(command, msg, nla.nla_type, nla_data) + nla_name, nla_data = self._Decode(command, msg, nla.nla_type, nla_data, nested) if nla_name in attributes and nla_name not in DUP_ATTRS_OK: raise ValueError("Duplicate attribute %s" % nla_name) @@ -159,6 +181,14 @@ class NetlinkSocket(object): self.sock = self._OpenNetlinkSocket(family, groups) self.pid = self.sock.getsockname()[1] + def close(self): + self.sock.close() + self.sock = None + + def __del__(self): + if self.sock: + self.close() + def MaybeDebugCommand(self, command, flags, data): # Default no-op implementation to be overridden by subclasses. pass @@ -183,9 +213,9 @@ class NetlinkSocket(object): # Find the error code. hdr, data = cstruct.Read(response, NLMsgHdr) if hdr.type == NLMSG_ERROR: - error = NLMsgErr(data).error + error = -NLMsgErr(data).error if error: - raise IOError(-error, os.strerror(-error)) + raise IOError(error, os.strerror(error)) else: raise ValueError("Expected ACK, got type %d" % hdr.type) @@ -220,7 +250,7 @@ class NetlinkSocket(object): # Parse the attributes in the nlmsg. attrlen = nlmsghdr.length - len(nlmsghdr) - len(nlmsg) - attributes = self._ParseAttributes(nlmsghdr.type, nlmsg, data[:attrlen]) + attributes = self._ParseAttributes(nlmsghdr.type, nlmsg, data[:attrlen], []) data = data[attrlen:] return (nlmsg, attributes), data @@ -241,7 +271,7 @@ class NetlinkSocket(object): self._ExpectDone() return out - def _Dump(self, command, msg, msgtype, attrs): + def _Dump(self, command, msg, msgtype, attrs=b""): """Sends a dump request and returns a list of decoded messages. Args: @@ -256,7 +286,7 @@ class NetlinkSocket(object): """ # Create a netlink dump request containing the msg. flags = NLM_F_DUMP | NLM_F_REQUEST - msg = "" if msg is None else msg.Pack() + msg = b"" if msg is None else msg.Pack() length = len(NLMsgHdr) + len(msg) + len(attrs) nlmsghdr = NLMsgHdr((length, command, flags, self.seq, self.pid)) diff --git a/net/test/netlink_test.py b/net/test/netlink_test.py new file mode 100755 index 0000000..98de3ae --- /dev/null +++ b/net/test/netlink_test.py @@ -0,0 +1,42 @@ +#!/usr/bin/python3 +# +# Copyright 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import iproute +import netlink +import sock_diag +import tcp_metrics + + +class NetlinkTest(unittest.TestCase): + + def _CheckConstant(self, expected, module, value, prefix): + self.assertEqual( + expected, + netlink.NetlinkSocket._GetConstantName(module.__name__, value, prefix)) + + def testGetConstantName(self): + self._CheckConstant("INET_DIAG_INFO", sock_diag, 2, "INET_DIAG_") + self._CheckConstant("INET_DIAG_BC_S_GE", sock_diag, 2, "INET_DIAG_BC_") + self._CheckConstant("IFA_ADDRESS", iproute, 1, "IFA_") + self._CheckConstant("IFA_F_SECONDARY", iproute, 1, "IFA_F_") + self._CheckConstant("TCP_METRICS_ATTR_AGE", tcp_metrics, 3, + "TCP_METRICS_ATTR_") + + +if __name__ == "__main__": + unittest.main() diff --git a/net/test/nf_test.py b/net/test/nf_test.py index cd6c976..2583c9a 100755 --- a/net/test/nf_test.py +++ b/net/test/nf_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2018 The Android Open Source Project # @@ -55,6 +55,7 @@ class NetilterRejectTargetTest(multinetwork_base.MultiNetworkBaseTest): sock.connect((addr, 53)) except IOError: pass + sock.close() def testRejectTcp4(self): self.CheckRejectedTcp(4, _TEST_IP4_ADDR) @@ -74,6 +75,7 @@ class NetilterRejectTargetTest(multinetwork_base.MultiNetworkBaseTest): sock.sendto(net_test.UDP_PAYLOAD, (addr, 53)) except IOError: pass + sock.close() def testRejectUdp4(self): self.CheckRejectedUdp(4, _TEST_IP4_ADDR) diff --git a/net/test/packets.py b/net/test/packets.py index 87a72f9..2a2ca1e 100644 --- a/net/test/packets.py +++ b/net/test/packets.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2015 The Android Open Source Project # @@ -32,7 +32,7 @@ TCP_WINDOW = 14400 PTB_MTU = 1280 PING_IDENT = 0xff19 -PING_PAYLOAD = "foobarbaz" +PING_PAYLOAD = b"foobarbaz" PING_SEQ = 3 PING_TOS = 0x83 @@ -107,7 +107,7 @@ def SYNACK(version, srcaddr, dstaddr, packet): ack=original.seq + 1, seq=None, flags=TCP_SYN | TCP_ACK, window=None)) -def ACK(version, srcaddr, dstaddr, packet, payload=""): +def ACK(version, srcaddr, dstaddr, packet, payload=b""): ip = _GetIpLayer(version) original = packet.getlayer("TCP") was_syn_or_fin = (original.flags & (TCP_SYN | TCP_FIN)) != 0 @@ -156,7 +156,7 @@ def ICMPPacketTooBig(version, srcaddr, dstaddr, packet): if version == 4: desc = "ICMPv4 fragmentation needed" pkt = (scapy.IP(src=srcaddr, dst=dstaddr, proto=1) / - scapy.ICMPerror(type=3, code=4) / str(packet)[:64]) + scapy.ICMPerror(type=3, code=4) / bytes(packet)[:64]) # Only newer versions of scapy understand that since RFC 1191, the last two # bytes of a fragmentation needed ICMP error contain the MTU. if hasattr(scapy.ICMP, "nexthopmtu"): @@ -167,7 +167,7 @@ def ICMPPacketTooBig(version, srcaddr, dstaddr, packet): else: return ("ICMPv6 Packet Too Big", scapy.IPv6(src=srcaddr, dst=dstaddr) / - scapy.ICMPv6PacketTooBig(mtu=PTB_MTU) / str(packet)[:1232]) + scapy.ICMPv6PacketTooBig(mtu=PTB_MTU) / bytes(packet)[:1232]) def ICMPEcho(version, srcaddr, dstaddr): ip = _GetIpLayer(version) @@ -184,15 +184,13 @@ def ICMPReply(version, srcaddr, dstaddr, packet): icmp = {4: icmpv4_reply, 6: scapy.ICMPv6EchoReply}[version] packet = (ip(src=srcaddr, dst=dstaddr) / icmp(id=PING_IDENT, seq=PING_SEQ) / PING_PAYLOAD) - # IPv6 only started copying the tclass to echo replies in 3.14. - if version == 4 or net_test.LINUX_VERSION >= (3, 14): - _SetPacketTos(packet, PING_TOS) + _SetPacketTos(packet, PING_TOS) return ("ICMPv%d echo reply" % version, packet) def NS(srcaddr, tgtaddr, srcmac): solicited = inet_pton(AF_INET6, tgtaddr) - last3bytes = tuple([ord(b) for b in solicited[-3:]]) - solicited = "ff02::1:ff%02x:%02x%02x" % last3bytes + last3bytes = tuple([net_test.ByteToHex(b) for b in solicited[-3:]]) + solicited = "ff02::1:ff%s:%s%s" % last3bytes packet = (scapy.IPv6(src=srcaddr, dst=solicited) / scapy.ICMPv6ND_NS(tgt=tgtaddr) / scapy.ICMPv6NDOptSrcLLAddr(lladdr=srcmac)) @@ -203,4 +201,3 @@ def NA(srcaddr, dstaddr, srcmac): scapy.ICMPv6ND_NA(tgt=srcaddr, R=0, S=1, O=1) / scapy.ICMPv6NDOptDstLLAddr(lladdr=srcmac)) return ("ICMPv6 NA", packet) - diff --git a/net/test/parameterization_test.py b/net/test/parameterization_test.py index 8f9e130..3b1951e 100755 --- a/net/test/parameterization_test.py +++ b/net/test/parameterization_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2018 The Android Open Source Project # diff --git a/net/test/pf_key.py b/net/test/pf_key.py index 3136a85..ca6689e 100755 --- a/net/test/pf_key.py +++ b/net/test/pf_key.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -186,7 +186,7 @@ def ParseExtension(exttype, data): if struct_type: ext, attrs = cstruct.Read(data, struct_type) else: - ext, attrs, = data, "" + ext, attrs = data, b"" return exttype, ext, attrs @@ -200,6 +200,13 @@ class PfKey(object): net_test.SetNonBlocking(self.sock) self.seq = 0 + def close(self): + self.sock.close() + self.sock = None + + def __del__(self): + if self.sock: self.close() + def Recv(self): reply = self.sock.recv(4096) msg = SadbMsg(reply) @@ -212,16 +219,16 @@ class PfKey(object): self.seq += 1 msg.seq = self.seq msg.pid = os.getpid() - msg.len = (len(SadbMsg) + len(extensions)) / 8 + msg.len = (len(SadbMsg) + len(extensions)) // 8 self.sock.send(msg.Pack() + extensions) # print("SEND: " + self.DecodeSadbMsg(msg)) return self.Recv() def PackPfKeyExtensions(self, extlist): - extensions = "" + extensions = b"" for exttype, extstruct, attrs in extlist: extdata = extstruct.Pack() - ext = SadbExt(((len(extdata) + len(SadbExt) + len(attrs)) / 8, exttype)) + ext = SadbExt(((len(extdata) + len(SadbExt) + len(attrs)) // 8, exttype)) extensions += ext.Pack() + extdata + attrs return extensions @@ -233,7 +240,7 @@ class PfKey(object): prefixlen = {AF_INET: 32, AF_INET6: 128}[addr.family] packed = addr.Pack() padbytes = (len(SadbExt) + len(SadbAddress) + len(packed)) % 8 - packed += "\x00" * padbytes + packed += b"\x00" * padbytes return (exttype, SadbAddress((0, prefixlen)), packed) def AddSa(self, src, dst, spi, satype, mode, reqid, encryption, @@ -243,10 +250,10 @@ class PfKey(object): replay = 4 extlist = [ (SADB_EXT_SA, SadbSa((htonl(spi), replay, SADB_SASTATE_MATURE, - auth, encryption, 0)), ""), + auth, encryption, 0)), b""), self.MakeSadbExtAddr(SADB_EXT_ADDRESS_SRC, src), self.MakeSadbExtAddr(SADB_EXT_ADDRESS_DST, dst), - (SADB_X_EXT_SA2, SadbXSa2((mode, 0, reqid)), ""), + (SADB_X_EXT_SA2, SadbXSa2((mode, 0, reqid)), b""), (SADB_EXT_KEY_AUTH, SadbKey((len(auth_key) * 8,)), auth_key), (SADB_EXT_KEY_ENCRYPT, SadbKey((len(encryption_key) * 8,)), encryption_key) @@ -258,7 +265,7 @@ class PfKey(object): msg = self.MakeSadbMsg(SADB_DELETE, satype) extlist = [ (SADB_EXT_SA, SadbSa((htonl(spi), 4, SADB_SASTATE_MATURE, - 0, 0, 0)), ""), + 0, 0, 0)), b""), self.MakeSadbExtAddr(SADB_EXT_ADDRESS_SRC, src), self.MakeSadbExtAddr(SADB_EXT_ADDRESS_DST, dst), ] @@ -302,7 +309,7 @@ class PfKey(object): """Returns a list of (SadbMsg, [(extension, attr), ...], ...) tuples.""" dump = [] msg = self.MakeSadbMsg(SADB_DUMP, SADB_TYPE_UNSPEC) - received = self.SendAndRecv(msg, "") + received = self.SendAndRecv(msg, b"") while received: msg, data = cstruct.Read(received, SadbMsg) extlen = self.ExtensionsLength(msg, SadbMsg) diff --git a/net/test/pf_key_test.py b/net/test/pf_key_test.py index 317ec7e..7791bd1 100755 --- a/net/test/pf_key_test.py +++ b/net/test/pf_key_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -18,13 +18,14 @@ from socket import * import unittest +import binascii import csocket import pf_key import xfrm -ENCRYPTION_KEY = ("308146eb3bd84b044573d60f5a5fd159" - "57c7d4fe567a2120f35bae0f9869ec22".decode("hex")) -AUTH_KEY = "af442892cdcd0ef650e9c299f9a8436a".decode("hex") +ENCRYPTION_KEY = binascii.unhexlify("308146eb3bd84b044573d60f5a5fd159" + "57c7d4fe567a2120f35bae0f9869ec22") +AUTH_KEY = binascii.unhexlify("af442892cdcd0ef650e9c299f9a8436a") class PfKeyTest(unittest.TestCase): @@ -33,6 +34,10 @@ class PfKeyTest(unittest.TestCase): self.pf_key = pf_key.PfKey() self.xfrm = xfrm.Xfrm() + def tearDown(self): + self.pf_key.close() + self.pf_key = None + def testAddDelSa(self): src4 = csocket.Sockaddr(("192.0.2.1", 0)) dst4 = csocket.Sockaddr(("192.0.2.2", 1)) @@ -72,28 +77,40 @@ class PfKeyTest(unittest.TestCase): # The algorithm names are null-terminated, but after that contain garbage. # Kernel bug? - aes_name = "cbc(aes)\x00" - sha256_name = "hmac(sha256)\x00" + aes_name = b"cbc(aes)\x00" + sha256_name = b"hmac(sha256)\x00" self.assertTrue(attrs4["XFRMA_ALG_CRYPT"].name.startswith(aes_name)) self.assertTrue(attrs6["XFRMA_ALG_CRYPT"].name.startswith(aes_name)) self.assertTrue(attrs4["XFRMA_ALG_AUTH"].name.startswith(sha256_name)) self.assertTrue(attrs6["XFRMA_ALG_AUTH"].name.startswith(sha256_name)) self.assertEqual(256, attrs4["XFRMA_ALG_CRYPT"].key_len) - self.assertEqual(256, attrs4["XFRMA_ALG_CRYPT"].key_len) - self.assertEqual(256, attrs6["XFRMA_ALG_AUTH"].key_len) + self.assertEqual(256, attrs6["XFRMA_ALG_CRYPT"].key_len) + self.assertEqual(256, attrs4["XFRMA_ALG_AUTH"].key_len) self.assertEqual(256, attrs6["XFRMA_ALG_AUTH"].key_len) + self.assertEqual(256, attrs4["XFRMA_ALG_AUTH_TRUNC"].key_len) self.assertEqual(256, attrs6["XFRMA_ALG_AUTH_TRUNC"].key_len) - self.assertEqual(256, attrs6["XFRMA_ALG_AUTH_TRUNC"].key_len) - self.assertEqual(128, attrs4["XFRMA_ALG_AUTH_TRUNC"].trunc_len) - self.assertEqual(128, attrs4["XFRMA_ALG_AUTH_TRUNC"].trunc_len) + if attrs4["XFRMA_ALG_AUTH_TRUNC"].trunc_len == 96: + missing4 = True + else: + self.assertEqual(128, attrs4["XFRMA_ALG_AUTH_TRUNC"].trunc_len) + missing4 = False + + if attrs6["XFRMA_ALG_AUTH_TRUNC"].trunc_len == 96: + missing6 = True + else: + self.assertEqual(128, attrs6["XFRMA_ALG_AUTH_TRUNC"].trunc_len) + missing6 = False self.pf_key.DelSa(src4, dst4, 0xdeadbeef, pf_key.SADB_TYPE_ESP) self.assertEqual(1, len(self.xfrm.DumpSaInfo())) self.pf_key.DelSa(src6, dst6, 0xbeefdead, pf_key.SADB_TYPE_ESP) self.assertEqual(0, len(self.xfrm.DumpSaInfo())) + if missing4 or missing6: + self.assertFalse("missing b8a72fd7c4e9 ANDROID: net: xfrm: make PF_KEY SHA256 use RFC-compliant truncation.") + if __name__ == "__main__": unittest.main() diff --git a/net/test/ping6_test.py b/net/test/ping6_test.py index d551b5f..af2e4c5 100755 --- a/net/test/ping6_test.py +++ b/net/test/ping6_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2014 The Android Open Source Project # @@ -16,6 +16,7 @@ # pylint: disable=g-bad-todo +import binascii import errno import os import posix @@ -34,8 +35,6 @@ import multinetwork_base import net_test -HAVE_PROC_NET_ICMP6 = os.path.isfile("/proc/net/icmp6") - ICMP_ECHO = 8 ICMP_ECHOREPLY = 0 ICMPV6_ECHO_REQUEST = 128 @@ -56,17 +55,17 @@ class PingReplyThread(threading.Thread): def __init__(self, tun, mymac, routermac, routeraddr): super(PingReplyThread, self).__init__() self._tun = tun - self._started = False - self._stopped = False + self._started_flag = False + self._stopped_flag = False self._mymac = mymac self._routermac = routermac self._routeraddr = routeraddr def IsStarted(self): - return self._started + return self._started_flag def Stop(self): - self._stopped = True + self._stopped_flag = True def ChecksumValid(self, packet): # Get and clear the checksums. @@ -94,7 +93,7 @@ class PingReplyThread(threading.Thread): # Serialize the packet, so scapy recalculates the checksums, and compare # them with the ones in the packet. - packet = packet.__class__(str(packet)) + packet = packet.__class__(bytes(packet)) for name in layers: layer = packet.getlayer(name) if layer and GetChecksum(layer) != sums[name]: @@ -122,7 +121,7 @@ class PingReplyThread(threading.Thread): self.SendPacket( scapy.IPv6(src=self.INTERMEDIATE_IPV6, dst=src) / scapy.ICMPv6PacketTooBig(mtu=self.LINK_MTU) / - str(packet)[:datalen]) + bytes(packet)[:datalen]) def IPv4Packet(self, ip): icmp = ip.getlayer(scapy.ICMP) @@ -184,14 +183,14 @@ class PingReplyThread(threading.Thread): def SendPacket(self, packet): packet = scapy.Ether(src=self._routermac, dst=self._mymac) / packet try: - posix.write(self._tun.fileno(), str(packet)) + posix.write(self._tun.fileno(), bytes(packet)) except Exception as e: - if not self._stopped: + if not self._stopped_flag: raise e def run(self): - self._started = True - while not self._stopped: + self._started_flag = True + while not self._stopped_flag: try: packet = posix.read(self._tun.fileno(), 4096) except OSError as e: @@ -200,7 +199,7 @@ class PingReplyThread(threading.Thread): else: break except ValueError as e: - if not self._stopped: + if not self._stopped_flag: raise e ether = scapy.Ether(packet) @@ -277,9 +276,9 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): # Check the data being sent is valid. self.assertGreater(len(data), 7, "Not enough data for ping packet") if family == AF_INET: - self.assertTrue(data.startswith("\x08\x00"), "Not an IPv4 echo request") + self.assertTrue(data.startswith(b"\x08\x00"), "Not an IPv4 echo request") elif family == AF_INET6: - self.assertTrue(data.startswith("\x80\x00"), "Not an IPv6 echo request") + self.assertTrue(data.startswith(b"\x80\x00"), "Not an IPv6 echo request") else: self.fail("Unknown socket address family %d" * s.family) @@ -287,11 +286,11 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): if family == AF_INET: addr, unused_port = src self.assertGreaterEqual(len(addr), len("1.1.1.1")) - self.assertTrue(rcvd.startswith("\x00\x00"), "Not an IPv4 echo reply") + self.assertTrue(rcvd.startswith(b"\x00\x00"), "Not an IPv4 echo reply") else: addr, unused_port, flowlabel, scope_id = src # pylint: disable=unbalanced-tuple-unpacking self.assertGreaterEqual(len(addr), len("::")) - self.assertTrue(rcvd.startswith("\x81\x00"), "Not an IPv6 echo reply") + self.assertTrue(rcvd.startswith(b"\x81\x00"), "Not an IPv6 echo reply") # Check that the flow label is zero and that the scope ID is sane. self.assertEqual(flowlabel, 0) if addr.startswith("fe80::"): @@ -304,7 +303,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): # Check the sequence number and the data. self.assertEqual(len(data), len(rcvd)) - self.assertEqual(data[6:].encode("hex"), rcvd[6:].encode("hex")) + self.assertEqual(binascii.hexlify(data[6:]), binascii.hexlify(rcvd[6:])) @staticmethod def IsAlmostEqual(expected, actual, delta): @@ -316,7 +315,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): "%s:%04X" % (net_test.FormatSockStatAddress(dstaddr), dstport), "%02X" % state, "%08X:%08X" % (txmem, rxmem), - str(os.getuid()), "2", "0"] + str(os.getuid()), "ref", "0"] for actual in self.ReadProcNetSocket(name): # Check that rxmem and txmem don't differ too much from each other. actual_txmem, actual_rxmem = expected[3].split(":") @@ -327,6 +326,8 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): # Check all the parameters except rxmem and txmem. expected[3] = actual[3] + # also do not check ref, it's always 2 on older kernels, but 1 for 'raw6' on 6.0+ + expected[5] = actual[5] if expected == actual: return @@ -335,35 +336,41 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): def testIPv4SendWithNoConnection(self): s = net_test.IPv4PingSocket() self.assertRaisesErrno(errno.EDESTADDRREQ, s.send, net_test.IPV4_PING) + s.close() def testIPv6SendWithNoConnection(self): s = net_test.IPv6PingSocket() self.assertRaisesErrno(errno.EDESTADDRREQ, s.send, net_test.IPV6_PING) + s.close() def testIPv4LoopbackPingWithConnect(self): s = net_test.IPv4PingSocket() s.connect(("127.0.0.1", 55)) - data = net_test.IPV4_PING + "foobarbaz" + data = net_test.IPV4_PING + b"foobarbaz" s.send(data) self.assertValidPingResponse(s, data) + s.close() def testIPv6LoopbackPingWithConnect(self): s = net_test.IPv6PingSocket() s.connect(("::1", 55)) s.send(net_test.IPV6_PING) self.assertValidPingResponse(s, net_test.IPV6_PING) + s.close() def testIPv4PingUsingSendto(self): s = net_test.IPv4PingSocket() written = s.sendto(net_test.IPV4_PING, (net_test.IPV4_ADDR, 55)) self.assertEqual(len(net_test.IPV4_PING), written) self.assertValidPingResponse(s, net_test.IPV4_PING) + s.close() def testIPv6PingUsingSendto(self): s = net_test.IPv6PingSocket() written = s.sendto(net_test.IPV6_PING, (net_test.IPV6_ADDR, 55)) self.assertEqual(len(net_test.IPV6_PING), written) self.assertValidPingResponse(s, net_test.IPV6_PING) + s.close() def testIPv4NoCrash(self): # Python 2.x does not provide either read() or recvmsg. @@ -373,6 +380,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): fd = s.fileno() reply = posix.read(fd, 4096) self.assertEqual(written, len(reply)) + s.close() def testIPv6NoCrash(self): # Python 2.x does not provide either read() or recvmsg. @@ -382,6 +390,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): fd = s.fileno() reply = posix.read(fd, 4096) self.assertEqual(written, len(reply)) + s.close() def testCrossProtocolCrash(self): # Checks that an ICMP error containing a ping packet that matches the ID @@ -410,6 +419,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): _, port = s.getsockname() scapy.send(GetIPv6Unreachable(port), verbose=False) # No crash? Good. + s.close() def testCrossProtocolCalls(self): """Tests that passing in the wrong family returns EAFNOSUPPORT. @@ -437,7 +447,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): ipv4sockaddr = csocket.Sockaddr((net_test.IPV4_ADDR, 53)) ipv4sockaddr = csocket.SockaddrIn6( ipv4sockaddr.Pack() + - "\x00" * (len(csocket.SockaddrIn6) - len(csocket.SockaddrIn))) + b"\x00" * (len(csocket.SockaddrIn6) - len(csocket.SockaddrIn))) s4 = net_test.IPv4PingSocket() s6 = net_test.IPv6PingSocket() @@ -453,12 +463,15 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s4, ipv6sockaddr, net_test.IPV4_PING, None, 0) CheckEAFNoSupport(csocket.Sendmsg, s6, ipv4sockaddr, net_test.IPV6_PING, None, 0) + s4.close() + s6.close() def testIPv4Bind(self): # Bind to unspecified address. s = net_test.IPv4PingSocket() s.bind(("0.0.0.0", 544)) self.assertEqual(("0.0.0.0", 544), s.getsockname()) + s.close() # Bind to loopback. s = net_test.IPv4PingSocket() @@ -467,6 +480,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): # Binding twice is not allowed. self.assertRaisesErrno(errno.EINVAL, s.bind, ("127.0.0.1", 22)) + s.close() # But binding two different sockets to the same ID is allowed. s2 = net_test.IPv4PingSocket() @@ -475,6 +489,8 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s3 = net_test.IPv4PingSocket() s3.bind(("127.0.0.1", 99)) self.assertEqual(("127.0.0.1", 99), s3.getsockname()) + s2.close() + s3.close() # If two sockets bind to the same port, the first one to call read() gets # the response. @@ -492,16 +508,22 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s4.setsockopt(SOL_SOCKET, SO_REUSEADDR, 0) self.assertRaisesErrno(errno.EADDRINUSE, s6.bind, ("0.0.0.0", 167)) + s4.close() + s5.close() + s6.close() + # Can't bind after sendto. s = net_test.IPv4PingSocket() s.sendto(net_test.IPV4_PING, (net_test.IPV4_ADDR, 9132)) self.assertRaisesErrno(errno.EINVAL, s.bind, ("0.0.0.0", 5429)) + s.close() def testIPv6Bind(self): # Bind to unspecified address. s = net_test.IPv6PingSocket() s.bind(("::", 769)) self.assertEqual(("::", 769, 0, 0), s.getsockname()) + s.close() # Bind to loopback. s = net_test.IPv6PingSocket() @@ -510,6 +532,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): # Binding twice is not allowed. self.assertRaisesErrno(errno.EINVAL, s.bind, ("::1", 22)) + s.close() # But binding two different sockets to the same ID is allowed. s2 = net_test.IPv6PingSocket() @@ -518,17 +541,22 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s3 = net_test.IPv6PingSocket() s3.bind(("::1", 99)) self.assertEqual(("::1", 99, 0, 0), s3.getsockname()) + s2.close() + s3.close() # Binding both IPv4 and IPv6 to the same socket works. s4 = net_test.IPv4PingSocket() s6 = net_test.IPv6PingSocket() s4.bind(("0.0.0.0", 444)) s6.bind(("::", 666, 0, 0)) + s4.close() + s6.close() # Can't bind after sendto. s = net_test.IPv6PingSocket() s.sendto(net_test.IPV6_PING, (net_test.IPV6_ADDR, 9132)) self.assertRaisesErrno(errno.EINVAL, s.bind, ("::", 5429)) + s.close() def testIPv4InvalidBind(self): s = net_test.IPv4PingSocket() @@ -545,6 +573,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): except IOError as e: if e.errno == errno.EACCES: pass # We're not root. let it go for now. + s.close() def testIPv6InvalidBind(self): s = net_test.IPv6PingSocket() @@ -560,6 +589,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): except IOError as e: if e.errno == errno.EACCES: pass # We're not root. let it go for now. + s.close() def testAfUnspecBind(self): # Binding to AF_UNSPEC is treated as IPv4 if the address is 0.0.0.0. @@ -573,12 +603,14 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): sockaddr = csocket.Sockaddr(("127.0.0.1", 58234)) sockaddr.family = AF_UNSPEC self.assertRaisesErrno(errno.EAFNOSUPPORT, csocket.Bind, s4, sockaddr) + s4.close() # This doesn't work for IPv6. s6 = net_test.IPv6PingSocket() sockaddr = csocket.Sockaddr(("::1", 58997)) sockaddr.family = AF_UNSPEC self.assertRaisesErrno(errno.EAFNOSUPPORT, csocket.Bind, s6, sockaddr) + s6.close() def testIPv6ScopedBind(self): # Can't bind to a link-local address without a scope ID. @@ -587,32 +619,40 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s.bind, (self.lladdr, 1026, 0, 0)) # Binding to a link-local address with a scope ID works, and the scope ID is - # returned by a subsequent getsockname. Interestingly, Python's getsockname - # returns "fe80:1%foo", even though it does not understand it. - expected = self.lladdr + "%" + self.ifname + # returned by a subsequent getsockname. On Python 2, getsockname returns + # "fe80:1%foo". Strip it off, since the ifindex field in the return value is + # what matters. s.bind((self.lladdr, 4646, 0, self.ifindex)) - self.assertEqual((expected, 4646, 0, self.ifindex), s.getsockname()) + sockname = s.getsockname() + expected = self.lladdr + if "%" in sockname[0]: + expected += "%" + self.ifname + self.assertEqual((expected, 4646, 0, self.ifindex), sockname) # Of course, for the above to work the address actually has to be configured # on the machine. self.assertRaisesErrno(errno.EADDRNOTAVAIL, s.bind, ("fe80::f00", 1026, 0, 1)) + s.close() # Scope IDs on non-link-local addresses are silently ignored. s = net_test.IPv6PingSocket() s.bind(("::1", 1234, 0, 1)) self.assertEqual(("::1", 1234, 0, 0), s.getsockname()) + s.close() def testBindAffectsIdentifier(self): s = net_test.IPv6PingSocket() s.bind((self.globaladdr, 0xf976)) s.sendto(net_test.IPV6_PING, (net_test.IPV6_ADDR, 55)) - self.assertEqual("\xf9\x76", s.recv(32768)[4:6]) + self.assertEqual(b"\xf9\x76", s.recv(32768)[4:6]) + s.close() s = net_test.IPv6PingSocket() s.bind((self.globaladdr, 0xace)) s.sendto(net_test.IPV6_PING, (net_test.IPV6_ADDR, 55)) - self.assertEqual("\x0a\xce", s.recv(32768)[4:6]) + self.assertEqual(b"\x0a\xce", s.recv(32768)[4:6]) + s.close() def testLinkLocalAddress(self): s = net_test.IPv6PingSocket() @@ -623,6 +663,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): # doesn't understand the "fe80::1%lo" format, even though it returns it. s.sendto(net_test.IPV6_PING, ("fe80::1", 55, 0, self.ifindex)) # No exceptions? Good. + s.close() def testLinkLocalOif(self): """Checks that ping to link-local addresses works correctly. @@ -668,6 +709,8 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s2.connect((dst, 123, 0, scopeid)) s2.send(net_test.IPV6_PING) self.assertValidPingResponse(s2, net_test.IPV6_PING) + s2.close() + s.close() def testMappedAddressFails(self): s = net_test.IPv6PingSocket() @@ -677,6 +720,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): self.assertValidPingResponse(s, net_test.IPV6_PING) self.assertRaisesErrno(errno.EINVAL, s.sendto, net_test.IPV6_PING, ("::ffff:192.0.2.1", 55)) + s.close() @unittest.skipUnless(False, "skipping: does not work yet") def testFlowLabel(self): @@ -704,6 +748,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): _, src = s.recvfrom(32768) _, _, flowlabel, _ = src self.assertEqual(0xdead, flowlabel & 0xfffff) + s.close() def testIPv4Error(self): s = net_test.IPv4PingSocket() @@ -713,6 +758,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): # We can't check the actual error because Python 2.7 doesn't implement # recvmsg, but we can at least check that the socket returns an error. self.assertRaisesErrno(errno.EHOSTUNREACH, s.recv, 32768) # No response. + s.close() def testIPv6Error(self): s = net_test.IPv6PingSocket() @@ -722,6 +768,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): # We can't check the actual error because Python 2.7 doesn't implement # recvmsg, but we can at least check that the socket returns an error. self.assertRaisesErrno(errno.EHOSTUNREACH, s.recv, 32768) # No response. + s.close() def testIPv6MulticastPing(self): s = net_test.IPv6PingSocket() @@ -731,20 +778,22 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s.sendto(net_test.IPV6_PING, ("ff02::1", 55, 0, self.ifindex)) self.assertValidPingResponse(s, net_test.IPV6_PING) self.assertValidPingResponse(s, net_test.IPV6_PING) + s.close() def testIPv4LargePacket(self): s = net_test.IPv4PingSocket() - data = net_test.IPV4_PING + 20000 * "a" + data = net_test.IPV4_PING + 20000 * b"a" s.sendto(data, ("127.0.0.1", 987)) self.assertValidPingResponse(s, data) + s.close() def testIPv6LargePacket(self): s = net_test.IPv6PingSocket() s.bind(("::", 0xace)) - data = net_test.IPV6_PING + "\x01" + 19994 * "\x00" + "aaaaa" + data = net_test.IPV6_PING + b"\x01" + 19994 * b"\x00" + b"aaaaa" s.sendto(data, ("::1", 953)) + s.close() - @unittest.skipUnless(HAVE_PROC_NET_ICMP6, "skipping: no /proc/net/icmp6") def testIcmpSocketsNotInIcmp6(self): numrows = len(self.ReadProcNetSocket("icmp")) numrows6 = len(self.ReadProcNetSocket("icmp6")) @@ -753,8 +802,8 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s.connect(("127.0.0.1", 0xbeef)) self.assertEqual(numrows + 1, len(self.ReadProcNetSocket("icmp"))) self.assertEqual(numrows6, len(self.ReadProcNetSocket("icmp6"))) + s.close() - @unittest.skipUnless(HAVE_PROC_NET_ICMP6, "skipping: no /proc/net/icmp6") def testIcmp6SocketsNotInIcmp(self): numrows = len(self.ReadProcNetSocket("icmp")) numrows6 = len(self.ReadProcNetSocket("icmp6")) @@ -763,14 +812,15 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s.connect(("::1", 0xbeef)) self.assertEqual(numrows, len(self.ReadProcNetSocket("icmp"))) self.assertEqual(numrows6 + 1, len(self.ReadProcNetSocket("icmp6"))) + s.close() def testProcNetIcmp(self): s = net_test.Socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP) s.bind(("127.0.0.1", 0xace)) s.connect(("127.0.0.1", 0xbeef)) self.CheckSockStatFile("icmp", "127.0.0.1", 0xace, "127.0.0.1", 0xbeef, 1) + s.close() - @unittest.skipUnless(HAVE_PROC_NET_ICMP6, "skipping: no /proc/net/icmp6") def testProcNetIcmp6(self): numrows6 = len(self.ReadProcNetSocket("icmp6")) s = net_test.IPv6PingSocket() @@ -787,6 +837,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): self.assertEqual(0, len(self.ReadProcNetSocket("icmp6"))) s.sendto(net_test.IPV6_PING, (net_test.IPV6_ADDR, 12345)) self.assertEqual(1, len(self.ReadProcNetSocket("icmp6"))) + s.close() # Can't bind after sendto, apparently. s = net_test.IPv6PingSocket() @@ -805,18 +856,21 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): self.assertValidPingResponse(s, net_test.IPV6_PING) self.CheckSockStatFile("icmp6", self.lladdr, 0xd00d, "ff02::1", 0xdead, 1, txmem=0, rxmem=0) + s.close() def testProcNetUdp6(self): s = net_test.Socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP) s.bind(("::1", 0xace)) s.connect(("::1", 0xbeef)) self.CheckSockStatFile("udp6", "::1", 0xace, "::1", 0xbeef, 1) + s.close() def testProcNetRaw6(self): s = net_test.Socket(AF_INET6, SOCK_RAW, IPPROTO_RAW) s.bind(("::1", 0xace)) s.connect(("::1", 0xbeef)) self.CheckSockStatFile("raw6", "::1", 0xff, "::1", 0, 1) + s.close() def testIPv6MTU(self): """Tests IPV6_RECVERR and path MTU discovery on ping sockets. @@ -830,7 +884,7 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): s.setsockopt(net_test.SOL_IPV6, csocket.IPV6_MTU_DISCOVER, 2) s.setsockopt(net_test.SOL_IPV6, net_test.IPV6_RECVERR, 1) s.connect((net_test.IPV6_ADDR, 55)) - pkt = net_test.IPV6_PING + (PingReplyThread.LINK_MTU + 100) * "a" + pkt = net_test.IPV6_PING + (PingReplyThread.LINK_MTU + 100) * b"a" s.send(pkt) self.assertRaisesErrno(errno.EMSGSIZE, s.recv, 32768) data, addr, cmsg = csocket.Recvmsg(s, 4096, 1024, csocket.MSG_ERRQUEUE) @@ -840,18 +894,11 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): # the one we received. ident = struct.pack("!H", s.getsockname()[1]) pkt = pkt[:4] + ident + pkt[6:] - data = data[:2] + "\x00\x00" + pkt[4:] + data = data[:2] + b"\x00\x00" + pkt[4:] self.assertEqual(pkt, data) # Check the address that the packet was sent to. - # ... except in 4.1, where it just returns an AF_UNSPEC, like this: - # recvmsg(9, {msg_name(0)={sa_family=AF_UNSPEC, - # sa_data="\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"}, - # msg_iov(1)=[{"\x80\x00\x04\x6b\x00\xc4\x00\x03\x61\x61\x61\x61\x61\x61"..., 4096}], - # msg_controllen=64, {cmsg_len=60, cmsg_level=SOL_IPV6, cmsg_type=, ...}, - # msg_flags=MSG_ERRQUEUE}, MSG_ERRQUEUE) = 1232 - if net_test.LINUX_VERSION != (4, 1, 0): - self.assertEqual(csocket.Sockaddr(("2001:4860:4860::8888", 0)), addr) + self.assertEqual(csocket.Sockaddr(("2001:4860:4860::8888", 0)), addr) # Check the cmsg data, including the link MTU. mtu = PingReplyThread.LINK_MTU @@ -863,13 +910,8 @@ class Ping6Test(multinetwork_base.MultiNetworkBaseTest): csocket.Sockaddr((src, 0)))) ] - # IP[V6]_RECVERR in 3.10 appears to return incorrect data for the port. - # The fix might have been in 676d236, but we don't have that in 3.10 and it - # touches code all over the tree. Instead, just don't check the port. - if net_test.LINUX_VERSION <= (3, 14, 0): - msglist[0][2][1].port = cmsg[0][2][1].port - self.assertEqual(msglist, cmsg) + s.close() if __name__ == "__main__": diff --git a/net/test/policy_crash_test.py b/net/test/policy_crash_test.py index ad1b92a..dbd6892 100755 --- a/net/test/policy_crash_test.py +++ b/net/test/policy_crash_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2019 The Android Open Source Project # @@ -70,6 +70,7 @@ # ---------------------------------------------------------------------- +import binascii import os import socket import unittest @@ -126,8 +127,8 @@ class RemovedFeatureTest(net_test.NetworkTest): + pkt2_frag_payload) s = socket.socket(socket.AF_INET6, socket.SOCK_RAW, socket.IPPROTO_RAW) - s.sendto(pkt1.decode('hex'), ('::1', 0)) - s.sendto(pkt2.decode('hex'), ('::1', 0)) + s.sendto(binascii.unhexlify(pkt1), ('::1', 0)) + s.sendto(binascii.unhexlify(pkt2), ('::1', 0)) s.close() diff --git a/net/test/qtaguid_test.py b/net/test/qtaguid_test.py deleted file mode 100755 index c121df2..0000000 --- a/net/test/qtaguid_test.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/python -# -# Copyright 2017 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for xt_qtaguid.""" - -import errno -from socket import * # pylint: disable=wildcard-import -import unittest -import os - -import net_test -import packets -import tcp_test - -CTRL_PROCPATH = "/proc/net/xt_qtaguid/ctrl" -OTHER_UID_GID = 12345 -HAVE_QTAGUID = os.path.exists(CTRL_PROCPATH) - - -@unittest.skipUnless(HAVE_QTAGUID, "xt_qtaguid not supported") -class QtaguidTest(tcp_test.TcpBaseTest): - - def RunIptablesCommand(self, args): - self.assertFalse(net_test.RunIptablesCommand(4, args)) - self.assertFalse(net_test.RunIptablesCommand(6, args)) - - def setUp(self): - self.RunIptablesCommand("-N qtaguid_test_OUTPUT") - self.RunIptablesCommand("-A OUTPUT -j qtaguid_test_OUTPUT") - - def tearDown(self): - self.RunIptablesCommand("-D OUTPUT -j qtaguid_test_OUTPUT") - self.RunIptablesCommand("-F qtaguid_test_OUTPUT") - self.RunIptablesCommand("-X qtaguid_test_OUTPUT") - - def WriteToCtrl(self, command): - ctrl_file = open(CTRL_PROCPATH, 'w') - ctrl_file.write(command) - ctrl_file.close() - - def CheckTag(self, tag, uid): - for line in open(CTRL_PROCPATH, 'r').readlines(): - if "tag=0x%x (uid=%d)" % ((tag|uid), uid) in line: - return True - return False - - def SetIptablesRule(self, version, is_add, is_gid, my_id, inverted): - add_del = "-A" if is_add else "-D" - uid_gid = "--gid-owner" if is_gid else "--uid-owner" - if inverted: - args = "%s qtaguid_test_OUTPUT -m owner ! %s %d -j DROP" % (add_del, uid_gid, my_id) - else: - args = "%s qtaguid_test_OUTPUT -m owner %s %d -j DROP" % (add_del, uid_gid, my_id) - self.assertFalse(net_test.RunIptablesCommand(version, args)) - - def AddIptablesRule(self, version, is_gid, myId): - self.SetIptablesRule(version, True, is_gid, myId, False) - - def AddIptablesInvertedRule(self, version, is_gid, myId): - self.SetIptablesRule(version, True, is_gid, myId, True) - - def DelIptablesRule(self, version, is_gid, myId): - self.SetIptablesRule(version, False, is_gid, myId, False) - - def DelIptablesInvertedRule(self, version, is_gid, myId): - self.SetIptablesRule(version, False, is_gid, myId, True) - - def CheckSocketOutput(self, version, is_gid): - myId = os.getgid() if is_gid else os.getuid() - self.AddIptablesRule(version, is_gid, myId) - family = {4: AF_INET, 6: AF_INET6}[version] - s = socket(family, SOCK_DGRAM, 0) - addr = {4: "127.0.0.1", 6: "::1"}[version] - s.bind((addr, 0)) - addr = s.getsockname() - self.assertRaisesErrno(errno.EPERM, s.sendto, "foo", addr) - self.DelIptablesRule(version, is_gid, myId) - s.sendto("foo", addr) - data, sockaddr = s.recvfrom(4096) - self.assertEqual("foo", data) - self.assertEqual(sockaddr, addr) - - def CheckSocketOutputInverted(self, version, is_gid): - # Load a inverted iptable rule on current uid/gid 0, traffic from other - # uid/gid should be blocked and traffic from current uid/gid should pass. - myId = os.getgid() if is_gid else os.getuid() - self.AddIptablesInvertedRule(version, is_gid, myId) - family = {4: AF_INET, 6: AF_INET6}[version] - s = socket(family, SOCK_DGRAM, 0) - addr1 = {4: "127.0.0.1", 6: "::1"}[version] - s.bind((addr1, 0)) - addr1 = s.getsockname() - s.sendto("foo", addr1) - data, sockaddr = s.recvfrom(4096) - self.assertEqual("foo", data) - self.assertEqual(sockaddr, addr1) - with net_test.RunAsUidGid(0 if is_gid else 12345, - 12345 if is_gid else 0): - s2 = socket(family, SOCK_DGRAM, 0) - addr2 = {4: "127.0.0.1", 6: "::1"}[version] - s2.bind((addr2, 0)) - addr2 = s2.getsockname() - self.assertRaisesErrno(errno.EPERM, s2.sendto, "foo", addr2) - self.DelIptablesInvertedRule(version, is_gid, myId) - s.sendto("foo", addr1) - data, sockaddr = s.recvfrom(4096) - self.assertEqual("foo", data) - self.assertEqual(sockaddr, addr1) - - def SendRSTOnClosedSocket(self, version, netid, expect_rst): - self.IncomingConnection(version, tcp_test.TCP_ESTABLISHED, netid) - self.accepted.setsockopt(net_test.SOL_TCP, net_test.TCP_LINGER2, -1) - net_test.EnableFinWait(self.accepted) - self.accepted.shutdown(SHUT_WR) - desc, fin = self.FinPacket() - self.ExpectPacketOn(netid, "Closing FIN_WAIT1 socket", fin) - finversion = 4 if version == 5 else version - desc, finack = packets.ACK(finversion, self.remoteaddr, self.myaddr, fin) - self.ReceivePacketOn(netid, finack) - try: - self.ExpectPacketOn(netid, "Closing FIN_WAIT1 socket", fin) - except AssertionError: - pass - self.accepted.close() - desc, rst = packets.RST(version, self.myaddr, self.remoteaddr, self.last_packet) - if expect_rst: - msg = "closing socket with linger2, expecting %s: " % desc - self.ExpectPacketOn(netid, msg, rst) - else: - msg = "closing socket with linger2, expecting no packets" - self.ExpectNoPacketsOn(netid, msg) - - def CheckUidGidCombination(self, version, invert_gid, invert_uid): - my_uid = os.getuid() - my_gid = os.getgid() - if invert_gid: - self.AddIptablesInvertedRule(version, True, my_gid) - else: - self.AddIptablesRule(version, True, OTHER_UID_GID) - if invert_uid: - self.AddIptablesInvertedRule(version, False, my_uid) - else: - self.AddIptablesRule(version, False, OTHER_UID_GID) - for netid in self.NETIDS: - self.SendRSTOnClosedSocket(version, netid, not invert_gid) - if invert_gid: - self.DelIptablesInvertedRule(version, True, my_gid) - else: - self.DelIptablesRule(version, True, OTHER_UID_GID) - if invert_uid: - self.AddIptablesInvertedRule(version, False, my_uid) - else: - self.DelIptablesRule(version, False, OTHER_UID_GID) - - def testCloseWithoutUntag(self): - self.dev_file = open("/dev/xt_qtaguid", "r"); - sk = socket(AF_INET, SOCK_DGRAM, 0) - uid = os.getuid() - tag = 0xff00ff00 << 32 - command = "t %d %d %d" % (sk.fileno(), tag, uid) - self.WriteToCtrl(command) - self.assertTrue(self.CheckTag(tag, uid)) - sk.close(); - self.assertFalse(self.CheckTag(tag, uid)) - self.dev_file.close(); - - def testTagWithoutDeviceOpen(self): - sk = socket(AF_INET, SOCK_DGRAM, 0) - uid = os.getuid() - tag = 0xff00ff00 << 32 - command = "t %d %d %d" % (sk.fileno(), tag, uid) - self.WriteToCtrl(command) - self.assertTrue(self.CheckTag(tag, uid)) - self.dev_file = open("/dev/xt_qtaguid", "r") - sk.close() - self.assertFalse(self.CheckTag(tag, uid)) - self.dev_file.close(); - - def testUidGidMatch(self): - self.CheckSocketOutput(4, False) - self.CheckSocketOutput(6, False) - self.CheckSocketOutput(4, True) - self.CheckSocketOutput(6, True) - self.CheckSocketOutputInverted(4, True) - self.CheckSocketOutputInverted(6, True) - self.CheckSocketOutputInverted(4, False) - self.CheckSocketOutputInverted(6, False) - - def testCheckNotMatchGid(self): - self.assertIn("match_no_sk_gid", open(CTRL_PROCPATH, 'r').read()) - - def testRstPacketNotDropped(self): - my_uid = os.getuid() - self.AddIptablesInvertedRule(4, False, my_uid) - for netid in self.NETIDS: - self.SendRSTOnClosedSocket(4, netid, True) - self.DelIptablesInvertedRule(4, False, my_uid) - self.AddIptablesInvertedRule(6, False, my_uid) - for netid in self.NETIDS: - self.SendRSTOnClosedSocket(6, netid, True) - self.DelIptablesInvertedRule(6, False, my_uid) - - def testUidGidCombineMatch(self): - self.CheckUidGidCombination(4, invert_gid=True, invert_uid=True) - self.CheckUidGidCombination(4, invert_gid=True, invert_uid=False) - self.CheckUidGidCombination(4, invert_gid=False, invert_uid=True) - self.CheckUidGidCombination(4, invert_gid=False, invert_uid=False) - self.CheckUidGidCombination(6, invert_gid=True, invert_uid=True) - self.CheckUidGidCombination(6, invert_gid=True, invert_uid=False) - self.CheckUidGidCombination(6, invert_gid=False, invert_uid=True) - self.CheckUidGidCombination(6, invert_gid=False, invert_uid=False) - - -if __name__ == "__main__": - unittest.main() diff --git a/net/test/removed_feature_test.py b/net/test/removed_feature_test.py index e58b4e3..d47824b 100755 --- a/net/test/removed_feature_test.py +++ b/net/test/removed_feature_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2016 The Android Open Source Project # @@ -28,7 +28,7 @@ class RemovedFeatureTest(net_test.NetworkTest): @classmethod def loadKernelConfig(cls): cls.KCONFIG = {} - with gzip.open('/proc/config.gz') as f: + with gzip.open("/proc/config.gz", mode="rt") as f: for line in f: line = line.strip() parts = line.split("=") @@ -68,19 +68,26 @@ class RemovedFeatureTest(net_test.NetworkTest): self.assertFeatureEnabled("CONFIG_IP6_NF_TARGET_REJECT") self.assertFeatureAbsent("CONFIG_IP6_NF_TARGET_REJECT_SKERR") - @unittest.skipUnless(net_test.LINUX_VERSION >= (4, 19, 0), "removed in 4.14-r") def testRemovedAndroidParanoidNetwork(self): - """Verify that ANDROID_PARANOID_NETWORK is gone.""" + """Verify that ANDROID_PARANOID_NETWORK is gone. + On a 4.14-q kernel you can achieve this by simply + changing the ANDROID_PARANOID_NETWORK default y to n + in your kernel source code in net/Kconfig: + + @@ -94,3 +94,3 @@ endif # if INET + config ANDROID_PARANOID_NETWORK + bool "Only allow certain groups to create sockets" + - default y + + default n + """ AID_NET_RAW = 3004 with net_test.RunAsUidGid(12345, AID_NET_RAW): self.assertRaisesErrno(errno.EPERM, socket, AF_PACKET, SOCK_RAW, 0) - @unittest.skipUnless(net_test.LINUX_VERSION >= (4, 19, 0), "exists in 4.14-P") def testRemovedQtaguid(self): self.assertRaisesErrno(errno.ENOENT, open, "/proc/net/xt_qtaguid") - @unittest.skipUnless(net_test.LINUX_VERSION >= (4, 19, 0), "exists in 4.14-P") def testRemovedTcpMemSysctls(self): self.assertRaisesErrno(errno.ENOENT, open, "/sys/kernel/ipv4/tcp_rmem_def") self.assertRaisesErrno(errno.ENOENT, open, "/sys/kernel/ipv4/tcp_rmem_max") diff --git a/net/test/resilient_rs_test.py b/net/test/resilient_rs_test.py index be3210b..f53217d 100755 --- a/net/test/resilient_rs_test.py +++ b/net/test/resilient_rs_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -77,12 +77,15 @@ class ResilientRouterSolicitationTest(multinetwork_base.MultiNetworkBaseTest): @classmethod def isIPv6RouterSolicitation(cls, packet): + def ToByte(c): + return c if isinstance(c, int) else ord(c) + return ((len(packet) >= 14 + 40 + 1) and # Use net_test.ETH_P_IPV6 here - (ord(packet[12]) == 0x86) and - (ord(packet[13]) == 0xdd) and - (ord(packet[14]) >> 4 == 6) and - (ord(packet[14 + 40]) == cls.ROUTER_SOLICIT)) + (ToByte(packet[12]) == 0x86) and + (ToByte(packet[13]) == 0xdd) and + (ToByte(packet[14]) >> 4 == 6) and + (ToByte(packet[14 + 40]) == cls.ROUTER_SOLICIT)) def makeTunInterface(self, netid): defaultDisableIPv6Path = self._PROC_NET_TUNABLE % ("default", "disable_ipv6") @@ -168,5 +171,7 @@ class ResilientRouterSolicitationTest(multinetwork_base.MultiNetworkBaseTest): self.assertLess(min_exp, t) self.assertGreater(max_exp, t) + tun.close() + if __name__ == "__main__": unittest.main() diff --git a/net/test/rootfs/bullseye-common.sh b/net/test/rootfs/bullseye-common.sh index 39f31d9..bd784ae 100644 --- a/net/test/rootfs/bullseye-common.sh +++ b/net/test/rootfs/bullseye-common.sh @@ -97,26 +97,29 @@ install_and_cleanup_iptables() { } setup_and_build_cuttlefish() { + if [ "$(uname -m)" = "aarch64" ]; then + apt-get install -y libc6:amd64 + fi + get_installed_packages >/root/originally-installed - # Install everything needed from bullseye to build cuttlefish-common + # Install everything needed from bullseye to build android-cuttlefish apt-get install -y \ cdbs \ - config-package-dev \ debhelper \ + devscripts \ dpkg-dev \ - git \ - golang - - if [ "$(uname -m)" = "arm64" ]; then - apt-get install -y libc6-dev:amd64 - fi + equivs \ + git - # Fetch cuttlefish and build it for cuttlefish-common + # Fetch android-cuttlefish and build it git clone https://github.com/google/android-cuttlefish.git /usr/src/$cuttlefish - cd /usr/src/$cuttlefish - dpkg-buildpackage -d -uc -us - cd - + for subdir in base frontend; do + cd /usr/src/$cuttlefish/$subdir + mk-build-deps --install --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' debian/control + dpkg-buildpackage -d -uc -us + cd - + done get_installed_packages >/root/installed remove_installed_packages /root/originally-installed /root/installed @@ -124,11 +127,14 @@ setup_and_build_cuttlefish() { } install_and_cleanup_cuttlefish() { - # Install and clean up cuttlefish-common - cd /usr/src + # Install and clean up cuttlefish host packages + cd /usr/src/$cuttlefish + apt-get install -y -f ./cuttlefish-base_*.deb + apt-get install -y -f ./cuttlefish-user_*.deb + apt-get install -y -f ./cuttlefish-integration_*.deb apt-get install -y -f ./cuttlefish-common_*.deb - rm -rf $cuttlefish cuttlefish*.{buildinfo,changes,deb,dsc} cd - + rm -rf /usr/src/$cuttlefish } bullseye_cleanup() { diff --git a/net/test/rootfs/bullseye-cuttlefish.sh b/net/test/rootfs/bullseye-cuttlefish.sh index 4ac5248..b805331 100755 --- a/net/test/rootfs/bullseye-cuttlefish.sh +++ b/net/test/rootfs/bullseye-cuttlefish.sh @@ -24,7 +24,7 @@ SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) setup_dynamic_networking "eth1" "br0" -update_apt_sources bullseye +update_apt_sources bullseye "" setup_cuttlefish_user @@ -32,12 +32,12 @@ setup_and_build_cuttlefish setup_and_build_iptables install_and_cleanup_cuttlefish -sed -i "s,^#\(bridge_interface=\),\1br0," /etc/default/cuttlefish-common +sed -i "s,^#\(bridge_interface=\),\1br0," /etc/default/cuttlefish-host-resources install_and_cleanup_iptables create_systemd_getty_symlinks ttyS0 hvc1 -setup_grub "net.ifnames=0 8250.nr_uarts=1" +setup_grub "quiet net.ifnames=0 8250.nr_uarts=1" apt-get purge -y vim-tiny bullseye_cleanup diff --git a/net/test/rootfs/bullseye-rockpi.sh b/net/test/rootfs/bullseye-rockpi.sh index 7df36a7..39b8519 100755 --- a/net/test/rootfs/bullseye-rockpi.sh +++ b/net/test/rootfs/bullseye-rockpi.sh @@ -26,7 +26,7 @@ sed -i "s,debian,rockpi," /etc/hosts sed -i "s,debian,rockpi," /etc/hostname # Build U-Boot FIT based on the Debian initrd -if [ -n "${embed_kernel_initrd_dtb}" ]; then +if [[ "${embed_kernel_initrd_dtb}" = "1" ]]; then mkimage -f auto -A arm64 -O linux -T kernel -C none -a 0x02080000 \ -d /boot/vmlinuz-$(uname -r) -i /boot/initrd.img-$(uname -r) \ -b /boot/dtb/rockchip/rk3399-rock-pi-4b.dtb /boot/boot.fit @@ -51,6 +51,8 @@ if dhcp ${scriptaddr} manifest.txt; then run manifest1 elif test "$ManifestVersion" = "2"; then run manifest2 + elif test "$ManifestVersion" = "3"; then + run manifest3 else run manifestX fi @@ -94,6 +96,28 @@ if test "$DFUethaddr" = "$ethaddr" || test "$DFUethaddr" = ""; then else echo "Update ${Sha} is not for me. Booting..." fi' +setenv manifest3 ' +env import -t ${scriptaddr} 0x8000 +if test "$DFUethaddr" = "$ethaddr" || test "$DFUethaddr" = ""; then + if test "$Sha" != "$OldSha"; then + setenv serverip ${TftpServer} + setenv loadaddr 0x00200000 + mmc dev 0 0; + setenv file $TplSplImg; offset=0x40; size=0x1f80; run tftpget1; setenv TplSplImg + setenv file $UbootEnv; offset=0x1fc0; size=0x40; run tftpget1; setenv UbootEnv + setenv file $UbootItb; offset=0x4000; size=0x2000; run tftpget1; setenv UbootItb + setenv file $TrustImg; offset=0x6000; size=0x2000; run tftpget1; setenv TrustImg + setenv file $EspImg; offset=0x8000; size=0x40000; run tftpget1; setenv EspImg + setenv file $RootfsImg; offset=0x48000; size=0; run tftpget1; setenv RootfsImg + mw.b ${scriptaddr} 0 0x8000 + env export -b ${scriptaddr} 0x8000 + mmc write ${scriptaddr} 0x1fc0 0x40 + else + echo "Already have ${Sha}. Booting..." + fi +else + echo "Update ${Sha} is not for me. Booting..." +fi' setenv tftpget1 ' if test "$file" != ""; then mw.b ${loadaddr} 0 0x400000 @@ -128,14 +152,27 @@ fi' if mmc dev 1 0; then; else run bootcmd_dhcp; fi +if bcb load 0 misc; then + # valid BCB found + if bcb test command = bootonce-bootloader; then + bcb clear command; bcb store + setenv autoload no; dhcp + fastboot udp + reset + elif bcb test command = boot-recovery; then + bcb clear command; bcb store + # we don't have recovery, reboot. + reset + fi +fi if test -e mmc ${devnum}:${distro_bootpart} /boot/rootfs.gz; then setenv loadaddr 0x00200000 mw.b ${loadaddr} 0 0x400000 load mmc ${devnum}:${distro_bootpart} ${loadaddr} /boot/rootfs.gz - gzwrite mmc ${devnum} ${loadaddr} 0x${filesize} 100000 0x1000000 + gzwrite mmc ${devnum} ${loadaddr} 0x${filesize} 100000 0x9100000 fi load mmc ${devnum}:${distro_bootpart} 0x06080000 /boot/boot.fit -setenv bootargs "8250.nr_uarts=4 earlycon=uart8250,mmio32,0xff1a0000 console=ttyS2,1500000n8 loglevel=7 sdhci.debug_quirks=0x20000000 root=LABEL=ROOT" +setenv bootargs "net.ifnames=0 8250.nr_uarts=4 earlycon=uart8250,mmio32,0xff1a0000 console=ttyS2,1500000n8 loglevel=7 kvm-arm.mode=nvhe sdhci.debug_quirks=0x20000000 root=LABEL=ROOT" bootm 0x06080000 EOF mkimage -C none -A arm -T script -d /boot/boot.cmd /boot/boot.scr @@ -261,17 +298,20 @@ led 0 src_dev=mmcblk0 dest_dev=mmcblk1 -part_num=p5 +part_num=p7 -if [ -e /dev/mmcblk0p5 ] && [ -e /dev/mmcblk1p5 ]; then +if [ -e "/dev/${src_dev}" ] && [ -e "/dev/${dest_dev}" ]; then led 1 - sgdisk -Z -a1 /dev/${dest_dev} - sgdisk -a1 -n:1:64:8127 -t:1:8301 -c:1:loader1 /dev/${dest_dev} - sgdisk -a1 -n:2:8128:8191 -t:2:8301 -c:2:env /dev/${dest_dev} - sgdisk -a1 -n:3:16384:24575 -t:3:8301 -c:3:loader2 /dev/${dest_dev} - sgdisk -a1 -n:4:24576:32767 -t:4:8301 -c:4:trust /dev/${dest_dev} - sgdisk -a1 -n:5:32768:- -A:5:set:2 -t:5:8305 -c:5:rootfs /dev/${dest_dev} + sgdisk -Z /dev/${dest_dev} + + sgdisk -a1 -n:1:64:8127 -t:1:8301 -c:1:idbloader /dev/${dest_dev} + sgdisk -a1 -n:2:8128:+64 -t:2:8301 -c:2:uboot_env /dev/${dest_dev} + sgdisk -n:3:8M:+4M -t:3:8301 -c:3:uboot /dev/${dest_dev} + sgdisk -n:4:12M:+4M -t:4:8301 -c:4:trust /dev/${dest_dev} + sgdisk -n:5:16M:+1M -t:5:8301 -c:5:misc /dev/${dest_dev} + sgdisk -n:6:17M:+128M -t:6:ef00 -c:6:esp -A:6:set:0 /dev/${dest_dev} + sgdisk -n:7:145M:0 -t:7:8305 -c:7:rootfs -A:7:set:2 /dev/${dest_dev} src_block_count=$(tune2fs -l /dev/${src_dev}${part_num} | grep "Block count:" | sed 's/.*: *//') src_block_size=$(tune2fs -l /dev/${src_dev}${part_num} | grep "Block size:" | sed 's/.*: *//') @@ -282,6 +322,7 @@ if [ -e /dev/mmcblk0p5 ] && [ -e /dev/mmcblk1p5 ]; then dd if=/dev/${src_dev}p2 of=/dev/${dest_dev}p2 conv=sync,noerror status=progress dd if=/dev/${src_dev}p3 of=/dev/${dest_dev}p3 conv=sync,noerror status=progress dd if=/dev/${src_dev}p4 of=/dev/${dest_dev}p4 conv=sync,noerror status=progress + dd if=/dev/${src_dev}p5 of=/dev/${dest_dev}p5 conv=sync,noerror status=progress echo "Writing ${src_fs_size_m} MB: /dev/${src_dev} -> /dev/${dest_dev}..." dd if=/dev/${src_dev}${part_num} of=/dev/${dest_dev}${part_num} bs=1M conv=sync,noerror status=progress @@ -364,9 +405,9 @@ systemctl enable poe systemctl enable led systemctl enable sd-dupe -setup_dynamic_networking "en*" "" +setup_dynamic_networking "eth0" "" -update_apt_sources bullseye +update_apt_sources bullseye "" setup_cuttlefish_user @@ -376,7 +417,10 @@ setup_and_build_iptables install_and_cleanup_cuttlefish install_and_cleanup_iptables -create_systemd_getty_symlinks ttyS0 hvc1 +create_systemd_getty_symlinks ttyS2 + +setup_grub "net.ifnames=0 8250.nr_uarts=4 earlycon=uart8250,mmio32,0xff1a0000 console=ttyS2,1500000n8 loglevel=7 kvm-arm.mode=nvhe sdhci.debug_quirks=0x20000000" apt-get purge -y vim-tiny +rm -f /etc/network/interfaces.d/eth0.conf bullseye_cleanup diff --git a/net/test/rootfs/bullseye-server.list b/net/test/rootfs/bullseye-server.list new file mode 100644 index 0000000..adea7b8 --- /dev/null +++ b/net/test/rootfs/bullseye-server.list @@ -0,0 +1,14 @@ +#include "bullseye-cuttlefish.list" +aapt +bzip2 +eject +gcc +gdisk +libglvnd0 +make +ntpdate +ntpstat +screen +software-properties-common +unzip +xz-utils diff --git a/net/test/rootfs/bullseye-server.sh b/net/test/rootfs/bullseye-server.sh new file mode 100755 index 0000000..c5343de --- /dev/null +++ b/net/test/rootfs/bullseye-server.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# +# Copyright (C) 2022 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e +set -u + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) + +. $SCRIPT_DIR/bullseye-common.sh + +arch=$(uname -m) +nvidia_arch=${arch} +[ "${arch}" = "x86_64" ] && arch=amd64 +[ "${arch}" = "aarch64" ] && arch=arm64 + +# Workaround for unnecessary firmware warning on ampere/gigabyte +mkdir -p /lib/firmware +touch /lib/firmware/ast_dp501_fw.bin + +setup_dynamic_networking "eth0" "" + +# NVIDIA driver needs dkms which requires /dev/fd +if [ ! -d /dev/fd ]; then + ln -s /proc/self/fd /dev/fd +fi + +update_apt_sources "bullseye bullseye-backports" "non-free" + +setup_cuttlefish_user + +# Install JRE +apt-get install -y openjdk-17-jre + +# Ubuntu compatibility +cat >>/etc/skel/.profile << EOF +PATH="/usr/sbin:\$PATH" +EOF + +# Get kernel and QEMU from backports +for package in linux-image-${arch} qemu-system-arm qemu-system-x86; do + apt-get install -y -t bullseye-backports ${package} +done + +# Install firmware package for AMD graphics +apt-get install -y firmware-amd-graphics + +get_installed_packages >/root/originally-installed + +# Using "Depends:" is more reliable than "Version:", because it works for +# backported ("bpo") kernels as well. NOTE: "Package" can be used instead +# if we don't install the metapackage ("linux-image-${arch}") but a +# specific version in the future +kmodver=$(dpkg -s linux-image-${arch} | grep ^Depends: | \ + cut -d: -f2 | cut -d" " -f2 | sed 's/linux-image-//') + +# Install headers from backports, to match the linux-image (removed below) +apt-get install -y -t bullseye-backports $(echo linux-headers-${kmodver}) + +# Dependencies for nvidia-installer (removed below) +apt-get install -y dkms libglvnd-dev libc6-dev pkg-config + +nvidia_version=525.60.13 +wget -q https://us.download.nvidia.com/tesla/${nvidia_version}/NVIDIA-Linux-${nvidia_arch}-${nvidia_version}.run +chmod a+x NVIDIA-Linux-${nvidia_arch}-${nvidia_version}.run +./NVIDIA-Linux-${nvidia_arch}-${nvidia_version}.run -x +cd NVIDIA-Linux-${nvidia_arch}-${nvidia_version} +if [[ "${nvidia_arch}" = "x86_64" ]]; then + installer_flags="--no-install-compat32-libs" +else + installer_flags="" +fi +./nvidia-installer ${installer_flags} --silent --no-backup --no-wine-files \ + --install-libglvnd --dkms -k "${kmodver}" +cd - +rm -rf NVIDIA-Linux-${nvidia_arch}-${nvidia_version}* + +get_installed_packages >/root/installed + +remove_installed_packages /root/originally-installed /root/installed + +setup_and_build_cuttlefish + +install_and_cleanup_cuttlefish + +# ttyAMA0 for ampere/gigabyte +# ttyS0 for GCE t2a +create_systemd_getty_symlinks ttyAMA0 ttyS0 + +setup_grub "net.ifnames=0 console=ttyAMA0 8250.nr_uarts=1 console=ttyS0 loglevel=4 amdgpu.runpm=0 amdgpu.dc=0" + +# Set up NTP using Google time servers and switch to UTC for uniformity +# NOTE: Installing ntp removes systemd-timesyncd +apt-get install -y ntp +sed -i -e 's,^\(pool .*debian.*\)$,# \1,' /etc/ntp.conf +cat >>/etc/ntp.conf <<EOF +pool time1.google.com iburst +pool time2.google.com iburst +pool time3.google.com iburst +pool time4.google.com iburst +# time.google.com as backup +pool time.google.com iburst +EOF +timedatectl set-timezone UTC + +# Switch to NetworkManager. To disrupt the bootstrapping the least, do this +# right at the end.. +rm -f /etc/network/interfaces.d/eth0.conf +apt-get install -y network-manager +apt-get purge -y vim-tiny +bullseye_cleanup diff --git a/net/test/rootfs/bullseye.list b/net/test/rootfs/bullseye.list index e908a11..7ef07b3 100644 --- a/net/test/rootfs/bullseye.list +++ b/net/test/rootfs/bullseye.list @@ -29,6 +29,7 @@ python2 python3-scapy strace systemd-sysv +systemd-timesyncd tcpdump traceroute udev diff --git a/net/test/rootfs/bullseye.sh b/net/test/rootfs/bullseye.sh index d959fca..e3496bb 100755 --- a/net/test/rootfs/bullseye.sh +++ b/net/test/rootfs/bullseye.sh @@ -24,7 +24,7 @@ SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) setup_static_networking -update_apt_sources bullseye +update_apt_sources bullseye "" # Disable the root password passwd -d root diff --git a/net/test/rootfs/common.sh b/net/test/rootfs/common.sh index c935250..211c6f8 100644 --- a/net/test/rootfs/common.sh +++ b/net/test/rootfs/common.sh @@ -17,13 +17,18 @@ trap "echo 3 >${exitcode}" ERR -# $1 - Suite name for apt sources +# $1 - Suite names for apt sources +# $2 - Additional repos, if any update_apt_sources() { # Add the needed debian sources - cat >/etc/apt/sources.list <<EOF -deb http://ftp.debian.org/debian bullseye main -deb-src http://ftp.debian.org/debian bullseye main + cat >/etc/apt/sources.list << EOF EOF + for source in $1; do + cat >>/etc/apt/sources.list <<EOF +deb http://ftp.debian.org/debian $source main $2 +deb-src http://ftp.debian.org/debian $source main $2 +EOF + done # Disable the automatic installation of recommended packages cat >/etc/apt/apt.conf.d/90recommends <<EOF @@ -61,8 +66,8 @@ setup_static_networking() { echo "nameserver 8.8.4.4" >>/etc/resolv.conf } -# $1 - Network interface for bridge (or NetworkManager DHCP) -# $2 - Bridge name. If set to the empty string, NetworkManager is used +# $1 - Network interface for bridge (or traditional DHCP) +# $2 - Bridge name. If not specified, no bridge is configured setup_dynamic_networking() { # So isc-dhcp-client can work with a read-only rootfs.. cat >>/etc/fstab <<EOF @@ -77,15 +82,10 @@ EOF # Set up automatic DHCP for *future* boots if [ -z "$2" ]; then - cat >/etc/systemd/network/dhcp.network <<EOF -[Match] -Name=$1 - -[Network] -DHCP=yes + cat >/etc/network/interfaces.d/$1.conf <<EOF +auto $1 +iface $1 inet dhcp EOF - # Mask the NetworkManager-wait-online service to prevent hangs - systemctl mask NetworkManager-wait-online.service else cat >/etc/network/interfaces.d/$2.conf <<EOF auto $2 @@ -119,19 +119,35 @@ create_systemd_getty_symlinks() { # $1 - Additional default command line setup_grub() { - if [ -n "${embed_kernel_initrd_dtb}" ]; then - # For testing the image with a virtual device + if [[ "${embed_kernel_initrd_dtb}" = "0" && "${install_grub}" = "0" ]]; then + return + fi + + if [[ "${install_grub}" = "1" ]]; then + # Mount fstab entry added by stage2 + mount /boot/efi + + # Install GRUB EFI (removable, for Cloud) + apt-get install -y grub-efi + grub_arch="$(uname -m)" + # Remap some mismatches with uname -m + [ "${grub_arch}" = "i686" ] && grub_arch=i386 + [ "${grub_arch}" = "aarch64" ] && grub_arch=arm64 + grub-install --target "${grub_arch}-efi" --removable + else + # Install common grub components apt-get install -y grub2-common - cat >/etc/default/grub <<EOF + mkdir /boot/grub + fi + + cat >/etc/default/grub <<EOF GRUB_DEFAULT=0 GRUB_TIMEOUT=5 GRUB_DISTRIBUTOR=Debian -GRUB_CMDLINE_LINUX_DEFAULT="quiet" +GRUB_CMDLINE_LINUX_DEFAULT="" GRUB_CMDLINE_LINUX="\\\$cmdline $1" EOF - mkdir /boot/grub - update-grub - fi + update-grub } cleanup() { @@ -139,15 +155,19 @@ cleanup() { mkdir -p /var/lib/systemd/{coredump,linger,rfkill,timesync} chown systemd-timesync:systemd-timesync /var/lib/systemd/timesync - # If embedding isn't enabled, remove the embedded modules and initrd and - # uninstall the tools to regenerate the initrd, as they're unlikely to - # ever be used - if [ -z "${embed_kernel_initrd_dtb}" ]; then - apt-get purge -y initramfs-tools initramfs-tools-core klibc-utils kmod + + # If embedding isn't enabled, remove the embedded modules and initrd + if [[ "${embed_kernel_initrd_dtb}" = "0" ]]; then rm -f "/boot/initrd.img-$(uname -r)" rm -rf "/lib/modules/$(uname -r)" fi + # If embedding isn't enabled *and* GRUB isn't being installed, uninstall + # the tools to regenerate the initrd, as they're unlikely to ever be used + if [[ "${embed_kernel_initrd_dtb}" = "0" && "${install_grub}" = "0" ]]; then + apt-get purge -y initramfs-tools initramfs-tools-core klibc-utils kmod + fi + # Miscellaneous cleanup rm -rf /var/lib/apt/lists/* || true rm -f /root/* || true diff --git a/net/test/rootfs/stage1.sh b/net/test/rootfs/stage1.sh index ccf54f1..0c60ffb 100755 --- a/net/test/rootfs/stage1.sh +++ b/net/test/rootfs/stage1.sh @@ -29,8 +29,14 @@ ln -s /tmp/bin/kmod /tmp/insmod # Load just enough to get the rootfs from virtio_blk module_dir=/lib/modules/$(uname -r)/kernel -# virtio_pci_modern_dev was split out in 5.12 -/tmp/insmod ${module_dir}/drivers/virtio/virtio_pci_modern_dev.ko || true +# virtio_pci_modern_dev.ko for 5.12-5.19 kernel +if [ -e "${module_dir}/drivers/virtio/virtio_pci_modern_dev.ko" ]; then + /tmp/insmod ${module_dir}/drivers/virtio/virtio_pci_modern_dev.ko || sh +fi +# virtio_pci_legacy_dev.ko for 6.0+ kernel +if [ -e "${module_dir}/drivers/virtio/virtio_pci_legacy_dev.ko" ]; then + /tmp/insmod ${module_dir}/drivers/virtio/virtio_pci_legacy_dev.ko +fi /tmp/insmod ${module_dir}/drivers/virtio/virtio_pci.ko /tmp/insmod ${module_dir}/drivers/block/virtio_blk.ko /tmp/insmod ${module_dir}/drivers/char/hw_random/virtio-rng.ko @@ -39,7 +45,14 @@ module_dir=/lib/modules/$(uname -r)/kernel mount -t devtmpfs devtmpfs /dev # Mount /dev/vda over the top of /root -mount /dev/vda /root +rm -f /dev/ram0 +mount -n -t proc proc /proc +mount LABEL=ROOT /root +if [[ "${install_grub}" = "1" ]]; then + mkdir -p /root/boot/efi + mount LABEL=SYSTEM /root/boot/efi +fi +umount /proc # Switch to the new root and start stage 2 mount -n --move /dev /root/dev diff --git a/net/test/rootfs/stage2.sh b/net/test/rootfs/stage2.sh index 84fc8ea..0ca1d56 100755 --- a/net/test/rootfs/stage2.sh +++ b/net/test/rootfs/stage2.sh @@ -32,12 +32,19 @@ rm -rf /debootstrap /var/lib/apt/lists/* # Read-only root breaks booting via init cat >/etc/fstab << EOF -LABEL=ROOT / ext4 defaults,discard 0 1 -tmpfs /tmp tmpfs defaults 0 0 -tmpfs /var/log tmpfs defaults 0 0 -tmpfs /var/tmp tmpfs defaults 0 0 +LABEL=ROOT / ext4 defaults,discard 0 1 +tmpfs /tmp tmpfs defaults 0 0 +tmpfs /var/log tmpfs defaults 0 0 +tmpfs /var/tmp tmpfs defaults 0 0 EOF +# If we're installing grub, add the EFI partition +if [[ "${install_grub}" = "1" ]]; then + cat >>/etc/fstab << EOF +LABEL=SYSTEM /boot/efi vfat umask=0077 0 1 +EOF +fi + # systemd will attempt to re-create this symlink if it does not exist, # which fails if it is booting from a read-only root filesystem (which # is normally the case). The syslink must be relative, not absolute, @@ -60,7 +67,9 @@ find /var/log -type f -exec rm -f '{}' ';' find /var/tmp -type f -exec rm -f '{}' ';' # Create an empty initramfs to be combined with modules later -sed -i 's,^COMPRESS=gzip,COMPRESS=lz4,' /etc/initramfs-tools/initramfs.conf +sed -i -e 's,^MODULES=dep,MODULES=most,' \ + -e 's,^COMPRESS=gzip,COMPRESS=lz4,' \ + /etc/initramfs-tools/initramfs.conf depmod -a $(uname -r) update-initramfs -c -k $(uname -r) dd if=/boot/initrd.img-$(uname -r) of=/dev/vdb conv=fsync diff --git a/net/test/run_net_test.sh b/net/test/run_net_test.sh index 1bf876d..8d44cf3 100755 --- a/net/test/run_net_test.sh +++ b/net/test/run_net_test.sh @@ -12,7 +12,8 @@ EOF } # Common kernel options -OPTIONS=" ANDROID DEBUG_SPINLOCK DEBUG_ATOMIC_SLEEP DEBUG_MUTEXES DEBUG_RT_MUTEXES" +OPTIONS=" ANDROID GKI_NET_XFRM_HACKS" +OPTIONS="$OPTIONS DEBUG_SPINLOCK DEBUG_ATOMIC_SLEEP DEBUG_MUTEXES DEBUG_RT_MUTEXES" OPTIONS="$OPTIONS WARN_ALL_UNSEEDED_RANDOM IKCONFIG IKCONFIG_PROC" OPTIONS="$OPTIONS DEVTMPFS DEVTMPFS_MOUNT FHANDLE" OPTIONS="$OPTIONS IPV6 IPV6_ROUTER_PREF IPV6_MULTIPLE_TABLES IPV6_ROUTE_INFO" @@ -64,13 +65,32 @@ OPTIONS="$OPTIONS BLK_DEV_UBD HOSTFS" # QEMU specific options OPTIONS="$OPTIONS PCI VIRTIO VIRTIO_PCI VIRTIO_BLK NET_9P NET_9P_VIRTIO 9P_FS" OPTIONS="$OPTIONS CRYPTO_DEV_VIRTIO SERIAL_8250 SERIAL_8250_PCI" +OPTIONS="$OPTIONS SERIAL_8250_CONSOLE PCI_HOST_GENERIC SERIAL_AMBA_PL011" +OPTIONS="$OPTIONS SERIAL_AMBA_PL011_CONSOLE" # Obsolete options present at some time in Android kernels OPTIONS="$OPTIONS IP_NF_TARGET_REJECT_SKERR IP6_NF_TARGET_REJECT_SKERR" +# b/262323440 - UML *sometimes* seems to have issues with: +# UPSTREAM: hardening: Clarify Kconfig text for auto-var-init +# which is in 4.14.~299/4.19.~266 LTS and which does: +# prompt "Initialize kernel stack variables at function entry" +# default GCC_PLUGIN_STRUCTLEAK_BYREF_ALL if COMPILE_TEST && GCC_PLUGINS +# default INIT_STACK_ALL_PATTERN if COMPILE_TEST && CC_HAS_AUTO_VAR_INIT_PATTERN +# + default INIT_STACK_ALL_ZERO if CC_HAS_AUTO_VAR_INIT_PATTERN +# default INIT_STACK_NONE +# and thus presumably switches from INIT_STACK_NONE to INIT_STACK_ALL_ZERO +# +# My guess it that this is triggering some sort of UML and/or compiler bug... +# Let's just turn it off... we don't care that much. +OPTIONS="$OPTIONS INIT_STACK_NONE" + # These two break the flo kernel due to differences in -Werror on recent GCC. DISABLE_OPTIONS=" REISERFS_FS ANDROID_PMEM" +# Disable frame size warning on arm64. GCC 10 generates >1k stack frames. +DISABLE_OPTIONS="$DISABLE_OPTIONS FRAME_WARN" + # How many TAP interfaces to create to provide the VM with real network access # via the host. This requires privileges (e.g., root access) on the host. # @@ -84,7 +104,7 @@ DISABLE_OPTIONS=" REISERFS_FS ANDROID_PMEM" NUMTAPINTERFACES=0 # The root filesystem disk image we'll use. -ROOTFS=${ROOTFS:-net_test.rootfs.20150203} +ROOTFS=${ROOTFS:-net_test.rootfs.20221014} COMPRESSED_ROOTFS=$ROOTFS.xz URL=https://dl.google.com/dl/android/$COMPRESSED_ROOTFS @@ -108,6 +128,12 @@ nowrite=1 nobuild=0 norun=0 +KVER_MAJOR="$(sed -rn 's@^ *VERSION *= *([0-9]+)$@\1@p' < "${KERNEL_DIR}/Makefile")" +KVER_MINOR="$(sed -rn 's@^ *PATCHLEVEL *= *([0-9]+)$@\1@p' < "${KERNEL_DIR}/Makefile")" +KVER_LEVEL="$(sed -rn 's@^ *SUBLEVEL *= *([0-9]+)$@\1@p' < "${KERNEL_DIR}/Makefile")" +KVER="${KVER_MAJOR}.${KVER_MINOR}.${KVER_LEVEL}" +echo "Detected kernel version ${KVER}" + if [[ -z "${DEFCONFIG:-}" ]]; then case "${ARCH}" in um) @@ -400,7 +426,7 @@ else # Map the --readonly flag to a QEMU block device flag if ((nowrite > 0)); then - blockdevice=",readonly" + blockdevice=",readonly=on" else blockdevice= fi diff --git a/net/test/sock_diag.py b/net/test/sock_diag.py index 03d5587..ae59897 100755 --- a/net/test/sock_diag.py +++ b/net/test/sock_diag.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2015 The Android Open Source Project # @@ -72,6 +72,9 @@ INET_DIAG_BC_D_COND = 8 INET_DIAG_BC_DEV_COND = 9 INET_DIAG_BC_MARK_COND = 10 +CONSTANT_PREFIXES = netlink.MakeConstantPrefixes([ + "INET_DIAG_", "INET_DIAG_REQ_", "INET_DIAG_BC_"]) + # Data structure formats. # These aren't constants, they're classes. So, pylint: disable=invalid-name InetDiagSockId = cstruct.Struct( @@ -114,13 +117,13 @@ class SockDiag(netlink.NetlinkSocket): def __init__(self): super(SockDiag, self).__init__(netlink.NETLINK_SOCK_DIAG) - def _Decode(self, command, msg, nla_type, nla_data): + def _Decode(self, command, msg, nla_type, nla_data, nested): """Decodes netlink attributes to Python types.""" if msg.family == AF_INET or msg.family == AF_INET6: if isinstance(msg, InetDiagReqV2): - prefix = "INET_DIAG_REQ" + prefix = "INET_DIAG_REQ_" else: - prefix = "INET_DIAG" + prefix = "INET_DIAG_" name = self._GetConstantName(__name__, nla_type, prefix) else: # Don't know what this is. Leave it as an integer. @@ -130,7 +133,7 @@ class SockDiag(netlink.NetlinkSocket): "INET_DIAG_SKV6ONLY"]: data = ord(nla_data) elif name == "INET_DIAG_CONG": - data = nla_data.strip("\x00") + data = nla_data.strip(b"\x00") elif name == "INET_DIAG_MEMINFO": data = InetDiagMeminfo(nla_data) elif name == "INET_DIAG_INFO": @@ -168,7 +171,7 @@ class SockDiag(netlink.NetlinkSocket): @staticmethod def _EmptyInetDiagSockId(): - return InetDiagSockId(("\x00" * len(InetDiagSockId))) + return InetDiagSockId((b"\x00" * len(InetDiagSockId))) @staticmethod def PackBytecode(instructions): @@ -220,10 +223,10 @@ class SockDiag(netlink.NetlinkSocket): raise ValueError("Jumps must be > 0") if op in [INET_DIAG_BC_NOP, INET_DIAG_BC_JMP, INET_DIAG_BC_AUTO]: - arg = "" + arg = b"" elif op in [INET_DIAG_BC_S_GE, INET_DIAG_BC_S_LE, INET_DIAG_BC_D_GE, INET_DIAG_BC_D_LE]: - arg = "\x00\x00" + struct.pack("=H", arg) + arg = b"\x00\x00" + struct.pack("=H", arg) elif op in [INET_DIAG_BC_S_COND, INET_DIAG_BC_D_COND]: addr, prefixlen, port = arg family = AF_INET6 if ":" in addr else AF_INET @@ -248,7 +251,7 @@ class SockDiag(netlink.NetlinkSocket): # print(positions) - packed = "" + packed = b"" for i, (op, yes, no, arg) in enumerate(instructions): yes = positions[i + yes] - positions[i] no = positions[i + no] - positions[i] @@ -346,7 +349,7 @@ class SockDiag(netlink.NetlinkSocket): """Converts an IP address string to binary format for InetDiagSockId.""" padded = SockDiag.RawAddress(addr) if len(padded) < 16: - padded += "\x00" * (16 - len(padded)) + padded += b"\x00" * (16 - len(padded)) return padded @staticmethod @@ -354,12 +357,9 @@ class SockDiag(netlink.NetlinkSocket): """Creates an InetDiagReqV2 that matches the specified socket.""" family = s.getsockopt(net_test.SOL_SOCKET, net_test.SO_DOMAIN) protocol = s.getsockopt(net_test.SOL_SOCKET, net_test.SO_PROTOCOL) - if net_test.LINUX_VERSION >= (3, 8): - iface = s.getsockopt(SOL_SOCKET, net_test.SO_BINDTODEVICE, - net_test.IFNAMSIZ) - iface = GetInterfaceIndex(iface) if iface else 0 - else: - iface = 0 + iface = s.getsockopt(SOL_SOCKET, net_test.SO_BINDTODEVICE, + net_test.IFNAMSIZ) + iface = GetInterfaceIndex(iface) if iface else 0 src, sport = s.getsockname()[:2] try: dst, dport = s.getpeername()[:2] @@ -371,7 +371,7 @@ class SockDiag(netlink.NetlinkSocket): raise e src = SockDiag.PaddedAddress(src) dst = SockDiag.PaddedAddress(dst) - sock_id = InetDiagSockId((sport, dport, src, dst, iface, "\x00" * 8)) + sock_id = InetDiagSockId((sport, dport, src, dst, iface, b"\x00" * 8)) return InetDiagReqV2((family, protocol, 0, 0xffffffff, sock_id)) @staticmethod @@ -387,7 +387,7 @@ class SockDiag(netlink.NetlinkSocket): # the inode number to ensure we don't mistakenly match another socket on # the same port but with a different IP address. inode = os.fstat(s.fileno()).st_ino - results = self.Dump(req, "") + results = self.Dump(req, b"") if len(results) == 0: raise ValueError("Dump of %s returned no sockets" % req) for diag_msg, attrs in results: @@ -423,11 +423,10 @@ class SockDiag(netlink.NetlinkSocket): if __name__ == "__main__": n = SockDiag() n.DEBUG = True - bytecode = "" sock_id = n._EmptyInetDiagSockId() sock_id.dport = 443 ext = 1 << (INET_DIAG_TOS - 1) | 1 << (INET_DIAG_TCLASS - 1) states = 0xffffffff - diag_msgs = n.DumpAllInetSockets(IPPROTO_TCP, "", + diag_msgs = n.DumpAllInetSockets(IPPROTO_TCP, b"", sock_id=sock_id, ext=ext, states=states) print(diag_msgs) diff --git a/net/test/sock_diag_test.py b/net/test/sock_diag_test.py index beda5e4..aa14343 100755 --- a/net/test/sock_diag_test.py +++ b/net/test/sock_diag_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2015 The Android Open Source Project # @@ -16,6 +16,7 @@ # pylint: disable=g-bad-todo,g-bad-file-header,wildcard-import from errno import * # pylint: disable=wildcard-import +import binascii import os import random import select @@ -36,42 +37,12 @@ import tcp_test TcpInfo = cstruct.Struct("TcpInfo", "64xI", "tcpi_rcv_ssthresh") NUM_SOCKETS = 30 -NO_BYTECODE = "" -LINUX_4_9_OR_ABOVE = net_test.LINUX_VERSION >= (4, 9, 0) +NO_BYTECODE = b"" LINUX_4_19_OR_ABOVE = net_test.LINUX_VERSION >= (4, 19, 0) IPPROTO_SCTP = 132 -def HaveUdpDiag(): - """Checks if the current kernel has config CONFIG_INET_UDP_DIAG enabled. - - This config is required for device running 4.9 kernel that ship with P, In - this case always assume the config is there and use the tests to check if the - config is enabled as required. - - For all ther other kernel version, there is no way to tell whether a dump - succeeded: if the appropriate handler wasn't found, __inet_diag_dump just - returns an empty result instead of an error. So, just check to see if a UDP - dump returns no sockets when we know it should return one. If not, some tests - will be skipped. - - Returns: - True if the kernel is 4.9 or above, or the CONFIG_INET_UDP_DIAG is enabled. - False otherwise. - """ - if LINUX_4_9_OR_ABOVE: - return True; - s = socket(AF_INET6, SOCK_DGRAM, 0) - s.bind(("::", 0)) - s.connect((s.getsockname())) - sd = sock_diag.SockDiag() - have_udp_diag = len(sd.DumpAllInetSockets(IPPROTO_UDP, "")) > 0 - s.close() - return have_udp_diag - def HaveSctp(): - if net_test.LINUX_VERSION < (4, 7, 0): - return False try: s = socket(AF_INET, SOCK_STREAM, IPPROTO_SCTP) s.close() @@ -79,7 +50,6 @@ def HaveSctp(): except IOError: return False -HAVE_UDP_DIAG = HaveUdpDiag() HAVE_SCTP = HaveSctp() @@ -251,10 +221,16 @@ class SockDiagTest(SockDiagBaseTest): info = self.sock_diag.GetSockInfo(req) self.assertSockInfoMatchesSocket(sock, info) + def assertItemsEqual(self, expected, actual): + try: + super(SockDiagTest, self).assertItemsEqual(expected, actual) + except AttributeError: + # This was renamed in python3 but has the same behaviour. + super(SockDiagTest, self).assertCountEqual(expected, actual) + def testFindsAllMySocketsTcp(self): self.CheckFindsAllMySockets(SOCK_STREAM, IPPROTO_TCP) - @unittest.skipUnless(HAVE_UDP_DIAG, "INET_UDP_DIAG not enabled") def testFindsAllMySocketsUdp(self): self.CheckFindsAllMySockets(SOCK_DGRAM, IPPROTO_UDP) @@ -274,16 +250,16 @@ class SockDiagTest(SockDiagBaseTest): # pylint: enable=bad-whitespace bytecode = self.PackAndCheckBytecode(instructions) expected = ( - "0208500000000000" - "050848000000ffff" - "071c20000a800000ffffffff00000000000000000000000000000001" - "01041c00" - "0718200002200000ffffffff7f000001" - "0508100000006566" - "00040400" + b"0208500000000000" + b"050848000000ffff" + b"071c20000a800000ffffffff00000000000000000000000000000001" + b"01041c00" + b"0718200002200000ffffffff7f000001" + b"0508100000006566" + b"00040400" ) states = 1 << tcp_test.TCP_ESTABLISHED - self.assertMultiLineEqual(expected, bytecode.encode("hex")) + self.assertEqual(expected, binascii.hexlify(bytecode)) self.assertEqual(76, len(bytecode)) self.socketpairs = self._CreateLotsOfSockets(SOCK_STREAM) filteredsockets = self.sock_diag.DumpAllInetSockets(IPPROTO_TCP, bytecode, @@ -322,7 +298,7 @@ class SockDiagTest(SockDiagBaseTest): # sockets other than the ones it creates itself. Make the bytecode more # specific and remove it. states = 1 << tcp_test.TCP_ESTABLISHED - self.assertFalse(self.sock_diag.DumpAllInetSockets(IPPROTO_TCP, "", + self.assertFalse(self.sock_diag.DumpAllInetSockets(IPPROTO_TCP, NO_BYTECODE, states=states)) unused_pair4 = net_test.CreateSocketPair(AF_INET, SOCK_STREAM, "127.0.0.1") @@ -376,7 +352,7 @@ class SockDiagTest(SockDiagBaseTest): sock_id = self.sock_diag._EmptyInetDiagSockId() req = sock_diag.InetDiagReqV2((AF_INET6, IPPROTO_TCP, 0, 0xffffffff, sock_id)) - self.sock_diag._Dump(code, req, sock_diag.InetDiagMsg, "") + self.sock_diag._Dump(code, req, sock_diag.InetDiagMsg) op = sock_diag.SOCK_DIAG_BY_FAMILY DiagDump(op) # No errors? Good. @@ -390,12 +366,10 @@ class SockDiagTest(SockDiagBaseTest): cookie = sock.getsockopt(net_test.SOL_SOCKET, net_test.SO_COOKIE, 8) self.assertEqual(diag_msg.id.cookie, cookie) - @unittest.skipUnless(LINUX_4_9_OR_ABOVE, "SO_COOKIE not supported") def testGetsockoptcookie(self): self.CheckSocketCookie(AF_INET, "127.0.0.1") self.CheckSocketCookie(AF_INET6, "::1") - @unittest.skipUnless(HAVE_UDP_DIAG, "INET_UDP_DIAG not enabled") def testDemonstrateUdpGetSockIdBug(self): # TODO: this is because udp_dump_one mistakenly uses __udp[46]_lib_lookup # by passing the source address as the source address argument. @@ -414,10 +388,7 @@ class SockDiagTest(SockDiagBaseTest): # Create a fully-specified diag req from our socket, including cookie if # we can get it. req = self.sock_diag.DiagReqFromSocket(s) - if LINUX_4_9_OR_ABOVE: - req.id.cookie = s.getsockopt(net_test.SOL_SOCKET, net_test.SO_COOKIE, 8) - else: - req.id.cookie = "\xff" * 16 # INET_DIAG_NOCOOKIE[2] + req.id.cookie = s.getsockopt(net_test.SOL_SOCKET, net_test.SO_COOKIE, 8) # As is, this request does not find anything. with self.assertRaisesErrno(ENOENT): @@ -580,6 +551,7 @@ class TcpRcvWindowTest(tcp_test.TcpBaseTest, SockDiagBaseTest): return f.write("60") + f.close() def checkInitRwndSize(self, version, netid): self.IncomingConnection(version, tcp_test.TCP_ESTABLISHED, netid) @@ -684,21 +656,19 @@ class SockDestroyTcpTest(tcp_test.TcpBaseTest, SockDiagBaseTest): diag_msg, attrs = self.sock_diag.GetSockInfo(diag_req) self.ReceivePacketOn(self.netid, finack) - # See if we can find the resulting FIN_WAIT2 socket. This does not appear - # to work on 3.10. - if net_test.LINUX_VERSION >= (3, 18): - diag_req.states = 1 << tcp_test.TCP_FIN_WAIT2 - infos = self.sock_diag.Dump(diag_req, "") - self.assertTrue(any(diag_msg.state == tcp_test.TCP_FIN_WAIT2 - for diag_msg, attrs in infos), - "Expected to find FIN_WAIT2 socket in %s" % infos) + # See if we can find the resulting FIN_WAIT2 socket. + diag_req.states = 1 << tcp_test.TCP_FIN_WAIT2 + infos = self.sock_diag.Dump(diag_req, NO_BYTECODE) + self.assertTrue(any(diag_msg.state == tcp_test.TCP_FIN_WAIT2 + for diag_msg, attrs in infos), + "Expected to find FIN_WAIT2 socket in %s" % infos) def FindChildSockets(self, s): """Finds the SYN_RECV child sockets of a given listening socket.""" d = self.sock_diag.FindSockDiagFromFd(self.s) req = self.sock_diag.DiagReqFromDiagMsg(d, IPPROTO_TCP) req.states = 1 << tcp_test.TCP_SYN_RECV | 1 << tcp_test.TCP_ESTABLISHED - req.id.cookie = "\x00" * 8 + req.id.cookie = b"\x00" * 8 bad_bytecode = self.PackAndCheckBytecode( [(sock_diag.INET_DIAG_BC_MARK_COND, 1, 2, (0xffff, 0xffff))]) @@ -723,19 +693,10 @@ class SockDestroyTcpTest(tcp_test.TcpBaseTest, SockDiagBaseTest): is_established = (state == tcp_test.TCP_NOT_YET_ACCEPTED) expected_state = tcp_test.TCP_ESTABLISHED if is_established else state - # The new TCP listener code in 4.4 makes SYN_RECV sockets live in the - # regular TCP hash tables, and inet_diag_find_one_icsk can find them. - # Before 4.4, we can see those sockets in dumps, but we can't fetch - # or close them. - can_close_children = is_established or net_test.LINUX_VERSION >= (4, 4) - for child in children: - if can_close_children: - diag_msg, attrs = self.sock_diag.GetSockInfo(child) - self.assertEqual(diag_msg.state, expected_state) - self.assertMarkIs(self.netid, attrs) - else: - self.assertRaisesErrno(ENOENT, self.sock_diag.GetSockInfo, child) + diag_msg, attrs = self.sock_diag.GetSockInfo(child) + self.assertEqual(diag_msg.state, expected_state) + self.assertMarkIs(self.netid, attrs) def CloseParent(expect_reset): msg = "Closing parent IPv%d %s socket %s child" % ( @@ -762,13 +723,12 @@ class SockDestroyTcpTest(tcp_test.TcpBaseTest, SockDiagBaseTest): CloseParent(is_established) if is_established: CheckChildrenClosed() - elif can_close_children: + else: CloseChildren() CheckChildrenClosed() self.s.close() else: - if can_close_children: - CloseChildren() + CloseChildren() CloseParent(False) self.s.close() @@ -785,10 +745,10 @@ class SockDestroyTcpTest(tcp_test.TcpBaseTest, SockDiagBaseTest): self.IncomingConnection(version, tcp_test.TCP_LISTEN, self.netid) self.assertRaisesErrno(ENOTCONN, self.s.recv, 4096) self.CloseDuringBlockingCall(self.s, lambda sock: sock.accept(), EINVAL) - self.assertRaisesErrno(ECONNABORTED, self.s.send, "foo") + self.assertRaisesErrno(ECONNABORTED, self.s.send, b"foo") self.assertRaisesErrno(EINVAL, self.s.accept) # TODO: this should really return an error such as ENOTCONN... - self.assertEqual("", self.s.recv(4096)) + self.assertEqual(b"", self.s.recv(4096)) def testReadInterrupted(self): """Tests that read() is interrupted by SOCK_DESTROY.""" @@ -797,9 +757,9 @@ class SockDestroyTcpTest(tcp_test.TcpBaseTest, SockDiagBaseTest): self.CloseDuringBlockingCall(self.accepted, lambda sock: sock.recv(4096), ECONNABORTED) # Writing returns EPIPE, and reading returns EOF. - self.assertRaisesErrno(EPIPE, self.accepted.send, "foo") - self.assertEqual("", self.accepted.recv(4096)) - self.assertEqual("", self.accepted.recv(4096)) + self.assertRaisesErrno(EPIPE, self.accepted.send, b"foo") + self.assertEqual(b"", self.accepted.recv(4096)) + self.assertEqual(b"", self.accepted.recv(4096)) def testConnectInterrupted(self): """Tests that connect() is interrupted by SOCK_DESTROY.""" @@ -867,9 +827,9 @@ class PollOnCloseTest(tcp_test.TcpBaseTest, SockDiagBaseTest): self.assertRaisesErrno(errno, self.accepted.recv, 4096) # Subsequent operations behave as normal. - self.assertRaisesErrno(EPIPE, self.accepted.send, "foo") - self.assertEqual("", self.accepted.recv(4096)) - self.assertEqual("", self.accepted.recv(4096)) + self.assertRaisesErrno(EPIPE, self.accepted.send, b"foo") + self.assertEqual(b"", self.accepted.recv(4096)) + self.assertEqual(b"", self.accepted.recv(4096)) def CheckPollDestroy(self, mask, expected, ignoremask): """Interrupts a poll() with SOCK_DESTROY.""" @@ -892,15 +852,7 @@ class PollOnCloseTest(tcp_test.TcpBaseTest, SockDiagBaseTest): self.assertSocketErrors(ECONNRESET) def testReadPollRst(self): - # Until 3d4762639d ("tcp: remove poll() flakes when receiving RST"), poll() - # would sometimes return POLLERR and sometimes POLLIN|POLLERR|POLLHUP. This - # is due to a race inside the kernel and thus is not visible on the VM, only - # on physical hardware. - if net_test.LINUX_VERSION < (4, 14, 0): - ignoremask = select.POLLIN | select.POLLHUP - else: - ignoremask = 0 - self.CheckPollRst(select.POLLIN, self.POLLIN_ERR_HUP, ignoremask) + self.CheckPollRst(select.POLLIN, self.POLLIN_ERR_HUP, 0) def testWritePollRst(self): self.CheckPollRst(select.POLLOUT, select.POLLOUT, 0) @@ -920,7 +872,6 @@ class PollOnCloseTest(tcp_test.TcpBaseTest, SockDiagBaseTest): self.CheckPollDestroy(self.POLLIN_OUT, select.POLLOUT, 0) -@unittest.skipUnless(HAVE_UDP_DIAG, "INET_UDP_DIAG not enabled") class SockDestroyUdpTest(SockDiagBaseTest): """Tests SOCK_DESTROY on UDP sockets. @@ -1005,13 +956,13 @@ class SockDestroyUdpTest(SockDiagBaseTest): # Check that reads on connected sockets are interrupted. s.connect((addr, 53)) - self.assertEqual(3, s.send("foo")) + self.assertEqual(3, s.send(b"foo")) self.CloseDuringBlockingCall(s, lambda sock: sock.recv(4096), ECONNABORTED) # A destroyed socket is no longer connected, but still usable. - self.assertRaisesErrno(EDESTADDRREQ, s.send, "foo") - self.assertEqual(3, s.sendto("foo", (addr, 53))) + self.assertRaisesErrno(EDESTADDRREQ, s.send, b"foo") + self.assertEqual(3, s.sendto(b"foo", (addr, 53))) # Check that reads on unconnected sockets are also interrupted. self.CloseDuringBlockingCall(s, lambda sock: sock.recv(4096), @@ -1037,7 +988,6 @@ class SockDestroyPermissionTest(SockDiagBaseTest): self.assertRaises(ValueError, self.sock_diag.CloseSocketFromFd, s) - @unittest.skipUnless(HAVE_UDP_DIAG, "INET_UDP_DIAG not enabled") def testUdp(self): self.CheckPermissions(SOCK_DGRAM) @@ -1170,12 +1120,11 @@ class SockDiagMarkTest(tcp_test.TcpBaseTest, SockDiagBaseTest): # Other TCP states are tested in SockDestroyTcpTest. # UDP sockets. - if HAVE_UDP_DIAG: - s = socket(family, SOCK_DGRAM, 0) - mark = self.SetRandomMark(s) - s.connect(("", 53)) - self.assertSocketMarkIs(s, mark) - s.close() + s = socket(family, SOCK_DGRAM, 0) + mark = self.SetRandomMark(s) + s.connect(("", 53)) + self.assertSocketMarkIs(s, mark) + s.close() # Basic test for SCTP. sctp_diag was only added in 4.7. if HAVE_SCTP: diff --git a/net/test/srcaddr_selection_test.py b/net/test/srcaddr_selection_test.py index 1e7a107..f515c47 100755 --- a/net/test/srcaddr_selection_test.py +++ b/net/test/srcaddr_selection_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2014 The Android Open Source Project # @@ -109,7 +109,7 @@ class IPv6SourceAddressSelectionTest(multinetwork_base.MultiNetworkBaseTest): pktinfo = multinetwork_base.MakePktInfo(6, address, 0) cmsgs = [(net_test.SOL_IPV6, IPV6_PKTINFO, pktinfo)] s = self.BuildSocket(6, net_test.UDPSocket, netid, "mark") - return csocket.Sendmsg(s, (dest, 53), "Hello", cmsgs, 0) + return csocket.Sendmsg(s, (dest, 53), b"Hello", cmsgs, 0) def assertAddressUsable(self, address, netid): self.BindToAddress(address) @@ -211,10 +211,7 @@ class OptimisticAddressTest(MultiInterfaceSourceAddressSelectionTest): self.test_ip, self.test_ifindex, iproute.IFA_F_OPTIMISTIC) # Optimistic addresses are usable but are not selected. - if net_test.LINUX_VERSION >= (3, 18, 0): - # The version checked in to android kernels <= 3.10 requires the - # use_optimistic sysctl to be turned on. - self.assertAddressUsable(self.test_ip, self.test_netid) + self.assertAddressUsable(self.test_ip, self.test_netid) self.assertAddressNotSelected(self.test_ip, self.test_netid) # Busy wait for DAD to complete (should be less than 1 second). @@ -327,14 +324,11 @@ class NoNsFromOptimisticTest(MultiInterfaceSourceAddressSelectionTest): self.OnlinkPrefix(6, self.test_netid)) self.SendWithSourceAddress(self.test_ip, self.test_netid, onlink_dest) - if net_test.LINUX_VERSION >= (3, 18, 0): - # Older versions will actually choose the optimistic address to - # originate Neighbor Solications (RFC violation). - expected_ns = packets.NS( - self.test_lladdr, - onlink_dest, - self.MyMacAddress(self.test_netid))[1] - self.ExpectPacketOn(self.test_netid, "link-local NS", expected_ns) + expected_ns = packets.NS( + self.test_lladdr, + onlink_dest, + self.MyMacAddress(self.test_netid))[1] + self.ExpectPacketOn(self.test_netid, "link-local NS", expected_ns) # TODO(ek): add tests listening for netlink events. diff --git a/net/test/sysctls_test.py b/net/test/sysctls_test.py index cb608f6..a4d8d66 100755 --- a/net/test/sysctls_test.py +++ b/net/test/sysctls_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2021 The Android Open Source Project # @@ -22,19 +22,23 @@ import net_test class SysctlsTest(net_test.NetworkTest): def check(self, f): - algs = open(f).readline().strip().split(' ') + with open(f) as algs_file: + algs = algs_file.readline().strip().split(' ') bad_algs = [a for a in algs if a not in ['cubic', 'reno']] msg = ("Obsolete TCP congestion control algorithm found. These " "algorithms will decrease real-world networking performance for " "users and must be disabled. Found: %s" % bad_algs) self.assertEqual(bad_algs, [], msg) + @unittest.skipUnless(net_test.LINUX_VERSION >= (5, 7, 0), "not yet namespaced") def testAllowedCongestionControl(self): self.check('/proc/sys/net/ipv4/tcp_allowed_congestion_control') + @unittest.skipUnless(net_test.LINUX_VERSION >= (5, 7, 0), "not yet namespaced") def testAvailableCongestionControl(self): self.check('/proc/sys/net/ipv4/tcp_available_congestion_control') + @unittest.skipUnless(net_test.LINUX_VERSION >= (4, 15, 0), "not yet namespaced") def testCongestionControl(self): self.check('/proc/sys/net/ipv4/tcp_congestion_control') diff --git a/net/test/tcp_fastopen_test.py b/net/test/tcp_fastopen_test.py index 9c777c6..f5fc00f 100755 --- a/net/test/tcp_fastopen_test.py +++ b/net/test/tcp_fastopen_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -66,8 +66,6 @@ class TcpFastOpenTest(multinetwork_base.MultiNetworkBaseTest): self.tcp_metrics.GetMetrics(saddr, daddr) def clearBlackhole(self): - if net_test.LINUX_VERSION < (4, 14, 0): - return # Prior to 4.15 this sysctl is not namespace aware. if net_test.LINUX_VERSION < (4, 15, 0) and not os.path.exists(BH_TIMEOUT_SYSCTL): return @@ -98,7 +96,7 @@ class TcpFastOpenTest(multinetwork_base.MultiNetworkBaseTest): syn.getlayer("TCP").options = [(TCPOPT_FASTOPEN, "")] msg = "Fastopen connect: expected %s" % desc syn = self.ExpectPacketOn(netid, msg, syn) - syn = ip_layer(str(syn)) + syn = ip_layer(bytes(syn)) # Receive a SYN+ACK with a TFO cookie and expect the connection to proceed # as normal. @@ -106,7 +104,7 @@ class TcpFastOpenTest(multinetwork_base.MultiNetworkBaseTest): synack.getlayer("TCP").options = [ (TCPOPT_FASTOPEN, "helloT"), ("NOP", None), ("NOP", None)] self.ReceivePacketOn(netid, synack) - synack = ip_layer(str(synack)) + synack = ip_layer(bytes(synack)) desc, ack = packets.ACK(version, myaddr, remoteaddr, synack) msg = "First connect: got SYN+ACK, expected %s" % desc self.ExpectPacketOn(netid, msg, ack) @@ -133,11 +131,9 @@ class TcpFastOpenTest(multinetwork_base.MultiNetworkBaseTest): msg = "TFO write, expected %s" % desc self.ExpectPacketOn(netid, msg, syn) - @unittest.skipUnless(net_test.LINUX_VERSION >= (4, 9, 0), "not yet backported") def testConnectOptionIPv4(self): self.CheckConnectOption(4) - @unittest.skipUnless(net_test.LINUX_VERSION >= (4, 9, 0), "not yet backported") def testConnectOptionIPv6(self): self.CheckConnectOption(6) diff --git a/net/test/tcp_metrics.py b/net/test/tcp_metrics.py index 03f604f..87c753a 100755 --- a/net/test/tcp_metrics.py +++ b/net/test/tcp_metrics.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -19,6 +19,7 @@ from socket import * # pylint: disable=wildcard-import import struct +import binascii import cstruct import genetlink import net_test @@ -61,7 +62,7 @@ class TcpMetrics(genetlink.GenericNetlink): ctrl = genetlink.GenericNetlinkControl() self.family = ctrl.GetFamily(TCP_METRICS_GENL_NAME) - def _Decode(self, command, msg, nla_type, nla_data): + def _Decode(self, command, msg, nla_type, nla_data, nested): """Decodes TCP metrics netlink attributes to human-readable format.""" name = self._GetConstantName(__name__, nla_type, "TCP_METRICS_ATTR_") @@ -79,7 +80,7 @@ class TcpMetrics(genetlink.GenericNetlink): elif name == "TCP_METRICS_ATTR_FOPEN_COOKIE": data = nla_data else: - data = nla_data.encode("hex") + data = binascii.hexlify(nla_data) return name, data diff --git a/net/test/tcp_nuke_addr_test.py b/net/test/tcp_nuke_addr_test.py index e5d17b2..6010d5f 100755 --- a/net/test/tcp_nuke_addr_test.py +++ b/net/test/tcp_nuke_addr_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -25,7 +25,7 @@ import net_test IPV4_LOOPBACK_ADDR = "127.0.0.1" IPV6_LOOPBACK_ADDR = "::1" -LOOPBACK_DEV = "lo" +LOOPBACK_DEV = b"lo" LOOPBACK_IFINDEX = 1 SIOCKILLADDR = 0x8939 @@ -68,7 +68,6 @@ def CreateIPv6SocketPair(): return net_test.CreateSocketPair(AF_INET6, SOCK_STREAM, IPV6_LOOPBACK_ADDR) -@unittest.skipUnless(net_test.LINUX_VERSION >= (4, 4, 0), "grace period") class TcpNukeAddrTest(net_test.NetworkTest): """Tests that SIOCKILLADDR no longer exists. @@ -86,7 +85,7 @@ class TcpNukeAddrTest(net_test.NetworkTest): def CheckNukeAddrUnsupported(self, socketpair, addr): s1, s2 = socketpair self.assertRaisesErrno(errno.ENOTTY, KillAddrIoctl, addr) - data = "foo" + data = b"foo" try: self.assertEqual(len(data), s1.send(data)) self.assertEqual(data, s2.recv(4096)) diff --git a/net/test/tcp_repair_test.py b/net/test/tcp_repair_test.py index e0b156e..cc5ed41 100755 --- a/net/test/tcp_repair_test.py +++ b/net/test/tcp_repair_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2019 The Android Open Source Project # @@ -138,7 +138,7 @@ class TcpRepairTest(multinetwork_base.MultiNetworkBaseTest): sock.setsockopt(SOL_TCP, TCP_REPAIR, TCP_REPAIR_ON) # In repair mode with NO_QUEUE, writes fail... - self.assertRaisesErrno(EINVAL, sock.send, "write test") + self.assertRaisesErrno(EINVAL, sock.send, b"write test") # remote data is coming. TEST_RECEIVED = net_test.UDP_PAYLOAD diff --git a/net/test/tcp_test.py b/net/test/tcp_test.py index 5043d46..5a073e6 100644 --- a/net/test/tcp_test.py +++ b/net/test/tcp_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2015 The Android Open Source Project # @@ -122,7 +122,7 @@ class TcpBaseTest(multinetwork_base.MultiNetworkBaseTest): self.ExpectPacketOn(netid, msg + ": expecting %s" % desc, data) desc, fin = packets.FIN(version, remoteaddr, myaddr, data) - fin = packets._GetIpLayer(version)(str(fin)) + fin = packets._GetIpLayer(version)(bytes(fin)) ack_desc, ack = packets.ACK(version, myaddr, remoteaddr, fin) msg = "Received %s, expected to see reply %s" % (desc, ack_desc) diff --git a/net/test/tun_twister.py b/net/test/tun_twister.py index f42d789..07f4982 100644 --- a/net/test/tun_twister.py +++ b/net/test/tun_twister.py @@ -61,7 +61,7 @@ class TunTwister(object): sock.settimeout(1.0) sock.sendto("hello", ("1.2.3.4", 8080)) data, addr = sock.recvfrom(1024) - self.assertEqual("hello", data) + self.assertEqual(b"hello", data) self.assertEqual(("1.2.3.4", 8080), addr) """ @@ -94,11 +94,11 @@ class TunTwister(object): def __exit__(self, *args): # Signal thread exit. - os.write(self._signal_write, "bye") + os.write(self._signal_write, b"bye") os.close(self._signal_write) self._thread.join(TunTwister._POLL_TIMEOUT_SEC) os.close(self._signal_read) - if self._thread.isAlive(): + if self._thread.is_alive(): raise RuntimeError("Timed out waiting for thread exit") # Re-raise any error thrown from our thread. if isinstance(self._error, Exception): diff --git a/net/test/vts_kernel_net_tests.xml b/net/test/vts_kernel_net_tests.xml index 34540c6..1be8357 100644 --- a/net/test/vts_kernel_net_tests.xml +++ b/net/test/vts_kernel_net_tests.xml @@ -23,10 +23,6 @@ <option name="cleanup" value="true" /> </target_preparer> - <target_preparer class="com.android.tradefed.targetprep.DeviceSetup"> - <option name="airplane-mode" value="ON" /> - </target_preparer> - <test class="com.android.tradefed.testtype.binary.ExecutableTargetTest" > <option name="per-binary-timeout" value="10m" /> <option name="test-command-line" key="vts_kernel_net_tests" value="/data/local/tmp/vts_kernel_net_tests/kernel_net_tests_bin" /> diff --git a/net/test/xfrm.py b/net/test/xfrm.py index 83437bd..3d003b6 100755 --- a/net/test/xfrm.py +++ b/net/test/xfrm.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2016 The Android Open Source Project # @@ -122,16 +122,16 @@ XFRM_POLICY_ICMP = 2 XFRM_STATE_AF_UNSPEC = 32 # XFRM algorithm names, as defined in net/xfrm/xfrm_algo.c. -XFRM_EALG_CBC_AES = "cbc(aes)" -XFRM_EALG_CTR_AES = "rfc3686(ctr(aes))" -XFRM_AALG_HMAC_MD5 = "hmac(md5)" -XFRM_AALG_HMAC_SHA1 = "hmac(sha1)" -XFRM_AALG_HMAC_SHA256 = "hmac(sha256)" -XFRM_AALG_HMAC_SHA384 = "hmac(sha384)" -XFRM_AALG_HMAC_SHA512 = "hmac(sha512)" -XFRM_AALG_AUTH_XCBC_AES = "xcbc(aes)" -XFRM_AEAD_GCM_AES = "rfc4106(gcm(aes))" -XFRM_AEAD_CHACHA20_POLY1305 = "rfc7539esp(chacha20,poly1305)" +XFRM_EALG_CBC_AES = b"cbc(aes)" +XFRM_EALG_CTR_AES = b"rfc3686(ctr(aes))" +XFRM_AALG_HMAC_MD5 = b"hmac(md5)" +XFRM_AALG_HMAC_SHA1 = b"hmac(sha1)" +XFRM_AALG_HMAC_SHA256 = b"hmac(sha256)" +XFRM_AALG_HMAC_SHA384 = b"hmac(sha384)" +XFRM_AALG_HMAC_SHA512 = b"hmac(sha512)" +XFRM_AALG_AUTH_XCBC_AES = b"xcbc(aes)" +XFRM_AEAD_GCM_AES = b"rfc4106(gcm(aes))" +XFRM_AEAD_CHACHA20_POLY1305 = b"rfc7539esp(chacha20,poly1305)" # Data structure formats. # These aren't constants, they're classes. So, pylint: disable=invalid-name @@ -213,7 +213,7 @@ UDP_ENCAP_ESPINUDP = 2 _INF = 2 ** 64 -1 NO_LIFETIME_CFG = XfrmLifetimeCfg((_INF, _INF, _INF, _INF, 0, 0, 0, 0)) -NO_LIFETIME_CUR = "\x00" * len(XfrmLifetimeCur) +NO_LIFETIME_CUR = b"\x00" * len(XfrmLifetimeCur) # IPsec constants. IPSEC_PROTO_ANY = 255 @@ -243,7 +243,7 @@ def PaddedAddress(addr): """Converts an IP address string to binary format for InetDiagSockId.""" padded = RawAddress(addr) if len(padded) < 16: - padded += "\x00" * (16 - len(padded)) + padded += b"\x00" * (16 - len(padded)) return padded @@ -368,7 +368,7 @@ class Xfrm(netlink.NetlinkSocket): else: print("%s" % cmdname) - def _Decode(self, command, unused_msg, nla_type, nla_data): + def _Decode(self, command, unused_msg, nla_type, nla_data, nested): """Decodes netlink attributes to Python types.""" name = self._GetConstantName(nla_type, "XFRMA_") @@ -516,7 +516,7 @@ class Xfrm(netlink.NetlinkSocket): xfrm_id = XfrmId((PaddedAddress(dst), spi, proto)) family = AF_INET6 if ":" in dst else AF_INET - nlattrs = "" + nlattrs = b"" if encryption is not None: enc, key = encryption nlattrs += self._NlAttr(XFRMA_ALG_CRYPT, enc.Pack() + key) @@ -602,7 +602,7 @@ class Xfrm(netlink.NetlinkSocket): min_spi: The minimum value of the acceptable SPI range (inclusive). max_spi: The maximum value of the acceptable SPI range (inclusive). """ - spi = XfrmUserSpiInfo("\x00" * len(XfrmUserSpiInfo)) + spi = XfrmUserSpiInfo(b"\x00" * len(XfrmUserSpiInfo)) spi.min = min_spi spi.max = max_spi spi.info.id.daddr = PaddedAddress(dst) @@ -618,15 +618,15 @@ class Xfrm(netlink.NetlinkSocket): if nl_hdr.type == XFRM_MSG_NEWSA: return XfrmUsersaInfo(data) if nl_hdr.type == netlink.NLMSG_ERROR: - error = netlink.NLMsgErr(data).error - raise IOError(error, os.strerror(-error)) + error = -netlink.NLMsgErr(data).error + raise IOError(error, os.strerror(error)) raise ValueError("Unexpected netlink message type: %d" % nl_hdr.type) def DumpSaInfo(self): - return self._Dump(XFRM_MSG_GETSA, None, XfrmUsersaInfo, "") + return self._Dump(XFRM_MSG_GETSA, None, XfrmUsersaInfo) def DumpPolicyInfo(self): - return self._Dump(XFRM_MSG_GETPOLICY, None, XfrmUserpolicyInfo, "") + return self._Dump(XFRM_MSG_GETPOLICY, None, XfrmUserpolicyInfo) def FindSaInfo(self, spi): sainfo = [sa for sa, attrs in self.DumpSaInfo() if sa.id.spi == spi] @@ -635,7 +635,7 @@ class Xfrm(netlink.NetlinkSocket): def FlushPolicyInfo(self): """Send a Netlink Request to Flush all records from the SPD""" flags = netlink.NLM_F_REQUEST | netlink.NLM_F_ACK - self._SendNlRequest(XFRM_MSG_FLUSHPOLICY, "", flags) + self._SendNlRequest(XFRM_MSG_FLUSHPOLICY, b"", flags) def FlushSaInfo(self): usersa_flush = XfrmUsersaFlush((IPSEC_PROTO_ANY,)) @@ -753,9 +753,12 @@ class Xfrm(netlink.NetlinkSocket): net_test.GetAddressFamily(net_test.GetAddressVersion(new_saddr)))) nlattrs.append((XFRMA_MIGRATE, xfrmMigrate)) + if xfrm_if_id is not None: + nlattrs.append((XFRMA_IF_ID, struct.pack("=I", xfrm_if_id))) + for selector in selectors: - self.SendXfrmNlRequest(XFRM_MSG_MIGRATE, - XfrmUserpolicyId(sel=selector, dir=direction), nlattrs) + self.SendXfrmNlRequest(XFRM_MSG_MIGRATE, + XfrmUserpolicyId(sel=selector, dir=direction), nlattrs) # UPDSA is called exclusively to update the set_mark=new_output_mark. self.AddSaInfo(new_saddr, new_daddr, spi, XFRM_MODE_TUNNEL, 0, encryption, diff --git a/net/test/xfrm_algorithm_test.py b/net/test/xfrm_algorithm_test.py index 8a50fde..8466953 100755 --- a/net/test/xfrm_algorithm_test.py +++ b/net/test/xfrm_algorithm_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -20,7 +20,6 @@ import os import itertools from scapy import all as scapy from socket import * # pylint: disable=wildcard-import -import subprocess import threading import unittest @@ -94,7 +93,7 @@ AEAD_ALGOS = [ def GenerateKey(key_len): if key_len % 8 != 0: raise ValueError("Invalid key length in bits: " + str(key_len)) - return os.urandom(key_len / 8) + return os.urandom(key_len // 8) # Does the kernel support this algorithm? def HaveAlgo(crypt_algo, auth_algo, aead_algo): @@ -143,13 +142,13 @@ def AlgoEnforcedOrEnabled(crypt, auth, aead, target_algo, target_kernel): # Return true if this algorithm should be enforced or is enabled on this kernel def AuthEnforcedOrEnabled(authCase): auth = authCase[0] - crypt = xfrm.XfrmAlgo(("ecb(cipher_null)", 0)) + crypt = xfrm.XfrmAlgo((b"ecb(cipher_null)", 0)) return AlgoEnforcedOrEnabled(crypt, auth, None, auth.name, authCase[1]) # Return true if this algorithm should be enforced or is enabled on this kernel def CryptEnforcedOrEnabled(cryptCase): crypt = cryptCase[0] - auth = xfrm.XfrmAlgoAuth(("digest_null", 0, 0)) + auth = xfrm.XfrmAlgoAuth((b"digest_null", 0, 0)) return AlgoEnforcedOrEnabled(crypt, auth, None, crypt.name, cryptCase[1]) # Return true if this algorithm should be enforced or is enabled on this kernel @@ -183,16 +182,16 @@ class XfrmAlgorithmTest(xfrm_base.XfrmLazyTest): param_string = "" if cryptCase is not None: crypt = cryptCase[0] - param_string += "%s_%d_" % (crypt.name, crypt.key_len) + param_string += "%s_%d_" % (crypt.name.decode(), crypt.key_len) if authCase is not None: auth = authCase[0] - param_string += "%s_%d_%d_" % (auth.name, auth.key_len, + param_string += "%s_%d_%d_" % (auth.name.decode(), auth.key_len, auth.trunc_len) if aeadCase is not None: aead = aeadCase[0] - param_string += "%s_%d_%d_" % (aead.name, aead.key_len, + param_string += "%s_%d_%d_" % (aead.name.decode(), aead.key_len, aead.icv_len) param_string += "%s_%s" % ("IPv4" if version == 4 else "IPv6", @@ -233,17 +232,17 @@ class XfrmAlgorithmTest(xfrm_base.XfrmLazyTest): local_addr = self.MyAddress(version, netid) remote_addr = self.GetRemoteSocketAddress(version) auth_left = (xfrm.XfrmAlgoAuth((auth.name, auth.key_len, auth.trunc_len)), - os.urandom(auth.key_len / 8)) if auth else None + os.urandom(auth.key_len // 8)) if auth else None auth_right = (xfrm.XfrmAlgoAuth((auth.name, auth.key_len, auth.trunc_len)), - os.urandom(auth.key_len / 8)) if auth else None + os.urandom(auth.key_len // 8)) if auth else None crypt_left = (xfrm.XfrmAlgo((crypt.name, crypt.key_len)), - os.urandom(crypt.key_len / 8)) if crypt else None + os.urandom(crypt.key_len // 8)) if crypt else None crypt_right = (xfrm.XfrmAlgo((crypt.name, crypt.key_len)), - os.urandom(crypt.key_len / 8)) if crypt else None + os.urandom(crypt.key_len // 8)) if crypt else None aead_left = (xfrm.XfrmAlgoAead((aead.name, aead.key_len, aead.icv_len)), - os.urandom(aead.key_len / 8)) if aead else None + os.urandom(aead.key_len // 8)) if aead else None aead_right = (xfrm.XfrmAlgoAead((aead.name, aead.key_len, aead.icv_len)), - os.urandom(aead.key_len / 8)) if aead else None + os.urandom(aead.key_len // 8)) if aead else None spi_left = 0xbeefface spi_right = 0xcafed00d req_ids = [100, 200, 300, 400] # Used to match templates and SAs. @@ -341,8 +340,8 @@ class XfrmAlgorithmTest(xfrm_base.XfrmLazyTest): self.assertEqual(remote_addr, peer[0]) self.assertEqual(client_port, peer[1]) data = accepted.recv(2048) - self.assertEqual("hello request", data) - accepted.send("hello response") + self.assertEqual(b"hello request", data) + accepted.send(b"hello response") except Exception as e: server_error = e finally: @@ -354,8 +353,8 @@ class XfrmAlgorithmTest(xfrm_base.XfrmLazyTest): data, peer = sock.recvfrom(2048) self.assertEqual(remote_addr, peer[0]) self.assertEqual(client_port, peer[1]) - self.assertEqual("hello request", data) - sock.sendto("hello response", peer) + self.assertEqual(b"hello request", data) + sock.sendto(b"hello response", peer) except Exception as e: server_error = e finally: @@ -382,11 +381,12 @@ class XfrmAlgorithmTest(xfrm_base.XfrmLazyTest): with TapTwister(fd=self.tuns[netid].fileno(), validator=AssertEncrypted): sock_left.connect((remote_addr, right_port)) - sock_left.send("hello request") + sock_left.send(b"hello request") data = sock_left.recv(2048) - self.assertEqual("hello response", data) + self.assertEqual(b"hello response", data) sock_left.close() - server.join() + server.join(timeout=2.0) + self.assertFalse(server.is_alive(), "Timed out waiting for server exit") if server_error: raise server_error diff --git a/net/test/xfrm_base.py b/net/test/xfrm_base.py index e61322e..e5aadf3 100644 --- a/net/test/xfrm_base.py +++ b/net/test/xfrm_base.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -16,6 +16,7 @@ from socket import * # pylint: disable=wildcard-import from scapy import all as scapy +import binascii import struct import csocket @@ -25,15 +26,15 @@ import net_test import util import xfrm -_ENCRYPTION_KEY_256 = ("308146eb3bd84b044573d60f5a5fd159" - "57c7d4fe567a2120f35bae0f9869ec22".decode("hex")) -_AUTHENTICATION_KEY_128 = "af442892cdcd0ef650e9c299f9a8436a".decode("hex") +_ENCRYPTION_KEY_256 = binascii.unhexlify("308146eb3bd84b044573d60f5a5fd159" + "57c7d4fe567a2120f35bae0f9869ec22") +_AUTHENTICATION_KEY_128 = binascii.unhexlify("af442892cdcd0ef650e9c299f9a8436a") -_ALGO_AUTH_NULL = (xfrm.XfrmAlgoAuth(("digest_null", 0, 0)), "") +_ALGO_AUTH_NULL = (xfrm.XfrmAlgoAuth((b"digest_null", 0, 0)), b"") _ALGO_HMAC_SHA1 = (xfrm.XfrmAlgoAuth((xfrm.XFRM_AALG_HMAC_SHA1, 128, 96)), _AUTHENTICATION_KEY_128) -_ALGO_CRYPT_NULL = (xfrm.XfrmAlgo(("ecb(cipher_null)", 0)), "") +_ALGO_CRYPT_NULL = (xfrm.XfrmAlgo((b"ecb(cipher_null)", 0)), b"") _ALGO_CBC_AES_256 = (xfrm.XfrmAlgo((xfrm.XFRM_EALG_CBC_AES, 256)), _ENCRYPTION_KEY_256) @@ -88,12 +89,12 @@ def _GetCryptParameters(crypt_alg): Returns: A tuple of the block size, and IV length """ - cryptParameters = { - _ALGO_CRYPT_NULL: (4, 0), - _ALGO_CBC_AES_256: (16, 16) - } + if crypt_alg == _ALGO_CRYPT_NULL: + return (4, 0) + if crypt_alg == _ALGO_CBC_AES_256: + return (16, 16) + return (0, 0) - return cryptParameters.get(crypt_alg, (0, 0)) def GetEspPacketLength(mode, version, udp_encap, payload, auth_alg, crypt_alg): @@ -120,7 +121,7 @@ def GetEspPacketLength(mode, version, udp_encap, payload, # Size constants esp_hdr_len = len(xfrm.EspHdr) # SPI + Seq number - icv_len = auth_trunc_len / 8 + icv_len = auth_trunc_len // 8 # Add inner IP header if tunnel mode if mode == xfrm.XFRM_MODE_TUNNEL: @@ -142,6 +143,18 @@ def GetEspPacketLength(mode, version, udp_encap, payload, return payload_len +def GetEspTrailer(length, nexthdr): + # ESP padding per RFC 4303 section 2.4. + # For a null cipher with a block size of 1, padding is only necessary to + # ensure that the 1-byte Pad Length and Next Header fields are right aligned + # on a 4-byte boundary. + esplen = length + 2 # Packet length plus Pad Length and Next Header. + padlen = util.GetPadLength(4, esplen) + # The pad bytes are consecutive integers starting from 0x01. + padding = "".join((chr(i) for i in range(1, padlen + 1))).encode("utf-8") + return padding + struct.pack("BB", padlen, nexthdr) + + def EncryptPacketWithNull(packet, spi, seq, tun_addrs): """Apply null encryption to a packet. @@ -151,7 +164,7 @@ def EncryptPacketWithNull(packet, spi, seq, tun_addrs): The input packet is assumed to be a UDP packet. The input packet *MUST* have its length and checksum fields in IP and UDP headers set appropriately. This can be done by "rebuilding" the scapy object. e.g., - ip6_packet = scapy.IPv6(str(ip6_packet)) + ip6_packet = scapy.IPv6(bytes(ip6_packet)) TODO: Support TCP @@ -188,21 +201,12 @@ def EncryptPacketWithNull(packet, spi, seq, tun_addrs): inner_layer = udp_layer esp_nexthdr = IPPROTO_UDP - - # ESP padding per RFC 4303 section 2.4. - # For a null cipher with a block size of 1, padding is only necessary to - # ensure that the 1-byte Pad Length and Next Header fields are right aligned - # on a 4-byte boundary. - esplen = (len(inner_layer) + 2) # UDP length plus Pad Length and Next Header. - padlen = util.GetPadLength(4, esplen) - # The pad bytes are consecutive integers starting from 0x01. - padding = "".join((chr(i) for i in range(1, padlen + 1))) - trailer = padding + struct.pack("BB", padlen, esp_nexthdr) + trailer = GetEspTrailer(len(inner_layer), esp_nexthdr) # Assemble the packet. esp_packet.payload = scapy.Raw(inner_layer) packet = new_ip_layer if new_ip_layer else packet - packet.payload = scapy.Raw(str(esp_packet) + trailer) + packet.payload = scapy.Raw(bytes(esp_packet) + trailer) # TODO: Can we simplify this and avoid the initial copy()? # Fix the IPv4/IPv6 headers. @@ -210,13 +214,13 @@ def EncryptPacketWithNull(packet, spi, seq, tun_addrs): packet.nh = IPPROTO_ESP # Recompute plen. packet.plen = None - packet = scapy.IPv6(str(packet)) + packet = scapy.IPv6(bytes(packet)) elif type(packet) is scapy.IP: packet.proto = IPPROTO_ESP # Recompute IPv4 len and checksum. packet.len = None packet.chksum = None - packet = scapy.IP(str(packet)) + packet = scapy.IP(bytes(packet)) else: raise ValueError("First layer in packet should be IPv4 or IPv6: " + repr(packet)) return packet @@ -237,7 +241,7 @@ def DecryptPacketWithNull(packet): Returns: A tuple of decrypted packet (scapy.IPv6 or scapy.IP) and EspHdr """ - esp_hdr, esp_data = cstruct.Read(str(packet.payload), xfrm.EspHdr) + esp_hdr, esp_data = cstruct.Read(bytes(packet.payload), xfrm.EspHdr) # Parse and strip ESP trailer. pad_len, esp_nexthdr = struct.unpack("BB", esp_data[-2:]) trailer_len = pad_len + 2 # Add the size of the pad_len and next_hdr fields. @@ -256,12 +260,12 @@ def DecryptPacketWithNull(packet): if type(packet) is scapy.IPv6: packet.nh = IPPROTO_UDP packet.plen = None # Recompute packet length. - packet = scapy.IPv6(str(packet)) + packet = scapy.IPv6(bytes(packet)) elif type(packet) is scapy.IP: packet.proto = IPPROTO_UDP packet.len = None # Recompute packet length. packet.chksum = None # Recompute IPv4 checksum. - packet = scapy.IP(str(packet)) + packet = scapy.IP(bytes(packet)) else: raise ValueError("First layer in packet should be IPv4 or IPv6: " + repr(packet)) return packet, esp_hdr @@ -305,7 +309,7 @@ class XfrmBaseTest(multinetwork_base.MultiNetworkBaseTest): if src_addr is not None: self.assertEqual(src_addr, packet.src) # extract the ESP header - esp_hdr, _ = cstruct.Read(str(packet.payload), xfrm.EspHdr) + esp_hdr, _ = cstruct.Read(bytes(packet.payload), xfrm.EspHdr) self.assertEqual(xfrm.EspHdr((spi, seq)), esp_hdr) return packet @@ -323,3 +327,4 @@ class XfrmLazyTest(XfrmBaseTest): super(XfrmBaseTest, self).tearDown() self.xfrm.FlushSaInfo() self.xfrm.FlushPolicyInfo() + self.xfrm.close() diff --git a/net/test/xfrm_test.py b/net/test/xfrm_test.py index 439a2d2..4c5bff5 100755 --- a/net/test/xfrm_test.py +++ b/net/test/xfrm_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -18,6 +18,7 @@ from errno import * # pylint: disable=wildcard-import from scapy import all as scapy from socket import * # pylint: disable=wildcard-import +import binascii import struct import subprocess import threading @@ -53,11 +54,12 @@ TEST_SPI2 = 0x1235 class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): def assertIsUdpEncapEsp(self, packet, spi, seq, length): - self.assertEqual(IPPROTO_UDP, packet.proto) + protocol = packet.nh if packet.version == 6 else packet.proto + self.assertEqual(IPPROTO_UDP, protocol) udp_hdr = packet[scapy.UDP] self.assertEqual(4500, udp_hdr.dport) self.assertEqual(length, len(udp_hdr)) - esp_hdr, _ = cstruct.Read(str(udp_hdr.payload), xfrm.EspHdr) + esp_hdr, _ = cstruct.Read(bytes(udp_hdr.payload), xfrm.EspHdr) # FIXME: this file currently swaps SPI byte order manually, so SPI needs to # be double-swapped here. self.assertEqual(xfrm.EspHdr((spi, seq)), esp_hdr) @@ -79,10 +81,10 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): "\tauth-trunc hmac(sha1) 0x%s 96\n" "\tenc cbc(aes) 0x%s\n" "\tsel src ::/0 dst ::/0 \n" % ( - xfrm_base._AUTHENTICATION_KEY_128.encode("hex"), - xfrm_base._ENCRYPTION_KEY_256.encode("hex"))) + binascii.hexlify(xfrm_base._AUTHENTICATION_KEY_128).decode("utf-8"), + binascii.hexlify(xfrm_base._ENCRYPTION_KEY_256).decode("utf-8"))) - actual = subprocess.check_output("ip xfrm state".split()) + actual = subprocess.check_output("ip xfrm state".split()).decode("utf-8") # Newer versions of IP also show anti-replay context. Don't choke if it's # missing. actual = actual.replace( @@ -193,11 +195,15 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): xfrm_base.SetPolicySockopt(s, family, None) s.sendto(net_test.UDP_PAYLOAD, (remotesockaddr, 53)) self.ExpectPacketOn(netid, "Send after clear 2, expected %s" % desc, pkt) + s.close() # Clearing if a policy was never set is safe. s = socket(AF_INET6, SOCK_DGRAM, 0) xfrm_base.SetPolicySockopt(s, family, None) + s.close() + s2.close() + def testSocketPolicyIPv4(self): self._TestSocketPolicy(4) @@ -208,36 +214,38 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): self._TestSocketPolicy(5) # Sets up sockets and marks to correct netid - def _SetupUdpEncapSockets(self): + def _SetupUdpEncapSockets(self, version): netid = self.RandomNetid() - myaddr = self.MyAddress(4, netid) - remoteaddr = self.GetRemoteAddress(4) + myaddr = self.MyAddress(version, netid) + remoteaddr = self.GetRemoteAddress(version) + family = net_test.GetAddressFamily(version) # Reserve a port on which to receive UDP encapsulated packets. Sending # packets works without this (and potentially can send packets with a source # port belonging to another application), but receiving requires the port to # be bound and the encapsulation socket option enabled. - encap_sock = net_test.Socket(AF_INET, SOCK_DGRAM, 0) + encap_sock = net_test.Socket(family, SOCK_DGRAM, 0) encap_sock.bind((myaddr, 0)) encap_port = encap_sock.getsockname()[1] encap_sock.setsockopt(IPPROTO_UDP, xfrm.UDP_ENCAP, xfrm.UDP_ENCAP_ESPINUDP) # Open a socket to send traffic. - s = socket(AF_INET, SOCK_DGRAM, 0) + # TODO: test with a different family than the encap socket. + s = socket(family, SOCK_DGRAM, 0) self.SelectInterface(s, netid, "mark") s.connect((remoteaddr, 53)) return netid, myaddr, remoteaddr, encap_sock, encap_port, s # Sets up SAs and applies socket policy to given socket - def _SetupUdpEncapSaPair(self, myaddr, remoteaddr, in_spi, out_spi, + def _SetupUdpEncapSaPair(self, version, myaddr, remoteaddr, in_spi, out_spi, encap_port, s, use_null_auth): in_reqid = 123 out_reqid = 456 # Create inbound and outbound SAs that specify UDP encapsulation. encaptmpl = xfrm.XfrmEncapTmpl((xfrm.UDP_ENCAP_ESPINUDP, htons(encap_port), - htons(4500), 16 * "\x00")) + htons(4500), 16 * b"\x00")) self.CreateNewSa(myaddr, remoteaddr, out_spi, out_reqid, encaptmpl, use_null_auth) @@ -247,21 +255,22 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): use_null_auth) # Apply socket policies to s. - xfrm_base.ApplySocketPolicy(s, AF_INET, xfrm.XFRM_POLICY_OUT, out_spi, + family = net_test.GetAddressFamily(version) + xfrm_base.ApplySocketPolicy(s, family, xfrm.XFRM_POLICY_OUT, out_spi, out_reqid, None) # TODO: why does this work without a per-socket policy applied? # The received packet obviously matches an SA, but don't inbound packets # need to match a policy as well? (b/71541609) - xfrm_base.ApplySocketPolicy(s, AF_INET, xfrm.XFRM_POLICY_IN, in_spi, + xfrm_base.ApplySocketPolicy(s, family, xfrm.XFRM_POLICY_IN, in_spi, in_reqid, None) # Uncomment for debugging. # subprocess.call("ip xfrm state".split()) # Check that packets can be sent and received. - def _VerifyUdpEncapSocket(self, netid, remoteaddr, myaddr, encap_port, sock, - in_spi, out_spi, null_auth, seq_num): + def _VerifyUdpEncapSocket(self, version, netid, remoteaddr, myaddr, encap_port, + sock, in_spi, out_spi, null_auth, seq_num): # Now send a packet. sock.sendto(net_test.UDP_PAYLOAD, (remoteaddr, 53)) srcport = sock.getsockname()[1] @@ -274,8 +283,8 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): auth_algo = ( xfrm_base._ALGO_AUTH_NULL if null_auth else xfrm_base._ALGO_HMAC_SHA1) expected_len = xfrm_base.GetEspPacketLength( - xfrm.XFRM_MODE_TRANSPORT, 4, True, net_test.UDP_PAYLOAD, auth_algo, - xfrm_base._ALGO_CBC_AES_256) + xfrm.XFRM_MODE_TRANSPORT, version, True, net_test.UDP_PAYLOAD, + auth_algo, xfrm_base._ALGO_CBC_AES_256) self.assertIsUdpEncapEsp(packet, out_spi, seq_num, expected_len) # Now test the receive path. Because we don't know how to decrypt packets, @@ -286,13 +295,14 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): # So the source and destination ports are swapped and the packet appears to # be sent from srcport to port 53. Open another socket on that port, and # apply the inbound policy to it. - twisted_socket = socket(AF_INET, SOCK_DGRAM, 0) + family = net_test.GetAddressFamily(version) + twisted_socket = socket(family, SOCK_DGRAM, 0) csocket.SetSocketTimeout(twisted_socket, 100) - twisted_socket.bind(("0.0.0.0", 53)) + twisted_socket.bind((net_test.GetWildcardAddress(version), 53)) # Save the payload of the packet so we can replay it back to ourselves, and # replace the SPI with our inbound SPI. - payload = str(packet.payload)[8:] + payload = bytes(packet.payload)[8:] spi_seq = xfrm.EspHdr((in_spi, seq_num)).Pack() payload = spi_seq + payload[len(spi_seq):] @@ -300,9 +310,10 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): start_integrity_failures = sainfo.stats.integrity_failed # Now play back the valid packet and check that we receive it. - incoming = (scapy.IP(src=remoteaddr, dst=myaddr) / + ip = {4: scapy.IP, 6: scapy.IPv6}[version] + incoming = (ip(src=remoteaddr, dst=myaddr) / scapy.UDP(sport=4500, dport=encap_port) / payload) - incoming = scapy.IP(str(incoming)) + incoming = ip(bytes(incoming)) self.ReceivePacketOn(netid, incoming) sainfo = self.xfrm.FindSaInfo(in_spi) @@ -319,41 +330,45 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): else: data, src = twisted_socket.recvfrom(4096) self.assertEqual(net_test.UDP_PAYLOAD, data) - self.assertEqual((remoteaddr, srcport), src) + self.assertEqual((remoteaddr, srcport), src[:2]) self.assertEqual(start_integrity_failures, sainfo.stats.integrity_failed) # Check that unencrypted packets on twisted_socket are not received. unencrypted = ( - scapy.IP(src=remoteaddr, dst=myaddr) / scapy.UDP( + ip(src=remoteaddr, dst=myaddr) / scapy.UDP( sport=srcport, dport=53) / net_test.UDP_PAYLOAD) self.assertRaisesErrno(EAGAIN, twisted_socket.recv, 4096) - def _RunEncapSocketPolicyTest(self, in_spi, out_spi, use_null_auth): + twisted_socket.close() + + def _RunEncapSocketPolicyTest(self, version, in_spi, out_spi, use_null_auth): netid, myaddr, remoteaddr, encap_sock, encap_port, s = \ - self._SetupUdpEncapSockets() + self._SetupUdpEncapSockets(version) - self._SetupUdpEncapSaPair(myaddr, remoteaddr, in_spi, out_spi, encap_port, - s, use_null_auth) + self._SetupUdpEncapSaPair(version, myaddr, remoteaddr, in_spi, out_spi, + encap_port, s, use_null_auth) # Check that UDP encap sockets work with socket policy and given SAs - self._VerifyUdpEncapSocket(netid, remoteaddr, myaddr, encap_port, s, in_spi, - out_spi, use_null_auth, 1) + self._VerifyUdpEncapSocket(version, netid, remoteaddr, myaddr, encap_port, + s, in_spi, out_spi, use_null_auth, 1) + encap_sock.close() + s.close() # TODO: Add tests for ESP (non-encap) sockets. def testUdpEncapSameSpisNullAuth(self): # Use the same SPI both inbound and outbound because this lets us receive # encrypted packets by simply replaying the packets the kernel sends # without having to disable authentication - self._RunEncapSocketPolicyTest(TEST_SPI, TEST_SPI, True) + self._RunEncapSocketPolicyTest(4, TEST_SPI, TEST_SPI, True) def testUdpEncapSameSpis(self): - self._RunEncapSocketPolicyTest(TEST_SPI, TEST_SPI, False) + self._RunEncapSocketPolicyTest(4, TEST_SPI, TEST_SPI, False) def testUdpEncapDifferentSpisNullAuth(self): - self._RunEncapSocketPolicyTest(TEST_SPI, TEST_SPI2, True) + self._RunEncapSocketPolicyTest(4, TEST_SPI, TEST_SPI2, True) def testUdpEncapDifferentSpis(self): - self._RunEncapSocketPolicyTest(TEST_SPI, TEST_SPI2, False) + self._RunEncapSocketPolicyTest(4, TEST_SPI, TEST_SPI2, False) def testUdpEncapRekey(self): # Select the two SPIs that will be used @@ -362,31 +377,31 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): # Setup sockets netid, myaddr, remoteaddr, encap_sock, encap_port, s = \ - self._SetupUdpEncapSockets() + self._SetupUdpEncapSockets(4) # The SAs must use null authentication, since we change SPIs on the fly # Without null authentication, this would result in an ESP authentication # error since the SPI is part of the authenticated section. The packet # would then be dropped - self._SetupUdpEncapSaPair(myaddr, remoteaddr, start_spi, start_spi, + self._SetupUdpEncapSaPair(4, myaddr, remoteaddr, start_spi, start_spi, encap_port, s, True) # Check that UDP encap sockets work with socket policy and given SAs - self._VerifyUdpEncapSocket(netid, remoteaddr, myaddr, encap_port, s, + self._VerifyUdpEncapSocket(4, netid, remoteaddr, myaddr, encap_port, s, start_spi, start_spi, True, 1) # Rekey this socket using the make-before-break paradigm. First we create # new SAs, update the per-socket policies, and only then remove the old SAs # # This allows us to switch to the new SA without breaking the outbound path. - self._SetupUdpEncapSaPair(myaddr, remoteaddr, rekey_spi, rekey_spi, + self._SetupUdpEncapSaPair(4, myaddr, remoteaddr, rekey_spi, rekey_spi, encap_port, s, True) # Check that UDP encap socket works with updated socket policy, sending # using new SA, but receiving on both old and new SAs - self._VerifyUdpEncapSocket(netid, remoteaddr, myaddr, encap_port, s, + self._VerifyUdpEncapSocket(4, netid, remoteaddr, myaddr, encap_port, s, rekey_spi, rekey_spi, True, 1) - self._VerifyUdpEncapSocket(netid, remoteaddr, myaddr, encap_port, s, + self._VerifyUdpEncapSocket(4, netid, remoteaddr, myaddr, encap_port, s, start_spi, rekey_spi, True, 2) # Delete old SAs @@ -394,8 +409,92 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): self.xfrm.DeleteSaInfo(myaddr, start_spi, IPPROTO_ESP) # Check that UDP encap socket works with updated socket policy and new SAs - self._VerifyUdpEncapSocket(netid, remoteaddr, myaddr, encap_port, s, + self._VerifyUdpEncapSocket(4, netid, remoteaddr, myaddr, encap_port, s, rekey_spi, rekey_spi, True, 3) + encap_sock.close() + s.close() + + def _CheckUDPEncapRecv(self, version, mode): + netid, myaddr, remoteaddr, encap_sock, encap_port, s = \ + self._SetupUdpEncapSockets(version) + + # Create inbound and outbound SAs that specify UDP encapsulation. + reqid = 123 + encaptmpl = xfrm.XfrmEncapTmpl((xfrm.UDP_ENCAP_ESPINUDP, htons(encap_port), + htons(4500), 16 * b"\x00")) + self.xfrm.AddSaInfo(remoteaddr, myaddr, TEST_SPI, mode, reqid, + xfrm_base._ALGO_CRYPT_NULL, xfrm_base._ALGO_AUTH_NULL, None, + encaptmpl, None, None) + + sainfo = self.xfrm.FindSaInfo(TEST_SPI) + self.assertEqual(0, sainfo.curlft.packets) + self.assertEqual(0, sainfo.curlft.bytes) + self.assertEqual(0, sainfo.stats.integrity_failed) + + IpType = {4: scapy.IP, 6: scapy.IPv6}[version] + if mode == xfrm.XFRM_MODE_TRANSPORT: + # Due to a bug in the IPv6 UDP encap code, there must be at least 32 + # bytes after the ESP header or the packet will be dropped. + # 8 (UDP header) + 18 (payload) + 2 (ESP trailer) = 28, dropped + # 8 (UDP header) + 19 (payload) + 4 (ESP trailer) = 32, received + # There is a similar bug in IPv4 encap, but the minimum is only 12 bytes, + # which is much less likely to occur. This doesn't affect tunnel mode + # because IP headers are always at least 20 bytes long. + data = 19 * b"a" + datalen = len(data) + data += xfrm_base.GetEspTrailer(len(data), IPPROTO_UDP) + self.assertEqual(32, len(data) + 8) + # TODO: update scapy and use scapy.ESP instead of manually generating ESP header. + inner_pkt = xfrm.EspHdr(spi=TEST_SPI, seqnum=1).Pack() + bytes( + scapy.UDP(sport=443, dport=32123) / data) + input_pkt = (IpType(src=remoteaddr, dst=myaddr) / + scapy.UDP(sport=4500, dport=encap_port) / + inner_pkt) + else: + # TODO: test IPv4 in IPv6 encap and vice versa. + data = b"" # Empty UDP payload + datalen = len(data) + {4: 20, 6: 40}[version] + data += xfrm_base.GetEspTrailer(len(data), IPPROTO_UDP) + # TODO: update scapy and use scapy.ESP instead of manually generating ESP header. + inner_pkt = xfrm.EspHdr(spi=TEST_SPI, seqnum=1).Pack() + bytes( + IpType(src=remoteaddr, dst=myaddr) / + scapy.UDP(sport=443, dport=32123) / data) + input_pkt = (IpType(src=remoteaddr, dst=myaddr) / + scapy.UDP(sport=4500, dport=encap_port) / + inner_pkt) + + # input_pkt.show2() + self.ReceivePacketOn(netid, input_pkt) + + sainfo = self.xfrm.FindSaInfo(TEST_SPI) + self.assertEqual(1, sainfo.curlft.packets) + self.assertEqual(datalen + 8, sainfo.curlft.bytes) + self.assertEqual(0, sainfo.stats.integrity_failed) + + # Uncomment for debugging. + # subprocess.call("ip -s xfrm state".split()) + + encap_sock.close() + s.close() + + def testIPv4UDPEncapRecvTransport(self): + self._CheckUDPEncapRecv(4, xfrm.XFRM_MODE_TRANSPORT) + + def testIPv4UDPEncapRecvTunnel(self): + self._CheckUDPEncapRecv(4, xfrm.XFRM_MODE_TUNNEL) + + # IPv6 UDP encap is broken between: + # 4db4075f92af ("esp6: fix check on ipv6_skip_exthdr's return value") and + # 5f9c55c8066b ("ipv6: check return value of ipv6_skip_exthdr") + @unittest.skipUnless(net_test.KernelAtLeast([(5, 10, 108), (5, 15, 31)]), + reason="Unsupported or broken on current kernel") + def testIPv6UDPEncapRecvTransport(self): + self._CheckUDPEncapRecv(6, xfrm.XFRM_MODE_TRANSPORT) + + @unittest.skipUnless(net_test.KernelAtLeast([(5, 10, 108), (5, 15, 31)]), + reason="Unsupported or broken on current kernel") + def testIPv6UDPEncapRecvTunnel(self): + self._CheckUDPEncapRecv(6, xfrm.XFRM_MODE_TUNNEL) def testAllocSpecificSpi(self): spi = 0xABCD @@ -466,6 +565,7 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): with self.assertRaisesErrno(EAGAIN): s.send(net_test.UDP_PAYLOAD) self.ExpectNoPacketsOn(netid, "Packet not blocked by policy") + s.close() def _CheckNullEncryptionTunnelMode(self, version): family = net_test.GetAddressFamily(version) @@ -507,28 +607,29 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): IpType = {4: scapy.IP, 6: scapy.IPv6}[version] input_pkt = (IpType(src=remote_addr, dst=local_addr) / scapy.UDP(sport=remote_port, dport=local_port) / - "input hello") - input_pkt = IpType(str(input_pkt)) # Compute length, checksum. + b"input hello") + input_pkt = IpType(bytes(input_pkt)) # Compute length, checksum. input_pkt = xfrm_base.EncryptPacketWithNull(input_pkt, 0x9876, 1, (tun_remote, tun_local)) self.ReceivePacketOn(netid, input_pkt) msg, addr = sock.recvfrom(1024) - self.assertEqual("input hello", msg) + self.assertEqual(b"input hello", msg) self.assertEqual((remote_addr, remote_port), addr[:2]) # Send and capture a packet. - sock.sendto("output hello", (remote_addr, remote_port)) + sock.sendto(b"output hello", (remote_addr, remote_port)) packets = self.ReadAllPacketsOn(netid) self.assertEqual(1, len(packets)) output_pkt = packets[0] output_pkt, esp_hdr = xfrm_base.DecryptPacketWithNull(output_pkt) - self.assertEqual(output_pkt[scapy.UDP].len, len("output_hello") + 8) + self.assertEqual(output_pkt[scapy.UDP].len, len(b"output_hello") + 8) self.assertEqual(remote_addr, output_pkt.dst) self.assertEqual(remote_port, output_pkt[scapy.UDP].dport) # length of the payload plus the UDP header - self.assertEqual("output hello", str(output_pkt[scapy.UDP].payload)) + self.assertEqual(b"output hello", bytes(output_pkt[scapy.UDP].payload)) self.assertEqual(0xABCD, esp_hdr.spi) + sock.close() def testNullEncryptionTunnelMode(self): """Verify null encryption in tunnel mode. @@ -571,27 +672,28 @@ class XfrmFunctionalTest(xfrm_base.XfrmLazyTest): IpType = {4: scapy.IP, 6: scapy.IPv6}[version] input_pkt = (IpType(src=remote_addr, dst=local_addr) / scapy.UDP(sport=remote_port, dport=local_port) / - "input hello") - input_pkt = IpType(str(input_pkt)) # Compute length, checksum. + b"input hello") + input_pkt = IpType(bytes(input_pkt)) # Compute length, checksum. input_pkt = xfrm_base.EncryptPacketWithNull(input_pkt, 0x9876, 1, None) self.ReceivePacketOn(netid, input_pkt) msg, addr = sock.recvfrom(1024) - self.assertEqual("input hello", msg) + self.assertEqual(b"input hello", msg) self.assertEqual((remote_addr, remote_port), addr[:2]) # Send and capture a packet. - sock.sendto("output hello", (remote_addr, remote_port)) + sock.sendto(b"output hello", (remote_addr, remote_port)) packets = self.ReadAllPacketsOn(netid) self.assertEqual(1, len(packets)) output_pkt = packets[0] output_pkt, esp_hdr = xfrm_base.DecryptPacketWithNull(output_pkt) # length of the payload plus the UDP header - self.assertEqual(output_pkt[scapy.UDP].len, len("output_hello") + 8) + self.assertEqual(output_pkt[scapy.UDP].len, len(b"output_hello") + 8) self.assertEqual(remote_addr, output_pkt.dst) self.assertEqual(remote_port, output_pkt[scapy.UDP].dport) - self.assertEqual("output hello", str(output_pkt[scapy.UDP].payload)) + self.assertEqual(b"output hello", bytes(output_pkt[scapy.UDP].payload)) self.assertEqual(0xABCD, esp_hdr.spi) + sock.close() def testNullEncryptionTransportMode(self): """Verify null encryption in transport mode. @@ -731,6 +833,8 @@ class XfrmOutputMarkTest(xfrm_base.XfrmLazyTest): with self.assertRaisesErrno(ENETUNREACH): s.sendto(net_test.UDP_PAYLOAD, (remoteaddr, 53)) + s.close() + def testTunnelModeOutputMarkIPv4(self): for netid in self.NETIDS: tunsrc = self.MyAddress(4, netid) @@ -768,9 +872,9 @@ class XfrmOutputMarkTest(xfrm_base.XfrmLazyTest): self.assertEqual(mark, attributes["XFRMA_OUTPUT_MARK"]) def testInvalidAlgorithms(self): - key = "af442892cdcd0ef650e9c299f9a8436a".decode("hex") - invalid_auth = (xfrm.XfrmAlgoAuth(("invalid(algo)", 128, 96)), key) - invalid_crypt = (xfrm.XfrmAlgo(("invalid(algo)", 128)), key) + key = binascii.unhexlify("af442892cdcd0ef650e9c299f9a8436a") + invalid_auth = (xfrm.XfrmAlgoAuth((b"invalid(algo)", 128, 96)), key) + invalid_crypt = (xfrm.XfrmAlgo((b"invalid(algo)", 128)), key) with self.assertRaisesErrno(ENOSYS): self.xfrm.AddSaInfo(TEST_ADDR1, TEST_ADDR2, 0x1234, xfrm.XFRM_MODE_TRANSPORT, 0, xfrm_base._ALGO_CBC_AES_256, @@ -900,5 +1004,7 @@ class XfrmOutputMarkTest(xfrm_base.XfrmLazyTest): self.xfrm.DeleteSaInfo(remote, TEST_SPI, IPPROTO_ESP, mark) self.xfrm.DeletePolicyInfo(sel, xfrm.XFRM_POLICY_OUT, mark) + s.close() + if __name__ == "__main__": unittest.main() diff --git a/net/test/xfrm_tunnel_test.py b/net/test/xfrm_tunnel_test.py index e319a7d..4efb46a 100755 --- a/net/test/xfrm_tunnel_test.py +++ b/net/test/xfrm_tunnel_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # # Copyright 2017 The Android Open Source Project # @@ -23,6 +23,7 @@ import itertools import struct import unittest +from net_test import LINUX_VERSION from scapy import all as scapy from tun_twister import TunTwister import csocket @@ -39,9 +40,10 @@ _TEST_XFRM_IFNAME = "ipsec42" _TEST_XFRM_IF_ID = 42 _TEST_SPI = 0x1234 -# Does the kernel support xfrmi interfaces? +# Does the kernel support CONFIG_XFRM_INTERFACE? def HaveXfrmInterfaces(): - if net_test.LINUX_VERSION >= (4, 19, 0): + # 4.19+ must have CONFIG_XFRM_INTERFACE enabled + if LINUX_VERSION >= (4, 19, 0): return True try: @@ -60,9 +62,30 @@ def HaveXfrmInterfaces(): HAVE_XFRM_INTERFACES = HaveXfrmInterfaces() -# Does the kernel support CONFIG_XFRM_MIGRATE? +# Two kernel fixes have been added in 5.17 to allow XFRM_MIGRATE to work correctly +# when (1) there are multiple tunnels with the same selectors; and (2) addresses +# are updated to a different IP family. These two fixes were pulled into upstream +# LTS releases 4.14.273, 4.19.236, 5.4.186, 5.10.107 and 5.15.30, from whence they +# flowed into the Android Common Kernel (via standard LTS merges). +# As such we require 4.14.273+, 4.19.236+, 5.4.186+, 5.10.107+, 5.15.30+ or 5.17+ +# to have these fixes. +def HasXfrmMigrateFixes(): + return ( + ((LINUX_VERSION >= (4, 14, 273)) and (LINUX_VERSION < (4, 19, 0))) or + ((LINUX_VERSION >= (4, 19, 236)) and (LINUX_VERSION < (5, 4, 0))) or + ((LINUX_VERSION >= (5, 4, 186)) and (LINUX_VERSION < (5, 10, 0))) or + ((LINUX_VERSION >= (5, 10, 107)) and (LINUX_VERSION < (5, 15, 0))) or + (LINUX_VERSION >= (5, 15, 30)) + ) + + +# Does the kernel support CONFIG_XFRM_MIGRATE and include the kernel fixes? def SupportsXfrmMigrate(): - if net_test.LINUX_VERSION >= (5, 10, 0): + if not HasXfrmMigrateFixes(): + return False + + # 5.10+ must have CONFIG_XFRM_MIGRATE enabled + if LINUX_VERSION >= (5, 10, 0): return True # XFRM_MIGRATE depends on xfrmi interfaces @@ -134,7 +157,7 @@ def _GetNullAuthCryptTunnelModePkt(inner_version, src_inner, src_outer, input_pkt = ( IpType(**ip_hdr_options) / scapy.UDP(sport=src_port, dport=dst_port) / net_test.UDP_PAYLOAD) - input_pkt = IpType(str(input_pkt)) # Compute length, checksum. + input_pkt = IpType(bytes(input_pkt)) # Compute length, checksum. input_pkt = xfrm_base.EncryptPacketWithNull(input_pkt, spi, seq_num, (src_outer, dst_outer)) @@ -168,7 +191,7 @@ def InjectTests(): InjectParameterizedTests(XfrmTunnelTest) InjectParameterizedTests(XfrmInterfaceTest) InjectParameterizedTests(XfrmVtiTest) - InjectParameterizedTests(XfrmInterfaceMigrateTest) + InjectParameterizedMigrateTests(XfrmInterfaceMigrateTest) def InjectParameterizedTests(cls): @@ -180,6 +203,15 @@ def InjectParameterizedTests(cls): util.InjectParameterizedTest(cls, param_list, NameGenerator) +def InjectParameterizedMigrateTests(cls): + VERSIONS = (4, 6) + param_list = itertools.product(VERSIONS, VERSIONS, VERSIONS) + + def NameGenerator(*args): + return "IPv%d_in_IPv%d_to_outer_IPv%d" % tuple(args) + + util.InjectParameterizedTest(cls, param_list, NameGenerator) + class XfrmTunnelTest(xfrm_base.XfrmLazyTest): @@ -266,7 +298,6 @@ class XfrmTunnelTest(xfrm_base.XfrmLazyTest): xfrm.XFRM_POLICY_OUT, True) -@unittest.skipUnless(net_test.LINUX_VERSION >= (3, 18, 0), "VTI Unsupported") class XfrmAddDeleteVtiTest(xfrm_base.XfrmBaseTest): def _VerifyVtiInfoData(self, vti_info_data, version, local_addr, remote_addr, ikey, okey): @@ -558,17 +589,12 @@ class XfrmInterface(IpSecBaseInterface): self.local = new_local self.remote = new_remote + self.version = net_test.GetAddressVersion(new_local) self.underlying_netid = new_underlying_netid class XfrmTunnelBase(xfrm_base.XfrmBaseTest): - # Subclass that does not allow multiple tunnels (e.g. XfrmInterfaceMigrateTest) - # should override this method. - @classmethod - def allowMultipleTunnels(cls): - return True - @classmethod def setUpClass(cls): xfrm_base.XfrmBaseTest.setUpClass() @@ -582,9 +608,6 @@ class XfrmTunnelBase(xfrm_base.XfrmBaseTest): cls.tunnelsV4 = {} cls.tunnelsV6 = {} - if not cls.allowMultipleTunnels(): - return - for i, underlying_netid in enumerate(cls.tuns): for version in 4, 6: netid = _BASE_TUNNEL_NETID[version] + _TUNNEL_NETID_OFFSET + i @@ -752,11 +775,11 @@ class XfrmTunnelBase(xfrm_base.XfrmBaseTest): # workaround in this manner if inner_version == 4: ip_hdr_options = { - 'id': scapy.IP(str(pkt.payload)[8:]).id, - 'flags': scapy.IP(str(pkt.payload)[8:]).flags + 'id': scapy.IP(bytes(pkt.payload)[8:]).id, + 'flags': scapy.IP(bytes(pkt.payload)[8:]).flags } else: - ip_hdr_options = {'fl': scapy.IPv6(str(pkt.payload)[8:]).fl} + ip_hdr_options = {'fl': scapy.IPv6(bytes(pkt.payload)[8:]).fl} expected = _GetNullAuthCryptTunnelModePkt( inner_version, local_inner, tunnel.local, local_port, remote_inner, @@ -771,7 +794,7 @@ class XfrmTunnelBase(xfrm_base.XfrmBaseTest): self.assertEqual(len(expected), len(pkt)) # Check everything else - self.assertEqual(str(expected.payload), str(pkt.payload)) + self.assertEqual(bytes(expected.payload), bytes(pkt.payload)) def _CheckTunnelEncryption(self, tunnel, inner_version, local_inner, remote_inner): @@ -790,7 +813,7 @@ class XfrmTunnelBase(xfrm_base.XfrmBaseTest): tunnel.remote) # Check that packet is not sent in plaintext - self.assertTrue(str(net_test.UDP_PAYLOAD) not in str(pkt)) + self.assertTrue(bytes(net_test.UDP_PAYLOAD) not in bytes(pkt)) # Check src/dst self.assertEqual(tunnel.local, pkt.src) @@ -866,26 +889,29 @@ class XfrmTunnelBase(xfrm_base.XfrmBaseTest): self._CheckTunnelEncryption(tunnel, inner_version, local_inner, remote_inner) + def _RebuildTunnel(self, tunnel, use_null_crypt): + # Some tests require that the out_seq_num and in_seq_num are the same + # (Specifically encrypted tests), rebuild SAs to ensure seq_num is 1 + # + # Until we get better scapy support, the only way we can build an + # encrypted packet is to send it out, and read the packet from the wire. + # We then generally use this as the "inbound" encrypted packet, injecting + # it into the interface for which it is expected on. + # + # As such, this is required to ensure that encrypted packets (which we + # currently have no way to easily modify) are not considered replay + # attacks by the inbound SA. (eg: received 3 packets, seq_num_in = 3, + # sent only 1, # seq_num_out = 1, inbound SA would consider this a replay + # attack) + tunnel.TeardownXfrm() + tunnel.SetupXfrm(use_null_crypt) + def _TestTunnel(self, inner_version, outer_version, func, use_null_crypt): """Bootstrap method to setup and run tests for the given parameters.""" tunnel = self.randomTunnel(outer_version) try: - # Some tests require that the out_seq_num and in_seq_num are the same - # (Specifically encrypted tests), rebuild SAs to ensure seq_num is 1 - # - # Until we get better scapy support, the only way we can build an - # encrypted packet is to send it out, and read the packet from the wire. - # We then generally use this as the "inbound" encrypted packet, injecting - # it into the interface for which it is expected on. - # - # As such, this is required to ensure that encrypted packets (which we - # currently have no way to easily modify) are not considered replay - # attacks by the inbound SA. (eg: received 3 packets, seq_num_in = 3, - # sent only 1, # seq_num_out = 1, inbound SA would consider this a replay - # attack) - tunnel.TeardownXfrm() - tunnel.SetupXfrm(use_null_crypt) + self._RebuildTunnel(tunnel, use_null_crypt) local_inner = tunnel.addrs[inner_version] remote_inner = _GetRemoteInnerAddress(inner_version) @@ -959,7 +985,6 @@ class XfrmTunnelBase(xfrm_base.XfrmBaseTest): tunnel.SetupXfrm(False) -@unittest.skipUnless(net_test.LINUX_VERSION >= (3, 18, 0), "VTI Unsupported") class XfrmVtiTest(XfrmTunnelBase): INTERFACE_CLASS = VtiInterface @@ -1012,15 +1037,23 @@ class XfrmInterfaceTest(XfrmTunnelBase): def ParamTestXfrmIntfRekey(self, inner_version, outer_version): self._TestTunnelRekey(inner_version, outer_version) -@unittest.skipUnless(SUPPORTS_XFRM_MIGRATE, "XFRM migration unsupported") +############################################################################## +# +# Test for presence of CONFIG_XFRM_MIGRATE and kernel patches +# +# xfrm: Check if_id in xfrm_migrate +# Upstream commit: c1aca3080e382886e2e58e809787441984a2f89b +# +# xfrm: Fix xfrm migrate issues when address family changes +# Upstream commit: e03c3bba351f99ad932e8f06baa9da1afc418e02 +# +# Those two upstream 5.17 fixes above were pulled in to LTS in kernel versions +# 4.14.273, 4.19.236, 5.4.186, 5.10.107, 5.15.30. +# +@unittest.skipUnless(SUPPORTS_XFRM_MIGRATE, + "XFRM migration unsupported or fixes not included") class XfrmInterfaceMigrateTest(XfrmTunnelBase): - # TODO: b/172497215 There is a kernel issue that XFRM_MIGRATE cannot work correctly - # when there are multiple tunnels with the same selectors. Thus before this issue - # is fixed, #allowMultipleTunnels must be overridden to avoid setting up multiple - # tunnels. This need to be removed after the kernel issue is fixed. - @classmethod - def allowMultipleTunnels(cls): - return False + INTERFACE_CLASS = XfrmInterface def setUpTunnel(self, outer_version, use_null_crypt): underlying_netid = self.RandomNetid() @@ -1043,9 +1076,17 @@ class XfrmInterfaceMigrateTest(XfrmTunnelBase): self._SetupTunnelNetwork(tunnel, False) tunnel.Teardown() - def _TestTunnel(self, inner_version, outer_version, func, use_null_crypt): + def _TestTunnel(self, inner_version, outer_version, new_outer_version, func, + use_null_crypt): + tunnel = self.randomTunnel(outer_version) + + old_underlying_netid = tunnel.underlying_netid + old_local = tunnel.local + old_remote = tunnel.remote + + try: - tunnel = self.setUpTunnel(outer_version, use_null_crypt) + self._RebuildTunnel(tunnel, use_null_crypt) # Verify functionality before migration local_inner = tunnel.addrs[inner_version] @@ -1053,39 +1094,54 @@ class XfrmInterfaceMigrateTest(XfrmTunnelBase): func(tunnel, inner_version, local_inner, remote_inner) # Migrate tunnel - # TODO:b/169170981 Add tests that migrate 4 -> 6 and 6 -> 4 new_underlying_netid = self.RandomNetid(exclude=tunnel.underlying_netid) - new_local = self.MyAddress(outer_version, new_underlying_netid) - new_remote = net_test.IPV4_ADDR2 if outer_version == 4 else net_test.IPV6_ADDR2 + new_version = new_outer_version + new_local = self.MyAddress(new_version, new_underlying_netid) + new_remote = net_test.IPV4_ADDR2 if new_version == 4 else net_test.IPV6_ADDR2 tunnel.Migrate(new_underlying_netid, new_local, new_remote) # Verify functionality after migration func(tunnel, inner_version, local_inner, remote_inner) finally: - self.tearDownTunnel(tunnel) + # Reset the tunnel to the original configuration + tunnel.TeardownXfrm() - def ParamTestMigrateXfrmIntfInput(self, inner_version, outer_version): - self._TestTunnel(inner_version, outer_version, self._CheckTunnelInput, True) + self.local = old_local + self.remote = old_remote + self.underlying_netid = old_underlying_netid + tunnel.SetupXfrm(False) - def ParamTestMigrateXfrmIntfOutput(self, inner_version, outer_version): - self._TestTunnel(inner_version, outer_version, self._CheckTunnelOutput, - True) - def ParamTestMigrateXfrmIntfInOutEncrypted(self, inner_version, outer_version): - self._TestTunnel(inner_version, outer_version, self._CheckTunnelEncryption, - False) + def ParamTestMigrateXfrmIntfInput(self, inner_version, outer_version, + new_outer_version): + self._TestTunnel(inner_version, outer_version, new_outer_version, + self._CheckTunnelInput, True) - def ParamTestMigrateXfrmIntfIcmp(self, inner_version, outer_version): - self._TestTunnel(inner_version, outer_version, self._CheckTunnelIcmp, False) + def ParamTestMigrateXfrmIntfOutput(self, inner_version, outer_version, + new_outer_version): + self._TestTunnel(inner_version, outer_version, new_outer_version, + self._CheckTunnelOutput, True) - def ParamTestMigrateXfrmIntfEncryptionWithIcmp(self, inner_version, outer_version): - self._TestTunnel(inner_version, outer_version, + def ParamTestMigrateXfrmIntfInOutEncrypted(self, inner_version, outer_version, + new_outer_version): + self._TestTunnel(inner_version, outer_version, new_outer_version, + self._CheckTunnelEncryption, False) + + def ParamTestMigrateXfrmIntfIcmp(self, inner_version, outer_version, + new_outer_version): + self._TestTunnel(inner_version, outer_version, new_outer_version, + self._CheckTunnelIcmp, False) + + def ParamTestMigrateXfrmIntfEncryptionWithIcmp(self, inner_version, outer_version, + new_outer_version): + self._TestTunnel(inner_version, outer_version, new_outer_version, self._CheckTunnelEncryptionWithIcmp, False) - def ParamTestMigrateXfrmIntfRekey(self, inner_version, outer_version): - self._TestTunnel(inner_version, outer_version, self._CheckTunnelRekey, - True) + def ParamTestMigrateXfrmIntfRekey(self, inner_version, outer_version, + new_outer_version): + self._TestTunnel(inner_version, outer_version, new_outer_version, + self._CheckTunnelRekey, True) if __name__ == "__main__": InjectTests() |