diff options
| author | Eric Jeong <ericjeong@google.com> | 2022-03-22 04:28:06 +0000 |
|---|---|---|
| committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2022-03-22 04:28:06 +0000 |
| commit | 7a7428aefa66365af6168f26f75fed9030841a6a (patch) | |
| tree | 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | |
| parent | 2ca2e67d4c75eb2adca03cc3526c77031d436e5a (diff) | |
| parent | dc74a47c79da6580abbdf8b116f147794e42e4b7 (diff) | |
| download | iorap-7a7428aefa66365af6168f26f75fed9030841a6a.tar.gz | |
[Re-land] Remove iorap daemon codes am: dc74a47c79
Original change: https://android-review.googlesource.com/c/platform/system/iorap/+/2025264
Change-Id: Id3b5fcafeef88579c5c22f55e3bf96dc884ac318
109 files changed, 0 insertions, 20784 deletions
diff --git a/Android.bp b/Android.bp deleted file mode 100644 index 1d181bd..0000000 --- a/Android.bp +++ /dev/null @@ -1,744 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package { - default_applicable_licenses: ["Android-Apache-2.0"], -} - -filegroup { - name: "iorap-aidl", - srcs: [ - // note: using **/* doesn't work, so list each file one by one: - // see also b/70046217 - - // note: list only 'interface' aidl files, otherwise - // aidl generates an error "Refusing to generate code with unstructured parcelables." - "binder/com/google/android/startop/iorap/IIorap.aidl", - "binder/com/google/android/startop/iorap/ITaskListener.aidl", - ], - path: "binder", -} - -cc_defaults { - name: "iorap-default-flags", - - cflags: [ - "-Wall", - "-Werror", - "-Wextra", - "-Wno-missing-field-initializers", - "-Wno-unused-parameter", - "-Wno-unused-variable", - ], - - local_include_dirs: [ - "include", - "src", - ], - // TODO: shouldn't need this but there's a soong/cmake generator bug. - export_include_dirs: [ - "include", - "src", - ], - - /* - TODO: Header refactoring cleanup: - - Option 1): Move src/$component/file_name.h to src/$component/include/$component/file_name.h - Option 2): Symlink src/$component/include/$component to src/$component - - Set export_include_dirs to '$component/include' for that component. - - Also delete the 'include' directory unless we have code other non-iorap - targets are allowed to reference. - */ - - clang: true, - static_libs: ["libc++fs"], - shared_libs: ["libbase"], - - // build all ioraps for host. - host_supported: true, - target: { - darwin: { - enabled: false, - } - }, -} - -cc_defaults { - name: "iorap-default-dependencies", - - static_libs: [ - "libiorap-binder", - "libplatformprotos", // android framework C++ protos. - ], - shared_libs: [ - "libbinder", - "libutils", - "libcutils", // tracing. - - "libfruit", // dependency injection. - // TODO: remove these annoying dependencies by hiding them in the main library code. - - // dependency for libplatformprotos - // "libprotobuf-cpp-lite", - - // libplatformprotos has an indirect dependency on full, causing compilation/linking - // errors if we use lite - "libprotobuf-cpp-full", - - "packagemanager_aidl-cpp", - - // phenotype flags support - "server_configurable_flags", - ], - - // srcs: [":libprotobuf-internal-protos"], - // commented out because it causes compilation errors - // TODO: can we use the lite library somehow? - - header_libs: ["librxcpp"], -} - -cc_library_static { - name: "libiorap-binder", - defaults: ["iorap-default-flags"], - - srcs: [ - ":iorap-aidl", - "src/binder/iiorap_impl.cc", - "src/binder/package_change_observer.cc", - "src/binder/package_manager_remote.cc", - "src/binder/package_version_map.cc", - ], - shared_libs: [ - "libbinder", - "libutils", - "libcutils", // tracing. - "packagemanager_aidl-cpp", - ], - aidl: { - local_include_dirs: ["binder"], - include_dirs: ["frameworks/native/aidl/binder"], - export_aidl_headers: true, - }, - - static_libs: [ - "libplatformprotos", // android framework C++ protos. - ], -} - -cc_defaults { - name: "libiorap-manager-default-dependencies", - static_libs: [ - "libiorap-binder", - "libiorap-perfetto", - "libiorap-prefetcher", - "libiorap-db", - "libiorap-maintenance", - ], - defaults: [ - "libiorap-perfetto-default-dependencies", - "libiorap-prefetcher-default-dependencies", - "libiorap-db-default-dependencies", - ], - // Users of 'libiorap-manager' also need to include these defaults to avoid - // linking errors. -} - -cc_library_static { - name: "libiorap-manager", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-manager-default-dependencies", - ], - - srcs: [ - "src/manager/**/*.cc", - ], -} - -cc_binary { - name: "iorapd", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-manager-default-dependencies", - ], - srcs: [ - "src/iorapd/main.cc", - ], - static_libs: [ - "libiorap-manager", - ], - init_rc: [ - "iorapd.rc", - ], - // iorapd fork+execs into iorap.prefetcherd and iorap.cmd.compiler - // maintenance used by tests - required: [ - "iorap.cmd.compiler", - "iorap.prefetcherd", - "iorap.cmd.maintenance", - ], -} - -cc_library_static { - name: "libiorap-inode2filename", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - ], - - srcs: [ - "src/inode2filename/**/*.cc", - ], -} - -cc_binary { - name: "iorap.inode2filename", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - ], - srcs: [ - "src/inode2filename/**/*.cc", - ], - // Easier debugging. TODO: make a separate debug config. - // XX: Using -O0 seems to completely hide some errors. - cflags: ["-O2", "-UNDEBUG", "-DIORAP_INODE2FILENAME_MAIN=1"], - sanitize: { - undefined: true, - all_undefined: true, - // Pretty print when ubsan detects a problem. - // Otherwise it just calls abort(). - -/* - diag: { - undefined: true, - }, - */ // don't use the diag when you want it to crash. - }, -} - -cc_test { - name: "iorapd-tests", - test_suites: ["device-tests"], - gtest: false, // we use gtest *and* gmock. - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-compiler-default-dependencies", - ], - srcs: [ - "tests/src/binder/*.cc", - "tests/src/inode2filename/*.cc", - "tests/src/log/*.cc", - "tests/src/tmp/*.cc", - ], - data: [ - "tests/src/compiler/testdata/*", - ], - cflags: ["-O2", "-UNDEBUG"], - - // TODO: we should probably have per-component tests. - static_libs: ["libgmock_main", "libgmock", "libgtest", "libiorap-inode2filename"], - -} - - -cc_test_host { - name: "iorapd-host-tests", - gtest: false, // we use gtest *and* gmock. - target: { - darwin: { - enabled: false, - } - }, - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-compiler-default-dependencies", - "libiorap-maintenance-default-dependencies", - ], - srcs: [ - "tests/src/compiler/*.cc", - "tests/src/db/*.cc", - "tests/src/maintenance/*.cc", - ], - data: [ - "tests/src/compiler/testdata/*", - "tests/src/maintenance/testdata/*", - ], - cflags: ["-O2", "-UNDEBUG"], - - // TODO: we should probably have per-component tests. - static_libs: [ - "libgmock_main", - "libgmock", - "libgtest", - "libiorap-compiler", - "libiorap-maintenance", - ], -} - -filegroup { - name: "libiorap-perfetto-protos", - srcs: [ - ], -} - -// Static libraries cannot export their dependencies, -// the current convention is to use an extra 'defaults' rule for statics -// to bring in all the dependencies. -cc_defaults { - name: "libiorap-perfetto-default-dependencies", - - // Some of the libperfetto header typedefs leak out into iorap. - // Avoids compilation #include errors. - // TODO: clean this up, the headers should not leak out (maybe all we need is a PerfettoConsumer - // forward declaration?). - include_dirs: ["external/perfetto/include"], - // Various perfetto protos are used directly by iorap. - // - // Furthermore, we need this regardless to avoid linking errors when linking - // libiorap-perfetto.a into the main cc_binary rule. - static_libs: [ - "perfetto_trace_protos", - ], - - shared_libs: [ - // Not part of true dependencies: Users of 'libiorap-perfetto' do not link against - // libperfetto. - // We only put this to avoid linking errors when building iorapd. - // TODO: can we split iorapd into libiorapd-main that doesn't link against libperfetto? - // only the last cc_binary should need the full transitive closure of the dependency graph. - "libperfetto", - ] -} - -cc_library_static { - name: "libiorap-perfetto", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-perfetto-default-dependencies", - ], - - srcs: [ - "src/perfetto/**/*.cc", - ], -} - -cc_binary { - name: "iorap.cmd.perfetto", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - ], - shared_libs: ["libperfetto"], - include_dirs: ["external/perfetto/include"], - srcs: [ - "src/perfetto/**/*.cc", - ], - // Easier debugging. TODO: make a separate debug config. - // XX: Using -O0 seems to completely hide some errors. - cflags: ["-O2", "-UNDEBUG", "-DIORAP_PERFETTO_MAIN=1"], - sanitize: { - undefined: true, - all_undefined: true, - // Pretty print when ubsan detects a problem. - // Otherwise it just calls abort(). - -/* - diag: { - undefined: true, - }, - */ // don't use the diag when you want it to crash. - }, - - static_libs: [ - "perfetto_trace_protos", - ], -} - -// Static libraries cannot export their dependencies, -// the current convention is to use an extra 'defaults' rule for statics -// to bring in all the dependencies. -cc_defaults { - name: "libiorap-compiler-default-dependencies", - - defaults: [ - // use the perfetto namespace - "libiorap-perfetto-default-dependencies", - // use the inode2filename namespace - "libiorap-serialize-default-dependencies", // uses but does not re-export serialize. - ], - - // Some of the libperfetto header typedefs leak out into iorap. - // Avoids compilation #include errors. - // TODO: clean this up, the headers should not leak out (maybe all we need is a PerfettoConsumer - // forward declaration?). - include_dirs: [], - // Various perfetto protos are used directly by iorap. - // - // Furthermore, we need this regardless to avoid linking errors when linking - // libiorap-compiler.a into the main cc_binary rule. - static_libs: [ - "libiorap-perfetto", - // "perfetto_trace_protos", - "libiorap-inode2filename", - "libiorap-serialize", - ], - - shared_libs: [ - // Not part of true dependencies: Users of 'libiorap-compiler' do not link against - // libperfetto. - // We only put this to avoid linking errors when building iorapd. - // TODO: can we split iorapd into libiorapd-main that doesn't link against libperfetto? - // only the last cc_binary should need the full transitive closure of the dependency graph. - ] -} - -cc_library_static { - name: "libiorap-compiler", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-compiler-default-dependencies", - ], - - srcs: [ - "src/compiler/**/*.cc", - ], -} - -cc_binary { - name: "iorap.cmd.compiler", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-compiler-default-dependencies", - ], - shared_libs: [], - include_dirs: [], - srcs: [ - "src/compiler/**/*.cc", - ], - // Easier debugging. TODO: make a separate debug config. - // XX: Using -O0 seems to completely hide some errors. - cflags: ["-O2", "-UNDEBUG", "-DIORAP_COMPILER_MAIN=1"], - sanitize: { - undefined: true, - all_undefined: true, - // Pretty print when ubsan detects a problem. - // Otherwise it just calls abort(). - -/* - diag: { - undefined: true, - }, - */ // don't use the diag when you want it to crash. - }, - - static_libs: [ - ], - required: [ - "iorap.inode2filename", - ], -} - -// Static libraries cannot export their dependencies, -// the current convention is to use an extra 'defaults' rule for statics -// to bring in all the dependencies. -cc_defaults { - name: "libiorap-serialize-default-dependencies", - - defaults: [ - ], - - include_dirs: [], - static_libs: [ - ], - shared_libs: [ - ], - // Above intentionally left empty. - srcs: [ - "src/serialize/**/*.proto", - ], -} - -cc_library_static { - name: "libiorap-serialize", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-serialize-default-dependencies", - ], - - srcs: [ - "src/serialize/**/*.cc", - ], -} - - -// Static libraries cannot export their dependencies, -// the current convention is to use an extra 'defaults' rule for statics -// to bring in all the dependencies. -cc_defaults { - name: "libiorap-prefetcher-default-dependencies", - - defaults: [ - ], - - include_dirs: [], - static_libs: [ - "libiorap-serialize", - ], - shared_libs: [ - "libminijail", - ], - - // disable mac builds because libminijail doesn't work there - target: { - darwin: { - enabled: false, - }, - }, -} - -cc_library_static { - name: "libiorap-prefetcher", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-prefetcher-default-dependencies", - "libiorap-serialize-default-dependencies", - ], - - srcs: [ - "src/prefetcher/**/*.cc", - ], -} - -cc_binary { - name: "iorap.prefetcherd", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-prefetcher-default-dependencies", - "libiorap-serialize-default-dependencies", - ], - shared_libs: [], - include_dirs: [], - srcs: [ - "src/prefetcher/**/*.cc", - ], - // Easier debugging. TODO: make a separate debug config. - // XX: Using -O0 seems to completely hide some errors. - cflags: ["-O2", "-UNDEBUG", "-DIORAP_PREFETCHER_MAIN=1"], - sanitize: { - undefined: true, - all_undefined: true, - // Pretty print when ubsan detects a problem. - // Otherwise it just calls abort(). - -/* - diag: { - undefined: true, - }, - */ // don't use the diag when you want it to crash. - }, - - static_libs: [ - ], -} - -cc_binary { - name: "iorap.cmd.prefetcher.client", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-prefetcher-default-dependencies", - "libiorap-serialize-default-dependencies", - ], - shared_libs: [], - include_dirs: [], - srcs: [ - "src/prefetcher/**/*.cc", - ], - // Easier debugging. TODO: make a separate debug config. - // XX: Using -O0 seems to completely hide some errors. - cflags: ["-O2", "-UNDEBUG", "-DIORAP_PREFETCHER_MAIN_CLIENT=1"], - sanitize: { - undefined: true, - all_undefined: true, - // Pretty print when ubsan detects a problem. - // Otherwise it just calls abort(). - -/* - diag: { - undefined: true, - }, - */ // don't use the diag when you want it to crash. - }, - - static_libs: [ - ], -} - -prebuilt_etc { - name: "iorap.prefetcherd.policy", - sub_dir: "seccomp_policy", - arch: { - arm: { - src: "seccomp_policy/prefetcherd.arm.policy" - }, - arm64: { - src: "seccomp_policy/prefetcherd.arm64.policy" - }, - x86: { - src: "seccomp_policy/prefetcherd.x86.policy" - }, - x86_64: { - src: "seccomp_policy/prefetcherd.x86_64.policy" - }, - }, - required: [ - "crash_dump.policy", - ], -} - -// Static libraries cannot export their dependencies, -// the current convention is to use an extra 'defaults' rule for statics -// to bring in all the dependencies. -cc_defaults { - name: "libiorap-db-default-dependencies", - - defaults: [ - ], - - include_dirs: [], - static_libs: [ - ], - shared_libs: [ - "libsqlite", - ], -} - -cc_library_static { - name: "libiorap-db", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-db-default-dependencies", - ], - - srcs: [ - "src/db/**/*.cc", - ], -} - -cc_binary { - name: "iorap.cmd.db", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-db-default-dependencies", - ], - shared_libs: [], - include_dirs: [], - srcs: [ - "src/db/**/*.cc", - ], - // Easier debugging. TODO: make a separate debug config. - // XX: Using -O0 seems to completely hide some errors. - cflags: ["-O2", "-UNDEBUG", "-DIORAP_DB_MAIN=1"], - sanitize: { - undefined: true, - all_undefined: true, - // Pretty print when ubsan detects a problem. - // Otherwise it just calls abort(). - -/* - diag: { - undefined: true, - }, - */ // don't use the diag when you want it to crash. - }, - - static_libs: [ - ], -} - -cc_defaults { - name: "libiorap-maintenance-default-dependencies", - - defaults: [ - "libiorap-compiler-default-dependencies", - "libiorap-db-default-dependencies", - "libiorap-prefetcher-default-dependencies", - ], - - include_dirs: [], - - static_libs: [ - "libiorap-binder", - "libiorap-compiler", - "libiorap-db", - "libiorap-prefetcher", - ], - - shared_libs: [] -} - -cc_library_static { - name: "libiorap-maintenance", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-maintenance-default-dependencies", - ], - - srcs: [ - "src/maintenance/*.cc", - ], -} - -cc_binary { - name: "iorap.cmd.maintenance", - defaults: [ - "iorap-default-flags", - "iorap-default-dependencies", - "libiorap-maintenance-default-dependencies", - ], - shared_libs: [], - include_dirs: [], - srcs: [ - "src/maintenance/*.cc", - ], - cflags: ["-O2", "-UNDEBUG", "-DIORAP_MAINTENANCE_MAIN=1"], - sanitize: { - undefined: true, - all_undefined: true, - - diag: { - undefined: true, - }, - }, - - static_libs: [], -} diff --git a/Android.mk b/Android.mk deleted file mode 100644 index 4e9f5fd..0000000 --- a/Android.mk +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (C) 2019 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -LOCAL_PATH := $(call my-dir) - -# Build every binary target in system/iorap -.PHONY: iorap-nall -iorap-nall: \ - iorapd iorap.inode2filename iorapd-tests iorap.cmd.perfetto \ - iorap.cmd.compiler - -# Build every binary target required for -# frameworks/base/startop/scripts/app_startup_runner -# to work with iorap. -.PHONY: iorap-app-startup-runner -iorap-app-startup-runner: \ - iorapd iorap.inode2filename \ - iorap.cmd.compiler - - diff --git a/binder/com/google/android/startop/iorap/AppIntentEvent.aidl b/binder/com/google/android/startop/iorap/AppIntentEvent.aidl deleted file mode 100644 index dbf1d02..0000000 --- a/binder/com/google/android/startop/iorap/AppIntentEvent.aidl +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -/** @hide */ -parcelable AppIntentEvent cpp_header "binder/app_intent_event.h"; diff --git a/binder/com/google/android/startop/iorap/AppLaunchEvent.aidl b/binder/com/google/android/startop/iorap/AppLaunchEvent.aidl deleted file mode 100644 index ea3f9f1..0000000 --- a/binder/com/google/android/startop/iorap/AppLaunchEvent.aidl +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -/** @hide */ -parcelable AppLaunchEvent cpp_header "binder/app_launch_event.h"; diff --git a/binder/com/google/android/startop/iorap/DexOptEvent.aidl b/binder/com/google/android/startop/iorap/DexOptEvent.aidl deleted file mode 100644 index a4e175b..0000000 --- a/binder/com/google/android/startop/iorap/DexOptEvent.aidl +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -/** @hide */ -parcelable DexOptEvent cpp_header "binder/dexopt_event.h"; diff --git a/binder/com/google/android/startop/iorap/IIorap.aidl b/binder/com/google/android/startop/iorap/IIorap.aidl deleted file mode 100644 index 05faefe..0000000 --- a/binder/com/google/android/startop/iorap/IIorap.aidl +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -import com.google.android.startop.iorap.ITaskListener; - -import com.google.android.startop.iorap.AppIntentEvent; -import com.google.android.startop.iorap.AppLaunchEvent; -import com.google.android.startop.iorap.DexOptEvent; -import com.google.android.startop.iorap.JobScheduledEvent; -import com.google.android.startop.iorap.PackageEvent; -import com.google.android.startop.iorap.RequestId; -import com.google.android.startop.iorap.SystemServiceEvent; -import com.google.android.startop.iorap.SystemServiceUserEvent; - -/** -* IIOrap is a client interface to the input/output readahead and pin daemon (iorapd). -* -* The aim is to speed-up the cold start-up time of certain use-cases like application startup -* by utilizing trace-based pinning or readahead. -* -* Programatically, the behavior of iorapd should be treated like a black box. There is no -* "correctness", but only performance. By sending the right events at the appropriate time, -* we can squeeze more performance out of the system. -* -* If some events are not appropriately wired up, system performance may (temporarily) degrade. -* -* {@hide} */ -oneway interface IIorap { - /** - * Set an ITaskListener which will be used to deliver notifications of in-progress/completition - * for the onXEvent method calls below this.<br /><br /> - * - * iorapd does all the work asynchronously and may deliver one or more onProgress events after - * the event begins to be processed. It will always send back one onComplete that is considered - * terminal.<br /><br /> - * - * onProgress/onComplete are matched to the original event by the requestId. Once an onComplete - * occurs for any given requestId, no further callbacks with the same requestId will occur. - * It is illegal for the caller to reuse the same requestId on different invocations of IIorap. - * <br /><br /> - * - * onXEvent(id1) must be well-ordered w.r.t. onXEvent(id2), the assumption is that later - * calls happen-after earlier calls and that id2 > id1. Decreasing request IDs will - * immediately get rejected. - * <br /><br /> - * - * Sequence diagram of stereotypical successful event delivery and response notification: - * - * <pre> - * - * ┌─────────────┐ ┌──────┐ - * │system_server│ │iorapd│ - * └──────┬──────┘ └──┬───┘ - * Request [01]: onSomeEvent ┌┴┐ - * │────────────────────────>│ │ - * │ │ │ - * │ │ │ ╔════════════════════════╗ - * │ │ │ ║start processing event ░║ - * │ │ │ ╚════════════════════════╝ - * │ │ │ - * ╔═══════╤════════╪═════════════════════════╪═╪══════════════════════════════╗ - * ║ LOOP │ 1 or more times │ │ ║ - * ╟───────┘ │ │ │ ║ - * ║ │Request [01]: onProgress │ │ ║ - * ║ │<────────────────────────│ │ ║ - * ║ │ │ │ ║ - * ║ │ │ │────┐ ║ - * ║ │ │ │ │ workload in progress ║ - * ║ │ │ │<───┘ ║ - * ╚════════════════╪═════════════════════════╪═╪══════════════════════════════╝ - * │ └┬┘ - * . . - * . . - * . . - * . . - * . . - * │ ┌┴┐ ╔═════════════════════════╗ - * │ │ │ ║finish processing event ░║ - * │ │ │ ╚═════════════════════════╝ - * │Request [01]: onComplete │ │ - * │<────────────────────────│ │ - * ┌──────┴──────┐ ┌─└┬┘──┐ - * │system_server│ │iorapd│ - * └─────────────┘ └──────┘ - * - * </pre> <!-- system/iorap/docs/binder/IIorap_setTaskListener.plantuml --> - */ - void setTaskListener(ITaskListener listener); - - // All callbacks will be done via the ITaskListener. - // The RequestId passed in is the same RequestId sent back via the ITaskListener. - // See above for more details. - - // Note: For each ${Type}Event, see the ${Type}Event.java for more documentation - // in frameworks/base/startop/src/com/google/android/startop/iorap/${Type}Event.java - - // void onActivityHintEvent(in RequestId request, in ActivityHintEvent event); - void onAppLaunchEvent(in RequestId request, in AppLaunchEvent event); - void onDexOptEvent(in RequestId request, in DexOptEvent event); - void onJobScheduledEvent(in RequestId request, in JobScheduledEvent event); - void onPackageEvent(in RequestId request, in PackageEvent event); - void onAppIntentEvent(in RequestId request, in AppIntentEvent event); - void onSystemServiceEvent(in RequestId request, in SystemServiceEvent event); - void onSystemServiceUserEvent(in RequestId request, in SystemServiceUserEvent event); -} diff --git a/binder/com/google/android/startop/iorap/ITaskListener.aidl b/binder/com/google/android/startop/iorap/ITaskListener.aidl deleted file mode 100644 index 0b5f4a6..0000000 --- a/binder/com/google/android/startop/iorap/ITaskListener.aidl +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -import com.google.android.startop.iorap.TaskResult; -import com.google.android.startop.iorap.RequestId; - -/** -* Provide callbacks to the {@code IIorap} client in response to method invocations.<br /><br /> -* -* @see com.google.android.startop.iorap.IIorap -* -* {@hide} */ -oneway interface ITaskListener { - void onProgress(in RequestId requestId, in TaskResult result); - void onComplete(in RequestId requestId, in TaskResult result); -} - -// TODO: we can probably delete the multiple methods. one is likely sufficient? diff --git a/binder/com/google/android/startop/iorap/JobScheduledEvent.aidl b/binder/com/google/android/startop/iorap/JobScheduledEvent.aidl deleted file mode 100644 index d4b5454..0000000 --- a/binder/com/google/android/startop/iorap/JobScheduledEvent.aidl +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -/** @hide */ -parcelable JobScheduledEvent cpp_header "binder/job_scheduled_event.h"; diff --git a/binder/com/google/android/startop/iorap/PackageEvent.aidl b/binder/com/google/android/startop/iorap/PackageEvent.aidl deleted file mode 100644 index 8f9d18e..0000000 --- a/binder/com/google/android/startop/iorap/PackageEvent.aidl +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -/** @hide */ -parcelable PackageEvent cpp_header "binder/package_event.h"; diff --git a/binder/com/google/android/startop/iorap/RequestId.aidl b/binder/com/google/android/startop/iorap/RequestId.aidl deleted file mode 100644 index 895af87..0000000 --- a/binder/com/google/android/startop/iorap/RequestId.aidl +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -/** @hide */ -parcelable RequestId cpp_header "binder/request_id.h"; diff --git a/binder/com/google/android/startop/iorap/SystemServiceEvent.aidl b/binder/com/google/android/startop/iorap/SystemServiceEvent.aidl deleted file mode 100644 index 005676f..0000000 --- a/binder/com/google/android/startop/iorap/SystemServiceEvent.aidl +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -/** @hide */ -parcelable SystemServiceEvent cpp_header "binder/system_service_event.h"; diff --git a/binder/com/google/android/startop/iorap/SystemServiceUserEvent.aidl b/binder/com/google/android/startop/iorap/SystemServiceUserEvent.aidl deleted file mode 100644 index c165dc4..0000000 --- a/binder/com/google/android/startop/iorap/SystemServiceUserEvent.aidl +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -/** @hide */ -parcelable SystemServiceUserEvent cpp_header "binder/system_service_user_event.h"; diff --git a/binder/com/google/android/startop/iorap/TaskResult.aidl b/binder/com/google/android/startop/iorap/TaskResult.aidl deleted file mode 100644 index 76cce8f..0000000 --- a/binder/com/google/android/startop/iorap/TaskResult.aidl +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.startop.iorap; - -/** @hide */ -parcelable TaskResult cpp_header "binder/task_result.h"; diff --git a/docs/binder/ActivityHint.dot b/docs/binder/ActivityHint.dot deleted file mode 100644 index 94b5a84..0000000 --- a/docs/binder/ActivityHint.dot +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Convert with `graph-easy --as=boxart` to get textual rendering. - */ -digraph finite_state_machine { -rankdir=LR; -size="8,5" -node [shape = circle ]; STARTED; -node [shape = doublecircle]; - -"" -> STARTED; -STARTED -> CANCELLED; -STARTED -> COMPLETED; -COMPLETED -> POST_COMPLETED; -COMPLETED -> CANCELLED; -} diff --git a/docs/binder/IIorap_setTaskListener.plantuml b/docs/binder/IIorap_setTaskListener.plantuml deleted file mode 100644 index 5cdfac7..0000000 --- a/docs/binder/IIorap_setTaskListener.plantuml +++ /dev/null @@ -1,53 +0,0 @@ -@startuml -' Copyright (C) 2018 The Android Open Source Project -' -' Licensed under the Apache License, Version 2.0 (the "License"); -' you may not use this file except in compliance with the License. -' You may obtain a copy of the License at -' -' http://www.apache.org/licenses/LICENSE-2.0 -' -' Unless required by applicable law or agreed to in writing, software -' distributed under the License is distributed on an "AS IS" BASIS, -' WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -' See the License for the specific language governing permissions and -' limitations under the License. - -' Compile with PlantUML: -' http://www.plantuml.com/plantuml/uml/ - - -'hide footbox - -participant "system_server" as ss -participant iorapd - -autonumber 1 0 "Request [00]:" - -ss -\ iorapd : onSomeEvent -activate iorapd -note right of iorapd - start processing event -end note - -loop 1 or more times -iorapd -\ ss : onProgress - -autonumber stop -iorapd -> iorapd : workload in progress -autonumber resume - -end - -' some time later... -... ... - -note right of iorapd - finish processing event -end note - - -iorapd -\ ss : onComplete - -deactivate iorapd -@enduml diff --git a/docs/binder/TaskResult.dot b/docs/binder/TaskResult.dot deleted file mode 100644 index 28d7819..0000000 --- a/docs/binder/TaskResult.dot +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Convert with `graph-easy --as=boxart` to get textual rendering. - */ -digraph finite_state_machine { -rankdir=LR; -size="8,5" -node [shape = circle ]; BEGAN ONGOING; -node [shape = doublecircle]; - -// graph-easy does not support multiple state syntax {}, write one-by-one - -"" -> BEGAN; -"" -> ERROR; -BEGAN -> ERROR; -ONGOING -> ERROR; - -BEGAN -> ONGOING; -BEGAN -> COMPLETED; -ONGOING -> COMPLETED; - -} diff --git a/include/binder/activity_info.h b/include/binder/activity_info.h deleted file mode 100644 index 0712f18..0000000 --- a/include/binder/activity_info.h +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_ACTIVITY_INFO_H_ -#define IORAP_BINDER_ACTIVITY_INFO_H_ - -#include "binder/common.h" -#include "binder/auto_parcelable.h" -#include "common/introspection.h" - -#include <string> - -namespace iorap { -namespace binder { - -struct ActivityInfo : public AutoParcelable<ActivityInfo> { - std::string package_name; - std::string activity_name; -}; - -IORAP_INTROSPECT_ADAPT_STRUCT(ActivityInfo, package_name, activity_name); - -} -} -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(ActivityInfo) - -#endif // IORAP_BINDER_ACTIVITY_INFO_H_ diff --git a/include/binder/app_intent_event.h b/include/binder/app_intent_event.h deleted file mode 100644 index bbd19ee..0000000 --- a/include/binder/app_intent_event.h +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_APP_INTENT_EVENT_H_ -#define IORAP_BINDER_APP_INTENT_EVENT_H_ - -#include "binder/common.h" -#include "binder/auto_parcelable.h" -#include "common/introspection.h" - -#include "binder/activity_info.h" - -namespace iorap { -namespace binder { - -struct AppIntentEvent : public AutoParcelable<AppIntentEvent> { - enum class Type : int32_t { - kDefaultIntentChanged = 0, - }; - - Type type; - ActivityInfo old_activity_info; - ActivityInfo new_activity_info; -}; - -IORAP_INTROSPECT_ADAPT_STRUCT(AppIntentEvent, type, old_activity_info, new_activity_info); - -} -} -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(AppIntentEvent) - -#endif // IORAP_BINDER_APP_INTENT_EVENT_H_ diff --git a/include/binder/app_launch_event.h b/include/binder/app_launch_event.h deleted file mode 100644 index 0127e09..0000000 --- a/include/binder/app_launch_event.h +++ /dev/null @@ -1,448 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_APP_LAUNCH_EVENT_H_ -#define IORAP_BINDER_APP_LAUNCH_EVENT_H_ - -#include "binder/common.h" -#include "common/introspection.h" -#include "common/expected.h" - -#include <binder/Parcel.h> -#include <binder/Parcelable.h> -#include <frameworks/base/core/proto/android/content/intent.pb.h> // IntentProto -#include <frameworks/base/core/proto/android/server/activitymanagerservice.pb.h> // ActivityRecord - -namespace iorap { -namespace binder { - -// These protos are part of the iorapd binder ABI, alias them for easier usage. -using IntentProto = ::android::content::IntentProto; -using ActivityRecordProto = ::com::android::server::wm::ActivityRecordProto; - -struct AppLaunchEvent : public ::android::Parcelable { - // Index position matters: Keep up-to-date with AppLaunchEvent.java sTypes field. - enum class Type : int32_t { - kUninitialized = -1, - kIntentStarted = 0, - kIntentFailed = 1, - kActivityLaunched = 2, - kActivityLaunchFinished = 3, - kActivityLaunchCancelled = 4, - kReportFullyDrawn = 5, - }; - - enum class Temperature : int32_t { - kUninitialized = -1, - kCold = 1, - kWarm = 2, - kHot = 3, - }; - - Type type{Type::kUninitialized}; - int64_t sequence_id{-1}; - // kIntentStarted only. - std::unique_ptr<IntentProto> intent_proto; - // kActivityLaunched only. - Temperature temperature{Temperature::kUninitialized}; - // kActivityLaunch*. Can be null in kActivityLaunchCancelled. - std::unique_ptr<ActivityRecordProto> activity_record_proto; - // kIntentStarted, kActivityLaunchFinished and kReportFullyDrawn only. - int64_t timestamp_nanos{-1}; - - AppLaunchEvent() = default; - AppLaunchEvent(Type type, - int64_t sequence_id, - std::unique_ptr<IntentProto> intent_proto = nullptr, - Temperature temperature = Temperature::kUninitialized, - std::unique_ptr<ActivityRecordProto> activity_record_proto = nullptr, - int64_t timestamp_nanos = -1) - : type(type), - sequence_id(sequence_id), - intent_proto(std::move(intent_proto)), - temperature(temperature), - activity_record_proto(std::move(activity_record_proto)), - timestamp_nanos(timestamp_nanos) { - } - - AppLaunchEvent(const AppLaunchEvent& other) { - *this = other; - } - - AppLaunchEvent& operator=(const AppLaunchEvent& other) { - if (&other == this) { - return *this; - } - - type = other.type; - sequence_id = other.sequence_id; - if (other.intent_proto != nullptr) { - intent_proto.reset(new IntentProto(*other.intent_proto)); - } - temperature = other.temperature; - if (other.activity_record_proto != nullptr) { - activity_record_proto.reset(new ActivityRecordProto(*other.activity_record_proto)); - } - timestamp_nanos = other.timestamp_nanos; - - return *this; - } - - ::android::status_t readFromParcel(const android::Parcel* parcel) override { - -# define PARCEL_READ_OR_RETURN(function, ...) \ - if (::android::status_t res = function(__VA_ARGS__); res != ::android::NO_ERROR) { \ - LOG(ERROR) << "AppLaunchEvent::readFromParcel failed"; \ - return res; \ - } - - int32_t type_int; - PARCEL_READ_OR_RETURN(parcel->readInt32, &type_int); - type = static_cast<Type>(type_int); - - LOG(VERBOSE) << "AppLaunchEvent::readFromParcel (type=" << type_int << ")"; - - PARCEL_READ_OR_RETURN(parcel->readInt64, &sequence_id); - - switch (type) { - case Type::kIntentStarted: - PARCEL_READ_OR_RETURN(readIntent, parcel); - PARCEL_READ_OR_RETURN(parcel->readInt64, ×tamp_nanos); - break; - case Type::kIntentFailed: - // No extra arguments. - break; - case Type::kActivityLaunched: { - PARCEL_READ_OR_RETURN(readActivityRecordProto, parcel); - int32_t temperature_int; - PARCEL_READ_OR_RETURN(parcel->readInt32, &temperature_int); - temperature = static_cast<Temperature>(temperature_int); - break; - } - case Type::kActivityLaunchFinished: - PARCEL_READ_OR_RETURN(readActivityRecordProto, parcel); - PARCEL_READ_OR_RETURN(parcel->readInt64, ×tamp_nanos); - break; - case Type::kActivityLaunchCancelled: - PARCEL_READ_OR_RETURN(readActivityRecordProtoNullable, parcel); - break; - case Type::kReportFullyDrawn: - PARCEL_READ_OR_RETURN(readActivityRecordProto, parcel); - PARCEL_READ_OR_RETURN(parcel->readInt64, ×tamp_nanos); - break; - default: - return android::BAD_VALUE; - } -# undef PARCEL_READ_OR_RETURN - - return ::android::NO_ERROR; - - // TODO: std::variant + protobuf implementation in AutoParcelable. - } - -#define PARCEL_WRITE_OR_RETURN(function, ...) \ - if (::android::status_t res = function(__VA_ARGS__); res != ::android::NO_ERROR) { \ - return res; \ - } - - ::android::status_t writeToParcel(android::Parcel* parcel) const override { - PARCEL_WRITE_OR_RETURN(parcel->writeInt32, static_cast<int32_t>(type)); - PARCEL_WRITE_OR_RETURN(parcel->writeInt64, sequence_id); - - switch (type) { - case Type::kIntentStarted: - PARCEL_WRITE_OR_RETURN(writeIntent, parcel); - PARCEL_WRITE_OR_RETURN(parcel->writeInt64, timestamp_nanos); - break; - case Type::kIntentFailed: - // No extra arguments. - break; - case Type::kActivityLaunched: - PARCEL_WRITE_OR_RETURN(writeActivityRecordProto, parcel); - PARCEL_WRITE_OR_RETURN(parcel->writeInt32, static_cast<int32_t>(temperature)); - break; - case Type::kActivityLaunchFinished: - PARCEL_WRITE_OR_RETURN(writeActivityRecordProto, parcel); - PARCEL_WRITE_OR_RETURN(parcel->writeInt64, timestamp_nanos); - break; - case Type::kActivityLaunchCancelled: - PARCEL_WRITE_OR_RETURN(writeActivityRecordProtoNullable, parcel); - break; - case Type::kReportFullyDrawn: - PARCEL_WRITE_OR_RETURN(writeActivityRecordProtoNullable, parcel); - PARCEL_WRITE_OR_RETURN(parcel->writeInt64, timestamp_nanos); - break; - default: - DCHECK(false) << "attempted to write an uninitialized AppLaunchEvent to Parcel"; - return android::BAD_VALUE; - } - -#undef PARCEL_WRITE_OR_RETURN - - return android::NO_ERROR; - } - - private: - // Using 'unique_ptr' here because protobufs don't have a move constructor. Is there - // a better way that is cheap to pass them around? - template <typename T> - static expected<std::unique_ptr<T>, ::android::status_t> - ReadProto(const android::Parcel* parcel) { - DCHECK(parcel != nullptr); - - ::android::status_t res; - - std::vector<uint8_t> byte_vector; - if ((res = parcel->readByteVector(/*out*/&byte_vector)) != ::android::NO_ERROR) { - return unexpected(res); - } - // TODO: we may want to do this without an extra copy, by parsing - // the protobuf directly out of the parcel. - - const uint8_t* data = byte_vector.data(); - const size_t size = byte_vector.size(); - - std::unique_ptr<T> proto_ptr{new T{}}; - - if (!proto_ptr) { - return unexpected(::android::NO_MEMORY); - } - - if (!proto_ptr->ParseFromArray(data, size)) { - return unexpected(::android::BAD_VALUE); - } - - return proto_ptr; - } - - template <typename T> - static expected<std::unique_ptr<T>, ::android::status_t> - ReadNullableProto(const android::Parcel* parcel) { - DCHECK(parcel != nullptr); - - bool value; - - ::android::status_t res; - res = parcel->readBool(/*out*/&value); - - if (res != ::android::NO_ERROR) { - return unexpected(res); - } - - if (!value) { - return std::unique_ptr<T>{nullptr}; - } - - return ReadProto<T>(parcel); - } - - template <typename T> - static ::android::status_t - WriteProto(android::Parcel* parcel, const std::unique_ptr<T>& proto) { - DCHECK(parcel != nullptr); - DCHECK(proto != nullptr); - - std::vector<uint8_t> byte_vector; - { - const int serialized_size = proto->ByteSize(); - byte_vector.resize(serialized_size); - if (!proto->SerializeToArray(byte_vector.data(), serialized_size)) { - return ::android::BAD_VALUE; - } - } - - ::android::status_t res; - if ((res = parcel->writeByteVector(/*in*/byte_vector)) != ::android::NO_ERROR) { - return res; - } - - return ::android::NO_ERROR; - } - - template <typename T> - static ::android::status_t - WriteNullableProto(android::Parcel* parcel, const std::unique_ptr<T>& maybe_proto) { - bool value = (maybe_proto != nullptr); - - ::android::status_t res; - res = parcel->writeBool(value); - - if (res != ::android::NO_ERROR) { - return res; - } - - if (!value) { - return ::android::NO_ERROR; - } - - return WriteProto<T>(parcel, maybe_proto); - } - - android::status_t readIntent(const android::Parcel* parcel) { - expected<std::unique_ptr<IntentProto>, ::android::status_t> maybe_intent = - ReadProto<IntentProto>(parcel); - - if (maybe_intent) { - intent_proto = std::move(maybe_intent.value()); - return ::android::NO_ERROR; - } else { - return maybe_intent.error(); - } - } - - android::status_t readActivityRecordProto(const android::Parcel* parcel) { - expected<std::unique_ptr<ActivityRecordProto>, ::android::status_t> maybe_record = - ReadProto<ActivityRecordProto>(parcel); - - if (maybe_record) { - activity_record_proto = std::move(maybe_record.value()); - return ::android::NO_ERROR; - } else { - return maybe_record.error(); - } - } - - android::status_t readActivityRecordProtoNullable(const android::Parcel* parcel) { - expected<std::unique_ptr<ActivityRecordProto>, ::android::status_t> maybe_record = - ReadNullableProto<ActivityRecordProto>(parcel); - - if (maybe_record) { - activity_record_proto = std::move(maybe_record.value()); - return ::android::NO_ERROR; - } else { - return maybe_record.error(); - } - } - - android::status_t writeIntent(android::Parcel* parcel) const { - return WriteProto<IntentProto>(parcel, intent_proto); - } - - android::status_t writeActivityRecordProto(android::Parcel* parcel) const { - return WriteProto<ActivityRecordProto>(parcel, activity_record_proto); - } - - android::status_t writeActivityRecordProtoNullable(android::Parcel* parcel) const { - return WriteNullableProto<ActivityRecordProto>(parcel, activity_record_proto); - } -}; - -inline std::ostream& operator<<(std::ostream& os, const AppLaunchEvent::Type& type) { - switch (type) { - case AppLaunchEvent::Type::kUninitialized: - os << "kUninitialized"; - break; - case AppLaunchEvent::Type::kIntentStarted: - os << "kIntentStarted"; - break; - case AppLaunchEvent::Type::kIntentFailed: - os << "kIntentFailed"; - break; - case AppLaunchEvent::Type::kActivityLaunched: - os << "kActivityLaunched"; - break; - case AppLaunchEvent::Type::kActivityLaunchCancelled: - os << "kActivityLaunchCancelled"; - break; - case AppLaunchEvent::Type::kActivityLaunchFinished: - os << "kActivityLaunchFinished"; - break; - case AppLaunchEvent::Type::kReportFullyDrawn: - os << "kReportFullyDrawn"; - break; - default: - os << "(unknown)"; - } - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const AppLaunchEvent::Temperature& type) { - switch (type) { - case AppLaunchEvent::Temperature::kUninitialized: - os << "kUninitialized"; - break; - case AppLaunchEvent::Temperature::kCold: - os << "kCold"; - break; - case AppLaunchEvent::Temperature::kWarm: - os << "kWarm"; - break; - case AppLaunchEvent::Temperature::kHot: - os << "kHot"; - break; - default: - os << "(unknown)"; - } - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const AppLaunchEvent& e) { - os << "AppLaunchEvent{"; - os << "type=" << e.type << ","; - os << "sequence_id=" << e.sequence_id << ","; - - os << "intent_proto="; - if (e.intent_proto == nullptr) { - os << "(nullptr)"; - } else { - os << "(action=" << e.intent_proto->action() << ","; - os << "component="; - if (e.intent_proto->has_component()) { - // $package/$class_name - os << e.intent_proto->component().package_name() << "/" - << e.intent_proto->component().class_name(); - } else { - os << "(no component)"; - } - os << ")"; - } - os << ","; - - os << "temperature=" << e.temperature << ","; - os << ","; - - os << "activity_record_proto="; - if (e.activity_record_proto == nullptr) { - os << "(nullptr)"; - } else { - // title or component name. - os << "'" << e.activity_record_proto->identifier().title() << "'"; - } - os << ","; - - os << "timestamp_nanos=" << e.timestamp_nanos << ","; - os << ","; - - os << "}"; - - return os; -} - -/* -IORAP_INTROSPECT_ADAPT_STRUCT(AppLaunchEvent, - type, - sequence_id, - intent_proto, - temperature, - activity_record_proto); -*/ - -} // namespace binder -} // namespace iorap - -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(AppLaunchEvent) - -#endif // IORAP_BINDER_APP_LAUNCH_EVENT_H_ diff --git a/include/binder/auto_parcelable.h b/include/binder/auto_parcelable.h deleted file mode 100644 index 357615c..0000000 --- a/include/binder/auto_parcelable.h +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_AUTO_PARCELABLE_H_ -#define IORAP_BINDER_AUTO_PARCELABLE_H_ - -#include "binder/Parcelable.h" -#include "binder/Parcel.h" -#include "binder/Status.h" - -#include "common/introspection.h" - -namespace iorap { -namespace binder { - -// -// Implements the android::Parcelable interface (readFromParcel, writeToParcel) -// automatically by using compile-time introspection on T. -// -// Requires that 'T' implements introspection by using the IORAP_INTROSPECT_ADAPT_STRUCT macro. -// -template <typename T> -struct AutoParcelable : public ::android::Parcelable { - private: - using Status = android::binder::Status; - using Parcel = android::Parcel; - using Parcelable = android::Parcelable; - using status_t = android::status_t; - - public: - // Write every (introspected) field to the parcel by automatically inferring the correct - // write method to invoke on the parcel from the member type. - status_t writeToParcel(Parcel* parcel) const override { - if (parcel == nullptr) { - return ::android::UNEXPECTED_NULL; - } - - status_t result = android::NO_ERROR; - ::iorap::introspect::for_each_member_field_value(*Self(), [&](auto&& value) { - if (result == android::NO_ERROR) { - result = writeAnyToParcel(/*inout*/parcel, value); - } - }); - - return result; - } - - // Read every (introspected) field to the parcel by automatically inferring the correct - // read method to invoke on the parcel from the member type. - // - // Resilient to partial read failures: A return code other than NO_ERROR means that - // the current value is left unmodified. - status_t readFromParcel(const Parcel* parcel) override { - if (parcel == nullptr) { - return ::android::UNEXPECTED_NULL; - } - - T tmp{*Self()}; - - // Unpack all the parcelable data into a temporary copy. - // Parceling could fail halfway through, in which case - // the original object is unaffected. - - status_t result = android::NO_ERROR; - ::iorap::introspect::for_each_member_field_set_value(tmp, [&](auto field_type) { - // type<?> field_type - - using ValueT = typename decltype(field_type)::type; - - if (result == android::NO_ERROR) { - auto&& [res, read_value] = readAnyFromParcel<ValueT>(/*inout*/parcel); - result = res; - return ::iorap::introspect::aliasing_forward<ValueT>(read_value); - } else { - // TODO: nice-to-have fold over members to early-out on failure. - return ValueT{}; - } - }); - - if (result != android::NO_ERROR) { - return result; - } - - // Success! Now we can copy all the data in a single step. - *Self() = std::move(tmp); - - // TODO: nice-to-have some kind of invariants-checking after reading the parcel data. - - return ::android::NO_ERROR; - } - - private: -#define AUTO_PARCELABLE_BINDER_MAPPING(FN) \ -FN(Byte,int8_t)\ -FN(Int32,int32_t)\ -FN(Uint32,uint32_t)\ -FN(Int64,int64_t)\ -FN(Uint64,uint64_t)\ -FN(Float,float)\ -FN(Double,double)\ -FN(Bool,bool)\ -FN(CString,const char*)\ -FN(String16,const String16&)\ -FN(String16,const std::unique_ptr<String16>&)\ -FN(StrongBinder,const sp<IBinder>&)\ - - template <typename F> - static status_t writeAnyToParcel(Parcel* parcel, const F& value) { - using namespace android; // NOLINT - - // 'F' is the original type of the field here, so it's safe to use it undecayed. - // However, to make matching easier we almost always want to match against the decayed type. - using D = std::decay_t<F>; // [const] [volatile] X[&][&] -> X - - if constexpr (std::is_base_of_v<Parcelable, D>) { - return value.writeToParcel(parcel); - } else if constexpr (std::is_enum_v<D>) { - return writeAnyToParcel(parcel, static_cast<std::underlying_type_t<F>>(value)); -#define AUTO_PARCELABLE_WRITE_TO_PARCEL(fn_name, type_name) \ - } else if constexpr (std::is_same_v<D, std::decay_t<type_name>>) { \ - return parcel->write ## fn_name (value); -AUTO_PARCELABLE_BINDER_MAPPING(AUTO_PARCELABLE_WRITE_TO_PARCEL) - } else if constexpr (std::is_same_v<D, std::string>) { - return parcel->writeUtf8AsUtf16(value); - } else { - STATIC_FAIL(D, "Unsupported type: Add more manual type conversions above^^^"); - } - -#undef AUTO_PARCELABLE_WRITE_TO_PARCEL - } - - template <typename F> - static auto readAnyFromParcel(const Parcel* parcel) { - // returns pair(status_t, ~F~) - using namespace android; - - // Since 'F' is almost always an lvalue reference (due to F=decltype(auto&&), - // we should lose the references, and also any consts. - using D = std::decay_t<F>; - - D value; - status_t result; - - if constexpr (std::is_base_of_v<Parcelable, D>) { - status_t result = value.readFromParcel(/*in*/parcel); - } else if constexpr (std::is_enum_v<D>) { - auto&& [res, val] = readAnyFromParcel<std::underlying_type_t<D>>(parcel); - result = res; - value = static_cast<D>(val); -#define AUTO_PARCELABLE_READ_FROM_PARCEL(fn_name, type_name) \ - } else if constexpr (std::is_same_v<D, std::decay_t<type_name>>) { \ - result = parcel->read ## fn_name (/*out*/&value); -AUTO_PARCELABLE_BINDER_MAPPING(AUTO_PARCELABLE_READ_FROM_PARCEL) - } else if constexpr (std::is_same_v<D, std::string>) { - result = parcel->readUtf8FromUtf16(/*out*/&value); - } else { - STATIC_FAIL(D, "Unsupported type: Add more manual type conversions above^^^"); - } -#undef AUTO_PARCELABLE_READ_FROM_PARCEL - - return std::make_pair(result, std::move(value)); - } - - T* Self() { - return static_cast<T*>(this); - } - const T* Self() const { - return static_cast<const T*>(this); - } -}; - -} // namespace binder -} // namespace iorap - -#endif // IORAP_BINDER_AUTO_PARCELABLE_H_ diff --git a/include/binder/common.h b/include/binder/common.h deleted file mode 100644 index f31e1e9..0000000 --- a/include/binder/common.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_COMMON_H_ -#define IORAP_BINDER_COMMON_H_ - -// AIDL has no way to specify a custom C++ namespace for parcelables. -// -// We want our parcelables to live in the ::iorap::binder namespace, -// not in com.google.android.startop.iorap -// So this macro is just a short-hand for doing an alias across namespaces. -#define IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(what) \ -namespace com { namespace google { namespace android { namespace startop { namespace iorap { using what = ::iorap::binder::what; } } } } } - -#endif // IORAP_BINDER_COMMON_H_ diff --git a/include/binder/dexopt_event.h b/include/binder/dexopt_event.h deleted file mode 100644 index cb506f4..0000000 --- a/include/binder/dexopt_event.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_DEXOPT_EVENT_H_ -#define IORAP_BINDER_DEXOPT_EVENT_H_ - -#include "binder/common.h" -#include "binder/auto_parcelable.h" -#include "common/introspection.h" - -namespace iorap { -namespace binder { - -struct DexOptEvent : public AutoParcelable<DexOptEvent> { - enum class Type : int32_t { - kPackageUpdate = 0, - }; - - Type type; - std::string package_name; -}; - -IORAP_INTROSPECT_ADAPT_STRUCT(DexOptEvent, type, package_name); - -} -} -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(DexOptEvent) - -#endif // IORAP_BINDER_DEXOPT_EVENT_H_ diff --git a/include/binder/job_scheduled_event.h b/include/binder/job_scheduled_event.h deleted file mode 100644 index 6b067d7..0000000 --- a/include/binder/job_scheduled_event.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_JOB_SCHEDULED_EVENT_H_ -#define IORAP_BINDER_JOB_SCHEDULED_EVENT_H_ - -#include "binder/common.h" -#include "binder/auto_parcelable.h" -#include "common/introspection.h" - -namespace iorap { -namespace binder { - -struct JobScheduledEvent : public AutoParcelable<JobScheduledEvent> { - enum class Type : int32_t { - kStartJob = 0, - kStopJob = 1, - }; - - Type type; - int32_t job_id; - std::string package_name; - bool should_update_versions; - - enum class Sort : int32_t { - kIdleMaintenance = 0, - }; - - Sort sort; - - constexpr bool operator==(const JobScheduledEvent& other) const { - return type == other.type - && job_id == other.job_id - && sort == other.sort - && package_name == other.package_name - && should_update_versions == other.should_update_versions; - } - - constexpr bool operator!=(const JobScheduledEvent& other) const { - return !(*this == other); - } -}; - -IORAP_INTROSPECT_ADAPT_STRUCT( - JobScheduledEvent, type, job_id, sort, package_name, should_update_versions); - -} -} -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(JobScheduledEvent) - -#endif // IORAP_BINDER_JOB_SCHEDULED_EVENT_H_ diff --git a/include/binder/package_event.h b/include/binder/package_event.h deleted file mode 100644 index d8d7933..0000000 --- a/include/binder/package_event.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_PACKAGE_EVENT_H_ -#define IORAP_BINDER_PACKAGE_EVENT_H_ - -#include "binder/common.h" -#include "binder/auto_parcelable.h" -#include "common/introspection.h" - -namespace iorap { -namespace binder { - -struct PackageEvent : public AutoParcelable<PackageEvent> { - enum class Type : int32_t { - kReplaced = 0, - }; - - Type type; - std::string package_uri; - std::string package_name; -}; - -IORAP_INTROSPECT_ADAPT_STRUCT(PackageEvent, type, package_uri, package_name); - -} -} -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(PackageEvent) - -#endif // IORAP_BINDER_PACKAGE_EVENT_H_ diff --git a/include/binder/request_id.h b/include/binder/request_id.h deleted file mode 100644 index 71a2edd..0000000 --- a/include/binder/request_id.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_REQUEST_ID_H_ -#define IORAP_BINDER_REQUEST_ID_H_ - -#include "binder/common.h" -#include "binder/auto_parcelable.h" - -namespace iorap { -namespace binder { - -struct RequestId : public AutoParcelable<RequestId> { - int64_t request_id; - - constexpr bool operator==(const RequestId& other) const { - return request_id == other.request_id; - } - - constexpr bool operator!=(const RequestId& other) const { - return !(*this == other); - } -}; - -IORAP_INTROSPECT_ADAPT_STRUCT(RequestId, request_id); - -} -} -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(RequestId) - -#endif // IORAP_BINDER_REQUEST_ID_H_ diff --git a/include/binder/system_service_event.h b/include/binder/system_service_event.h deleted file mode 100644 index a82c200..0000000 --- a/include/binder/system_service_event.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_SYSTEM_SERVICE_EVENT_H_ -#define IORAP_BINDER_SYSTEM_SERVICE_EVENT_H_ - -#include "binder/common.h" -#include "binder/auto_parcelable.h" -#include "common/introspection.h" - -namespace iorap { -namespace binder { - -struct SystemServiceEvent : public AutoParcelable<SystemServiceEvent> { - enum class Type : int32_t { - kBootPhase = 0, - kStart = 1, - }; - - Type type; -}; - -IORAP_INTROSPECT_ADAPT_STRUCT(SystemServiceEvent, type); - -} -} -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(SystemServiceEvent) - -#endif // IORAP_BINDER_SYSTEM_SERVICE_EVENT_H_ diff --git a/include/binder/system_service_user_event.h b/include/binder/system_service_user_event.h deleted file mode 100644 index 43f92ea..0000000 --- a/include/binder/system_service_user_event.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_SYSTEM_SERVICE_USER_EVENT_H_ -#define IORAP_BINDER_SYSTEM_SERVICE_USER_EVENT_H_ - -#include "binder/common.h" -#include "binder/auto_parcelable.h" -#include "common/introspection.h" - -namespace iorap { -namespace binder { - -struct SystemServiceUserEvent : public AutoParcelable<SystemServiceUserEvent> { - enum class Type : int32_t { - kStartUser = 0, - kUnlockUser, - kSwitchUser, - kStopUser, - kCleanupUser, - }; - - Type type; - int32_t user_handle; -}; - -IORAP_INTROSPECT_ADAPT_STRUCT(SystemServiceUserEvent, type, user_handle); - -} -} -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(SystemServiceUserEvent) - -#endif // IORAP_BINDER_SYSTEM_SERVICE_USER_EVENT_H_ diff --git a/include/binder/task_result.h b/include/binder/task_result.h deleted file mode 100644 index b060f3f..0000000 --- a/include/binder/task_result.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_TASK_RESULT_H_ -#define IORAP_BINDER_TASK_RESULT_H_ - -#include "binder/common.h" -#include "binder/auto_parcelable.h" -#include "common/introspection.h" - -namespace iorap { -namespace binder { - -struct TaskResult : public AutoParcelable<TaskResult> { - enum class State : int32_t { - kBegan = 0, - kOngoing = 1, - kCompleted = 2, - kError = 3, - kMax = kError, - }; - - State state; - - TaskResult() = default; - explicit TaskResult(State state) : state(state) {} - - constexpr bool operator==(const TaskResult& other) const { - return state == other.state; - } - - constexpr bool operator!=(const TaskResult& other) const { - return !(*this == other); - } -}; - -IORAP_INTROSPECT_ADAPT_STRUCT(TaskResult, state); - -} -} -IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(TaskResult) - -#endif // IORAP_BINDER_TASK_RESULT_H_ diff --git a/iorapd.rc b/iorapd.rc deleted file mode 100644 index 63f6b13..0000000 --- a/iorapd.rc +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (C) 2018 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -service iorapd /system/bin/iorapd - class main - disabled - user iorapd - group iorapd - capabilities DAC_READ_SEARCH - task_profiles ServiceCapacityLow -# TODO: should this be something else like performance/tasks? -# the main 'activity hint' thread needs to be fairly high priority to kick off -# tracing or prefetching asap. - -on property:ro.iorapd.enable=false - stop iorapd - -on post-fs-data - # Create directory for iorapd (see iorapd_data_file in selinux file_contexts). - mkdir /data/misc/iorapd/ 0750 iorapd system - -# Start iorapd when either prefetching or tracing is enabled. -on property:persist.device_config.runtime_native_boot.iorap_perfetto_enable=true && property:ro.iorapd.enable=true - start iorapd - -on property:persist.device_config.runtime_native_boot.iorap_readahead_enable=true && property:ro.iorapd.enable=true - start iorapd diff --git a/seccomp_policy/prefetcherd.arm.policy b/seccomp_policy/prefetcherd.arm.policy deleted file mode 120000 index a4642af..0000000 --- a/seccomp_policy/prefetcherd.arm.policy +++ /dev/null @@ -1 +0,0 @@ -prefetcherd.arm64.policy
\ No newline at end of file diff --git a/seccomp_policy/prefetcherd.arm64.policy b/seccomp_policy/prefetcherd.arm64.policy deleted file mode 100644 index a796ad1..0000000 --- a/seccomp_policy/prefetcherd.arm64.policy +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (C) 2019 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# main usage: fadvise to prefetch data -fadvise64: 1 - -# bionic's fdsan requires this -getrlimit: 1 - -@include /system/etc/seccomp_policy/crash_dump.arm64.policy diff --git a/seccomp_policy/prefetcherd.x86.policy b/seccomp_policy/prefetcherd.x86.policy deleted file mode 120000 index a4642af..0000000 --- a/seccomp_policy/prefetcherd.x86.policy +++ /dev/null @@ -1 +0,0 @@ -prefetcherd.arm64.policy
\ No newline at end of file diff --git a/seccomp_policy/prefetcherd.x86_64.policy b/seccomp_policy/prefetcherd.x86_64.policy deleted file mode 120000 index a4642af..0000000 --- a/seccomp_policy/prefetcherd.x86_64.policy +++ /dev/null @@ -1 +0,0 @@ -prefetcherd.arm64.policy
\ No newline at end of file diff --git a/src/binder/iiorap_def.h b/src/binder/iiorap_def.h deleted file mode 100644 index 5763380..0000000 --- a/src/binder/iiorap_def.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IIORAP_IFACE_DEF_H -#define IIORAP_IFACE_DEF_H - -// Provide an x-macro that defines the 'IIorap.aidl' interface through repeated -// macro invocation on the member functions and their respective parameters. -// -// Future changes in the AIDL file to add new methods or change parameters should avoid -// a boilerplate-ripple effect in the rest of the codebase. - -#define IIORAP_IFACE_DEF(FN_BEGIN, FN, FN_END) \ -FN_BEGIN(::com::google::android::startup::iorap::,IIorap) \ -/* name <see IORAP_BINDER_PARAM_JOIN> */ \ -FN(setTaskListener, (const ::android::sp<::com::google::android::startop::iorap::,ITaskListener,>&,listener)) /*NOLINT*/ \ -FN(onAppLaunchEvent,(const ::com::google::android::startop::iorap::,RequestId,&,request), \ - (const ::com::google::android::startop::iorap::,AppLaunchEvent,&,event)) \ -FN(onJobScheduledEvent, \ - (const ::com::google::android::startop::iorap::,RequestId,&,request), \ - (const ::com::google::android::startop::iorap::,JobScheduledEvent,&,event)) \ -FN(onPackageEvent, (const ::com::google::android::startop::iorap::,RequestId,&,request), \ - (const ::com::google::android::startop::iorap::,PackageEvent,&,event)) \ -FN(onAppIntentEvent,(const ::com::google::android::startop::iorap::,RequestId,&,request), \ - (const ::com::google::android::startop::iorap::,AppIntentEvent,&,event)) \ -FN(onSystemServiceEvent, \ - (const ::com::google::android::startop::iorap::,RequestId,&,request), \ - (const ::com::google::android::startop::iorap::,SystemServiceEvent,&,event)) \ -FN(onSystemServiceUserEvent, \ - (const ::com::google::android::startop::iorap::,RequestId,&,request), \ - (const ::com::google::android::startop::iorap::,SystemServiceUserEvent,&,event))\ -FN(onDexOptEvent, (const ::com::google::android::startop::iorap::,RequestId,&,request), \ - (const ::com::google::android::startop::iorap::,DexOptEvent,&,event)) \ -FN_END() \ - -// Convenience macros to unpack the 2nd parameter from IIORAP_IFACE_DEF#FN calls. - -#define IORAP_BINDER_PARAM_JOIN_ALL(arg) IORAP_BINDER_PARAM_JOIN_ALL_IMPL arg -#define IORAP_BINDER_PARAM_JOIN_ALL_IMPL(type_l, type, type_r, name) type_l type type_r name - -#define IORAP_BINDER_PARAM_JOIN_NAMES(arg) IORAP_BINDER_PARAM_JOIN_NAMES_IMPL arg -#define IORAP_BINDER_PARAM_JOIN_NAMES_IMPL(type_l, type, type_r, name) name - -#endif //IIORAP_IFACE_DEF_H diff --git a/src/binder/iiorap_impl.cc b/src/binder/iiorap_impl.cc deleted file mode 100644 index 13adcc0..0000000 --- a/src/binder/iiorap_impl.cc +++ /dev/null @@ -1,491 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "binder/iiorap_impl.h" -#include "binder/iiorap_def.h" -#include "common/macros.h" -#include "manager/event_manager.h" - -#include <android-base/logging.h> -#include <android-base/properties.h> -#include <binder/BinderService.h> -#include <binder/IPCThreadState.h> -#include <include/binder/request_id.h> -#include <utils/Printer.h> - -#include <codecvt> -#include <locale> -#include <utility> - -/* - * Definitions for the IIorap binder native service implementation. - * See also IIorap.aidl. - */ - -using Status = ::android::binder::Status; -using ITaskListener = ::com::google::android::startop::iorap::ITaskListener; - -namespace iorap { -namespace binder { - -namespace { -// Forward declarations. -template<typename ... Args> -Status Send(const char* function_name, Args&& ... args); -} - -// Join all parameter declarations by splitting each parameter with a comma. -// Types are used fully. -#define IIORAP_IMPL_ARG_DECLARATIONS(...) \ - IORAP_PP_MAP_SEP(IORAP_BINDER_PARAM_JOIN_ALL, IORAP_PP_COMMA, __VA_ARGS__) -#define IIORAP_IMPL_ARG_NAMES(...) \ - IORAP_PP_MAP_SEP(IORAP_BINDER_PARAM_JOIN_NAMES, IORAP_PP_COMMA, __VA_ARGS__) -#define IIORAP_IMPL_BODY(name, ...) \ - ::android::binder::Status IIorapImpl::name(IIORAP_IMPL_ARG_DECLARATIONS(__VA_ARGS__)) { \ - return Send(#name, impl_.get(), IIORAP_IMPL_ARG_NAMES(__VA_ARGS__)); \ - } - -IIORAP_IFACE_DEF(/*begin*/IORAP_PP_NOP, IIORAP_IMPL_BODY, /*end*/IORAP_PP_NOP); - -#undef IIORAP_IMPL_BODY -#undef IIORAP_IMPL_ARG_NAMES -#undef IIORAP_IMPL_ARGS - -namespace { - -struct ServiceParams { - bool fake_{false}; - std::shared_ptr<manager::EventManager> event_manager_; -}; - -static std::atomic<bool> s_service_started_{false}; -static std::atomic<bool> s_service_params_ready_{false}; - -// TODO: BinderService constructs IIorapImpl, -// but how do I get a pointer to it afterwards? -// -// This is a workaround for that, by using a global. -static ServiceParams s_service_params_; -static std::atomic<ServiceParams*> s_service_params_atomic_; - -// Convert an android::String16 (UTF-16) to a UTF-8 std::string. -static std::string String16ToStdString(const ::android::String16& s16) { - std::u16string u16{s16.string()}; - std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> convert; - - std::string res = convert.to_bytes(u16); - return res; -} - -} // namespace anonymous - -class IIorapImpl::Impl { - // ITaskListener implementation for iorap::manager::EventManager. - struct EventManagerTaskCallbacks : public iorap::manager::TaskResultCallbacks { - explicit EventManagerTaskCallbacks(iorap::borrowed<IIorapImpl::Impl*> impl) { - CHECK(impl != nullptr); - impl_ = impl; - } - - virtual void OnProgress(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) override { - impl_->ReplyWithResult(request_id, /*completed*/false, std::move(task_result)); - } - virtual void OnComplete(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) override { - impl_->ReplyWithResult(request_id, /*completed*/true, std::move(task_result)); - } - - virtual ~EventManagerTaskCallbacks() {} - - iorap::borrowed<IIorapImpl::Impl*> impl_; - }; - - public: - ~Impl() { - package_manager_->UnregisterPackageChangeObserver(package_change_observer_); - } - - void SetTaskListener(const ::android::sp<ITaskListener>& listener) { - ::android::sp<ITaskListener> old_listener = listener_; - if (old_listener != nullptr && listener != nullptr) { - LOG(WARNING) << "IIorap::setTaskListener: already had a task listener set"; - } - listener_ = listener; - } - - void ReplyWithResult(const RequestId& request_id, TaskResult::State result_state) { - ::android::sp<ITaskListener> listener = listener_; - if (listener == nullptr) { - // No listener. Cannot send anything back to the client. - // This could be normal, e.g. client had set listener to null before disconnecting. - LOG(DEBUG) << "Drop result, no listener registered."; - // TODO: print the result with ostream operator<< - return; - } - - TaskResult result; - result.state = result_state; - - // TODO: verbose, not info. - if (result_state == TaskResult::State::kCompleted) { - LOG(VERBOSE) << "ITaskListener::onComplete (request_id=" << request_id.request_id << ")"; - listener->onComplete(request_id, result); - } else { - LOG(VERBOSE) << "ITaskListener::onProgress (request_id=" << request_id.request_id << ")"; - listener->onProgress(request_id, result); - } - } - - void ReplyWithResult(const RequestId& request_id, bool completed, TaskResult result) { - ::android::sp<ITaskListener> listener = listener_; - if (listener == nullptr) { - // No listener. Cannot send anything back to the client. - // This could be normal, e.g. client had set listener to null before disconnecting. - LOG(DEBUG) << "Drop result, no listener registered."; - // TODO: print the result with ostream operator<< - return; - } - - // TODO: verbose, not info. - if (completed) { - LOG(VERBOSE) << "ITaskListener::onComplete (request_id=" << request_id.request_id << ")"; - listener->onComplete(request_id, result); - } else { - LOG(VERBOSE) << "ITaskListener::onProgress (request_id=" << request_id.request_id << ")"; - listener->onProgress(request_id, result); - } - } - - bool OnAppLaunchEvent(const RequestId& request_id, - const AppLaunchEvent& event) { - if (MaybeHandleFakeBehavior(request_id)) { - return true; - } - - return service_params_.event_manager_->OnAppLaunchEvent(request_id, event); - } - - bool OnDexOptEvent(const RequestId& request_id, const DexOptEvent& event) { - if (MaybeHandleFakeBehavior(request_id)) { - return true; - } - - return service_params_.event_manager_->OnDexOptEvent(request_id, event); - } - - bool OnJobScheduledEvent(const RequestId& request_id, - const JobScheduledEvent& event) { - if (MaybeHandleFakeBehavior(request_id)) { - return true; - } - - return service_params_.event_manager_->OnJobScheduledEvent(request_id, event); - } - - void Dump(/*borrow*/::android::Printer& printer, - const ::android::Vector<::android::String16>& args) { - - if (args.size() == 0) { - service_params_.event_manager_->Dump(/*borrow*/printer); - return; - } - - ::android::String16 arg_prev; - for (const ::android::String16& arg : args) { - bool unknown = false; - if (arg == ::android::String16("--all") || arg == ::android::String16("-a")) { - // using 'dumpsys' or 'bugreport' passes a single '-a' flag to this function. - service_params_.event_manager_->Dump(/*borrow*/printer); - } else if (arg == ::android::String16("--refresh-properties")) { - service_params_.event_manager_->RefreshSystemProperties(/*borrow*/printer); - printer.printLine("System properties refreshed."); - } else if (arg == ::android::String16("--compile-package")) { - // Intentionally left blank. - } else if (arg_prev == ::android::String16("--compile-package")) { - std::string package_name = String16ToStdString(arg); - - if (!service_params_.event_manager_->CompilePackage(/*borrow*/printer, package_name)) { - printer.printFormatLine("Failed to compile package %s.", package_name.c_str()); - } else { - printer.printFormatLine("Package %s compiled.", package_name.c_str()); - } - } else if (arg == ::android::String16("--purge-package")) { - // Intentionally left blank. - } else if (arg_prev == ::android::String16("--purge-package")) { - std::string package_name = String16ToStdString(arg); - - if (!service_params_.event_manager_->PurgePackage(/*borrow*/printer, package_name)) { - printer.printFormatLine("Failed to purge package %s.", package_name.c_str()); - } else { - printer.printFormatLine("Package %s purged.", package_name.c_str()); - } - } else { - unknown = true; - } - - if (unknown && arg != ::android::String16("--help")) { - printer.printLine("Invalid arguments."); - printer.printLine(""); - - printer.printLine("Arguments were:"); - for (const ::android::String16& arg16 : args) { - printer.printFormatLine(" '%s'", String16ToStdString(arg16).c_str()); - } - printer.printLine(""); - } - - if (unknown || arg == ::android::String16("--help")) { - printer.printLine("Iorapd dumpsys commands:"); - printer.printLine(" (none),--all,-a: Print state information for debugging iorapd."); - printer.printLine(" --help: Display this help menu"); - printer.printLine(" --compile-package <name>: Compile single package on device"); - printer.printLine(" --purge-package <name>: Delete database entries/files for package"); - printer.printLine(" --refresh-properties: Refresh system properties"); - return; - } - - arg_prev = arg; - } - } - - void HandleFakeBehavior(const RequestId& request_id) { - DCHECK(service_params_.fake_); - - // Send these dummy callbacks for testing only. - ReplyWithResult(request_id, TaskResult::State::kBegan); - ReplyWithResult(request_id, TaskResult::State::kOngoing); - ReplyWithResult(request_id, TaskResult::State::kCompleted); - } - - // TODO: Subclass IIorap with a separate fake implementation. - bool MaybeHandleFakeBehavior(const RequestId& request_id) { - if (service_params_.fake_) { - HandleFakeBehavior(request_id); - return true; - } - - return false; - } - - ::android::sp<ITaskListener> listener_; - - Impl(ServiceParams p) : service_params_{std::move(p)}, event_manager_callbacks_{new EventManagerTaskCallbacks{this}} { - CHECK(service_params_.event_manager_ != nullptr); - - service_params_.event_manager_->SetTaskResultCallbacks( - std::static_pointer_cast<manager::TaskResultCallbacks>(event_manager_callbacks_)); - - // Init the package change observer. - package_manager_ = PackageManagerRemote::Create(); - - if (package_manager_ == nullptr) { - LOG(ERROR) << "Failed to get package manager service in IIorapImpl::Impl." - << " Is system_server down?"; - exit(1); - } - - package_change_observer_ = - new PackageChangeObserver(service_params_.event_manager_); - package_manager_death_recipient_ = - new PackageManagerDeathRecipient(package_manager_, package_change_observer_); - - package_manager_->RegisterPackageChangeObserver(package_change_observer_); - package_manager_-> - RegisterPackageManagerDeathRecipient(package_manager_death_recipient_); - - } - - ServiceParams service_params_; - std::shared_ptr<EventManagerTaskCallbacks> event_manager_callbacks_; - android::sp<PackageChangeObserver> package_change_observer_; - android::sp<PackageManagerDeathRecipient> package_manager_death_recipient_; - std::shared_ptr<PackageManagerRemote> package_manager_; -}; - -using Impl = IIorapImpl::Impl; - -IIorapImpl::IIorapImpl() { - // Acquire edge of synchronizes-with IIorapImpl::Start(). - CHECK(s_service_params_ready_.load()); - // Do not turn this into a DCHECK, the above atomic load - // must happen-before the read of s_service_params_ready_. - impl_.reset(new Impl(std::move(s_service_params_))); -} - -namespace { - static bool started_ = false; -} -bool IIorapImpl::Start(std::shared_ptr<manager::EventManager> event_manager) { - if (s_service_started_.load()) { // Acquire-edge (see bottom of function). - // Note: Not meant to be idempotent. Two threads could race, and the second - // one would likely fail the publish. - - LOG(ERROR) << "service was already started"; - return false; // Already started - } - - CHECK(event_manager != nullptr); - - { - // This block of code needs to happen-before IIorapImpl::IIorapImpl. - - // TODO: There should be a simpler way of passing down - // this data which doesn't involve globals and memory synchronization. - ServiceParams* p = &s_service_params_; - // TODO: move all property reads to a dedicated Config class. - p->fake_ = ::android::base::GetBoolProperty("iorapd.binder.fake", /*default*/false); - p->event_manager_ = std::move(event_manager); - - // Release edge of synchronizes-with IIorapImpl::IIorapImpl. - s_service_params_ready_.store(true); - } - - ::android::IPCThreadState::self()->disableBackgroundScheduling(/*disable*/true); - ::android::status_t ret = android::BinderService<IIorapImpl>::publish(); - if (ret != android::OK) { - LOG(ERROR) << "BinderService::publish failed with error code: " << ret; - return false; - } - - android::sp<android::ProcessState> ps = android::ProcessState::self(); - // Reduce thread consumption by only using 1 thread. - // We should also be able to leverage this by avoiding locks, etc. - ps->setThreadPoolMaxThreadCount(/*maxThreads*/1); - ps->startThreadPool(); - ps->giveThreadPoolName(); - - // Release edge synchronizes-with the top of this function. - s_service_started_.store(true); - - // TODO: IIRC thread-start(t1) synchronizes-with t1.main, so we should be able - // to delete the majority of atomics for any pre-thread-start initialization... - - return true; -} - -::android::status_t IIorapImpl::dump(int fd, const ::android::Vector<::android::String16>& args) { - - ::android::FdPrinter printer{fd}; - - impl_->Dump(printer, args); - - return ::android::NO_ERROR; -} - -namespace { - -#define MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id) \ - if (self->MaybeHandleFakeBehavior(request_id)) { return ::android::binder::Status::ok(); } - -template <typename ... Args> -Status SendArgs(const char* function_name, - Impl* self, - const RequestId& request_id, - Args&&... /*rest*/) { - LOG(VERBOSE) << "IIorap::" << function_name << " (request_id = " << request_id.request_id << ")"; - - MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id); - - // TODO: implementation. - LOG(ERROR) << "IIorap::" << function_name << " -- not implemented for real code"; - return Status::fromStatusT(::android::INVALID_OPERATION); -} - -template <typename ... Args> -Status SendArgs(const char* function_name, Impl* self, Args&&... rest) { - DCHECK_EQ(std::string(function_name), "setTaskListener"); - LOG(VERBOSE) << "IIorap::setTaskListener"; - self->SetTaskListener(std::forward<Args&&>(rest)...); - - return Status::ok(); -} - -template <typename ... Args> -Status SendArgs(const char* function_name, - Impl* self, - const RequestId& request_id, - const AppLaunchEvent& app_launch_event) { - DCHECK_EQ(std::string(function_name), "onAppLaunchEvent"); - LOG(VERBOSE) << "IIorap::onAppLaunchEvent"; - - MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id); - - if (self->OnAppLaunchEvent(request_id, app_launch_event)) { - return Status::ok(); - } else { - // TODO: I suppose this should write out an exception back, - // like a service-specific error or something. - // - // It depends on whether or not we even have any synchronous - // errors. - // - // Most of the work here is done async, so it should handle - // async callbacks. - return Status::fromStatusT(::android::BAD_VALUE); - } -} - -template <typename ... Args> -Status SendArgs(const char* function_name, - Impl* self, - const RequestId& request_id, - const DexOptEvent& event) { - DCHECK_EQ(std::string(function_name), "onDexOptEvent"); - LOG(VERBOSE) << "IIorap::onDexOptEvent"; - - MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id); - - if (self->OnDexOptEvent(request_id, event)) { - return Status::ok(); - } else { - return Status::fromStatusT(::android::BAD_VALUE); - } -} - -template <typename ... Args> -Status SendArgs(const char* function_name, - Impl* self, - const RequestId& request_id, - const JobScheduledEvent& event) { - DCHECK_EQ(std::string(function_name), "onJobScheduledEvent"); - LOG(VERBOSE) << "IIorap::onJobScheduledEvent"; - - MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id); - - if (self->OnJobScheduledEvent(request_id, event)) { - return Status::ok(); - } else { - // TODO: I suppose this should write out an exception back, - // like a service-specific error or something. - // - // It depends on whether or not we even have any synchronous - // errors. - // - // Most of the work here is done async, so it should handle - // async callbacks. - return Status::fromStatusT(::android::BAD_VALUE); - } -} - -template <typename ... Args> -Status Send(const char* function_name, Args&&... args) { - LOG(VERBOSE) << "IIorap::Send(" << function_name << ")"; - - return SendArgs(function_name, std::forward<Args>(args)...); -} -} // namespace <anonymous> - -} // namespace binder -} // namespace iorap diff --git a/src/binder/iiorap_impl.h b/src/binder/iiorap_impl.h deleted file mode 100644 index ae1422f..0000000 --- a/src/binder/iiorap_impl.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_BINDER_IIORAP_IMPL_H -#define IORAP_BINDER_IIORAP_IMPL_H - -#include "binder/iiorap_def.h" -#include "binder/package_change_observer.h" -#include "binder/package_manager_remote.h" -#include "common/macros.h" - -#include "com/google/android/startop/iorap/BnIorap.h" - -#include <memory> - -namespace android { -template <typename Service> -class BinderService; -} - -namespace iorap::manager { - class EventManager; -}; - -namespace iorap { -namespace binder { - -// Implementation of the IIorap service. This enables remote clients -// to connect to this process via the binder service manager. -// -// See also IIorap.aidl. -class IIorapImpl : public ::com::google::android::startop::iorap::BnIorap { -public: - static bool Start(std::shared_ptr<iorap::manager::EventManager> event_manager); - static constexpr const char* getServiceName() { return "iorapd"; }; - - virtual ::android::status_t dump(int fd, - const ::android::Vector<::android::String16>& args) override; - -// Join all parameter declarations by splitting each parameter with a comma. -// Types are used fully. -#define IIORAP_IMPL_ARGS(...) \ - IORAP_PP_MAP_SEP(IORAP_BINDER_PARAM_JOIN_ALL, IORAP_PP_COMMA, __VA_ARGS__) -#define IIORAP_IMPL_BODY(name, ...) \ - ::android::binder::Status name(IIORAP_IMPL_ARGS(__VA_ARGS__)) override; - -IIORAP_IFACE_DEF(/*begin*/IORAP_PP_NOP, IIORAP_IMPL_BODY, /*end*/IORAP_PP_NOP); - -#undef IIORAP_IMPL_BODY -#undef IIORAP_IMPL_ARGS - - class Impl; -private: - IIorapImpl(); // open for BinderService::publish. - - friend class ::android::BinderService<IIorapImpl>; - - std::unique_ptr<Impl> impl_; -}; -} -} - - -#endif //IORAP_BINDER_IIORAP_IMPL_H diff --git a/src/binder/package_change_observer.cc b/src/binder/package_change_observer.cc deleted file mode 100644 index 1907af8..0000000 --- a/src/binder/package_change_observer.cc +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "package_change_observer.h" -#include "package_manager_remote.h" -#include "manager/event_manager.h" - -#include <android-base/logging.h> -#include <android-base/properties.h> - -namespace iorap::binder { - -PackageChangeObserver::PackageChangeObserver( - std::shared_ptr<iorap::manager::EventManager> event_manager) : - event_manager_(event_manager){} - -android::binder::Status PackageChangeObserver::onPackageChanged( - const android::content::pm::PackageChangeEvent& event) { - LOG(DEBUG) << "Received PackageChangeObserver::onPackageChanged"; - if (event_manager_->OnPackageChanged(event)) { - return android::binder::Status::ok(); - } else { - return android::binder::Status::fromStatusT(android::BAD_VALUE); - } -} -} // namespace iorap::binder diff --git a/src/binder/package_change_observer.h b/src/binder/package_change_observer.h deleted file mode 100644 index ac488ac..0000000 --- a/src/binder/package_change_observer.h +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_PACKAGE_CHANGE_OBSERVER_H_ -#define IORAP_SRC_PACKAGE_CHANGE_OBSERVER_H_ - -#include "binder/iiorap_def.h" - -#include <android/content/pm/BnPackageChangeObserver.h> -#include <android/content/pm/PackageChangeEvent.h> -#include <android/content/pm/IPackageManagerNative.h> - -namespace iorap::manager { - class EventManager; -}; - -namespace iorap::binder { - -class PackageChangeObserver : public android::content::pm::BnPackageChangeObserver { - public: - PackageChangeObserver(std::shared_ptr<iorap::manager::EventManager> event_manager); - - // Callback when the package is changed. - android::binder::Status onPackageChanged( - const ::android::content::pm::PackageChangeEvent& event) override; - private: - std::shared_ptr<iorap::manager::EventManager> event_manager_; -}; -} // namespace iorap::binder - -#endif // IORAP_SRC_PACKAGE_CHANGE_OBSERVER_H_ diff --git a/src/binder/package_manager_remote.cc b/src/binder/package_manager_remote.cc deleted file mode 100644 index 46671eb..0000000 --- a/src/binder/package_manager_remote.cc +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "package_manager_remote.h" - -#include <android-base/logging.h> -#include <android-base/properties.h> - -#include <chrono> -#include <thread> - -namespace iorap::binder { - -const constexpr int64_t kTimeoutMs = 60 * 1000; // 1 min -const constexpr int64_t kIntervalMs = 1000; // 1 sec - -std::shared_ptr<PackageManagerRemote> PackageManagerRemote::Create() { - std::shared_ptr<PackageManagerRemote> package_manager = - std::make_shared<PackageManagerRemote>(); - if (package_manager->ReconnectWithTimeout(kTimeoutMs)) { - return package_manager; - } - return nullptr; -} - -android::sp<IPackageManager> PackageManagerRemote::GetPackageService() { - android::sp<android::IBinder> binder = android::defaultServiceManager()->getService( - android::String16{"package_native"}); - if (binder == nullptr) { - LOG(ERROR) << "Cannot get package manager service!"; - return nullptr; - } - - return android::interface_cast<IPackageManager>(binder); -} - -std::optional<int64_t> PackageManagerRemote::GetPackageVersion( - const std::string& package_name) { - int64_t version_code; - android::binder::Status status = InvokeRemote( - [this, &version_code, &package_name]() { - return package_service_->getVersionCodeForPackage( - android::String16(package_name.c_str()), /*out*/&version_code); - }); - if (status.isOk()) { - return version_code; - } else { - LOG(WARNING) << "Failed to get version: " - << status.toString8().c_str() - << " for " << package_name - << ". Retry to connect package manager service."; - return std::nullopt; - } -} - -std::optional<VersionMap> PackageManagerRemote::GetPackageVersionMap() { - VersionMap package_version_map; - std::optional<std::vector<std::string>> packages = GetAllPackages(); - if (!packages) { - LOG(DEBUG) << "Failed to get all packages. The package manager may be down."; - return std::nullopt; - } - LOG(DEBUG) << "PackageManagerRemote::GetPackageVersionMap: " - << packages->size() - << " packages are found."; - - for (const std::string& package : *packages) { - std::optional<int64_t> version = GetPackageVersion(package); - if (!version) { - LOG(DEBUG) << "Cannot get version for " << package - << "Package manager may be down"; - return std::nullopt; - } - package_version_map[package] = *version; - } - - return package_version_map; -} - -std::optional<std::vector<std::string>> PackageManagerRemote::GetAllPackages() { - std::vector<std::string> packages; - android::binder::Status status = InvokeRemote( - [this, &packages]() { - return package_service_->getAllPackages(/*out*/&packages); - }); - - if (status.isOk()) { - return packages; - } - - LOG(ERROR) << "Failed to get all packages: " << status.toString8().c_str(); - return std::nullopt; - -} - -bool PackageManagerRemote::ReconnectWithTimeout(int64_t timeout_ms) { - int64_t count = 0; - package_service_ = nullptr; - std::chrono::duration interval = std::chrono::milliseconds(1000); - std::chrono::duration timeout = std::chrono::milliseconds(timeout_ms); - - while (package_service_ == nullptr) { - LOG(WARNING) << "Reconnect to package manager service: " << ++count << " times"; - package_service_ = GetPackageService(); - std::this_thread::sleep_for(interval); - if (count * interval >= timeout) { - LOG(ERROR) << "Fail to create version map in " - << timeout.count() - << " milliseconds." - << " Reason: Failed to connect to package manager service." - << " Is system_server down?"; - return false; - } - } - - return true; -} - -template <typename T> -android::binder::Status PackageManagerRemote::InvokeRemote(T&& lambda) { - android::binder::Status status = - static_cast<android::binder::Status>(lambda()); - if (status.isOk()) { - return status; - } - - if (!ReconnectWithTimeout(kTimeoutMs)) { - return status; - } - - return lambda(); -} - -void PackageManagerRemote::RegisterPackageChangeObserver( - android::sp<PackageChangeObserver> observer) { - LOG(DEBUG) << "Register package change observer."; - android::binder::Status status = InvokeRemote( - [this, &observer]() { - return package_service_->registerPackageChangeObserver(observer); - }); - - if (!status.isOk()) { - LOG(ERROR) << "Cannot register package change observer. Is system_server down?"; - exit(1); - } -} - -void PackageManagerRemote::UnregisterPackageChangeObserver( - android::sp<PackageChangeObserver> observer) { - LOG(DEBUG) << "Unregister package change observer."; - android::binder::Status status = InvokeRemote( - [this, &observer]() { - return package_service_->unregisterPackageChangeObserver(observer); - }); - - if (!status.isOk()) { - LOG(WARNING) << "Cannot unregister package change observer."; - } -} - -void PackageManagerRemote::RegisterPackageManagerDeathRecipient( - android::sp<PackageManagerDeathRecipient> death_recipient) { - LOG(DEBUG) << "Register package manager death recipient."; - android::status_t status = - android::IInterface::asBinder(package_service_.get())->linkToDeath(death_recipient); - - if (status == android::OK) { - return; - } - - if (!ReconnectWithTimeout(kTimeoutMs) || - android::OK != android::IInterface::asBinder( - package_service_.get())->linkToDeath(death_recipient)) { - LOG(ERROR) << "Failed to register package manager death recipient. Is system_server down?"; - exit(1); - } -} - -void PackageManagerDeathRecipient::binderDied(const android::wp<android::IBinder>& /* who */) { - LOG(DEBUG) << "PackageManagerDeathRecipient::binderDied try to re-register"; - package_manager_->RegisterPackageChangeObserver(observer_); - package_manager_-> - RegisterPackageManagerDeathRecipient(this); -} -} // namespace iorap::package_manager diff --git a/src/binder/package_manager_remote.h b/src/binder/package_manager_remote.h deleted file mode 100644 index 9b0dcd9..0000000 --- a/src/binder/package_manager_remote.h +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_ -#define IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_ - -#include "binder/package_change_observer.h" - -#include <android/content/pm/IPackageManagerNative.h> -#include <binder/IServiceManager.h> - -#include <optional> -#include <unordered_map> - -namespace iorap::binder { - -using IPackageManager = android::content::pm::IPackageManagerNative; -// A map between package name and its version. -using VersionMap = std::unordered_map<std::string, int64_t>; - -class PackageManagerRemote; - -class PackageManagerDeathRecipient : public android::IBinder::DeathRecipient { -public: - PackageManagerDeathRecipient(std::shared_ptr<PackageManagerRemote> package_manager, - android::sp<PackageChangeObserver> observer) : - package_manager_(package_manager), observer_(observer) {} - // android::IBinder::DeathRecipient override: - void binderDied(const android::wp<android::IBinder>& /* who */) override; - -private: - std::shared_ptr<PackageManagerRemote> package_manager_; - - android::sp<PackageChangeObserver> observer_; -}; - -class PackageManagerRemote { - public: - static std::shared_ptr<PackageManagerRemote> Create(); - - // Gets the package version based on the package name. - std::optional<int64_t> GetPackageVersion(const std::string& package_name); - - // Gets a map of package name and its version. - std::optional<VersionMap> GetPackageVersionMap(); - - void RegisterPackageChangeObserver(android::sp<PackageChangeObserver> observer); - - void UnregisterPackageChangeObserver(android::sp<PackageChangeObserver> observer); - - void RegisterPackageManagerDeathRecipient( - android::sp<PackageManagerDeathRecipient> death_recipient); - - private: - template <typename T> - android::binder::Status InvokeRemote(T&& lambda); - - // Reconnects to the package manager service. - // Retry until timeout. - bool ReconnectWithTimeout(int64_t timeout_ms); - - // Gets the package manager service. - static android::sp<IPackageManager> GetPackageService(); - - // Gets all package names. - std::optional<std::vector<std::string>> GetAllPackages(); - - android::sp<IPackageManager> package_service_; -}; -} // namespace iorap::package_manager - -#endif // IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_ diff --git a/src/binder/package_version_map.cc b/src/binder/package_version_map.cc deleted file mode 100644 index 784bb1b..0000000 --- a/src/binder/package_version_map.cc +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "package_version_map.h" - -#include <android-base/logging.h> -#include <android-base/properties.h> - -namespace iorap::binder { - -std::shared_ptr<PackageVersionMap> PackageVersionMap::Create() { - std::shared_ptr<PackageManagerRemote> package_manager = - PackageManagerRemote::Create(); - if (!package_manager) { - return std::make_shared<PackageVersionMap>(); - } - - std::optional<VersionMap> map = package_manager->GetPackageVersionMap(); - return std::make_shared<PackageVersionMap>(package_manager, map); -} - -void PackageVersionMap::UpdateAll() {std::lock_guard<std::mutex> lock(mutex_); - size_t old_size = version_map_->size(); - std::optional<VersionMap> new_version_map = - package_manager_->GetPackageVersionMap(); - if (!new_version_map) { - LOG(DEBUG) << "Failed to get the latest version map"; - return; - } - version_map_ = std::move(new_version_map); - LOG(DEBUG) << "Update for version is done. The size is from " << old_size - << " to " << version_map_->size(); -} - -bool PackageVersionMap::Update(std::string package_name, int64_t version) { - std::lock_guard<std::mutex> lock(mutex_); - if (!version_map_) { - LOG(DEBUG) << "The version map doesn't exist. " - << "The package manager may be down."; - return false; - } - - VersionMap::iterator it = version_map_->find(package_name); - if (it == version_map_->end()) { - LOG(DEBUG) << "New installed package " - << package_name - << " with version " - << version; - (*version_map_)[package_name] = version; - return true; - } - - if (it->second != version) { - LOG(DEBUG) << "New version package " - << package_name - << " with version " - << version; - (*version_map_)[package_name] = version; - return true; - } - - LOG(DEBUG) << "Same version package " - << package_name - << " with version " - << version; - return false; -} - -size_t PackageVersionMap::Size() { - if (!version_map_) { - LOG(DEBUG) << "The version map doesn't exist. " - << "The package manager may be down."; - return -1; - } - return version_map_->size(); -} - -std::optional<int64_t> PackageVersionMap::GetOrQueryPackageVersion( - const std::string& package_name) { - std::lock_guard<std::mutex> lock(mutex_); - if (!version_map_) { - LOG(DEBUG) << "The version map doesn't exist. " - << "The package manager may be down."; - return std::nullopt; - } - - VersionMap::iterator it = version_map_->find(package_name); - if (it == version_map_->end()) { - LOG(DEBUG) << "Cannot find version for: " << package_name - << " in the hash table"; - std::optional<int64_t> version = - package_manager_->GetPackageVersion(package_name); - if (version) { - LOG(VERBOSE) << "Find version for: " << package_name << " on the fly."; - (*version_map_)[package_name] = *version; - return *version; - } else { - LOG(ERROR) << "Cannot find version for: " << package_name - << " on the fly."; - return -1; - } - } - - return it->second; -} -} // namespace iorap::binder diff --git a/src/binder/package_version_map.h b/src/binder/package_version_map.h deleted file mode 100644 index f2048fe..0000000 --- a/src/binder/package_version_map.h +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_PACKAGE_VERSION_MAP_H_ -#define IORAP_SRC_PACKAGE_VERSION_MAP_H_ - -#include <android/content/pm/IPackageManagerNative.h> -#include <binder/IServiceManager.h> - -#include <optional> -#include <unordered_map> - -#include "package_manager_remote.h" - -namespace iorap::binder { - -class PackageVersionMap { - public: - static std::shared_ptr<PackageVersionMap> Create(); - - PackageVersionMap(std::shared_ptr<PackageManagerRemote> package_manager, - std::optional<VersionMap> version_map) - : package_manager_(package_manager), - version_map_(version_map) {} - - PackageVersionMap() - : package_manager_(nullptr), version_map_(std::nullopt) {} - - // Updates the version specified by 'package_name' to 'version'. - // - // Post-condition: Find(package_name) == version. - // * if the package is newly installed, insert and return true. - // * if the package version is changed, update the version to the - // given one and return true. - // * otherwise, return false. - bool Update(std::string package_name, int64_t version); - - void UpdateAll(); - - // Finds the version of the package in the hash table. - // -1 means the app is installed by unversioned. - // Empty means the app is not inside the RAM version map, maybe due to - // the app is newly installed. - std::optional<int64_t> Find(const std::string& package_name) { - VersionMap::iterator it = version_map_->find(package_name); - if (it == version_map_->end()) { - return std::nullopt; - } - return it->second; - } - - size_t Size(); - - // Gets or queries the version for the package. - // - // The method firstly access the hash map in the RAM, which is built when - // iorapd starts. If the version is not in the map, it tries the query - // the package manager via IPC, with a cost of ~0.6ms. - // - // If no version can be found for some reason, return -1, - // because when an app has no version the package manager returns -1. - std::optional<int64_t> GetOrQueryPackageVersion( - const std::string& package_name); - - private: - std::shared_ptr<PackageManagerRemote> package_manager_; - std::optional<VersionMap> version_map_; - std::mutex mutex_; -}; -} // namespace iorap::binder - -#endif // IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_ diff --git a/src/common/async_pool.h b/src/common/async_pool.h deleted file mode 100644 index 51cd38c..0000000 --- a/src/common/async_pool.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_COMMON_ASYNC_POOL_H_ -#define IORAP_SRC_COMMON_ASYNC_POOL_H_ - -#include <atomic> -#include <vector> -#include <deque> -#include <future> - -#include <condition_variable> -#include <future> - -namespace iorap::common { - -class AsyncPool { - std::atomic<bool> shutting_down_{false}; - std::deque<std::future<void>> futures_; - - std::mutex mutex_; - std::condition_variable cond_var_; - - public: - - // Any threads calling 'Join' should eventually unblock - // once all functors have run to completition. - void Shutdown() { - shutting_down_ = true; - - cond_var_.notify_all(); - } - - // Block forever until Shutdown is called *and* all - // functors passed to 'LaunchAsync' have run to completition. - void Join() { - std::unique_lock<std::mutex> lock(mutex_); - while (true) { - // Pop all items eagerly - while (true) { - auto it = futures_.begin(); - if (it == futures_.end()) { - break; - } - - std::future<void> future = std::move(*it); - futures_.pop_front(); - - lock.unlock(); // do not stall callers of LaunchAsync - future.get(); - lock.lock(); - } - - if (shutting_down_) { - break; - } - - // Wait until we either get more items or ask to be shutdown. - cond_var_.wait(lock); - } - } - - // Execute the functor 'u' in a new thread asynchronously. - // Using this spawns a new thread each time to immediately begin - // async execution. - template <typename T> - void LaunchAsync(T&& u) { - auto future = std::async(std::launch::async, std::forward<T>(u)); - - { - std::unique_lock<std::mutex> lock(mutex_); - futures_.push_back(std::move(future)); - } - cond_var_.notify_one(); - } -}; - -} // namespace iorap::common - -#endif // IORAP_SRC_COMMON_ASYNC_POOL_H_ diff --git a/src/common/cmd_utils.h b/src/common/cmd_utils.h deleted file mode 100644 index 4494119..0000000 --- a/src/common/cmd_utils.h +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_COMMON_CMD_UTILS_H_ -#define IORAP_SRC_COMMON_CMD_UTILS_H_ - -#include <android-base/parsebool.h> -#include <android-base/properties.h> - -#include <iostream> -#include <sstream> -#include <optional> -#include <vector> - -namespace iorap::common { -// Create execve-compatible argv. -// The lifetime is tied to that of vector. -inline std::unique_ptr<const char*[]> VecToArgv(const char* program_name, - const std::vector<std::string>& vector) { - // include program name in argv[0] - // include a NULL sentinel in the end. - std::unique_ptr<const char*[]> ptr{new const char*[vector.size() + 2]}; - - // program name - ptr[0] = program_name; - - // all the argv - for (size_t i = 0; i < vector.size(); ++i) { - ptr[i+1] = vector[i].c_str(); - } - - // null sentinel - ptr[vector.size() + 1] = nullptr; - - return ptr; -} - -// Appends an args to the argv. -template <class T> -void AppendArgs(std::vector<std::string>& argv, - const T& value) { - std::stringstream ss; - ss << value; - argv.push_back(ss.str()); -} - -// Appends an args to the argv. -template <class T, class T2> -void AppendArgs(std::vector<std::string>& argv, - const T& value, - const T2& value2) { - AppendArgs(argv, value); - AppendArgs(argv, value2); -} - -// Appends a named argument to the argv. -// -// For example if <name> is "--property" and <value> is int(200): -// the string "--property=200" is appended to the argv. -template <class T, class T2> -void AppendNamedArg(std::vector<std::string>& argv, - const T& name, - const T2& value) { - std::stringstream ss; - ss << name; - ss << "="; - ss << value; - - argv.push_back(ss.str()); -} - -// Appends args from a vector to the argv repeatedly to argv. -// -// For example, if <args> is "--timestamp" and <values> is [100, 200]. -// The "--timestamp 100" and "--timestamp 200" are appended. -template <class T> -void AppendArgsRepeatedly(std::vector<std::string>& argv, - std::string args, - const std::vector<T>& values) { - for (const T& v : values) { - AppendArgs(argv, args, v); - } -} - -// Appends args from a vector to the argv repeatedly to argv. -// -// For example, if values is [input1.pb, input2.pb], -// then the "input1.pb" and "input2.pb" are appended. -template <class T> -void AppendArgsRepeatedly(std::vector<std::string>& argv, - const std::vector<T>& values) { - for (const T& v : values) { - AppendArgs(argv, v); - } -} - -// Appends a named argument to the argv repeatedly with different values. -// -// For example if <name> is "--property" and <value> is [int(200), int(400)]: -// the strings "--property=200" and "--property=400" are both appended to the argv. -template <class T, class T2> -void AppendNamedArgRepeatedly(std::vector<std::string>& argv, - const T& name, - const std::vector<T2>& values) { - for (const T2& v :values) { - AppendNamedArg(argv, name, v); - } -} - -// Get the value of the property. -// Firstly, try to find the environment variable. If it does not exist, -// try to get the property. If neither, use the default value.. -// -// For example, for prop foo.bar.baz, it will first check for -// FOO_BAR_BAZ environment variable. -inline std::string GetEnvOrProperty(const std::string& prop, const std::string& default_val) { - std::string env_str = prop; - // a.b.c -> a_b_c - std::replace(env_str.begin(), env_str.end(), '.', '_'); - // a_b_c -> A_B_C - std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper); - char *env = getenv(env_str.c_str()); - if (env) { - return std::string(env); - } - return ::android::base::GetProperty(prop, default_val); -} - -// Get the boolean value of the property. -// Firstly, try to find the environment variable. If it does not exist, -// try to get the property. If neither, use the default value.. -// -// For example, for prop foo.bar.baz, it will first check for -// FOO_BAR_BAZ environment variable. -inline bool GetBoolEnvOrProperty(const std::string& prop, bool default_val) { - std::string env_str = prop; - // a.b.c -> a_b_c - std::replace(env_str.begin(), env_str.end(), '.', '_'); - // a_b_c -> A_B_C - std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper); - char *env = getenv(env_str.c_str()); - if (env) { - using ::android::base::ParseBoolResult; - - switch (::android::base::ParseBool(env)) { - case ParseBoolResult::kError: - break; - case ParseBoolResult::kFalse: - return false; - case ParseBoolResult::kTrue: - return true; - } - } - return ::android::base::GetBoolProperty(prop, default_val); -} - -} // namespace iorap::common - -#endif // IORAP_SRC_COMMON_CMD_UTILS_H_ diff --git a/src/common/debug.h b/src/common/debug.h deleted file mode 100644 index bc21c74..0000000 --- a/src/common/debug.h +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include <ostream> - -namespace iorap { - -// kIsDebugBuild is special. -// It gets to be in the 'iorap' namespace -// so that different modules don't need to qualify it. -#ifndef NDEBUG -static constexpr bool kIsDebugBuild = true; -#else -static constexpr bool kIsDebugBuild = false; -#endif - -namespace common { - -// TODO: move below code to helpers. -template <typename T, bool> -struct base_if_condition {}; - -template <typename T> -struct base_if_condition<T, true> : public T {}; - -template <typename T> -using base_if_debug = base_if_condition<T, kIsDebugBuild>; - -namespace detail { -// "if constexpr" doesn't allow us to exclude fields from a struct/class, -// and also "if constexpr" doesn't allow us to reference a field that does not -// exist. -// so we must move everything into a separate base class. -template <bool kIsDebug = kIsDebugBuild> -struct DebugCounterBase { - constexpr size_t value() const { - return counter; - } - - constexpr void set_value(size_t value) { - counter = value; - } - - size_t counter{1}; // Don't start with 0. -}; - -template <> -struct DebugCounterBase<false /*kIsDebug*/> { - constexpr size_t value() const { - return 0; - } - - constexpr void set_value(size_t value) { - } -}; -} // namespace detail - -// This counter does absolutely nothing, the code compiles to no-ops -// when debugging is disabled. -struct DebugCounter : detail::DebugCounterBase<> { - constexpr DebugCounter& operator++() { - set_value(value() + 1); - return *this; - } - - constexpr DebugCounter operator++(int) { - DebugCounter now = *this; - set_value(value() + 1); - return now; - } - - constexpr operator size_t() const { - return value(); - } - - friend std::ostream& operator<<(std::ostream& os, DebugCounter counter); -}; - -inline std::ostream& operator<<(std::ostream& os, DebugCounter counter) { - os << counter.value(); - return os; -} - -// TODO: refactor DebugCounter and base traits into their own files? - -} // namespace common -} // namespace iorap diff --git a/src/common/expected.h b/src/common/expected.h deleted file mode 100644 index c58e63f..0000000 --- a/src/common/expected.h +++ /dev/null @@ -1,410 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_COMMON_EXPECTED_H_ -#define IORAP_SRC_COMMON_EXPECTED_H_ - -#include <type_traits> -#include <utility> - -#include <android-base/logging.h> // CHECK/DCHECK. - -// Ignore the tautological-undefined-compare warning. -// We obviously want to do this to protect against undefined behavior -// that sets a reference to a null value. -#define DCHECK_UB_NOT_NULL(x) \ - DCHECK(reinterpret_cast<volatile decltype(x)>(x) != nullptr) - -/** - * Result<Value, Error>-like interface. - * - * Subset of the experimental standard C++ proposal (p0323r3) - * - * Example: - * - * expected<std::string, status_t> x = function_which_might_fail(); - * if (x) { - * std::string str = x.value(); - * } else { - * status_t err = x.error(); - * } - */ - -namespace iorap { -namespace detail { - // Use perfect forwarding for expected_data constructors with overloading. - struct expected_tag{}; - struct expected_tag_right : public expected_tag { - static constexpr bool is_right_v = true; - }; - struct expected_tag_error : public expected_tag { - static constexpr bool is_right_v = false; - }; - - template <typename T, typename E, bool DefineDestructor> - struct expected_data; - - // This doesn't always work because this code could be instantiated with a non-trivial T/E, - // and then the union becomes invalid. - template <typename T, typename E> - struct expected_data<T, E, /*DefineDestructor*/true> { - // Mark everything 'constexpr' to keep the code the same as the other partial specialization. - - template <typename U> - constexpr expected_data(U&& either, expected_tag_right) - : right_{std::forward<U>(either)}, is_right_{true} {} - - template <typename U> - constexpr expected_data(U&& either, expected_tag_error) - : error_{std::forward<U>(either)}, is_right_{false} {} - - constexpr bool has_value() const { - return is_right_; - } - - constexpr const T& value() const { - return right_; - } - - constexpr T& value() { - return right_; - } - - constexpr const E& error() const { - return error_; - } - - constexpr E& error() { - return error_; - } - - // Using an "anonymous union" here allows non-trivial types to be stored. - union { - T right_; - E error_; - }; - - bool is_right_; - - // Below code differs slightly by handling non-trivial constructors/destructors. - bool moved_from_{false}; - - // Note: Destructors cannot be templated, so it is illegal to use SFINAE to try to - // conditionalize this destructor somehow. - ~expected_data() { - if (moved_from_) { return; } - if (is_right_) { - right_.~T(); - } else { - error_.~E(); - } - } - - expected_data(expected_data&& other) - noexcept( - noexcept(T(std::move(other.right_))) && - noexcept(E(std::move(other.error_))) - ) { - DCHECK_UB_NOT_NULL(&other) << __PRETTY_FUNCTION__; - DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__; - if (other.is_right_) { - new (&right_) T(std::move(other.right_)); - } else { - new (&error_) E(std::move(other.error_)); - } - other.moved_from_ = true; - is_right_ = other.is_right_; - } - - expected_data(const expected_data& other) { - DCHECK_UB_NOT_NULL(&other) << __PRETTY_FUNCTION__; - DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__; - if (other.is_right_) { - new (&right_) T(other.right_); - } else { - new (&error_) E(other.error_); - } - is_right_ = other.is_right_; - } - - expected_data& operator=(const expected_data& other) { - DCHECK_UB_NOT_NULL(&other) << __PRETTY_FUNCTION__; - DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__; - - if (this == &other) { - return *this; - } - - if (other.is_right_) { - if (!is_right_) { - error_.~E(); - new (&right_) T(other.right_); - } else { - right_ = other.right_; - } - } else { - if (is_right_) { - right_.~T(); - new (&error_) E(other.error_); - } else { - error_ = other.error_; - } - } - is_right_ = other.is_right_; - - return *this; - } - - expected_data& operator=(expected_data&& other) { - DCHECK_UB_NOT_NULL(&other) << __PRETTY_FUNCTION__; - DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__; - - if (this == &other) { - return *this; - } - - if (other.is_right_) { - if (!is_right_) { - error_.~E(); - new (&right_) T(std::move(other.right_)); - } else { - right_ = std::move(other.right_); - } - } else { - if (is_right_) { - right_.~T(); - new (&error_) E(std::move(other.error_)); - } else { - error_ = std::move(other.error_); - } - } - - other.moved_from_ = true; - is_right_ = other.is_right_; - - return *this; - } - }; - - // Trivial-destructor copy of the above struct. - // - // A separate copy is required because otherwise compilation fails with an error about - // the union having an implicitly deleted constructor. - // - // Having this implementation gives us the property that - // - // (is_trivially_destructible<T> && is_trivially_destructible<E> - // ==> is_trivially_destructible<expected<T, E>>) - template <typename T, typename E> - struct expected_data<T, E, /*DefineDestructor*/false> { - template <typename U> - constexpr expected_data(U&& either, expected_tag_right) - : right_{std::forward<U>(either)}, is_right_{true} {} - - template <typename U> - constexpr expected_data(U&& either, expected_tag_error) - : error_{std::forward<U>(either)}, is_right_{false} {} - - constexpr bool has_value() const { - return is_right_; - } - - constexpr const T& value() const { - return right_; - } - - constexpr T& value() { - return right_; - } - - constexpr const E& error() const { - return error_; - } - - constexpr E& error() { - return error_; - } - - // Using an "anonymous union" here allows non-trivial types to be stored. - union { - T right_; - E error_; - }; - - bool is_right_; - - ~expected_data() = default; - }; - - // Select between trivial and non-trivial implementations. Trivial implementations - // are more optimized and constexpr-compatible. - template <typename T, typename E> - using expected_pick_data_t = - expected_data<T, E, - !(std::is_trivially_destructible_v<T> && std::is_trivially_destructible_v<E>) >; -} // namespace detail - -template <typename E> -struct unexpected; - -// Subset of std::experimental::expected proposal (p0323r3). -template <typename T, typename E> -struct expected { - // Never-empty: expected<T,E> values have either 'T' or 'E' in them. - template <typename U = T, typename _ = std::enable_if_t<std::is_default_constructible_v<U>>> - constexpr expected() noexcept(noexcept(T{})) : expected(T{}) {} - - constexpr expected(const T& value) : data_{value, detail::expected_tag_right{}} {} - constexpr expected(T&& value) : data_{std::move(value), detail::expected_tag_right{}} {} - constexpr expected(const E& error) : data_{error, detail::expected_tag_error{}} {} - constexpr expected(E&& error) : data_{std::move(error), detail::expected_tag_error{}} {} - - template <typename G = E> - constexpr expected(unexpected<G> const& u) : expected{u.value()} {} - - template <typename G = E> - constexpr expected(unexpected<G>&& u) : expected{std::move(u.value())} {} - - explicit constexpr operator bool() const { - return has_value(); - } - - constexpr bool has_value() const { - return data_.has_value(); - } - - constexpr const T& operator*() const { - return data_.value(); - } - - constexpr T& operator*() { - return data_.value(); - } - - constexpr const T* _Nonnull operator->() const { - return &data_.value(); - } - - constexpr T* _Nonnull operator->() { - return &data_.value(); - } - - constexpr T& value() & { - CHECK(has_value()); - return data_.value(); - } - - constexpr const T& value() const & { - CHECK(has_value()); - return data_.value(); - } - - constexpr T&& value() && { - CHECK(has_value()); - return std::move(data_.value()); - } - - constexpr const T& value() const && { - CHECK(has_value()); - return std::move(data_.value()); - } - - constexpr E& error() { - DCHECK(!has_value()); - return data_.error(); - } - - constexpr const E& error() const { - DCHECK(!has_value()); - return data_.error(); - } - - // TODO: other functions such as operator=, unexpected, etc. - private: - detail::expected_pick_data_t<T, E> data_; -}; - -// TODO: move to tests file -namespace { - struct TestType { - TestType() {} - ~TestType() {} - }; - struct TestType2 : TestType {}; - - static_assert(std::is_trivially_destructible_v<expected<int, /*error*/double> >); - static_assert(!std::is_trivially_destructible_v<expected<TestType, /*error*/double> >); - static_assert(!std::is_trivially_destructible_v<expected<int, /*error*/TestType> >); - static_assert(!std::is_trivially_destructible_v<expected<TestType, /*error*/TestType2> >); - - // Ensure expected is constexpr-compatible. - struct TestCase { - static constexpr auto t1 = expected<int, double>{}; - }; -} // namespace <anonymous> - -template <typename E> -struct unexpected { - unexpected() = delete; - constexpr explicit unexpected(const E& error) : error_{error} {} - constexpr explicit unexpected(E&& error) : error_{std::move(error)} {} - constexpr const E& value() const& { return error_; } - constexpr E& value() & { return error_; } - constexpr E&& value() && { return std::move(error_); } - constexpr E const&& value() const&& { return std::move(error_); } - private: - E error_; -}; - -template <class E> -constexpr bool operator==(const unexpected<E>& x, const unexpected<E>& y) { - return x.value() == y.value(); -} - -template <class E> -constexpr bool operator!=(const unexpected<E>& x, const unexpected<E>& y) { - return !(x == y); -} - -// TODO: move below codes to separate utils file -// -// future C++20 implementation of std::identity -struct identity { - template <typename U> - constexpr auto operator()(U&& v) const noexcept { - return std::forward<U>(v); - } -}; - -// Given a lambda [...](auto&& var) {...} -// apply std::forward to 'var' to achieve perfect forwarding. -// -// Note that this doesn't work when var is a template type, i.e. -// template <typename T> -// void func(T&& tvar) {...} -// -// It would be invalid to use this macro with 'tvar' in that context. -#define IORAP_FORWARD_LAMBDA(var) std::forward<decltype(var)>(var) - -// Borrowed non-null pointer, i.e. we do not own the lifetime. -// -// Function calls: This pointer is not used past the call. -// Struct fields: This pointer is not used past the lifetime of the struct. -template <class T, class = std::enable_if_t<std::is_pointer<T>::value>> -using borrowed = T _Nonnull; -// TODO: need a DCHECK or high warning levels, since null is technically well-defined. - -} // namespace iorap - -#endif // IORAP_SRC_COMMON_EXPECTED_H_ diff --git a/src/common/introspection.h b/src/common/introspection.h deleted file mode 100644 index af64a20..0000000 --- a/src/common/introspection.h +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_COMMON_INTROSPECTION_H -#define IORAP_COMMON_INTROSPECTION_H - -/* - * Provide zero-cost compile-time introspection of struct member fields. - * - * Example: - * - * // Declaration - * struct PackageEvent { - * - * int type; - * std::string package_uri; - * std::string package_name; - * }; - * - * IORAP_INTROSPECT_ADAPT_STRUCT(PackageEvent, type, package_uri, package_name); - * - * // Usage - * { - * std::stringstream str; - * for_each_member_field(PackageEvent{123,"hello","world"}, [&](auto&& val) { - * str << val << ","; - * } - * CHECK_EQ("123,hello,world,"s, str.str()); - * } - */ - -#include "common/macros.h" -#include "common/type.h" - -#include <tuple> - -namespace iorap { -namespace introspect { - -template <auto value> -struct member_type; - -// Compile-time introspection data for a member-to-pointer. -// -// Example: -// using package_uri_member_type = member_type<&PackageEvent::&package_uri> -// int type = package_uri_member_type::value(PackageEvent{123,"hello","world"}); -// CHECK_EQ(type, 123); -template <typename T, typename F, F T::*member> -struct member_type<member> { - // The type of the struct this field is located in, e.g. 'struct XYZ {...}' -> XYZ. - static constexpr auto struct_t = type_c<T>; - // The type of the field, e.g. 'struct XYZ { int x; }' -> int. - static constexpr auto type = type_c<F>; - - // Allow a 'const U', 'volatile U', 'U&' etc here. - // Returns the value inside of 'U'. - template <typename U> - static constexpr decltype(auto) value(U&& v) { - static_assert(std::is_same_v<T, std::decay_t<U>>, "U must be cvref of T"); - - using U_noref = std::remove_reference_t<U>; - - // This casts from the regular non-const pointer-to-member to a potentially const/volatile - // pointer-to-member. - F U_noref::*safer_member = member; - - // Now dereference it, - return v.*safer_member; - // TODO: are we properly returning && for rvalue, & for lvalue refs, etc? - } - - static constexpr void set_value(typename decltype(struct_t)::type& s, - typename decltype(type)::type&& value) { - s.*member = std::forward<typename decltype(type)::type>(value); - } -}; - -// Given a self : T, where T has introspection-enabled support, T has some -// members m1, m2, m3, ... , mN. -// -// Invokes fun(self.*m1); fun(self.*m2); fun(self.*m3); ... ; fun(self.*mN). -template <typename T, typename F> -static constexpr void for_each_member_field_value(T&& self, F&& fun) { - constexpr auto members = introspect_members(type_c<std::decay_t<T>>); - // std::tuple<member_type<A>, member_type<B>, ...> - - // Warning: Don't use 'v=std::forward<V>(v)' as that actually captures-by-value. - for_each(members, [&fun, &self](auto&& type) mutable { - // Note that 'type' is a member_type - fun(type.value(std::forward<T>(self))); - }); -} - -// Given a self : T, where T has introspection-enabled support, T has some -// members m1, m2, m3, ... , mN. The basic_type of each member is t1, t2, t3, ..., tN. -// -// Invokes -// self.*m1 = fun(self, t1); -// self.*m2 = fun(self, t2); -// self.*m3 = fun(self, t3); -// ...; -// self.*mN = fun(self, tN). -template <typename T, typename F> -static constexpr void for_each_member_field_set_value(T&& self, F&& fun) { - constexpr auto members = introspect_members(type_c<std::decay_t<T>>); - // std::tuple<member_type<A>, member_type<B>, ...> - - // Warning: Don't use 'v=std::forward<V>(v)' as that actually captures-by-value. - for_each(members, [&fun, &self](auto&& type) mutable { - // Note that 'type' is a member_type - type.set_value(std::forward<T>(self), fun(type.type)); - }); -} - -} -} - -// Add compile-time introspection capabilities to a pre-existing struct or class. -// -// Arguments: Name, [Member1, Member2, ... MemberN] -// -// Example: -// -// struct Rectangle { -// int height; -// int width; -// }; -// -// IORAP_INTROSPECT_ADAPT_STRUCT(Rectangle, height, width); -// -// See also for_each_member_field_value. -#define IORAP_INTROSPECT_ADAPT_STRUCT(/*name, [member1, member2, member3, ...]*/...) \ - IORAP_INTROSPECT_ADAPT_STRUCT_IMPL(IORAP_PP_NARG(__VA_ARGS__), __VA_ARGS__) - -#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL(N, ...) \ - IORAP_PP_CONCAT(IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_, N)(__VA_ARGS__) - -// This simple implementation relies on the 'introspect_members' function being overloaded -// for the type<T> values. ADL is then applied to resolve the exact overload for any T, -// thus allowing this function definition to be in any namespace. - -// The auto signature must conform to: -// introspect_members(type<T>) -> std::tuple<member_type1, member_type_2, ...> - -// TODO: it would be nice to capture the name of the member as a string literal. -#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_1(TYPE) \ - static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ - return std::make_tuple();\ - } -#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_2(TYPE, m1) \ - static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ - return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{}\ - );\ - } - -#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_3(TYPE, m1, m2) \ - static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ - return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{},\ - ::iorap::introspect::member_type<&TYPE::m2>{}\ - ); \ - } - -#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_4(TYPE, m1, m2, m3) \ - static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ - return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{},\ - ::iorap::introspect::member_type<&TYPE::m2>{},\ - ::iorap::introspect::member_type<&TYPE::m3>{}\ - ); \ - } - -#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_5(TYPE, m1, m2, m3, m4) \ - static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ - return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{},\ - ::iorap::introspect::member_type<&TYPE::m2>{},\ - ::iorap::introspect::member_type<&TYPE::m3>{},\ - ::iorap::introspect::member_type<&TYPE::m4>{}\ - ); \ - } - -#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_6(TYPE, m1, m2, m3, m4, m5) \ - static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ - return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{},\ - ::iorap::introspect::member_type<&TYPE::m2>{},\ - ::iorap::introspect::member_type<&TYPE::m3>{},\ - ::iorap::introspect::member_type<&TYPE::m4>{},\ - ::iorap::introspect::member_type<&TYPE::m5>{}\ - ); \ - } - -// TODO: Consider using IORAP_PP_MAP - - -#endif // IORAP_COMMON_INTROSPECTION_H diff --git a/src/common/loggers.h b/src/common/loggers.h deleted file mode 100644 index 1dbb2dc..0000000 --- a/src/common/loggers.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_SRC_COMMON_LOGGERS -#define IORAP_SRC_COMMON_LOGGERS - -#include <android-base/logging.h> - -namespace iorap { -namespace common { - -// Log to both Stderr and Logd for convenience when running this from the command line. -class StderrAndLogdLogger { - public: - explicit StderrAndLogdLogger(android::base::LogId default_log_id = android::base::MAIN) -#ifdef __ANDROID__ - : logd_(default_log_id) -#endif - { - } - - void operator()(::android::base::LogId id, - ::android::base::LogSeverity sev, - const char* tag, - const char* file, - unsigned int line, - const char* message) { -#ifdef __ANDROID__ - logd_(id, sev, tag, file, line, message); -#endif - StderrLogger(id, sev, tag, file, line, message); - } - - private: -#ifdef __ANDROID__ - ::android::base::LogdLogger logd_; -#endif -}; - -} // namespace iorap -} // namespace common - -#endif diff --git a/src/common/macros.h b/src/common/macros.h deleted file mode 100644 index 1b5b38b..0000000 --- a/src/common/macros.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_COMMON_MACROS_H_ -#define IORAP_COMMON_MACROS_H_ - -// Expands to the # of arguments passed to the macro. -// For example `IORAP_PP_NARG(a,b,c)` -> 3 -// For example `IORAP_PP_NARG(x)` -> 1 -// -// The # of arguments must be >0 and <64. -#define IORAP_PP_NARG(...) \ - IORAP_PP_NARG_IMPL(__VA_ARGS__, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,) // NOLINT - -// Implementation explanation: Equivalent of: -// lst = __VA_ARGS__ + 64..1 -// return lst[64] -// The variadic arguments logically shift-off the hardcoded 64..1 list, revealing -// the # of arguments that were shifted. -#define IORAP_PP_NARG_IMPL(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15, e16, e17, e18, e19, e20, e21, e22, e23, e24, e25, e26, e27, e28, e29, e30, e31, e32, e33, e34, e35, e36, e37, e38, e39, e40, e41, e42, e43, e44, e45, e46, e47, e48, e49, e50, e51, e52, e53, e54, e55, e56, e57, e58, e59, e60, e61, e62, e63, N, ...) N - -#define IORAP_PP_CONCAT(lhs, rhs) \ - IORAP_PP_CONCAT_IMPL(lhs, rhs) - -#define IORAP_PP_CONCAT_IMPL(lhs, rhs) \ - lhs ## rhs - -// Expands to call FN around every argument passed it. -// MAP(f, a1, a2, .., an) -> f(a1) f(a2) f(a3) ... f(an) -#define IORAP_PP_MAP(FN, ...) IORAP_PP_MAP_IMPL(IORAP_PP_NARG(__VA_ARGS__), /*sep=blank*/, FN, __VA_ARGS__) -// Expands to call FN around every argument passed it. -// Every non-first/non-last function call also has a separator prefixed before it. -// MAP_SEP(f, sep, a1, a2, .., an) -> f(a1) sep() f(a2) sep() f(a3) sep() ... sep() f(an-1) f(an) -#define IORAP_PP_MAP_SEP(FN, sep, ...) IORAP_PP_MAP_IMPL(IORAP_PP_NARG(__VA_ARGS__), FN, sep, __VA_ARGS__) -#define IORAP_PP_MAP_IMPL(N, FN, sep, ...) IORAP_PP_CONCAT(IORAP_PP_MAP_IMPL_, N)(FN, sep, __VA_ARGS__) -#define IORAP_PP_MAP_IMPL_1(FN, sep, a1) FN(a1) -#define IORAP_PP_MAP_IMPL_2(FN, sep, a1, a2) FN(a1) sep() FN(a2) -#define IORAP_PP_MAP_IMPL_3(FN, sep, a1, a2, a3) FN(a1) sep() FN(a2) sep() FN(a3) -#define IORAP_PP_MAP_IMPL_4(FN, sep, a1, a2, a3, a4) FN(a1) sep() FN(a2) sep() FN(a3) sep() FN(a4) - -// Deferred comma. Useful with the above MAP function when you need a comma separator. -#define IORAP_PP_COMMA() , -// All arguments are ignored. -#define IORAP_PP_NOP(...) - -#endif // IORAP_COMMON_MACROS_H_
\ No newline at end of file diff --git a/src/common/printer.h b/src/common/printer.h deleted file mode 100644 index 272e459..0000000 --- a/src/common/printer.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_UTILS_PRINTER_H_ -#define IORAP_UTILS_PRINTER_H_ - -#include <utils/Printer.h> - -#include <string.h> - -namespace iorap::common { - -class StderrLogPrinter : public ::android::Printer { -public: - // Create a printer using the specified logcat and log priority - // - Unless ignoreBlankLines is false, print blank lines to logcat - // (Note that the default ALOG behavior is to ignore blank lines) - StderrLogPrinter(const char* logtag, - android_LogPriority priority = ANDROID_LOG_DEBUG, - const char* prefix = nullptr, - bool ignore_blank_lines = false) - : log_printer_{logtag, priority, prefix, ignore_blank_lines} { - logtag_ = logtag; - prefix_ = prefix; - ignore_blank_lines_ = ignore_blank_lines; - } - - // Print the specified line to logcat. No \n at the end is necessary. - virtual void printLine(const char* string) override { - if (ignore_blank_lines_ && strlen(string) == 0) { - return; - } - std::cerr << logtag_ << ": "; - if (prefix_ != nullptr) { - std::cerr << prefix_; - } - std::cerr << string << std::endl; - log_printer_.printLine(string); - } - private: - ::android::LogPrinter log_printer_; - const char* logtag_; - const char* prefix_; - bool ignore_blank_lines_; -}; - -} // namespace iorap::common - -#endif // IORAP_UTILS_PRINTER_H_ diff --git a/src/common/property.h b/src/common/property.h deleted file mode 100644 index d910193..0000000 --- a/src/common/property.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_UTILS_PROPERTY_H_ -#define IORAP_UTILS_PROPERTY_H_ - -#include <android-base/properties.h> -#include <server_configurable_flags/get_flags.h> - -namespace iorap::common { - -constexpr const char* ph_namespace = "runtime_native_boot"; - -inline bool IsTracingEnabled(const std::string& default_value) { - return server_configurable_flags::GetServerConfigurableFlag( - ph_namespace, - "iorap_perfetto_enable", - ::android::base::GetProperty("iorapd.perfetto.enable", default_value)) == "true"; -} - -inline bool IsReadAheadEnabled(const std::string& default_value) { - return server_configurable_flags::GetServerConfigurableFlag( - ph_namespace, - "iorap_readahead_enable", - ::android::base::GetProperty("iorapd.readahead.enable", default_value)) == "true"; -} - -inline bool ExcludeDexFiles(bool default_value) { - return ::android::base::GetBoolProperty("iorapd.exclude_dex_files", default_value); -} - -} // namespace iorap::common - -#endif // IORAP_UTILS_PROPERTY_H_ diff --git a/src/common/rx_async.h b/src/common/rx_async.h deleted file mode 100644 index d16f5ae..0000000 --- a/src/common/rx_async.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_COMMON_RX_ASYNC_H_ -#define IORAP_SRC_COMMON_RX_ASYNC_H_ - -#include "common/async_pool.h" - -#include <rxcpp/rx.hpp> - -namespace iorap::common { - -// Helper functions to operate with rx chains asynchronously. -class RxAsync { - public: - // Subscribe to the observable on a new thread asynchronously. - // If no observe_on/subscribe_on is used, the chain will execute - // on that new thread. - // - // Returns the composite_subscription which can be used to - // unsubscribe from if we want to abort the chain early. - template <typename T, typename U> - static rxcpp::composite_subscription SubscribeAsync( - AsyncPool& async_pool, - T&& observable, - U&& subscriber) { - rxcpp::composite_subscription subscription; - - async_pool.LaunchAsync([subscription, // safe copy: ref-counted - observable=std::forward<T>(observable), - subscriber=std::forward<U>(subscriber)]() mutable { - observable - .as_blocking() - .subscribe(subscription, - std::forward<decltype(subscriber)>(subscriber)); - }); - - return subscription; - } - - template <typename T, typename U, typename E> - static rxcpp::composite_subscription SubscribeAsync( - AsyncPool& async_pool, - T&& observable, - U&& on_next, - E&& on_error) { - rxcpp::composite_subscription subscription; - - async_pool.LaunchAsync([subscription, // safe copy: ref-counted - observable=std::forward<T>(observable), - on_next=std::forward<U>(on_next), - on_error=std::forward<E>(on_error)]() mutable { - observable - .as_blocking() - .subscribe(subscription, - std::forward<decltype(on_next)>(on_next), - std::forward<decltype(on_error)>(on_error)); - }); - - return subscription; - } -}; - -} // namespace iorap::common - -#endif // IORAP_SRC_COMMON_RX_ASYNC_H_ diff --git a/src/common/trace.h b/src/common/trace.h deleted file mode 100644 index a07d7bc..0000000 --- a/src/common/trace.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_COMMON_TRACE_H_ -#define IORAP_COMMON_TRACE_H_ - -#include <cutils/trace.h> - -#include <cstdio> -#include <sstream> - -namespace iorap { - -// TODO: refactor into utils/Trace.h - -class ScopedFormatTrace { - public: - template <typename ... Args> - ScopedFormatTrace(uint64_t tag, const char* fmt, Args&&... args) : tag_{tag} { - char buffer[1024]; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat-security" - snprintf(buffer, sizeof(buffer), fmt, args...); -#pragma GCC diagnostic pop - atrace_begin(tag, buffer); - } - - ~ScopedFormatTrace() { - atrace_end(tag_); - } - private: - uint64_t tag_; -}; - -} // namespace iorap - -#endif // IORAP_COMMON_TRACE_H_ diff --git a/src/common/type.h b/src/common/type.h deleted file mode 100644 index e856d73..0000000 --- a/src/common/type.h +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_COMMON_TYPE_H -#define IORAP_COMMON_TYPE_H - -#include <cstdint> -#include <tuple> - -namespace iorap { -namespace introspect { - -/* - * Simple types-as-value abstractions. - * - * Allow types to be passed as regular function parameters instead of using 'template' type - * parameters. - * - * This enables the following, more concise, pattern: - * ---------------------------- - * - * Traditional metaprogramming with template parameters: - * - * template <typename ... Args> - * struct get_num_params { - * static constexpr size_t value = sizeof...(Args); - * }; - * - * typename get_num_params<decltype("hello"), decltype("world")>::value == 2 - * typename get_num_params<decltype(int), decltype(int), decltype(int), decltype(int)>::value == 4 - * - * Alternative metaprogramming with values: - * - * constexpr auto get_num_params = [](auto&&... val) { return sizeof...(val); }; - * - * get_num_params("hello", "world") == 2 - * get_num_params(0,0,0,0) == 4 - */ - -/* - * A fully instantiated type wrapper. - * - * basic_type<T> is intended for overloading functions between different basic_types. - * type_c<T> is intended for instantiating new type wrappers as a short-hand (and not requiring - * typename). - * - * For basic_type<T> in particular it allows one to overload on basic_type<T> to handle specific - * types, and there's no requirement that T be constexpr, be default constructible, and no - * template specializations is necessary. - * - * void foo(basic_type<int>) { - * printf("int"); - * } - * - * template <typename T> - * void foo(basic_type<T>) { - * printf("everything else"); - * } - * - * as opposed to this verbosity - * - * template <typename T> - * struct foo { - * void operator() { - * printf("everything else"); - * } - * }; - * - * template <> - * struct foo<int> { - * void operator() { - * printf("int"); - * } - * }; - * - * OR this super-hack which works in rare situations - * - * void foo(int) { - * printf("int"); - * } - * - * template <typename T> - * void foo(T&&) { - * printf("everything else") - * } - * - * Note that invoking the last foo(T&&) is particularly challenging. declval<T> fails at compilation - * with a static_assert, so a real value has to be constructed that is immediately discarded. - */ -template <typename T> -struct basic_type { - using type = T; -}; - -template <typename T> -struct type_impl { - struct _ : basic_type<T> { }; -}; - -template <typename T> -using type = basic_type<T>; // typename type_impl<T>::_; // subclass of basic_type<T> -// TODO: why doesn't using type_impl::_ work with ADL? - -template <typename T> -using type_t = type<T>; - -template <typename T> -constexpr auto type_c = type<T>{}; - -template <auto X> -struct value_constant { - static constexpr auto value = X; -}; - -template <int X> -constexpr auto int_c = value_constant<X>{}; - -template <typename T> -constexpr bool dependent_false_v = false; - -// Emit a static_assert(false) if the else branch in an 'if constexpr' is taken. -// Needs a type as the first parameter. -#define STATIC_FAIL(T, msg) static_assert(::iorap::introspect::dependent_false_v<T>, msg) -// Emit a static_assert(false) if an else branch in an 'if constexpr' is taken, used with -// (e.g. auto) values instead of types. -#define STATIC_FAIL_DT(var, msg) STATIC_FAIL(decltype(var), msg) - -template <size_t i, typename Tuple, typename F> -static constexpr void for_each_impl(Tuple&& t, F&& f) { - if constexpr (i == std::tuple_size<std::decay_t<Tuple>>::value) { - return; - } else { - f(std::get<i>(std::forward<Tuple>(t))); - for_each_impl<i+1>(std::forward<Tuple>(t), std::forward<F>(f)); - } -} - -// for each Tuple<a1,a2,...,aN> invoke { f(a1); f(a2); ... ; f(aN); } -template <typename Tuple, typename F> -static constexpr void for_each(Tuple&& t, F&& f) { - return for_each_impl<0u>(std::forward<Tuple>(t), std::forward<F>(f)); -} - -// Perfect forwarding for structured binding. -// -// Example: -// auto&& [a,b] = whatever; -// return aliasing_forward<T>(a); -template <typename T, typename U> -constexpr decltype(auto) aliasing_forward(U&& val) noexcept { - if constexpr (std::is_lvalue_reference_v<T>) { - return val; - } else { - return std::move(val); - } -} - - -} // namespace introspect -} // namespace iorap - -#endif // IORAP_COMMON_TYPE_H diff --git a/src/compiler/compiler.cc b/src/compiler/compiler.cc deleted file mode 100644 index a546c5c..0000000 --- a/src/compiler/compiler.cc +++ /dev/null @@ -1,992 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "compiler/compiler.h" - -#include "common/debug.h" -#include "common/expected.h" - -#include "perfetto/rx_producer.h" // TODO: refactor BinaryWireProtobuf to separate header. -#include "inode2filename/inode.h" -#include "inode2filename/search_directories.h" -#include "serialize/protobuf_io.h" - -#include <android-base/unique_fd.h> -#include <android-base/parseint.h> -#include <android-base/file.h> - -#include <perfetto/trace/trace.pb.h> // ::perfetto::protos::Trace -#include <perfetto/trace/trace_packet.pb.h> // ::perfetto::protos::TracePacket - -#include "rxcpp/rx.hpp" -#include <iostream> -#include <fstream> -#include <optional> -#include <utility> -#include <regex> - -#include <sched.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <syscall.h> -#include <fcntl.h> -#include <unistd.h> - -namespace iorap::compiler { - -using Inode = iorap::inode2filename::Inode; -using InodeResult = iorap::inode2filename::InodeResult; -using SearchDirectories = iorap::inode2filename::SearchDirectories; - -template <typename T> -using ProtobufPtr = iorap::perfetto::ProtobufPtr<T>; - -struct PerfettoTraceProtoInfo { - /* The perfetto trace proto. */ - ::iorap::perfetto::PerfettoTraceProto proto; - /* - * The timestamp limit of the trace. - * It's used to truncate the trace file. - */ - uint64_t timestamp_limit_ns; - /* - * The pid of the app. - * If positive, it's used to filter out other page cache events. - */ - int32_t pid; -}; - -struct PerfettoTracePtrInfo { - /* Deserialized protobuf data containing the perfetto trace. */ - ProtobufPtr<::perfetto::protos::Trace> trace_ptr; - /* - * The timestamp limit of the trace. - * It's used to truncate the trace file. - */ - uint64_t timestamp_limit_ns; - /* - * The pid of the app. - * If positive, it's used to filter out other page cache events. - */ - int32_t pid; -}; - -// Attempt to read protobufs from the filenames. -// Emits one (or none) protobuf for each filename, in the same order as the filenames. -// On any errors, the items are dropped (errors are written to the error LOG). -// -// All work is done on the same Coordinator as the Subscriber. -template <typename ProtoT /*extends MessageLite*/> -auto/*observable<PerfettoTracePtrInfo>*/ ReadProtosFromFileNames( - rxcpp::observable<CompilationInput> file_infos) { - using BinaryWireProtoT = ::iorap::perfetto::PerfettoTraceProto; - - return file_infos - .map([](const CompilationInput& file_info) -> - std::optional<PerfettoTraceProtoInfo> { - LOG(VERBOSE) << "compiler::ReadProtosFromFileNames " << file_info.filename - << " TimeStampLimit "<< file_info.timestamp_limit_ns - << " Pid " << file_info.pid << " [begin]"; - std::optional<BinaryWireProtoT> maybe_proto = - BinaryWireProtoT::ReadFullyFromFile(file_info.filename); - if (!maybe_proto) { - LOG(ERROR) << "Failed to read file: " << file_info.filename; - return std::nullopt; - } - return {{std::move(maybe_proto.value()), file_info.timestamp_limit_ns, file_info.pid}}; - }) - .filter([](const std::optional<PerfettoTraceProtoInfo>& proto_info) { - return proto_info.has_value(); - }) - .map([](std::optional<PerfettoTraceProtoInfo>& proto_info) -> - PerfettoTraceProtoInfo { - return proto_info.value(); - }) // TODO: refactor to something that flattens the optional, and logs in one operator. - .map([](PerfettoTraceProtoInfo& proto_info) -> - std::optional<PerfettoTracePtrInfo> { - std::optional<ProtobufPtr<ProtoT>> t = proto_info.proto.template MaybeUnserialize<ProtoT>(); - if (!t) { - LOG(ERROR) << "Failed to parse protobuf: "; // TODO: filename. - return std::nullopt; - } - return {{std::move(t.value()), proto_info.timestamp_limit_ns, proto_info.pid}}; - }) - .filter([](const std::optional<PerfettoTracePtrInfo>& trace_info) { - return trace_info.has_value(); - }) - .map([](std::optional<PerfettoTracePtrInfo>& trace_info) -> - PerfettoTracePtrInfo { - LOG(VERBOSE) << "compiler::ReadProtosFromFileNames [success]"; - return trace_info.value(); - // TODO: protobufs have no move constructor. this might be inefficient? - }); - -/* - return filenames - .map([](const std::string& filename) { - LOG(VERBOSE) << "compiler::ReadProtosFromFileNames " << filename << " [begin]"; - std::optional<BinaryWireProtoT> maybe_proto = - BinaryWireProtoT::ReadFullyFromFile(filename); - if (!maybe_proto) { - LOG(ERROR) << "Failed to read file: " << filename; - } - - std::unique_ptr<BinaryWireProtoT> ptr; - if (maybe_proto) { - ptr.reset(new BinaryWireProtoT{std::move(*maybe_proto)}); - } - return ptr; - }) - .filter([](const std::unique_ptr<BinaryWireProtoT>& proto) { - return proto != nullptr; - }) - .map([](std::unique_ptr<BinaryWireProtoT>& proto) { - std::optional<ProtoT> t = proto->template MaybeUnserialize<ProtoT>(); - if (!t) { - LOG(ERROR) << "Failed to parse protobuf: "; // TODO: filename. - } - return t; - }) - .filter([](const std::optional<ProtoT>& proto) { - return proto.has_value(); - }) - .map([](std::optional<ProtoT> proto) -> ProtoT { - LOG(VERBOSE) << "compiler::ReadProtosFromFileNames [success]"; - return std::move(proto.value()); - // TODO: protobufs have no move constructor. this might be inefficient? - }); - */ -} - -auto/*observable<PerfettoTracePtrInfo>*/ ReadPerfettoTraceProtos( - std::vector<CompilationInput> file_infos) { - auto filename_obs = rxcpp::observable<>::iterate(std::move(file_infos)); - rxcpp::observable<PerfettoTracePtrInfo> obs = - ReadProtosFromFileNames<::perfetto::protos::Trace>(std::move(filename_obs)); - return obs; -} - -// A flattened data representation of an MmFileMap*FtraceEvent. -// This representation is used for streaming processing. -// -// Note: Perfetto applies a 'union' over all possible fields on all possible devices -// (and uses the max sizeof per-field). -// -// Since all protobuf fields are optional, fields not present on a particular device are always -// null -struct PageCacheFtraceEvent { - /* - * Ftrace buffer-specific - */ - uint32_t cpu; // e.g. 0-7 for the cpu core number. - - /* - * Ftrace-event general data - */ - - // Nanoseconds since an epoch. - // Epoch is configurable by writing into trace_clock. - // By default this timestamp is CPU local. - uint64_t timestamp; - // Kernel pid (do not confuse with userspace pid aka tgid) - uint32_t pid; - - // Tagged by our code while parsing the ftraces: - uint64_t timestamp_relative; // timestamp relative to first ftrace within a Trace protobuf. - bool add_to_page_cache; // AddToPageCache=true, DeleteFromPageCache=false. - - /* - * mm_filemap-specific data - * - * Fields are common: - * - MmFilemapAddToPageCacheFtraceEvent - * - MmFilemapDeleteFromPageCacheFtraceEvent - */ - uint64_t pfn; // page frame number (physical) - null on some devices, e.g. marlin - uint64_t i_ino; // inode number (use in conjunction with s_dev) - uint64_t index; // offset into file: this is a multiple of the page size (usually 4096). - uint64_t s_dev; // (dev_t) device number - uint64_t page; // struct page*. - null on some devices, e.g. blueline. - - Inode inode() const { - return Inode::FromDeviceAndInode(static_cast<dev_t>(s_dev), - static_cast<ino_t>(i_ino)); - } -}; - -std::ostream& operator<<(std::ostream& os, const PageCacheFtraceEvent& e) { - os << "{"; - os << "cpu:" << e.cpu << ","; - os << "timestamp:" << e.timestamp << ","; - os << "pid:" << e.pid << ","; - os << "timestamp_relative:" << e.timestamp_relative << ","; - os << "add_to_page_cache:" << e.add_to_page_cache << ","; - os << "pfn:" << e.pfn << ","; - os << "i_ino:" << e.i_ino << ","; - os << "index:" << e.index << ","; - os << "s_dev:" << e.s_dev << ","; - os << "page:" << e.page; - os << "}"; - - return os; -} - -/* - * Gets the start timestamp. - * - * It is the minimium timestamp. - */ -std::optional<uint64_t> GetStartTimestamp(const ::perfetto::protos::Trace& trace) { - std::optional<uint64_t> timestamp_relative_start; - // Traverse each timestamp to get the minimium one. - for (const ::perfetto::protos::TracePacket& packet : trace.packet()) { - if (packet.has_timestamp()) { - timestamp_relative_start = timestamp_relative_start? - std::min(*timestamp_relative_start, packet.timestamp()) : packet.timestamp(); - } - if (!packet.has_ftrace_events()) { - continue; - } - const ::perfetto::protos::FtraceEventBundle& ftrace_event_bundle = - packet.ftrace_events(); - for (const ::perfetto::protos::FtraceEvent& event : ftrace_event_bundle.event()) { - if (event.has_timestamp()) { - timestamp_relative_start = timestamp_relative_start? - std::min(*timestamp_relative_start, event.timestamp()) : event.timestamp(); - } - } - } - return timestamp_relative_start; -} - -/* - * sample blueline output: - * - * $ adb shell cat /d/tracing/events/filemap/mm_filemap_add_to_page_cache/format - * - * name: mm_filemap_add_to_page_cache - * ID: 178 - * format: - * field:unsigned short common_type; offset:0; size:2; signed:0; - * field:unsigned char common_flags; offset:2; size:1; signed:0; - * field:unsigned char common_preempt_count; offset:3; size:1; signed:0; - * field:int common_pid; offset:4; size:4; signed:1; - * - * field:unsigned long pfn; offset:8; size:8; signed:0; - * field:unsigned long i_ino; offset:16; size:8; signed:0; - * field:unsigned long index; offset:24; size:8; signed:0; - * field:dev_t s_dev; offset:32; size:4; signed:0; - * - * print fmt: "dev %d:%d ino %lx page=%p pfn=%lu ofs=%lu", ((unsigned int) ((REC->s_dev) >> 20)), - * ((unsigned int) ((REC->s_dev) & ((1U << 20) - 1))), REC->i_ino, - * (((struct page *)(((0xffffffffffffffffUL) - ((1UL) << ((39) - 1)) + 1) - - * ((1UL) << ((39) - 12 - 1 + 6))) - (memstart_addr >> 12)) + (REC->pfn)), - * REC->pfn, REC->index << 12 - */ - -auto /*observable<PageCacheFtraceEvent>*/ SelectPageCacheFtraceEvents( - PerfettoTracePtrInfo trace_info) { - const ::perfetto::protos::Trace& trace = *(trace_info.trace_ptr); - - constexpr bool kDebugFunction = true; - - return rxcpp::observable<>::create<PageCacheFtraceEvent>( - [trace=std::move(trace), - timestamp_limit_ns=trace_info.timestamp_limit_ns, - app_pid=trace_info.pid] - (rxcpp::subscriber<PageCacheFtraceEvent> sub) { - uint64_t timestamp = 0; - uint64_t timestamp_relative = 0; - - std::optional<uint64_t> timestamp_relative_start = GetStartTimestamp(trace); - uint32_t cpu = 0; - uint32_t pid = 0; - bool add_to_page_cache = true; - - auto on_next_page_cache_event = [&](const auto& mm_event) { - PageCacheFtraceEvent out; - out.timestamp = timestamp; - out.cpu = cpu; - out.pid = pid; - - out.timestamp_relative = timestamp_relative; - out.add_to_page_cache = add_to_page_cache; - - out.pfn = mm_event.pfn(); - out.i_ino = mm_event.i_ino(); - out.index = mm_event.index(); - out.s_dev = mm_event.s_dev(); - out.page = mm_event.page(); - - sub.on_next(std::move(out)); - }; - - for (const ::perfetto::protos::TracePacket& packet : trace.packet()) { - // Break out of all loops if we are unsubscribed. - if (!sub.is_subscribed()) { - if (kDebugFunction) LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents unsubscribe"; - return; - } - - if (kDebugFunction) LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents TracePacket"; - - if (packet.has_timestamp()) { - timestamp_relative_start = timestamp_relative_start.value_or(packet.timestamp()); - timestamp = packet.timestamp(); // XX: should we call 'has_timestamp()' ? - } else { - timestamp = 0; - } - - if (packet.has_ftrace_events()) { - const ::perfetto::protos::FtraceEventBundle& ftrace_event_bundle = - packet.ftrace_events(); - - cpu = ftrace_event_bundle.cpu(); // XX: has_cpu ? - - for (const ::perfetto::protos::FtraceEvent& event : ftrace_event_bundle.event()) { - // Break out of all loops if we are unsubscribed. - if (!sub.is_subscribed()) { - return; - } - - if (app_pid >= 0 && - (!event.has_pid() || - event.pid() != static_cast<uint32_t>(app_pid))) { - continue; - } - - if (event.has_timestamp()) { - timestamp = event.timestamp(); - if(timestamp > timestamp_limit_ns) { - LOG(VERBOSE) << "The timestamp is " << timestamp << - ", which exceeds the limit "<< timestamp_limit_ns; - continue; - } - } else { - DCHECK(packet.has_timestamp() == false) - << "Timestamp in outer packet but not inner packet"; - // XX: use timestamp from the perfetto TracePacket ??? - // REVIEWERS: not sure if this is ok, does it use the same clock source and - // is the packet data going to be the same clock sample as the Ftrace event? - } - - if (timestamp_relative_start){ - timestamp_relative = timestamp - *timestamp_relative_start; - } else { - timestamp_relative = 0; - } - - pid = event.pid(); // XX: has_pid ? - - if (event.has_mm_filemap_add_to_page_cache()) { - add_to_page_cache = true; - - const ::perfetto::protos::MmFilemapAddToPageCacheFtraceEvent& mm_event = - event.mm_filemap_add_to_page_cache(); - - on_next_page_cache_event(mm_event); - } else if (event.has_mm_filemap_delete_from_page_cache()) { - add_to_page_cache = false; - - const ::perfetto::protos::MmFilemapDeleteFromPageCacheFtraceEvent& mm_event = - event.mm_filemap_delete_from_page_cache(); - - on_next_page_cache_event(mm_event); - } - } - } else { - if (kDebugFunction) { - LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents no ftrace event bundle"; - } - } - } - - if (kDebugFunction) { - LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents#on_completed"; - } - - // Let subscriber know there are no more items. - sub.on_completed(); - }); -} - -auto /*observable<Inode*/ SelectDistinctInodesFromTraces( - rxcpp::observable<PerfettoTracePtrInfo> traces) { - // Emit only unique (s_dev, i_ino) pairs from all Trace protos. - auto obs = traces - .flat_map([](PerfettoTracePtrInfo trace) { - rxcpp::observable<PageCacheFtraceEvent> obs = SelectPageCacheFtraceEvents(std::move(trace)); - // FIXME: dont check this in - // return obs; - //return obs.take(100); // for faster development - return obs; - }) // TODO: Upstream bug? using []()::perfetto::protos::Trace&) causes a compilation error. - .map([](const PageCacheFtraceEvent& event) -> Inode { - return Inode::FromDeviceAndInode(static_cast<dev_t>(event.s_dev), - static_cast<ino_t>(event.i_ino)); - }) - .tap([](const Inode& inode) { - LOG(VERBOSE) << "SelectDistinctInodesFromTraces (pre-distinct): " << inode; - }) - .distinct() // observable<Inode>*/ - ; - - return obs; -} -// TODO: static assert checks for convertible return values. - -auto/*observable<InodeResult>*/ ResolveInodesToFileNames( - rxcpp::observable<Inode> inodes, - inode2filename::InodeResolverDependencies dependencies) { - std::shared_ptr<inode2filename::InodeResolver> inode_resolver = - inode2filename::InodeResolver::Create(std::move(dependencies)); - return inode_resolver->FindFilenamesFromInodes(std::move(inodes)); -} - -using InodeMap = std::unordered_map<Inode, std::string /*filename*/>; -auto /*just observable<InodeMap>*/ ReduceResolvedInodesToMap( - rxcpp::observable<InodeResult> inode_results) { - return inode_results.reduce( - InodeMap{}, - [](InodeMap m, InodeResult result) { - if (result) { - LOG(VERBOSE) << "compiler::ReduceResolvedInodesToMap insert " << result; - m.insert({std::move(result.inode), std::move(result.data.value())}); - } else { - // TODO: side stats for how many of these are failed to resolve? - LOG(WARNING) << "compiler: Failed to resolve inode, " << result; - } - return m; - }, - [](InodeMap m) { - return m; // TODO: use an identity function - }); // emits exactly 1 InodeMap value. -} - -struct ResolvedPageCacheFtraceEvent { - std::string filename; - PageCacheFtraceEvent event; -}; - -std::ostream& operator<<(std::ostream& os, const ResolvedPageCacheFtraceEvent& e) { - os << "{"; - os << "filename:\"" << e.filename << "\","; - os << e.event; - os << "}"; - - return os; -} - -struct CombinedState { - CombinedState() = default; - explicit CombinedState(InodeMap inode_map) : inode_map{std::move(inode_map)} {} - explicit CombinedState(PageCacheFtraceEvent event) : ftrace_event{std::move(event)} {} - - CombinedState(InodeMap inode_map, PageCacheFtraceEvent event) - : inode_map(std::move(inode_map)), - ftrace_event{std::move(event)} {} - - std::optional<InodeMap> inode_map; - std::optional<PageCacheFtraceEvent> ftrace_event; - - bool HasAll() const { - return inode_map.has_value() && ftrace_event.has_value(); - } - - const InodeMap& GetInodeMap() const { - DCHECK(HasAll()); - return inode_map.value(); - } - - InodeMap& GetInodeMap() { - DCHECK(HasAll()); - return inode_map.value(); - } - - const PageCacheFtraceEvent& GetEvent() const { - DCHECK(HasAll()); - return ftrace_event.value(); - } - - PageCacheFtraceEvent& GetEvent() { - DCHECK(HasAll()); - return ftrace_event.value(); - } - - void Merge(CombinedState&& other) { - if (other.inode_map) { - inode_map = std::move(other.inode_map); - } - if (other.ftrace_event) { - ftrace_event = std::move(other.ftrace_event); - } - } -}; - -std::ostream& operator<<(std::ostream& os, const CombinedState& s) { - os << "CombinedState{inode_map:"; - if (s.inode_map) { - os << "|sz=" << (s.inode_map.value().size()) << "|"; - } else { - os << "(null)"; - } - os << ",event:"; - if (s.ftrace_event) { - //os << s.ftrace_event.value().timestamp << "ns"; - os << s.ftrace_event.value(); - } else { - os << "(null)"; - } - os << "}"; - return os; -} - -auto/*observable<ResolvedPageCacheFtraceEvent>*/ ResolvePageCacheEntriesFromProtos( - rxcpp::observable<PerfettoTracePtrInfo> traces, - inode2filename::InodeResolverDependencies dependencies) { - - // 1st chain = emits exactly 1 InodeMap. - - // [proto, proto, proto...] -> [inode, inode, inode, ...] - auto/*observable<Inode>*/ distinct_inodes = SelectDistinctInodesFromTraces(traces); - rxcpp::observable<Inode> distinct_inodes_obs = distinct_inodes.as_dynamic(); - // [inode, inode, inode, ...] -> [(inode, {filename|error}), ...] - auto/*observable<InodeResult>*/ inode_names = ResolveInodesToFileNames(distinct_inodes_obs, - std::move(dependencies)); - // rxcpp has no 'join' operators, so do a manual join with concat. - auto/*observable<InodeMap>*/ inode_name_map = ReduceResolvedInodesToMap(inode_names); - - // 2nd chain = emits all PageCacheFtraceEvent - auto/*observable<PageCacheFtraceEvent>*/ page_cache_ftrace_events = traces - .flat_map([](PerfettoTracePtrInfo trace) { - rxcpp::observable<PageCacheFtraceEvent> obs = SelectPageCacheFtraceEvents(std::move(trace)); - return obs; - }); - - auto inode_name_map_precombine = inode_name_map - .map([](InodeMap inode_map) { - LOG(VERBOSE) << "compiler::ResolvePageCacheEntriesFromProtos#inode_name_map_precombine "; - return CombinedState{std::move(inode_map)}; - }); - - auto page_cache_ftrace_events_precombine = page_cache_ftrace_events - .map([](PageCacheFtraceEvent event) { - LOG(VERBOSE) - << "compiler::ResolvePageCacheEntriesFromProtos#page_cache_ftrace_events_precombine " - << event; - return CombinedState{std::move(event)}; - }); - - // Combine 1st+2nd chain. - // - // concat subscribes to each observable, waiting until its completed, before subscribing - // to the next observable and waiting again. - // - // During all this, every #on_next is immediately forwarded to the downstream observables. - // In our case, we want to block until InodeNameMap is ready, and re-iterate all ftrace events. - auto/*observable<ResolvedPageCacheFtraceEvent>*/ resolved_events = inode_name_map_precombine - .concat(page_cache_ftrace_events_precombine) - .scan(CombinedState{}, - [](CombinedState current_state, CombinedState delta_state) { - LOG(VERBOSE) << "compiler::ResolvePageCacheEntriesFromProtos#scan " - << "current=" << current_state << "," - << "delta=" << delta_state; - // IT0 = (,) + (InodeMap,) - // IT1 = (InodeMap,) + (,Event) - // IT2..N = (InodeMap,Event1) + (,Event2) - current_state.Merge(std::move(delta_state)); - return current_state; - }) - .filter([](const CombinedState& state) { - return state.HasAll(); - }) - .map([](CombinedState& state) -> std::optional<ResolvedPageCacheFtraceEvent> { - PageCacheFtraceEvent& event = state.GetEvent(); - const InodeMap& inode_map = state.GetInodeMap(); - - auto it = inode_map.find(event.inode()); - if (it != inode_map.end()) { - std::string filename = it->second; - LOG(VERBOSE) << "compiler::ResolvePageCacheEntriesFromProtos combine_latest " << event; - return ResolvedPageCacheFtraceEvent{std::move(filename), std::move(event)}; - } else { - LOG(ERROR) << "compiler: FtraceEvent's inode did not have resolved filename: " << event; - return std::nullopt; - } - }) - .filter( - [](std::optional<ResolvedPageCacheFtraceEvent> maybe_event) { - return maybe_event.has_value(); - }) - .map([](std::optional<ResolvedPageCacheFtraceEvent> maybe_event) { - return std::move(maybe_event.value()); - }); - // -> observable<ResolvedPageCacheFtraceEvent> - - return resolved_events; -} - -namespace detail { -bool multiless_one(const std::string& a, const std::string& b) { - return std::lexicographical_compare(a.begin(), a.end(), - b.begin(), b.end()); -} - -template <typename T> -constexpr bool multiless_one(T&& a, T&& b) { // a < b - using std::less; // ADL - return less<std::decay_t<T>>{}(std::forward<T>(a), std::forward<T>(b)); -} - -constexpr bool multiless() { - return false; // [] < [] is always false. -} - -template <typename T, typename ... Args> -constexpr bool multiless(T&& a, T&& b, Args&&... args) { - if (a != b) { - return multiless_one(std::forward<T>(a), std::forward<T>(b)); - } else { - return multiless(std::forward<Args>(args)...); - } -} - -} // namespace detail - -// Return [A0...An] < [B0...Bn] ; vector-like scalar comparison of each field. -// Arguments are passed in the order A0,B0,A1,B1,...,An,Bn. -template <typename ... Args> -constexpr bool multiless(Args&&... args) { - return detail::multiless(std::forward<Args>(args)...); -} - -struct CompilerPageCacheEvent { - std::string filename; - uint64_t timestamp_relative; // use relative timestamp because absolute values aren't comparable - // across different trace protos. - // relative timestamps can be said to be 'approximately' comparable. - // assuming we compare the same application startup's trace times. - bool add_to_page_cache; // AddToPageCache=true, DeleteFromPageCache=false. - uint64_t index; // offset into file: this is a multiple of the page size (usually 4096). - - // All other data from the ftrace is dropped because we don't currently use it in the - // compiler algorithms. - - CompilerPageCacheEvent() = default; - CompilerPageCacheEvent(const ResolvedPageCacheFtraceEvent& resolved) - : CompilerPageCacheEvent(resolved.filename, resolved.event) { - } - - CompilerPageCacheEvent(ResolvedPageCacheFtraceEvent&& resolved) - : CompilerPageCacheEvent(std::move(resolved.filename), std::move(resolved.event)) { - } - - // Compare all fields (except the timestamp field). - static bool LessIgnoringTimestamp(const CompilerPageCacheEvent& a, - const CompilerPageCacheEvent& b) { - return multiless(a.filename, b.filename, - a.add_to_page_cache, b.add_to_page_cache, - a.index, b.index); - } - - // Compare all fields. Timestamps get highest precedence. - bool operator<(const CompilerPageCacheEvent& rhs) const { - return multiless(timestamp_relative, rhs.timestamp_relative, - filename, rhs.filename, - add_to_page_cache, rhs.add_to_page_cache, - index, rhs.index); - } - - private: - CompilerPageCacheEvent(std::string filename, const PageCacheFtraceEvent& event) - : filename(std::move(filename)), - timestamp_relative(event.timestamp_relative), - add_to_page_cache(event.add_to_page_cache), - index(event.index) { - } -}; - -std::ostream& operator<<(std::ostream& os, const CompilerPageCacheEvent& e) { - os << "{"; - os << "filename:\"" << e.filename << "\","; - os << "timestamp:" << e.timestamp_relative << ","; - os << "add_to_page_cache:" << e.add_to_page_cache << ","; - os << "index:" << e.index; - os << "}"; - return os; -} - -// Filter an observable chain of 'ResolvedPageCacheFtraceEvent' -// into an observable chain of 'ResolvedPageCacheFtraceEvent'. -// -// Any items emitted by the input chain that match the regular expression -// specified by blacklist_filter are not emitted into the output chain. -auto/*observable<ResolvedPageCacheFtraceEvent>*/ ApplyBlacklistToPageCacheEvents( - rxcpp::observable<ResolvedPageCacheFtraceEvent> resolved_events, - std::optional<std::string> blacklist_filter) { - bool has_re = blacklist_filter.has_value(); - // default regex engine is ecmascript. - std::regex reg_exp{blacklist_filter ? *blacklist_filter : std::string("")}; - - return resolved_events.filter( - [reg_exp, has_re](const ResolvedPageCacheFtraceEvent& event) { - if (!has_re) { - return true; - } - // Remove any entries that match the regex in --blacklist-filter/-bf. - bool res = std::regex_search(event.filename, reg_exp); - if (res) { - LOG(VERBOSE) << "Blacklist filter removed '" << event.filename << "' from chain."; - } - return !res; - }); -} - -// Compile an observable chain of 'ResolvedPageCacheFtraceEvent' into -// an observable chain of distinct, timestamp-ordered, CompilerPageCacheEvent. -// -// This is a reducing operation: No items are emitted until resolved_events is completed. -auto/*observable<CompilerPageCacheEvent>*/ CompilePageCacheEvents( - rxcpp::observable<ResolvedPageCacheFtraceEvent> resolved_events) { - - struct CompilerPageCacheEventIgnoringTimestampLess { - bool operator()(const CompilerPageCacheEvent& lhs, - const CompilerPageCacheEvent& rhs) const { - return CompilerPageCacheEvent::LessIgnoringTimestamp(lhs, rhs); - } - }; - - // Greedy O(N) compilation algorithm. - // - // This produces an inoptimal result (e.g. a small timestamp - // that might occur only 1% of the time nevertheless wins out), but the - // algorithm itself is quite simple, and doesn't require any heuristic tuning. - - // First pass: *Merge* into set that ignores the timestamp value for order, but retains - // the smallest timestamp value if the same key is re-inserted. - using IgnoreTimestampForOrderingSet = - std::set<CompilerPageCacheEvent, CompilerPageCacheEventIgnoringTimestampLess>; - // Second pass: *Sort* data by smallest timestamp first. - using CompilerPageCacheEventSet = - std::set<CompilerPageCacheEvent>; - - return resolved_events - .map( - [](ResolvedPageCacheFtraceEvent event) { - // Drop all the extra metadata like pid, cpu, etc. - // When we merge we could keep a list of the original data, but there is no advantage - // to doing so. - return CompilerPageCacheEvent{std::move(event)}; - } - ) - .reduce( - IgnoreTimestampForOrderingSet{}, - [](IgnoreTimestampForOrderingSet set, CompilerPageCacheEvent event) { - // Add each event to the set, keying by everything but the timestamp. - // If the key is already inserted, re-insert with the smaller timestamp value. - auto it = set.find(event); - - if (it == set.end()) { - // Need to insert new element. - set.insert(std::move(event)); - } else if (it->timestamp_relative > event.timestamp_relative) { - // Replace existing element: the new element has a smaller timestamp. - it = set.erase(it); - // Amortized O(1) time if insertion happens in the position before the hint. - set.insert(it, std::move(event)); - } // else: Skip insertion. Element already present with the minimum timestamp. - - return set; - }, - [](IgnoreTimestampForOrderingSet set) { - // Extract all elements from 'set', re-insert into 'ts_set'. - // The values are now ordered by timestamp (and then the rest of the fields). - CompilerPageCacheEventSet ts_set; - ts_set.merge(std::move(set)); - - - std::shared_ptr<CompilerPageCacheEventSet> final_set{ - new CompilerPageCacheEventSet{std::move(ts_set)}}; - return final_set; - // return ts_set; - }) // observable<CompilerPageCacheEventSet> (just) - .flat_map( - [](std::shared_ptr<CompilerPageCacheEventSet> final_set) { - // TODO: flat_map seems to make a copy of the parameter _every single iteration_ - // without the shared_ptr it would just make a copy of the set every time it went - // through the iterate function. - // Causing absurdly slow compile times x1000 slower than we wanted. - // TODO: file a bug upstream and/or fix upstream. - CompilerPageCacheEventSet& ts_set = *final_set; - // [](CompilerPageCacheEventSet& ts_set) { - LOG(DEBUG) << "compiler: Merge-pass completed (" << ts_set.size() << " entries)."; - //return rxcpp::sources::iterate(std::move(ts_set)); - return rxcpp::sources::iterate(ts_set).map([](CompilerPageCacheEvent e) { return e; }); - } - ); // observable<CompilerPageCacheEvent> -} - -/** Makes a vector of info that includes filename and timestamp limit. */ -std::vector<CompilationInput> MakeCompilationInputs( - std::vector<std::string> input_file_names, - std::vector<uint64_t> timestamp_limit_ns, - std::vector<int32_t> pids){ - // If the timestamp limit is empty, set the limit to max value - // for each trace file. - if (timestamp_limit_ns.empty()) { - for (size_t i = 0; i < input_file_names.size(); i++) { - timestamp_limit_ns.push_back(std::numeric_limits<uint64_t>::max()); - } - } - - // If the pids is empty, set all of them to -1. Because negative pid means any. - if (pids.empty()) { - for (size_t i = 0; i < input_file_names.size(); i++) { - pids.push_back(-1); - } - } - - DCHECK_EQ(input_file_names.size(), timestamp_limit_ns.size()); - std::vector<CompilationInput> file_infos; - for (size_t i = 0; i < input_file_names.size(); i++) { - file_infos.push_back({input_file_names[i], timestamp_limit_ns[i], pids[i]}); - } - return file_infos; -} - -bool PerformCompilation(std::vector<CompilationInput> perfetto_traces, - std::string output_file_name, - bool output_proto, - std::optional<std::string> blacklist_filter, - inode2filename::InodeResolverDependencies dependencies) { - auto trace_protos = ReadPerfettoTraceProtos(std::move(perfetto_traces)); - auto resolved_events = ResolvePageCacheEntriesFromProtos(std::move(trace_protos), - std::move(dependencies)); - auto filtered_events = - ApplyBlacklistToPageCacheEvents(std::move(resolved_events), blacklist_filter); - auto compiled_events = CompilePageCacheEvents(std::move(filtered_events)); - - std::ofstream ofs; - if (!output_file_name.empty()) { - - if (!output_proto) { - ofs.open(output_file_name); - - if (!ofs) { - LOG(ERROR) << "compiler: Failed to open output file for writing: " << output_file_name; - return false; - } - } - } - - auto trace_file_proto = serialize::ArenaPtr<serialize::proto::TraceFile>::Make(); - - // Fast lookup of filename -> FileIndex id. - std::unordered_map<std::string, int64_t /*file handle id*/> file_path_map; - int64_t file_handle_id = 0; - - int counter = 0; - compiled_events - // .as_blocking() - .tap([&](CompilerPageCacheEvent& event) { - if (!output_proto) { - return; - } - - if (!event.add_to_page_cache) { - // Skip DeleteFromPageCache events, they are only used for intermediate. - return; - } - - DCHECK(trace_file_proto->mutable_index() != nullptr); - serialize::proto::TraceFileIndex& index = *trace_file_proto->mutable_index(); - int64_t file_handle; - - // Add TraceFileIndexEntry if it doesn't exist. - - auto it = file_path_map.find(event.filename); - if (it == file_path_map.end()) { - file_handle = file_handle_id++; - file_path_map[event.filename] = file_handle; - - serialize::proto::TraceFileIndexEntry* entry = index.add_entries(); - DCHECK(entry != nullptr); - entry->set_id(file_handle); - entry->set_file_name(event.filename); - - if (kIsDebugBuild) { - int i = static_cast<int>(file_handle); - const serialize::proto::TraceFileIndexEntry& entry_ex = index.entries(i); - DCHECK_EQ(entry->id(), entry_ex.id()); - DCHECK_EQ(entry->file_name(), entry_ex.file_name()); - } - } else { - file_handle = it->second; - } - int kPageSize = 4096; // TODO: don't hardcode the page size. - - int entry_size = trace_file_proto->list().entries_size(); - bool merged = false; - if (entry_size > 0) { - serialize::proto::TraceFileEntry* entry = - trace_file_proto->mutable_list()->mutable_entries(entry_size-1); - if (entry->index_id() == file_handle && - entry->file_offset() + entry->file_length() == - static_cast<int64_t>(event.index) * kPageSize) { - entry->set_file_length(entry->file_length() + kPageSize); - merged = true; - } - } - - if (!merged) { - // Add TraceFileEntry. - DCHECK(trace_file_proto->mutable_list() != nullptr); - serialize::proto::TraceFileEntry* entry = trace_file_proto->mutable_list()->add_entries(); - DCHECK(entry != nullptr); - - entry->set_index_id(file_handle); - // Page index -> file offset in bytes. - entry->set_file_offset(static_cast<int64_t>(event.index) * kPageSize); - entry->set_file_length(kPageSize); - } - }) - .subscribe([&](CompilerPageCacheEvent event) { - if (!output_proto) { - if (output_file_name.empty()) { - LOG(INFO) << "CompilerPageCacheEvent" << event << std::endl; - } else { - ofs << event << "\n"; // TODO: write in proto format instead. - } - } - ++counter; - }); - - if (output_proto) { - LOG(DEBUG) << "compiler: WriteFully to begin into " << output_file_name; - ::google::protobuf::MessageLite& message = *trace_file_proto.get(); - if (auto res = serialize::ProtobufIO::WriteFully(message, output_file_name); !res) { - errno = res.error(); - PLOG(ERROR) << "compiler: Failed to write protobuf to file: " << output_file_name; - return false; - } else { - LOG(INFO) << "compiler: Wrote protobuf " << output_file_name; - } - } - - LOG(DEBUG) << "compiler: Compilation completed (" << counter << " events)."; - - return true; -} - -} // namespace iorap::compiler diff --git a/src/compiler/compiler.h b/src/compiler/compiler.h deleted file mode 100644 index 243ab36..0000000 --- a/src/compiler/compiler.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_COMPILER_COMPILER_H_ -#define IORAP_SRC_COMPILER_COMPILER_H_ - -#include "inode2filename/inode_resolver.h" - -#include <optional> -#include <string> -#include <vector> - -namespace iorap::compiler { -struct CompilationInput { - /* The name of the perfetto trace. */ - std::string filename; - /* - * The timestamp limit of the trace. - * It's used to truncate the trace file. - */ - uint64_t timestamp_limit_ns; - - /* - * The pid of the app. - * If positive, it's used to filter out other page cache events. - */ - int32_t pid; -}; - -// Compile one or more perfetto TracePacket protobufs that are stored on the filesystem -// by the filenames given with `input_file_names` and timestamp limit given with -// timestamp_limit_ns. -// -// For each input file name and timestamp limit, ignore any events from the input that -// exceed the associated timestamp limit. -// -// If blacklist_filter is non-null, ignore any file entries whose file path matches the -// regular expression in blacklist_filter. -// -// The result is stored into the file path specified by `output_file_name`. -// -// This is a blocking function which returns only when compilation finishes. The return value -// determines success/failure. -// -// Operation is transactional -- that is if there is a failure, `output_file_name` is untouched. -bool PerformCompilation(std::vector<iorap::compiler::CompilationInput> perfetto_traces, - std::string output_file_name, - bool output_proto, - std::optional<std::string> blacklist_filter, - inode2filename::InodeResolverDependencies dependencies); - -// The size and order of timestamp_limit_ns should match that of -// input_file_names, if not empty. -// If timestamp_limit_ns is empty, will use the max uint64_t. -std::vector<CompilationInput> MakeCompilationInputs( - std::vector<std::string> input_file_names, - std::vector<uint64_t> timestamp_limit_ns, - std::vector<int32_t> pids); -} - -#endif // IORAP_SRC_COMPILER_COMPILER_H_ diff --git a/src/compiler/main.cc b/src/compiler/main.cc deleted file mode 100644 index 3036936..0000000 --- a/src/compiler/main.cc +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/cmd_utils.h" -#include "common/debug.h" -#include "compiler/compiler.h" -#include "inode2filename/inode_resolver.h" - -#include <android-base/parseint.h> -#include <android-base/logging.h> - -#include <iostream> -#include <limits> -#include <optional> -#include <string> - -#if defined(IORAP_COMPILER_MAIN) - -namespace iorap::compiler { - -// Log everything to stderr. -// Log errors and higher to logd. -class StderrAndLogdErrorLogger { - public: - explicit StderrAndLogdErrorLogger(android::base::LogId default_log_id = android::base::MAIN) -#ifdef __ANDROID__ - : logd_(default_log_id) -#endif - { - } - - void operator()(::android::base::LogId id, - ::android::base::LogSeverity sev, - const char* tag, - const char* file, - unsigned int line, - const char* message) { -#ifdef __ANDROID__ - if (static_cast<int>(sev) >= static_cast<int>(::android::base::ERROR)) { - logd_(id, sev, tag, file, line, message); - } -#endif - StderrLogger(id, sev, tag, file, line, message); - } - - private: -#ifdef __ANDROID__ - ::android::base::LogdLogger logd_; -#endif -}; - -void Usage(char** argv) { - std::cerr << "Usage: " << argv[0] << " [--output-proto=output.pb] input1.pb [input2.pb ...]" << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Request a compilation of multiple inputs (format: PerfettoTraceProto)." << std::endl; - std::cerr << " The result is a TraceFile, representing a merged compiled trace with inodes resolved." << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Optional flags:" << std::endl; - std::cerr << " --help,-h Print this Usage." << std::endl; - std::cerr << " --denylist-filter,-df Specify regex acting as a denylist filter." - << std::endl; - std::cerr << " Filepaths matching this regex are removed from the output file." << std::endl; - std::cerr << " --output-text,-ot Output ascii text instead of protobuf (default off)." << std::endl; - std::cerr << " --output-proto $,-op $ TraceFile tracebuffer output file (default stdout)." << std::endl; - std::cerr << " --inode-textcache $,-it $ Resolve inode->filename from textcache (disables diskscan)." << std::endl; - std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; - std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; - std::cerr << " --pid,-p Set the pid for the compiled trace" << std::endl; - std::cerr << " --timestamp_limit_ns,-tl Set the limit timestamp in nanoseconds for the compiled trace. " - "The order and size of the timestamp should match that of " - "the input trace files. If not specified at all, All of" - "the timestamps are set to max."<< std::endl; - exit(1); -} - -int Main(int argc, char** argv) { - android::base::InitLogging(argv); - android::base::SetLogger(StderrAndLogdErrorLogger{}); - - bool wait_for_keystroke = false; - bool enable_verbose = false; - - std::optional<std::string> arg_blacklist_filter; - std::string arg_output_proto; - bool arg_output_text = false; - std::optional<std::string> arg_inode_textcache; - - std::vector<uint64_t> timestamp_limit_ns; - std::vector<int32_t> pids; - - if (argc == 1) { - // Need at least 1 input file to do anything. - Usage(argv); - } - - std::vector<std::string> arg_input_filenames; - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - bool has_arg_next = (arg+1)<argc; - std::string arg_next = has_arg_next ? argv[arg+1] : ""; - - if (argstr == "--help" || argstr == "-h") { - Usage(argv); - } else if (argstr == "--output-proto" || argstr == "-op") { - if (!has_arg_next) { - std::cerr << "Missing --output-proto <value>" << std::endl; - return 1; - } - arg_output_proto = arg_next; - ++arg; - } else if (argstr == "--output-text" || argstr == "-ot") { - arg_output_text = true; - } else if (argstr == "--inode-textcache" || argstr == "-it") { - if (!has_arg_next) { - std::cerr << "Missing --inode-textcache <value>" << std::endl; - return 1; - } - arg_inode_textcache = arg_next; - ++arg; - } else if (argstr == "--denylist-filter" || argstr == "-df") { - if (!has_arg_next) { - std::cerr << "Missing --denylist-filter <value>" << std::endl; - return 1; - } - arg_blacklist_filter = arg_next; - ++arg; - } - else if (argstr == "--verbose" || argstr == "-v") { - enable_verbose = true; - } else if (argstr == "--wait" || argstr == "-w") { - wait_for_keystroke = true; - } else if (argstr == "--pid" || argstr == "-p") { - if (!has_arg_next) { - std::cerr << "Missing --pid <value>" << std::endl; - return 1; - } - int32_t pid; - if (!::android::base::ParseInt<int32_t>(arg_next, &pid)) { - std::cerr << "Invalid --pid "<< arg_next << std::endl; - return 1; - } - pids.push_back(pid); - ++arg; - } else if (argstr == "--timestamp_limit_ns" || argstr == "-tl") { - if (!has_arg_next) { - std::cerr << "Missing --timestamp_limit_ns <value>" << std::endl; - return 1; - } - uint64_t timestamp; - if (!::android::base::ParseUint<uint64_t>(arg_next, ×tamp)) { - std::cerr << "Invalid --timestamp-limit-ns "<< arg_next << std::endl; - return 1; - } - timestamp_limit_ns.push_back(timestamp); - ++arg; - } else { - arg_input_filenames.push_back(argstr); - } - } - - if (!timestamp_limit_ns.empty() && - timestamp_limit_ns.size() != arg_input_filenames.size()) { - std::cerr << "The size of timestamp limits doesn't match the size of input files." - << std::endl; - return 1; - } - - if (!pids.empty() && pids.size() != arg_input_filenames.size()) { - std::cerr << "The size of pids doesn't match the size of input files." - << std::endl; - return 1; - } - if (enable_verbose) { - android::base::SetMinimumLogSeverity(android::base::VERBOSE); - - LOG(VERBOSE) << "Verbose check"; - LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; - } else { - android::base::SetMinimumLogSeverity(android::base::DEBUG); - } - - // Useful to attach a debugger... - // 1) $> iorap.cmd.compiler -w <args> - // 2) $> gdbclient <pid> - if (wait_for_keystroke) { - LOG(INFO) << "Self pid: " << getpid(); - LOG(INFO) << "Press any key to continue..."; - std::cin >> wait_for_keystroke; - } - - auto system_call = std::make_unique<SystemCallImpl>(); - - using namespace inode2filename; - - InodeResolverDependencies ir_dependencies; - // Passed from command-line. - if (arg_inode_textcache) { - ir_dependencies.data_source = DataSourceKind::kTextCache; - ir_dependencies.text_cache_filename = arg_inode_textcache; - ir_dependencies.verify = VerifyKind::kNone; // required for determinism. - } else { - ir_dependencies.data_source = DataSourceKind::kDiskScan; - LOG(WARNING) << "--inode-textcache unspecified. " - << "Inodes will be resolved by scanning the disk, which makes compilation " - << "non-deterministic."; - } - // TODO: add command line. - ir_dependencies.root_directories.push_back("/system"); - ir_dependencies.root_directories.push_back("/apex"); - ir_dependencies.root_directories.push_back("/data"); - ir_dependencies.root_directories.push_back("/vendor"); - ir_dependencies.root_directories.push_back("/product"); - ir_dependencies.root_directories.push_back("/metadata"); - // Hardcoded. - if (iorap::common::GetBoolEnvOrProperty("iorap.inode2filename.out_of_process", true)) { - ir_dependencies.process_mode = ProcessMode::kOutOfProcessIpc; - } else { - ir_dependencies.process_mode = ProcessMode::kInProcessDirect; - } - ir_dependencies.system_call = /*borrowed*/system_call.get(); - - int return_code = 0; - std::vector<CompilationInput> perfetto_traces = - MakeCompilationInputs(arg_input_filenames, timestamp_limit_ns, pids); - return_code = - !PerformCompilation(std::move(perfetto_traces), - std::move(arg_output_proto), - !arg_output_text, - arg_blacklist_filter, - std::move(ir_dependencies)); - - // Uncomment this if we want to leave the process around to inspect it from adb shell. - // sleep(100000); - - // 0 -> successfully wrote the proto out to file. - // 1 -> failed along the way (#on_error and also see the error logs). - return return_code; -} - -} // namespace iorap::compiler - -int main(int argc, char** argv) { - return ::iorap::compiler::Main(argc, argv); -} - -#endif // IORAP_COMPILER_MAIN diff --git a/src/db/app_component_name.h b/src/db/app_component_name.h deleted file mode 100644 index e364163..0000000 --- a/src/db/app_component_name.h +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_DB_APP_COMPONENT_NAME_H_ -#define IORAP_SRC_DB_APP_COMPONENT_NAME_H_ - -namespace iorap::db { - -struct AppComponentName { - std::string package; - std::string activity_name; - - // Turns the activity name to the fully qualified name. - // For example, if the activity name is ".MainActivity" and the package is - // foo.bar. Then the fully qualified name is foo.bar.MainActivity. - AppComponentName Canonicalize() const { - if (!activity_name.empty() && activity_name[0] == '.') { - return {package, package + activity_name}; - } - return {package, activity_name}; - } - - static bool HasAppComponentName(const std::string& s) { - return s.find('/') != std::string::npos; - } - - // "com.foo.bar/.A" -> {"com.foo.bar", ".A"} - static AppComponentName FromString(const std::string& s) { - constexpr const char delimiter = '/'; - - if (!HasAppComponentName(s)) { - return {std::move(s), ""}; - } - - std::string package = s.substr(0, s.find(delimiter)); - - std::string activity_name = s; - activity_name.erase(0, s.find(delimiter) + sizeof(delimiter)); - - return {std::move(package), std::move(activity_name)}; - } - - // {"com.foo.bar", ".A"} -> "com.foo.bar/.A" - std::string ToString() const { - return package + "/" + activity_name; - } - - /* - * '/' is encoded into %2F - * '%' is encoded into %25 - * - * This allows the component name to be be used as a file name - * ('/' is illegal due to being a path separator) with minimal - * munging. - */ - - // "com.foo.bar%2F.A%25" -> {"com.foo.bar", ".A%"} - static AppComponentName FromUrlEncodedString(const std::string& s) { - std::string cpy = s; - Replace(cpy, "%2F", "/"); - Replace(cpy, "%25", "%"); - - return FromString(cpy); - } - - // {"com.foo.bar", ".A%"} -> "com.foo.bar%2F.A%25" - std::string ToUrlEncodedString() const { - std::string s = ToString(); - Replace(s, "%", "%25"); - Replace(s, "/", "%2F"); - return s; - } - - /* - * '/' is encoded into @@ - * '%' is encoded into ^^ - * - * Two purpose: - * 1. This allows the package name to be used as a file name - * ('/' is illegal due to being a path separator) with minimal - * munging. - * 2. This allows the package name to be used in .mk file because - * '%' is a special char and cannot be easily escaped in Makefile. - * - * This is a workround for test purpose. - * Only package name is used because activity name varies on - * different testing framework. - * Hopefully, the double "@@" and "^^" are not used in other cases. - */ - // {"com.foo.bar", ".A%"} -> "com.foo.bar" - std::string ToMakeFileSafeEncodedPkgString() const { - std::string s = package; - Replace(s, "/", "@@"); - Replace(s, "%", "^^"); - return s; - } - - private: - static bool Replace(std::string& str, const std::string& from, const std::string& to) { - // TODO: call in a loop to replace all occurrences, not just the first one. - const size_t start_pos = str.find(from); - if (start_pos == std::string::npos) { - return false; - } - - str.replace(start_pos, from.length(), to); - - return true; -} -}; - -inline std::ostream& operator<<(std::ostream& os, const AppComponentName& name) { - os << name.ToString(); - return os; -} - -} // namespace iorap::db - -#endif // IORAP_SRC_DB_APP_COMPONENT_NAME_H_ diff --git a/src/db/clean_up.cc b/src/db/clean_up.cc deleted file mode 100644 index de1412d..0000000 --- a/src/db/clean_up.cc +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "db/clean_up.h" - -#include <android-base/file.h> - -#include <cstdio> -#include <filesystem> -#include <fstream> -#include <iostream> -#include <limits> -#include <optional> -#include <string> -#include <vector> - -#include "db/file_models.h" -#include "db/models.h" - -namespace iorap::db { - -void CleanUpFilesForActivity(const db::DbHandle& db, - const db::VersionedComponentName& vcn) { - LOG(DEBUG) << "Clean up files for activity " << vcn.GetActivity(); - // Remove perfetto traces. - std::vector<db::RawTraceModel> raw_traces = - db::RawTraceModel::SelectByVersionedComponentName(db, vcn); - for (db::RawTraceModel raw_trace : raw_traces) { - std::string file_path = raw_trace.file_path; - LOG(DEBUG) << "Remove file: " << file_path; - std::filesystem::remove(file_path.c_str()); - raw_trace.Delete(); - } - - // Remove compiled traces. - std::optional<db::PrefetchFileModel> prefetch_file = - db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn); - - if (prefetch_file) { - std::string file_path = prefetch_file->file_path; - LOG(DEBUG) << "Remove file: " << file_path; - std::filesystem::remove(file_path.c_str()); - prefetch_file->Delete(); - } -} - -void CleanUpFilesForPackage(const db::DbHandle& db, - int package_id, - const std::string& package_name, - int64_t version) { - LOG(DEBUG) << "Clean up files for package " << package_name << " with version " - << version; - std::vector<db::ActivityModel> activities = - db::ActivityModel::SelectByPackageId(db, package_id); - - for (db::ActivityModel activity : activities) { - db::VersionedComponentName vcn{package_name, activity.name, version}; - CleanUpFilesForActivity(db, vcn); - } -} - -void CleanUpFilesForPackage(const db::DbHandle& db, - const std::string& package_name, - int64_t version) { - std::optional<PackageModel> package = - PackageModel::SelectByNameAndVersion(db, package_name.c_str(), version); - - if (!package) { - LOG(DEBUG) << "No package to clean up " << package_name << " with version " << version; - return; - } - - CleanUpFilesForPackage(db, package->id, package_name, version); -} - -void CleanUpFilesForDb(const db::DbHandle& db) { - std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db); - for (db::PackageModel package : packages) { - CleanUpFilesForPackage(db, package.id, package.name, package.version); - } -} - -void CleanUpFilesForPackage(const std::string& db_path, - const std::string& package_name) { - iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path); - db::DbHandle db{db_schema.db()}; - CleanUpFilesForPackage(db, package_name); -} - - -void CleanUpFilesForPackage(const db::DbHandle& db, - const std::string& package_name) { - std::vector<PackageModel> packages = PackageModel::SelectByName(db, package_name.c_str());; - - for (PackageModel& package : packages) { - CleanUpFilesForPackage(db, package.id, package.name, package.version); - } - - if (packages.empty()) { - LOG(DEBUG) << "No package rows to clean up " << package_name; - } -} - -} // namespace iorap::db diff --git a/src/db/clean_up.h b/src/db/clean_up.h deleted file mode 100644 index 08b7bde..0000000 --- a/src/db/clean_up.h +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_DB_CLEANER_H_ -#define IORAP_SRC_DB_CLEANER_H_ - -#include <android/content/pm/IPackageManagerNative.h> - -#include <string> -#include <vector> - -#include "binder/package_version_map.h" -#include "db/file_models.h" - -namespace iorap::db { - -// Clean up perfetto traces and compiled traces in disk and rows -// in raw_traces and prefetch_files in the db. -void CleanUpFilesForDb(const db::DbHandle& db); - -// Clean up perfetto traces and compiled traces in disk and rows -// in raw_traces and prefetch_files in the db for a package id. -void CleanUpFilesForPackage(const db::DbHandle& db, - int package_id, - const std::string& package_name, - int64_t version); - -// Clean up all package rows (and files) associated with a package by name. -void CleanUpFilesForPackage(const std::string& db_path, - const std::string& package_name); - -// Clean up all package rows (and files) associated with a package by name. -void CleanUpFilesForPackage(const db::DbHandle& db, - const std::string& package_name); -// Clean up perfetto traces and compiled traces in disk and rows -// in raw_traces and prefetch_files in the db for a package name -// and version. -void CleanUpFilesForPackage(const db::DbHandle& db, - const std::string& package_name, - int64_t version); -} // namespace iorap::db - -#endif // IORAP_SRC_DB_CLEANER_H_ diff --git a/src/db/file_models.cc b/src/db/file_models.cc deleted file mode 100644 index d962d49..0000000 --- a/src/db/file_models.cc +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/cmd_utils.h" -#include "db/file_models.h" -#include "db/models.h" - -#include <android-base/file.h> -#include <android-base/properties.h> - -#include <algorithm> -#include <sstream> -#include <sys/stat.h> -#include <sys/types.h> -#include <time.h> -#include <unistd.h> - -namespace iorap::db { - -static constexpr const char* kRootPathProp = "iorapd.root.dir"; -static const unsigned int kPerfettoMaxTraces = - ::android::base::GetUintProperty("iorapd.perfetto.max_traces", /*default*/2u); - -static uint64_t GetTimeNanoseconds() { - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - - uint64_t now_ns = (now.tv_sec * 1000000000LL + now.tv_nsec); - return now_ns; -} - -static bool IsDir(const std::string& dirpath) { - struct stat st; - if (stat(dirpath.c_str(), &st) == 0) { - if (S_ISDIR(st.st_mode)) { - return true; - } - } - return false; -} - -// Given some path /a/b/c create all of /a, /a/b, /a/b/c/ recursively. -static bool MkdirWithParents(const std::string& path) { - size_t prev_end = 0; - while (prev_end < path.size()) { - size_t next_end = path.find('/', prev_end + 1); - - std::string dir_path = path.substr(0, next_end); - if (!IsDir(dir_path)) { -#if defined(_WIN32) - int ret = mkdir(dir_path.c_str()); -#else - mode_t old_mask = umask(0); - // The user permission is 5 to allow system server to - // read the files. No other users could do that because - // the upper directory only allows system server and iorapd - // to access. Also selinux rules prevent other users to - // read files here. - int ret = mkdir(dir_path.c_str(), 0755); - umask(old_mask); -#endif - if (ret != 0) { - PLOG(ERROR) << "failed to create dir " << dir_path; - return false; - } - } - prev_end = next_end; - - if (next_end == std::string::npos) { - break; - } - } - return true; -} - -FileModelBase::FileModelBase(VersionedComponentName vcn) - : vcn_{std::move(vcn)} { - root_path_ = common::GetEnvOrProperty(kRootPathProp, - /*default*/"/data/misc/iorapd"); -} - -std::string FileModelBase::BaseDir() const { - std::stringstream ss; - - ss << root_path_ << "/" << vcn_.GetPackage() << "/"; - ss << vcn_.GetVersion(); - ss << "/"; - ss << vcn_.GetActivity() << "/"; - ss << SubDir(); - - return ss.str(); -} - -std::string FileModelBase::FilePath() const { - std::stringstream ss; - ss << BaseDir(); - ss << "/"; - ss << BaseFile(); - - return ss.str(); -} - -bool FileModelBase::MkdirWithParents() { - LOG(VERBOSE) << "MkdirWithParents: " << BaseDir(); - return ::iorap::db::MkdirWithParents(BaseDir()); -} - -PerfettoTraceFileModel::PerfettoTraceFileModel(VersionedComponentName vcn, - uint64_t timestamp) - : FileModelBase{std::move(vcn)}, timestamp_{timestamp} { -} - -PerfettoTraceFileModel PerfettoTraceFileModel::CalculateNewestFilePath(VersionedComponentName vcn) { - uint64_t timestamp = GetTimeNanoseconds(); - return PerfettoTraceFileModel{vcn, timestamp}; -} - -std::string PerfettoTraceFileModel::BaseFile() const { - std::stringstream ss; - ss << timestamp_ << ".perfetto_trace.pb"; - return ss.str(); -} - -bool PerfettoTraceFileModel::NeedMorePerfettoTraces(DbHandle& db, - VersionedComponentName vcn) { - std::vector<RawTraceModel> raw_traces = - RawTraceModel::SelectByVersionedComponentName(db, vcn); - - size_t raw_traces_size = raw_traces.size(); - LOG(VERBOSE) << "The number of perfetto traces is " - << raw_traces_size - << " The cap is " - << kPerfettoMaxTraces ; - return raw_traces_size < kPerfettoMaxTraces; -} - -void PerfettoTraceFileModel::DeleteOlderFiles(DbHandle& db, VersionedComponentName vcn) { - std::vector<RawTraceModel> raw_traces = - RawTraceModel::SelectByVersionedComponentName(db, vcn); - - if (WOULD_LOG(VERBOSE)) { - size_t raw_traces_size = raw_traces.size(); - for (size_t i = 0; i < raw_traces_size; ++i) { - LOG(VERBOSE) << "DeleteOlderFiles - selected " << raw_traces[i]; - } - LOG(VERBOSE) << "DeleteOlderFiles - queried total " << raw_traces_size << " records"; - } - - size_t items_to_delete = 0; - if (raw_traces.size() > kPerfettoMaxTraces) { - items_to_delete = raw_traces.size() - kPerfettoMaxTraces; - } else { - LOG(VERBOSE) << "DeleteOlderFiles - don't delete older raw traces, too few files: " - << " wanted at least " << kPerfettoMaxTraces << ", but got " << raw_traces.size(); - } - - for (size_t i = 0; i < items_to_delete; ++i) { - RawTraceModel& raw_trace = raw_traces[i]; // sorted ascending -> items to delete are first. - std::string err_msg; - - if (!::android::base::RemoveFileIfExists(raw_trace.file_path, /*out*/&err_msg)) { - LOG(ERROR) << "Failed to remove raw trace file: " << raw_trace.file_path - << ", reason: " << err_msg; - } else { - raw_trace.Delete(); - LOG(DEBUG) << "Deleted raw trace for " << vcn << " at " << raw_trace.file_path; - } - } -} - -CompiledTraceFileModel::CompiledTraceFileModel(VersionedComponentName vcn) - : FileModelBase{std::move(vcn)} { -} - -CompiledTraceFileModel CompiledTraceFileModel::CalculateNewestFilePath(VersionedComponentName vcn) { - return CompiledTraceFileModel{vcn}; -} - -std::string CompiledTraceFileModel::BaseFile() const { - return "compiled_trace.pb"; -} - -} // namespace iorap::db diff --git a/src/db/file_models.h b/src/db/file_models.h deleted file mode 100644 index 984c206..0000000 --- a/src/db/file_models.h +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_DB_FILE_MODELS_H_ -#define IORAP_SRC_DB_FILE_MODELS_H_ - -#include <optional> -#include <ostream> -#include <string> - -namespace iorap::db { - -struct VersionedComponentName { - VersionedComponentName(std::string package, - std::string activity, - int64_t version) - : package_{std::move(package)}, - activity_{std::move(activity)}, - version_{version} { - } - - const std::string& GetPackage() const { - return package_; - } - - const std::string& GetActivity() const { - return activity_; - } - - int GetVersion() const { - return version_; - } - - private: - std::string package_; - std::string activity_; - int64_t version_; -}; - -inline std::ostream& operator<<(std::ostream& os, const VersionedComponentName& vcn) { - os << vcn.GetPackage() << "/" << vcn.GetActivity() << "@" << vcn.GetVersion(); - return os; -} - -class FileModelBase { - protected: - FileModelBase(VersionedComponentName vcn); - - virtual std::string SubDir() const = 0; - // Include the last file component only (/a/b/c.txt -> c.txt) - virtual std::string BaseFile() const = 0; - - virtual ~FileModelBase() {} - - public: - virtual std::string ModelName() const = 0; - // Return the absolute file path to this FileModel. - std::string FilePath() const; - - // Include the full path minus the basefile (/a/b/c.txt -> /a/b) - std::string BaseDir() const; - - // Create the base dir recursively (e.g. mkdir -p $basedir). - bool MkdirWithParents(); - private: - VersionedComponentName vcn_; - std::string subdir_; - std::string filename_; - std::string root_path_; -}; - -inline std::ostream& operator<<(std::ostream& os, const FileModelBase& fmb) { - os << fmb.ModelName() << "{'" << fmb.FilePath() << "'}"; - return os; -} - -class DbHandle; - -class PerfettoTraceFileModel : public FileModelBase { - protected: - virtual std::string ModelName() const override { return "PerfettoTrace"; } - virtual std::string SubDir() const override { return "raw_traces"; } - virtual std::string BaseFile() const override; - - PerfettoTraceFileModel(VersionedComponentName vcn, - uint64_t timestamp); - - public: - static PerfettoTraceFileModel CalculateNewestFilePath(VersionedComponentName vcn); - static void DeleteOlderFiles(DbHandle& db, VersionedComponentName vcn); - static bool NeedMorePerfettoTraces(DbHandle& db, VersionedComponentName vcn); - - virtual ~PerfettoTraceFileModel() {} - private: - uint64_t timestamp_; -}; - -class CompiledTraceFileModel : public FileModelBase { - protected: - virtual std::string ModelName() const override { return "CompiledTrace"; } - virtual std::string SubDir() const override { return "compiled_traces"; } - virtual std::string BaseFile() const override; - - CompiledTraceFileModel(VersionedComponentName vcn); - - public: - static CompiledTraceFileModel CalculateNewestFilePath(VersionedComponentName vcn); - - virtual ~CompiledTraceFileModel() {} -}; - -} // namespace iorap::db - -#endif // IORAP_SRC_DB_FILE_MODELS_H_ diff --git a/src/db/main.cc b/src/db/main.cc deleted file mode 100644 index 2edd961..0000000 --- a/src/db/main.cc +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/debug.h" -#include "common/loggers.h" -#include "db/app_component_name.h" -#include "db/models.h" - -#include <android-base/parseint.h> -#include <android-base/logging.h> - -#include <iostream> -#include <optional> -#include <string_view> -#include <string> -#include <vector> - -#include <sqlite3.h> - -#include <signal.h> - -namespace iorap::db { - -void Usage(char** argv) { - std::cerr << "Usage: " << argv[0] << " <path-to-sqlite.db>" << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Interface with the iorap sqlite database and issue commands." << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Optional flags:" << std::endl; - std::cerr << " --help,-h Print this Usage." << std::endl; - std::cerr << " --register-raw-trace,-rrt Register raw trace file path." << std::endl; - std::cerr << " --register-compiled-trace,-rct Register compiled trace file path." << std::endl; - std::cerr << " --insert-component,-ic Add component if it doesn't exist." << std::endl; - std::cerr << " --initialize,-i Initialize new database." << std::endl; - std::cerr << " --rescan,-rs Update all from canonical directories." << std::endl; - std::cerr << " --prune,-pr Remove any stale file paths." << std::endl; - std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; - std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; - exit(1); -} - -void error_log_sqlite_callback(void *pArg, int iErrCode, const char *zMsg) { - LOG(ERROR) << "SQLite error (" << iErrCode << "): " << zMsg; -} - -const constexpr int64_t kNoVersion = -1; - -int Main(int argc, char** argv) { - // Go to system logcat + stderr when running from command line. - android::base::InitLogging(argv, iorap::common::StderrAndLogdLogger{android::base::SYSTEM}); - - bool wait_for_keystroke = false; - bool enable_verbose = false; - - bool command_format_text = false; // false = binary. - - int arg_input_fd = -1; - int arg_output_fd = -1; - - std::vector<std::string> arg_input_filenames; - bool arg_use_sockets = false; - - std::vector<std::pair<std::string,std::string>> arg_register_raw_trace; - std::vector<std::pair<std::string,std::string>> arg_register_compiled_trace; - - std::vector<std::string> arg_insert_component; - - bool arg_initialize = false; - bool arg_rescan = false; - bool arg_prune = false; - - LOG(VERBOSE) << "argparse: argc=" << argc; - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - bool has_arg_next = (arg+1)<argc; - std::string arg_next = has_arg_next ? argv[arg+1] : ""; - - bool has_arg_next_next = (arg+2)<argc; - std::string arg_next_next = has_arg_next_next ? argv[arg+2] : ""; - - LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; - - if (argstr == "--help" || argstr == "-h") { - Usage(argv); - } else if (argstr == "--register-raw-trace" || argstr == "-rrt") { - if (!has_arg_next_next) { - LOG(ERROR) << "--register-raw-trace <component/name> <filepath>"; - Usage(argv); - } - arg_register_raw_trace.push_back({arg_next, arg_next_next}); - } else if (argstr == "--register-compiled-trace" || argstr == "-rct") { - if (!has_arg_next_next) { - LOG(ERROR) << "--register-compiled-trace <component/name> <filepath>"; - Usage(argv); - } - arg_register_compiled_trace.push_back({arg_next, arg_next_next}); - } else if (argstr == "--insert-component" || argstr == "-ic") { - if (!has_arg_next) { - LOG(ERROR) << "--insert-component <component/name>"; - Usage(argv); - } - arg_insert_component.push_back(arg_next); - } else if (argstr == "--initialize" || argstr == "-i") { - arg_initialize = true; - } else if (argstr == "--rescan" || argstr == "-rs") { - arg_rescan = true; - } else if (argstr == "--prune" || argstr == "-pr") { - arg_prune = true; - } else if (argstr == "--verbose" || argstr == "-v") { - enable_verbose = true; - } else if (argstr == "--wait" || argstr == "-w") { - wait_for_keystroke = true; - } else { - arg_input_filenames.push_back(argstr); - } - } - - if (arg_input_filenames.empty()) { - LOG(ERROR) << "Missing positional filename to a sqlite database."; - Usage(argv); - } - - if (enable_verbose) { - android::base::SetMinimumLogSeverity(android::base::VERBOSE); - - LOG(VERBOSE) << "Verbose check"; - LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; - } else { - android::base::SetMinimumLogSeverity(android::base::DEBUG); - } - - LOG(VERBOSE) << "argparse: argc=" << argc; - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - - LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; - } - - // Useful to attach a debugger... - // 1) $> iorap.cmd.readahead -w <args> - // 2) $> gdbclient <pid> - if (wait_for_keystroke) { - LOG(INFO) << "Self pid: " << getpid(); - - raise(SIGSTOP); - // LOG(INFO) << "Press any key to continue..."; - // std::cin >> wait_for_keystroke; - } - - // auto system_call = std::make_unique<SystemCallImpl>(); - // TODO: mock readahead calls? - // - // Uncomment this if we want to leave the process around to inspect it from adb shell. - // sleep(100000); - - int return_code = 0; - - LOG(VERBOSE) << "Hello world"; - - - do { - SchemaModel schema_model = SchemaModel::GetOrCreate(arg_input_filenames[0]); - DbHandle db = schema_model.db(); - if (arg_initialize) { - // Drop tables and restart from scratch. All rows are effectively dropped. - schema_model.Reinitialize(); - } - - for (const auto& component_and_file_name : arg_register_raw_trace) { - AppComponentName component_name = AppComponentName::FromString(component_and_file_name.first); - const std::string& file_path = component_and_file_name.second; - - LOG(VERBOSE) << "--register-raw-trace " << component_name << ", file_path: " << file_path; - - std::optional<ActivityModel> activity = - ActivityModel::SelectOrInsert(db, - component_name.package, - kNoVersion, - component_name.activity_name); - DCHECK(activity.has_value()); - LOG(DEBUG) << "Component selected/inserted: " << *activity; - } - - for (const std::string& component : arg_insert_component) { - AppComponentName component_name = AppComponentName::FromString(component); - - LOG(VERBOSE) << "raw component: " << component; - LOG(VERBOSE) << "package: " << component_name.package; - LOG(VERBOSE) << "activity name: " << component_name.activity_name; - - LOG(VERBOSE) << "--insert-component " << component_name; - - std::optional<ActivityModel> activity = - ActivityModel::SelectOrInsert(db, - component_name.package, - kNoVersion, - component_name.activity_name); - - DCHECK(activity.has_value()); - LOG(DEBUG) << "Component selected/inserted: " << *activity; - } - } while (false); - - LOG(VERBOSE) << "main: Terminating"; - - // 0 -> successfully executed all commands. - // 1 -> failed along the way (#on_error and also see the error logs). - return return_code; -} - -} // namespace iorap::db - -#if defined(IORAP_DB_MAIN) -int main(int argc, char** argv) { - return ::iorap::db::Main(argc, argv); -} -#endif // IORAP_DB_MAIN diff --git a/src/db/models.cc b/src/db/models.cc deleted file mode 100644 index 851a061..0000000 --- a/src/db/models.cc +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "db/models.h" - -namespace iorap::db { - -std::optional<DbHandle> SchemaModel::s_singleton_; - -} // namespace iorap::db diff --git a/src/db/models.h b/src/db/models.h deleted file mode 100644 index 79823fd..0000000 --- a/src/db/models.h +++ /dev/null @@ -1,1140 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_DB_MODELS_H_ -#define IORAP_SRC_DB_MODELS_H_ - -#include "clean_up.h" -#include "file_models.h" - -#include <android-base/logging.h> -#include <utils/String8.h> - -#include <filesystem> -#include <iostream> -#include <optional> -#include <ostream> -#include <string> -#include <sstream> -#include <type_traits> -#include <vector> - -#include <sqlite3.h> - -namespace iorap::db { - -const constexpr int kDbVersion = 3; - -struct SqliteDbDeleter { - void operator()(sqlite3* db) { - if (db != nullptr) { - LOG(VERBOSE) << "sqlite3_close for: " << db; - sqlite3_close(db); - } - } -}; - -class DbHandle { - public: - // Take over ownership of sqlite3 db. - explicit DbHandle(sqlite3* db) - : db_{std::shared_ptr<sqlite3>{db, SqliteDbDeleter{}}}, - mutex_{std::make_shared<std::mutex>()} { - } - - sqlite3* get() { - return db_.get(); - } - - operator sqlite3*() { - return db_.get(); - } - - std::mutex& mutex() { - return *mutex_.get(); - } - - private: - std::shared_ptr<sqlite3> db_; - std::shared_ptr<std::mutex> mutex_; -}; - -class ScopedLockDb { - public: - ScopedLockDb(std::mutex& mutex) : mutex(mutex) { - mutex.lock(); - } - - ScopedLockDb(DbHandle& handle) : ScopedLockDb(handle.mutex()) { - } - - ~ScopedLockDb() { - mutex.unlock(); - } - private: - std::mutex& mutex; -}; - -class DbStatement { - public: - template <typename ... Args> - static DbStatement Prepare(DbHandle db, const std::string& sql, Args&&... args) { - return Prepare(db, sql.c_str(), std::forward<Args>(args)...); - } - - template <typename ... Args> - static DbStatement Prepare(DbHandle db, const char* sql, Args&&... args) { - DCHECK(db.get() != nullptr); - DCHECK(sql != nullptr); - - // LOG(VERBOSE) << "Prepare DB=" << db.get(); - - sqlite3_stmt* stmt = nullptr; - int rc = sqlite3_prepare_v2(db.get(), sql, -1, /*out*/&stmt, nullptr); - - DbStatement db_stmt{db, stmt}; - DCHECK(db_stmt.CheckOk(rc)) << sql; - db_stmt.BindAll(std::forward<Args>(args)...); - - return db_stmt; - } - - DbStatement(DbHandle db, sqlite3_stmt* stmt) : db_(db), stmt_(stmt) { - } - - sqlite3_stmt* get() { - return stmt_; - } - - DbHandle db() { - return db_; - } - - // Successive BindAll calls *do not* start back at the 0th bind position. - template <typename T, typename ... Args> - void BindAll(T&& arg, Args&&... args) { - Bind(std::forward<T>(arg)); - BindAll(std::forward<Args>(args)...); - } - - void BindAll() {} - - template <typename T> - void Bind(const std::optional<T>& value) { - if (value) { - Bind(*value); - } else { - BindNull(); - } - } - - void Bind(bool value) { - CheckOk(sqlite3_bind_int(stmt_, bind_counter_++, value)); - } - - void Bind(int value) { - CheckOk(sqlite3_bind_int(stmt_, bind_counter_++, value)); - } - - void Bind(uint64_t value) { - CheckOk(sqlite3_bind_int64(stmt_, bind_counter_++, static_cast<int64_t>(value))); - } - - void Bind(const char* value) { - if (value != nullptr) { - //sqlite3_bind_text(stmt_, /*index*/bind_counter_++, value, -1, SQLITE_STATIC); - CheckOk(sqlite3_bind_text(stmt_, /*index*/bind_counter_++, value, -1, SQLITE_TRANSIENT)); - } else { - BindNull(); - } - } - - void Bind(const std::string& value) { - Bind(value.c_str()); - } - - template <typename E, typename = std::enable_if_t<std::is_enum_v<E>>> - void Bind(E value) { - Bind(static_cast<std::underlying_type_t<E>>(value)); - } - - void BindNull() { - CheckOk(sqlite3_bind_null(stmt_, bind_counter_++)); - } - - int Step() { - ++step_counter_; - return sqlite3_step(stmt_); - } - - bool Step(int expected) { - int rc = Step(); - if (rc != expected) { - LOG(ERROR) << "SQLite error: " << sqlite3_errmsg(db_.get()); - return false; - } - return true; - } - - // Successive ColumnAll calls start at the 0th column again. - template <typename T, typename ... Args> - void ColumnAll(T& first, Args&... rest) { - Column(first); - ColumnAll(rest...); - // Reset column counter back to 0 - column_counter_ = 0; - } - - void ColumnAll() {} - - template <typename T> - void Column(std::optional<T>& value) { - T tmp; - Column(/*out*/tmp); - - if (!tmp) { // disambiguate 0 and NULL - const unsigned char* text = sqlite3_column_text(stmt_, column_counter_ - 1); - if (text == nullptr) { - value = std::nullopt; - return; - } - } - value = std::move(tmp); - } - - template <typename E, typename = std::enable_if_t<std::is_enum_v<E>>> - void Column(E& value) { - std::underlying_type_t<E> tmp; - Column(/*out*/tmp); - value = static_cast<E>(tmp); - } - - void Column(bool& value) { - value = sqlite3_column_int(stmt_, column_counter_++); - } - - void Column(int& value) { - value = sqlite3_column_int(stmt_, column_counter_++); - } - - void Column(uint64_t& value) { - value = static_cast<uint64_t>(sqlite3_column_int64(stmt_, column_counter_++)); - } - - void Column(std::string& value) { - const unsigned char* text = sqlite3_column_text(stmt_, column_counter_++); - - DCHECK(text != nullptr) << "Column should be marked NOT NULL, otherwise use optional<string>"; - if (text == nullptr) { - LOG(ERROR) << "Got NULL back for column " << column_counter_-1 - << "; is this column marked NOT NULL?"; - value = "(((null)))"; // Don't segfault, keep going. - return; - } - - value = std::string{reinterpret_cast<const char*>(text)}; - } - - const char* ExpandedSql() { - char* p = sqlite3_expanded_sql(stmt_); - if (p == nullptr) { - return "(nullptr)"; - } - return p; - } - - const char* Sql() { - const char* p = sqlite3_sql(stmt_); - if (p == nullptr) { - return "(nullptr)"; - } - return p; - } - - - DbStatement(DbStatement&& other) - : db_{other.db_}, stmt_{other.stmt_}, bind_counter_{other.bind_counter_}, - step_counter_{other.step_counter_} { - other.db_ = DbHandle{nullptr}; - other.stmt_ = nullptr; - } - - ~DbStatement() { - if (stmt_ != nullptr) { - DCHECK_GT(step_counter_, 0) << " forgot to call Step()?"; - sqlite3_finalize(stmt_); - } - } - - private: - bool CheckOk(int rc, int expected = SQLITE_OK) { - if (rc != expected) { - LOG(ERROR) << "Got error for SQL query: '" << Sql() << "'" - << ", expanded: '" << ExpandedSql() << "'"; - LOG(ERROR) << "Failed SQLite api call (" << rc << "): " << sqlite3_errstr(rc); - } - return rc == expected; - } - - DbHandle db_; - sqlite3_stmt* stmt_; - int bind_counter_ = 1; - int step_counter_ = 0; - int column_counter_ = 0; -}; - -class DbQueryBuilder { - public: - // Returns the row ID that was inserted last. - template <typename... Args> - static std::optional<int> Insert(DbHandle db, const std::string& sql, Args&&... args) { - ScopedLockDb lock{db}; - - sqlite3_int64 last_rowid = sqlite3_last_insert_rowid(db.get()); - DbStatement stmt = DbStatement::Prepare(db, sql, std::forward<Args>(args)...); - - if (!stmt.Step(SQLITE_DONE)) { - return std::nullopt; - } - - last_rowid = sqlite3_last_insert_rowid(db.get()); - DCHECK_GT(last_rowid, 0); - - return static_cast<int>(last_rowid); - } - - template <typename... Args> - static bool Delete(DbHandle db, const std::string& sql, Args&&... args) { - return ExecuteOnce(db, sql, std::forward<Args>(args)...); - } - - template <typename... Args> - static bool Update(DbHandle db, const std::string& sql, Args&&... args) { - return ExecuteOnce(db, sql, std::forward<Args>(args)...); - } - - template <typename... Args> - static bool ExecuteOnce(DbHandle db, const std::string& sql, Args&&... args) { - ScopedLockDb lock{db}; - - DbStatement stmt = DbStatement::Prepare(db, sql, std::forward<Args>(args)...); - - if (!stmt.Step(SQLITE_DONE)) { - return false; - } - - return true; - } - - template <typename... Args> - static bool SelectOnce(DbStatement& stmt, Args&... args) { - int rc = stmt.Step(); - switch (rc) { - case SQLITE_ROW: - stmt.ColumnAll(/*out*/args...); - return true; - case SQLITE_DONE: - return false; - default: - LOG(ERROR) << "Failed to step (" << rc << "): " << sqlite3_errmsg(stmt.db()); - return false; - } - } -}; - -class Model { - public: - DbHandle db() const { - return db_; - } - - Model(DbHandle db) : db_{db} { - } - - private: - DbHandle db_; -}; - -class SchemaModel : public Model { - public: - static SchemaModel GetOrCreate(std::string location) { - int rc = sqlite3_config(SQLITE_CONFIG_LOG, ErrorLogCallback, /*data*/nullptr); - - if (rc != SQLITE_OK) { - LOG(FATAL) << "Failed to configure logging"; - } - - sqlite3* db = nullptr; - bool is_deprecated = false; - if (location != ":memory:") { - // Try to open DB if it already exists. - rc = sqlite3_open_v2(location.c_str(), /*out*/&db, SQLITE_OPEN_READWRITE, /*vfs*/nullptr); - - if (rc == SQLITE_OK) { - LOG(INFO) << "Opened existing database at '" << location << "'"; - SchemaModel schema{DbHandle{db}, location}; - if (schema.Version() == kDbVersion) { - return schema; - } else { - LOG(DEBUG) << "The version is old, reinit the db." - << " old version is " - << schema.Version() - << " and new version is " - << kDbVersion; - CleanUpFilesForDb(schema.db()); - is_deprecated = true; - } - } - } - - if (is_deprecated) { - // Remove the db and recreate it. - // TODO: migrate to a newer version without deleting the old one. - std::filesystem::remove(location.c_str()); - } - - // Create a new DB if one didn't exist already. - rc = sqlite3_open(location.c_str(), /*out*/&db); - - if (rc != SQLITE_OK) { - LOG(FATAL) << "Failed to open DB: " << sqlite3_errmsg(db); - } - - SchemaModel schema{DbHandle{db}, location}; - schema.Reinitialize(); - // TODO: migrate versions upwards when we rev the schema version - - int old_version = schema.Version(); - LOG(VERBOSE) << "Loaded schema version: " << old_version; - - return schema; - } - - void MarkSingleton() { - s_singleton_ = db(); - } - - static DbHandle GetSingleton() { - DCHECK(s_singleton_.has_value()); - return *s_singleton_; - } - - void Reinitialize() { - const char* sql_to_initialize = R"SQLC0D3( - DROP TABLE IF EXISTS schema_versions; - DROP TABLE IF EXISTS packages; - DROP TABLE IF EXISTS activities; - DROP TABLE IF EXISTS app_launch_histories; - DROP TABLE IF EXISTS raw_traces; - DROP TABLE IF EXISTS prefetch_files; -)SQLC0D3"; - char* err_msg = nullptr; - int rc = sqlite3_exec(db().get(), - sql_to_initialize, - /*callback*/nullptr, - /*arg*/0, - /*out*/&err_msg); - if (rc != SQLITE_OK) { - LOG(FATAL) << "Failed to drop tables: " << err_msg ? err_msg : "nullptr"; - } - - CreateSchema(); - LOG(INFO) << "Reinitialized database at '" << location_ << "'"; - } - - int Version() { - std::string query = "SELECT MAX(version) FROM schema_versions;"; - DbStatement stmt = DbStatement::Prepare(db(), query); - - int return_value = 0; - if (!DbQueryBuilder::SelectOnce(stmt, /*out*/return_value)) { - LOG(ERROR) << "Failed to query schema version"; - return -1; - } - - return return_value; - } - - protected: - SchemaModel(DbHandle db, std::string location) : Model{db}, location_(location) { - } - - private: - static std::optional<DbHandle> s_singleton_; - - void CreateSchema() { - const char* sql_to_initialize = R"SQLC0D3( - CREATE TABLE schema_versions( - version INTEGER NOT NULL - ); - - CREATE TABLE packages( - id INTEGER NOT NULL, - name TEXT NOT NULL, - version INTEGER NOT NULL, - - PRIMARY KEY(id) - ); - - CREATE TABLE activities( - id INTEGER NOT NULL, - name TEXT NOT NULL, - package_id INTEGER NOT NULL, - - PRIMARY KEY(id), - FOREIGN KEY (package_id) REFERENCES packages (id) ON DELETE CASCADE - ); - - CREATE TABLE app_launch_histories( - id INTEGER NOT NULL PRIMARY KEY, - activity_id INTEGER NOT NULL, - -- 1:Cold, 2:Warm, 3:Hot - temperature INTEGER CHECK (temperature IN (1, 2, 3)) NOT NULL, - trace_enabled INTEGER CHECK(trace_enabled in (TRUE, FALSE)) NOT NULL, - readahead_enabled INTEGER CHECK(trace_enabled in (TRUE, FALSE)) NOT NULL, - -- absolute timestamp since epoch - intent_started_ns INTEGER CHECK(intent_started_ns IS NULL or intent_started_ns >= 0), - -- absolute timestamp since epoch - total_time_ns INTEGER CHECK(total_time_ns IS NULL or total_time_ns >= 0), - -- absolute timestamp since epoch - report_fully_drawn_ns INTEGER CHECK(report_fully_drawn_ns IS NULL or report_fully_drawn_ns >= 0), - -- pid of the app - pid INTEGER CHECK(pid IS NULL or pid >= 0), - - FOREIGN KEY (activity_id) REFERENCES activities (id) ON DELETE CASCADE - ); - - CREATE TABLE raw_traces( - id INTEGER NOT NULL PRIMARY KEY, - history_id INTEGER NOT NULL, - file_path TEXT NOT NULL, - - FOREIGN KEY (history_id) REFERENCES app_launch_histories (id) ON DELETE CASCADE - ); - - CREATE TABLE prefetch_files( - id INTEGER NOT NULL PRIMARY KEY, - activity_id INTEGER NOT NULL, - file_path TEXT NOT NULL, - - FOREIGN KEY (activity_id) REFERENCES activities (id) ON DELETE CASCADE - ); -)SQLC0D3"; - - char* err_msg = nullptr; - int rc = sqlite3_exec(db().get(), - sql_to_initialize, - /*callback*/nullptr, - /*arg*/0, - /*out*/&err_msg); - - if (rc != SQLITE_OK) { - LOG(FATAL) << "Failed to create tables: " << err_msg ? err_msg : "nullptr"; - } - - const char* sql_to_insert_schema_version = R"SQLC0D3( - INSERT INTO schema_versions VALUES(%d) - )SQLC0D3"; - rc = sqlite3_exec(db().get(), - android::String8::format(sql_to_insert_schema_version, - kDbVersion), - /*callback*/nullptr, - /*arg*/0, - /*out*/&err_msg); - - if (rc != SQLITE_OK) { - LOG(FATAL) << "Failed to insert the schema version: " - << err_msg ? err_msg : "nullptr"; - } - } - - static void ErrorLogCallback(void *pArg, int iErrCode, const char *zMsg) { - LOG(ERROR) << "SQLite error (" << iErrCode << "): " << zMsg; - } - - std::string location_; -}; - -class PackageModel : public Model { - protected: - PackageModel(DbHandle db) : Model{db} { - } - - public: - static std::optional<PackageModel> SelectById(DbHandle db, int id) { - ScopedLockDb lock{db}; - int original_id = id; - - std::string query = "SELECT * FROM packages WHERE id = ?1 LIMIT 1;"; - DbStatement stmt = DbStatement::Prepare(db, query, id); - - PackageModel p{db}; - if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) { - return std::nullopt; - } - - return p; - } - - static std::vector<PackageModel> SelectByName(DbHandle db, const char* name) { - ScopedLockDb lock{db}; - - std::string query = "SELECT * FROM packages WHERE name = ?1;"; - DbStatement stmt = DbStatement::Prepare(db, query, name); - - std::vector<PackageModel> packages; - - PackageModel p{db}; - while (DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) { - packages.push_back(p); - } - - return packages; - } - - static std::optional<PackageModel> SelectByNameAndVersion(DbHandle db, - const char* name, - int version) { - ScopedLockDb lock{db}; - - std::string query = - "SELECT * FROM packages WHERE name = ?1 AND version = ?2 LIMIT 1;"; - DbStatement stmt = DbStatement::Prepare(db, query, name, version); - - PackageModel p{db}; - if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) { - return std::nullopt; - } - - return p; - } - - static std::vector<PackageModel> SelectAll(DbHandle db) { - ScopedLockDb lock{db}; - - std::string query = "SELECT * FROM packages;"; - DbStatement stmt = DbStatement::Prepare(db, query); - - std::vector<PackageModel> packages; - PackageModel p{db}; - while (DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) { - packages.push_back(p); - } - - return packages; - } - - static std::optional<PackageModel> Insert(DbHandle db, - std::string name, - int version) { - const char* sql = "INSERT INTO packages (name, version) VALUES (?1, ?2);"; - - std::optional<int> inserted_row_id = - DbQueryBuilder::Insert(db, sql, name, version); - if (!inserted_row_id) { - return std::nullopt; - } - - PackageModel p{db}; - p.name = name; - p.version = version; - p.id = *inserted_row_id; - - return p; - } - - bool Delete() { - const char* sql = "DELETE FROM packages WHERE id = ?"; - - return DbQueryBuilder::Delete(db(), sql, id); - } - - int id; - std::string name; - int version; -}; - -inline std::ostream& operator<<(std::ostream& os, const PackageModel& p) { - os << "PackageModel{id=" << p.id << ",name=" << p.name << ","; - os << "version="; - os << p.version; - os << "}"; - return os; -} - -class ActivityModel : public Model { - protected: - ActivityModel(DbHandle db) : Model{db} { - } - - public: - static std::optional<ActivityModel> SelectById(DbHandle db, int id) { - ScopedLockDb lock{db}; - int original_id = id; - - std::string query = "SELECT * FROM activities WHERE id = ? LIMIT 1;"; - DbStatement stmt = DbStatement::Prepare(db, query, id); - - ActivityModel p{db}; - if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.package_id)) { - return std::nullopt; - } - - return p; - } - - static std::optional<ActivityModel> SelectByNameAndPackageId(DbHandle db, - const char* name, - int package_id) { - ScopedLockDb lock{db}; - - std::string query = "SELECT * FROM activities WHERE name = ? AND package_id = ? LIMIT 1;"; - DbStatement stmt = DbStatement::Prepare(db, query, name, package_id); - - ActivityModel p{db}; - if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.package_id)) { - return std::nullopt; - } - - return p; - } - - static std::vector<ActivityModel> SelectByPackageId(DbHandle db, - int package_id) { - ScopedLockDb lock{db}; - - std::string query = "SELECT * FROM activities WHERE package_id = ?;"; - DbStatement stmt = DbStatement::Prepare(db, query, package_id); - - std::vector<ActivityModel> activities; - ActivityModel p{db}; - while (DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.package_id)) { - activities.push_back(p); - } - - return activities; - } - - static std::optional<ActivityModel> Insert(DbHandle db, - std::string name, - int package_id) { - const char* sql = "INSERT INTO activities (name, package_id) VALUES (?1, ?2);"; - - std::optional<int> inserted_row_id = - DbQueryBuilder::Insert(db, sql, name, package_id); - if (!inserted_row_id) { - return std::nullopt; - } - - ActivityModel p{db}; - p.id = *inserted_row_id; - p.name = name; - p.package_id = package_id; - - return p; - } - - // Try to select by package_name+activity_name, otherwise insert into both tables. - // Package version is ignored for selects. - static std::optional<ActivityModel> SelectOrInsert( - DbHandle db, - std::string package_name, - int package_version, - std::string activity_name) { - std::optional<PackageModel> package = PackageModel::SelectByNameAndVersion(db, - package_name.c_str(), - package_version); - if (!package) { - package = PackageModel::Insert(db, package_name, package_version); - DCHECK(package.has_value()); - } - - std::optional<ActivityModel> activity = - ActivityModel::SelectByNameAndPackageId(db, - activity_name.c_str(), - package->id); - if (!activity) { - activity = Insert(db, activity_name, package->id); - // XX: should we really return an optional here? This feels like it should never fail. - } - - return activity; - } - - int id; - std::string name; - int package_id; // PackageModel::id -}; - -inline std::ostream& operator<<(std::ostream& os, const ActivityModel& p) { - os << "ActivityModel{id=" << p.id << ",name=" << p.name << ","; - os << "package_id=" << p.package_id << "}"; - return os; -} - -class AppLaunchHistoryModel : public Model { - protected: - AppLaunchHistoryModel(DbHandle db) : Model{db} { - } - - public: - enum class Temperature : int32_t { - kUninitialized = -1, // Note: Not a valid SQL value. - kCold = 1, - kWarm = 2, - kHot = 3, - }; - - static std::optional<AppLaunchHistoryModel> SelectById(DbHandle db, int id) { - ScopedLockDb lock{db}; - int original_id = id; - - std::string query = "SELECT * FROM app_launch_histories WHERE id = ? LIMIT 1;"; - DbStatement stmt = DbStatement::Prepare(db, query, id); - - AppLaunchHistoryModel p{db}; - if (!DbQueryBuilder::SelectOnce(stmt, - p.id, - p.activity_id, - p.temperature, - p.trace_enabled, - p.readahead_enabled, - p.intent_started_ns, - p.total_time_ns, - p.report_fully_drawn_ns, - p.pid)) { - return std::nullopt; - } - - return p; - } - - // Selects the activity history for an activity id. - // The requirements are: - // * Should be cold run. - // * Pefetto trace is enabled. - // * intent_start_ns is *NOT* null. - static std::vector<AppLaunchHistoryModel> SelectActivityHistoryForCompile( - DbHandle db, - int activity_id) { - ScopedLockDb lock{db}; - std::string query = "SELECT * FROM app_launch_histories " - "WHERE activity_id = ?1 AND" - " temperature = 1 AND" - " trace_enabled = TRUE AND" - " intent_started_ns IS NOT NULL;"; - DbStatement stmt = DbStatement::Prepare(db, query, activity_id); - std::vector<AppLaunchHistoryModel> result; - - AppLaunchHistoryModel p{db}; - while (DbQueryBuilder::SelectOnce(stmt, - p.id, - p.activity_id, - p.temperature, - p.trace_enabled, - p.readahead_enabled, - p.intent_started_ns, - p.total_time_ns, - p.report_fully_drawn_ns, - p.pid)) { - result.push_back(p); - } - return result; - } - - static std::optional<AppLaunchHistoryModel> Insert(DbHandle db, - int activity_id, - AppLaunchHistoryModel::Temperature temperature, - bool trace_enabled, - bool readahead_enabled, - std::optional<uint64_t> intent_started_ns, - std::optional<uint64_t> total_time_ns, - std::optional<uint64_t> report_fully_drawn_ns, - int32_t pid) - { - const char* sql = "INSERT INTO app_launch_histories (activity_id, temperature, trace_enabled, " - "readahead_enabled, intent_started_ns, " - "total_time_ns, report_fully_drawn_ns, pid) " - "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8);"; - - std::optional<int> inserted_row_id = - DbQueryBuilder::Insert(db, - sql, - activity_id, - temperature, - trace_enabled, - readahead_enabled, - intent_started_ns, - total_time_ns, - report_fully_drawn_ns, - pid); - if (!inserted_row_id) { - return std::nullopt; - } - - AppLaunchHistoryModel p{db}; - p.id = *inserted_row_id; - p.activity_id = activity_id; - p.temperature = temperature; - p.trace_enabled = trace_enabled; - p.readahead_enabled = readahead_enabled; - p.intent_started_ns = intent_started_ns; - p.total_time_ns = total_time_ns; - p.report_fully_drawn_ns = report_fully_drawn_ns; - p.pid = pid; - - return p; - } - - static bool UpdateReportFullyDrawn(DbHandle db, - int history_id, - uint64_t report_fully_drawn_ns) - { - const char* sql = "UPDATE app_launch_histories " - "SET report_fully_drawn_ns = ?1 " - "WHERE id = ?2;"; - - bool result = DbQueryBuilder::Update(db, sql, report_fully_drawn_ns, history_id); - - if (!result) { - LOG(ERROR)<< "Failed to update history_id:"<< history_id - << ", report_fully_drawn_ns: " << report_fully_drawn_ns; - } - return result; - } - - int id; - int activity_id; // ActivityModel::id - Temperature temperature = Temperature::kUninitialized; - bool trace_enabled; - bool readahead_enabled; - std::optional<uint64_t> intent_started_ns; - std::optional<uint64_t> total_time_ns; - std::optional<uint64_t> report_fully_drawn_ns; - int32_t pid; -}; - -inline std::ostream& operator<<(std::ostream& os, const AppLaunchHistoryModel& p) { - os << "AppLaunchHistoryModel{id=" << p.id << "," - << "activity_id=" << p.activity_id << "," - << "temperature=" << static_cast<int>(p.temperature) << "," - << "trace_enabled=" << p.trace_enabled << "," - << "readahead_enabled=" << p.readahead_enabled << "," - << "intent_started_ns="; - if (p.intent_started_ns) { - os << *p.intent_started_ns; - } else { - os << "(nullopt)"; - } - os << ","; - os << "total_time_ns="; - if (p.total_time_ns) { - os << *p.total_time_ns; - } else { - os << "(nullopt)"; - } - os << ","; - os << "report_fully_drawn_ns="; - if (p.report_fully_drawn_ns) { - os << *p.report_fully_drawn_ns; - } else { - os << "(nullopt)"; - } - os << ","; - os << "pid=" << p.pid; - os << "}"; - return os; -} - -class RawTraceModel : public Model { - protected: - RawTraceModel(DbHandle db) : Model{db} { - } - - public: - - // Return raw_traces, sorted ascending by the id. - static std::vector<RawTraceModel> SelectByVersionedComponentName(DbHandle db, - VersionedComponentName vcn) { - ScopedLockDb lock{db}; - - const char* sql = - "SELECT raw_traces.id, raw_traces.history_id, raw_traces.file_path " - "FROM raw_traces " - "INNER JOIN app_launch_histories ON raw_traces.history_id = app_launch_histories.id " - "INNER JOIN activities ON activities.id = app_launch_histories.activity_id " - "INNER JOIN packages ON packages.id = activities.package_id " - "WHERE packages.name = ? AND activities.name = ? AND packages.version = ?" - "ORDER BY raw_traces.id ASC"; - - DbStatement stmt = DbStatement::Prepare(db, - sql, - vcn.GetPackage(), - vcn.GetActivity(), - vcn.GetVersion()); - - std::vector<RawTraceModel> results; - - RawTraceModel p{db}; - while (DbQueryBuilder::SelectOnce(stmt, p.id, p.history_id, p.file_path)) { - results.push_back(p); - } - - return results; - } - - static std::optional<RawTraceModel> SelectByHistoryId(DbHandle db, int history_id) { - ScopedLockDb lock{db}; - - const char* sql = - "SELECT id, history_id, file_path " - "FROM raw_traces " - "WHERE history_id = ?1 " - "LIMIT 1;"; - - DbStatement stmt = DbStatement::Prepare(db, sql, history_id); - - RawTraceModel p{db}; - if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.history_id, p.file_path)) { - return std::nullopt; - } - - return p; - } - - static std::optional<RawTraceModel> Insert(DbHandle db, - int history_id, - std::string file_path) { - const char* sql = "INSERT INTO raw_traces (history_id, file_path) VALUES (?1, ?2);"; - - std::optional<int> inserted_row_id = - DbQueryBuilder::Insert(db, sql, history_id, file_path); - if (!inserted_row_id) { - return std::nullopt; - } - - RawTraceModel p{db}; - p.id = *inserted_row_id; - p.history_id = history_id; - p.file_path = file_path; - - return p; - } - - bool Delete() { - const char* sql = "DELETE FROM raw_traces WHERE id = ?"; - - return DbQueryBuilder::Delete(db(), sql, id); - } - - int id; - int history_id; - std::string file_path; -}; - -inline std::ostream& operator<<(std::ostream& os, const RawTraceModel& p) { - os << "RawTraceModel{id=" << p.id << ",history_id=" << p.history_id << ","; - os << "file_path=" << p.file_path << "}"; - return os; -} - -class PrefetchFileModel : public Model { - protected: - PrefetchFileModel(DbHandle db) : Model{db} { - } - - public: - static std::optional<PrefetchFileModel> SelectByVersionedComponentName( - DbHandle db, - VersionedComponentName vcn) { - ScopedLockDb lock{db}; - - const char* sql = - "SELECT prefetch_files.id, prefetch_files.activity_id, prefetch_files.file_path " - "FROM prefetch_files " - "INNER JOIN activities ON activities.id = prefetch_files.activity_id " - "INNER JOIN packages ON packages.id = activities.package_id " - "WHERE packages.name = ? AND activities.name = ? AND packages.version = ?"; - - DbStatement stmt = DbStatement::Prepare(db, - sql, - vcn.GetPackage(), - vcn.GetActivity(), - vcn.GetVersion()); - - PrefetchFileModel p{db}; - - if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.activity_id, p.file_path)) { - return std::nullopt; - } - - return p; - } - - static std::vector<PrefetchFileModel> SelectAll(DbHandle db) { - ScopedLockDb lock{db}; - - std::string query = - "SELECT prefetch_files.id, prefetch_files.activity_id, prefetch_files.file_path " - "FROM prefetch_files"; - DbStatement stmt = DbStatement::Prepare(db, query); - - std::vector<PrefetchFileModel> prefetch_files; - PrefetchFileModel p{db}; - while (DbQueryBuilder::SelectOnce(stmt, p.id, p.activity_id, p.file_path)) { - prefetch_files.push_back(p); - } - - return prefetch_files; - } - - static std::optional<PrefetchFileModel> Insert(DbHandle db, - int activity_id, - std::string file_path) { - const char* sql = "INSERT INTO prefetch_files (activity_id, file_path) VALUES (?1, ?2);"; - - std::optional<int> inserted_row_id = - DbQueryBuilder::Insert(db, sql, activity_id, file_path); - if (!inserted_row_id) { - return std::nullopt; - } - - PrefetchFileModel p{db}; - p.id = *inserted_row_id; - p.activity_id = activity_id; - p.file_path = file_path; - - return p; - } - - bool Delete() { - const char* sql = "DELETE FROM prefetch_files WHERE id = ?"; - - return DbQueryBuilder::Delete(db(), sql, id); - } - - int id; - int activity_id; - std::string file_path; -}; - -inline std::ostream& operator<<(std::ostream& os, const PrefetchFileModel& p) { - os << "PrefetchFileModel{id=" << p.id << ",activity_id=" << p.activity_id << ","; - os << "file_path=" << p.file_path << "}"; - return os; -} - -} // namespace iorap::db - -#endif // IORAP_SRC_DB_MODELS_H_ diff --git a/src/inode2filename/data_source.cc b/src/inode2filename/data_source.cc deleted file mode 100644 index 5fcce73..0000000 --- a/src/inode2filename/data_source.cc +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "inode2filename/data_source.h" - -#include "common/cmd_utils.h" -#include "inode2filename/inode_resolver.h" -#include "inode2filename/search_directories.h" - -#include <android-base/logging.h> - -#include <fstream> -#include <stdio.h> - -namespace rx = rxcpp; - -namespace iorap::inode2filename { - -std::vector<std::string> ToArgs(DataSourceKind data_source_kind) { - const char* value = nullptr; - - switch (data_source_kind) { - case DataSourceKind::kDiskScan: - value = "diskscan"; - break; - case DataSourceKind::kTextCache: - value = "textcache"; - break; - case DataSourceKind::kBpf: - value = "bpf"; - break; - } - - std::vector<std::string> args; - iorap::common::AppendNamedArg(args, "--data-source", value); - return args; -} - -std::vector<std::string> ToArgs(const DataSourceDependencies& deps) { - std::vector<std::string> args; - - iorap::common::AppendArgsRepeatedly(args, ToArgs(deps.data_source)); - // intentionally skip system_call; it does not have a command line equivalent - iorap::common::AppendNamedArgRepeatedly(args, "--root", deps.root_directories); - - if (deps.text_cache_filename) { - iorap::common::AppendNamedArg(args, "--textcache", *(deps.text_cache_filename)); - } - - return args; -} - -class DiskScanDataSource : public DataSource { - public: - DiskScanDataSource(DataSourceDependencies dependencies) - : DataSource(std::move(dependencies)) { - DCHECK_NE(dependencies_.root_directories.size(), 0u) << "Root directories can't be empty"; - } - - virtual rxcpp::observable<InodeResult> EmitInodes() const override { - SearchDirectories searcher{/*borrow*/dependencies_.system_call}; - return searcher.ListAllFilenames(dependencies_.root_directories); - } - - // Since not all Inodes emitted are the ones searched for, doing additional stat(2) calls here - // would be redundant. - // - // We set the device number to a dummy value here (-1) so that InodeResolver - // can fill it in later with stat(2). This is effectively the same thing as always doing - // verification. - virtual bool ResultIncludesDeviceNumber() const { return false; }; - - virtual ~DiskScanDataSource() {} -}; - -static inline void LeftTrim(/*inout*/std::string& s) { - s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { - return !std::isspace(ch); - })); -} - -class TextCacheDataSource : public DataSource { - public: - TextCacheDataSource(DataSourceDependencies dependencies) - : DataSource(std::move(dependencies)) { - DCHECK(dependencies_.text_cache_filename.has_value()) << "Must have text cache filename"; - } - - virtual rxcpp::observable<InodeResult> EmitInodes() const override { - const std::string& file_name = *dependencies_.text_cache_filename; - - return rx::observable<>::create<InodeResult>( - [file_name](rx::subscriber<InodeResult> dest) { - LOG(VERBOSE) << "TextCacheDataSource (lambda)"; - - std::ifstream ifs{file_name}; - - if (!ifs.is_open()) { - dest.on_error(rxcpp::util::make_error_ptr( - std::ios_base::failure("Could not open specified text cache filename"))); - return; - } - - while (dest.is_subscribed() && ifs) { - LOG(VERBOSE) << "TextCacheDataSource (read line)"; - // TODO. - - uint64_t device_number = 0; - uint64_t inode_number = 0; - uint64_t file_size = 0; - - // Parse lines of form: - // "$device_number $inode $filesize $filename..." - // This format conforms to system/extras/pagecache/pagecache.py - - ifs >> device_number; - ifs >> inode_number; - ifs >> file_size; - (void)file_size; // Not used in iorapd. - - std::string value_filename; - std::getline(/*inout*/ifs, /*out*/value_filename); - - if (value_filename.empty()) { - // Ignore empty lines. - continue; - } - - // getline, unlike ifstream, does not ignore spaces. - // There's always at least 1 space in a textcache output file. - // However, drop *all* spaces on the left since filenames starting with a space are - // ambiguous to us. - LeftTrim(/*inout*/value_filename); - - Inode inode = Inode::FromDeviceAndInode(static_cast<dev_t>(device_number), - static_cast<ino_t>(inode_number)); - - LOG(VERBOSE) << "TextCacheDataSource (on_next) " << inode << "->" << value_filename; - dest.on_next(InodeResult::makeSuccess(inode, value_filename)); - } - - dest.on_completed(); - } - ); - - // TODO: plug into the filtering and verification graph. - } - - virtual ~TextCacheDataSource() {} -}; - -DataSource::DataSource(DataSourceDependencies dependencies) - : dependencies_{std::move(dependencies)} { - DCHECK(dependencies_.system_call != nullptr); -} - -std::shared_ptr<DataSource> DataSource::Create(DataSourceDependencies dependencies) { - switch (dependencies.data_source) { - case DataSourceKind::kDiskScan: - return std::shared_ptr<DataSource>{new DiskScanDataSource{std::move(dependencies)}}; - case DataSourceKind::kTextCache: - return std::shared_ptr<DataSource>{new TextCacheDataSource{std::move(dependencies)}}; - case DataSourceKind::kBpf: - // TODO: BPF-based data source. - LOG(FATAL) << "Not implemented yet"; - return nullptr; - default: - LOG(FATAL) << "Invalid data source value"; - return nullptr; - } -} - -void DataSource::StartRecording() {} -void DataSource::StopRecording() {} - -} // namespace iorap::inode2filename diff --git a/src/inode2filename/data_source.h b/src/inode2filename/data_source.h deleted file mode 100644 index 36c8383..0000000 --- a/src/inode2filename/data_source.h +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_INODE2FILENAME_DATA_SOURCE_H_ -#define IORAP_SRC_INODE2FILENAME_DATA_SOURCE_H_ - -#include "inode2filename/inode_result.h" -#include "inode2filename/system_call.h" - -#include <rxcpp/rx.hpp> - -#include <memory> -#include <optional> -#include <string> -#include <vector> -namespace iorap::inode2filename { - -enum class DataSourceKind { - kDiskScan, - kTextCache, - kBpf, -}; - -std::vector<std::string> ToArgs(DataSourceKind data_source_kind); - -struct DataSourceDependencies { - DataSourceKind data_source = DataSourceKind::kDiskScan; - borrowed<SystemCall*> system_call = nullptr; - - // kDiskScan-specific options. Other data sources ignore these fields. - std::vector<std::string> root_directories; - // kTextCache-specific options. Other data sources ignore these fields. - std::optional<std::string> text_cache_filename; -}; - -std::vector<std::string> ToArgs(const DataSourceDependencies& deps); - -class DataSource : std::enable_shared_from_this<DataSource> { - public: - static std::shared_ptr<DataSource> Create(DataSourceDependencies dependencies); - - virtual void StartRecording(); // XX: feels like this should be BPF-specific. - virtual void StopRecording(); - - // Emit all inode->filename mappings (i.e. an infinite lazy list). - // The specific order is determined by the extra dependency options. - // - // The work must terminate if all subscriptions are removed. - virtual rxcpp::observable<InodeResult> EmitInodes() const = 0; - - // Does the InodeResult include a valid device number? - // If returning false, the InodeResolver fills in the missing device number with stat(2). - virtual bool ResultIncludesDeviceNumber() const { return true; }; - - protected: - virtual ~DataSource() {} - DataSource(DataSourceDependencies dependencies); - DataSourceDependencies dependencies_; -}; - -} // namespace iorap::inode2filename - -#endif // IORAP_SRC_INODE2FILENAME_DATA_SOURCE_H_ diff --git a/src/inode2filename/inode.cc b/src/inode2filename/inode.cc deleted file mode 100644 index f0e956b..0000000 --- a/src/inode2filename/inode.cc +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "inode2filename/inode.h" - -#include <android-base/logging.h> -#include <android-base/parseint.h> -#include <android-base/strings.h> - -#include <string> -#include <vector> - -#include <sys/sysmacros.h> -#include <sys/types.h> - -using android::base::ParseUint; - -namespace iorap::inode2filename { - -// TODO: refactor to return expected<Inode, string> -bool Inode::Parse(const std::string& str, Inode* out, std::string* error_msg) { - DCHECK(out != nullptr); - DCHECK(error_msg != nullptr); - - // Major:minor:inode OR dev_t@inode - std::vector<std::string> lst_pair = android::base::Split(str, "@"); - if (lst_pair.size() == 2) { - size_t dev_whole = 0; - if (!ParseUint(lst_pair[0], &dev_whole)) { - *error_msg = "Failed to parse the whole device id as uint."; - return false; - } - - dev_t dev_w = static_cast<dev_t>(dev_whole); - out->device_major = major(dev_w); - out->device_minor = minor(dev_w); - - if (!ParseUint(lst_pair[1], &out->inode)) { - *error_msg = "Failed to parse inode as uint."; - return false; - } - - return true; - } - - std::vector<std::string> lst = android::base::Split(str, ":"); - - if (lst.size() != 3) { - *error_msg = "Too few : separated items"; - return false; - } - - if (!ParseUint(lst[0], &out->device_major)) { - *error_msg = "Failed to parse 0th element as a uint"; - return false; - } - - if (!ParseUint(lst[1], &out->device_minor)) { - *error_msg = "Failed to parse 1st element as a uint"; - return false; - } - - if (!ParseUint(lst[2], &out->inode)) { - *error_msg = "Failed to parse 2nd element as a uint"; - return false; - } - - return true; -} - -} // namespace iorap::inode2filename - -// Ensure our pollution-free aliases match the typedefs in the system headers. -static_assert(std::is_same_v<iorap::inode2filename::dev_t, dev_t>); -static_assert(std::is_same_v<iorap::inode2filename::ino_t, ino_t>); - -// TODO: consider some tests for major, minor, etc. diff --git a/src/inode2filename/inode.h b/src/inode2filename/inode.h deleted file mode 100644 index f336a14..0000000 --- a/src/inode2filename/inode.h +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_INODE2FILENAME_INODE_H_ -#define IORAP_SRC_INODE2FILENAME_INODE_H_ - -#include <functional> -#include <ostream> -#include <string> - -#include <stddef.h> - -namespace iorap::inode2filename { - -// Avoid polluting headers. -#if defined(__ANDROID__) -# if !defined(__LP64__) -/* This historical accident means that we had a 32-bit dev_t on 32-bit architectures. */ -using dev_t = uint32_t; -# else -using dev_t = uint64_t; -# endif -using ino_t = unsigned long; -#else -# if !defined(__x86_64__) -using dev_t = unsigned long long; -using ino_t = unsigned long long; -# else -using dev_t = unsigned long; -using ino_t = unsigned long; -# endif -#endif - -#ifdef makedev -#undef makedev -#endif - -/** Combines `major` and `minor` into a device number. */ -constexpr inline dev_t makedev(unsigned int major, unsigned int minor) { - return - (((major) & 0xfffff000ULL) << 32) | (((major) & 0xfffULL) << 8) | - (((minor) & 0xffffff00ULL) << 12) | (((minor) & 0xffULL)); -} - -#ifdef major -#undef major -#endif - -/** Extracts the major part of a device number. */ -constexpr inline unsigned int major(dev_t dev) { - return - ((unsigned) ((((unsigned long long) (dev) >> 32) & 0xfffff000) | (((dev) >> 8) & 0xfff))); -} - -#ifdef minor -#undef minor -#endif - -/** Extracts the minor part of a device number. */ -constexpr inline unsigned int minor(dev_t dev) { - return - ((unsigned) ((((dev) >> 12) & 0xffffff00) | ((dev) & 0xff))); -}; -// Note: above definitions copied from sysmacros.h, to avoid polluting global namespace in a header. - -/* - * A convenient datum representing a (dev_t, ino_t) tuple. - * - * ino_t values may be reused across different devices (e.g. different partitions), - * so we need the full tuple to uniquely identify an inode on a system. - */ -struct Inode { - size_t device_major; // dev_t = makedev(major, minor) - size_t device_minor; - size_t inode; // ino_t = inode - - // "Major:minor:inode" OR "dev_t@inode" - static bool Parse(const std::string& str, /*out*/Inode* out, /*out*/std::string* error_msg); - - constexpr bool operator==(const Inode& rhs) const { - return device_major == rhs.device_major && - device_minor == rhs.device_minor && - inode == rhs.inode; - } - - constexpr bool operator!=(const Inode& rhs) const { - return !(*this == rhs); - } - - Inode() = default; - constexpr Inode(size_t device_major, size_t device_minor, size_t inode) - : device_major{device_major}, device_minor{device_minor}, inode{inode} { - } - - static constexpr Inode FromDeviceAndInode(dev_t dev, ino_t inode) { - return Inode{major(dev), minor(dev), static_cast<size_t>(inode)}; - } - - constexpr dev_t GetDevice() const { - return makedev(device_major, device_minor); - } - - constexpr ino_t GetInode() const { - return static_cast<ino_t>(inode); - } -}; - -inline std::ostream& operator<<(std::ostream& os, const Inode& inode) { - os << inode.device_major << ":" << inode.device_minor << ":" << inode.inode; - return os; -} - -} // namespace iorap::inode2filename - -namespace std { - template <> - struct hash<iorap::inode2filename::Inode> { - using argument_type = iorap::inode2filename::Inode; - using result_type = size_t; - result_type operator()(argument_type const& s) const noexcept { - // Hash the inode by using only the inode#. Ignore devices, we are extremely unlikely - // to ever collide on the devices. - result_type const h1 = std::hash<size_t>{}(s.inode); - return h1; - } - }; -} // namespace std - -namespace rxcpp { -template <class T, typename> -struct filtered_hash; - -// support for the 'distinct' rx operator. -template <> -struct filtered_hash<iorap::inode2filename::Inode, void> : std::hash<iorap::inode2filename::Inode> { -}; -} // namespace rxcpp - -#endif // IORAP_SRC_INODE2FILENAME_INODE_H_ diff --git a/src/inode2filename/inode_resolver.cc b/src/inode2filename/inode_resolver.cc deleted file mode 100644 index 5e2945d..0000000 --- a/src/inode2filename/inode_resolver.cc +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "inode2filename/inode_resolver.h" - -#include "common/cmd_utils.h" -#include "inode2filename/out_of_process_inode_resolver.h" -#include "inode2filename/search_directories.h" - -#include <android-base/logging.h> - -#include <fstream> -#include <stdio.h> - -namespace rx = rxcpp; - -namespace iorap::inode2filename { - -std::vector<std::string> ToArgs(ProcessMode process_mode) { - const char* value = nullptr; - - switch (process_mode) { - case ProcessMode::kInProcessDirect: - value = "in"; - break; - case ProcessMode::kInProcessIpc: - value = "in-ipc"; - break; - case ProcessMode::kOutOfProcessIpc: - value = "out"; - break; - } - - std::vector<std::string> args; - iorap::common::AppendNamedArg(args, "--process-mode", value); - return args; -} - -std::vector<std::string> ToArgs(VerifyKind verify_kind) { - const char* value = nullptr; - - switch (verify_kind) { - case VerifyKind::kNone: - value = "none"; - break; - case VerifyKind::kStat: - value = "stat"; - break; - } - - std::vector<std::string> args; - iorap::common::AppendNamedArg(args, "--verify", value); - return args; -} - -std::vector<std::string> ToArgs(const InodeResolverDependencies& deps) { - std::vector<std::string> args = ToArgs(*static_cast<const DataSourceDependencies*>(&deps)); - iorap::common::AppendArgsRepeatedly(args, ToArgs(deps.process_mode)); - iorap::common::AppendArgsRepeatedly(args, ToArgs(deps.verify)); - - return args; -} - -struct InodeResolver::Impl { - Impl(InodeResolverDependencies dependencies) - : dependencies_{std::move(dependencies)} { - DCHECK(dependencies_.system_call != nullptr); - data_source_ = DataSource::Create(/*downcast*/dependencies_); - } - Impl(InodeResolverDependencies dependencies, std::shared_ptr<DataSource> data_source) - : dependencies_{std::move(dependencies)} { - DCHECK(dependencies_.system_call != nullptr); - data_source_ = std::move(data_source); - DCHECK(data_source_ != nullptr); - } - InodeResolverDependencies dependencies_; - std::shared_ptr<DataSource> data_source_; -}; - -InodeResolver::InodeResolver(InodeResolverDependencies dependencies) - : impl_(new InodeResolver::Impl{std::move(dependencies)}) { -} - -InodeResolver::InodeResolver(InodeResolverDependencies dependencies, - std::shared_ptr<DataSource> data_source) - : impl_(new InodeResolver::Impl{std::move(dependencies), std::move(data_source)}) { -} - -std::shared_ptr<InodeResolver> InodeResolver::Create(InodeResolverDependencies dependencies) { - if (dependencies.process_mode == ProcessMode::kInProcessDirect) { - return std::shared_ptr<InodeResolver>{ - new InodeResolver{std::move(dependencies)}}; - } else if (dependencies.process_mode == ProcessMode::kOutOfProcessIpc) { - return std::shared_ptr<InodeResolver>{ - new OutOfProcessInodeResolver{std::move(dependencies)}}; - } else { - CHECK(false); - } - return nullptr; -} - -std::shared_ptr<InodeResolver> InodeResolver::Create(InodeResolverDependencies dependencies, - std::shared_ptr<DataSource> data_source) { - if (dependencies.process_mode == ProcessMode::kInProcessDirect) { - return std::shared_ptr<InodeResolver>{ - new InodeResolver{std::move(dependencies), std::move(data_source)}}; - } else if (dependencies.process_mode == ProcessMode::kOutOfProcessIpc) { - CHECK(false); // directly providing a DataSource only makes sense in-process - } else { - CHECK(false); - } - return nullptr; -} - -rxcpp::observable<InodeResult> - InodeResolver::FindFilenamesFromInodes(rxcpp::observable<Inode> inodes) const { - - // It's inefficient to search for inodes until the full search list is available, - // so first reduce to a vector so we can access all the inodes simultaneously. - return inodes.reduce(std::vector<Inode>{}, - [](std::vector<Inode> vec, Inode inode) { - vec.push_back(inode); - return vec; - }, - [](std::vector<Inode> v) { - return v; // TODO: use an identity function - }) - .flat_map([self=shared_from_this()](std::vector<Inode> vec) { - // All borrowed values (e.g. SystemCall) must outlive the observable. - return self->FindFilenamesFromInodes(vec); - } - ); -} - -rxcpp::observable<InodeResult> -InodeResolver::FindFilenamesFromInodes(std::vector<Inode> inodes) const { - const DataSource& data_source = *impl_->data_source_; - const InodeResolverDependencies& dependencies = impl_->dependencies_; - - // Get lazy list of inodes from the data source. - rxcpp::observable<InodeResult> all_inodes = impl_->data_source_->EmitInodes(); - - // Filter it according to the source+dependency requirements. - // Unsubscribe from 'all_inodes' early if all inodes are matched early. - const bool needs_device_number = !data_source.ResultIncludesDeviceNumber(); - const bool needs_verification = dependencies.verify == VerifyKind::kStat; - SearchDirectories search{impl_->dependencies_.system_call}; - return search.FilterFilenamesForSpecificInodes(all_inodes, - inodes, - needs_device_number, - needs_verification); -} - -rxcpp::observable<InodeResult> -InodeResolver::EmitAll() const { - const DataSource& data_source = *impl_->data_source_; - const InodeResolverDependencies& dependencies = impl_->dependencies_; - - // Get lazy list of inodes from the data source. - rxcpp::observable<InodeResult> all_inodes = impl_->data_source_->EmitInodes(); - - // Apply verification and fill-in missing device numbers. - const bool needs_device_number = !data_source.ResultIncludesDeviceNumber(); - const bool needs_verification = dependencies.verify == VerifyKind::kStat; - SearchDirectories search{impl_->dependencies_.system_call}; - return search.EmitAllFilenames(all_inodes, - needs_device_number, - needs_verification); -} - -InodeResolver::~InodeResolver() { - // std::unique_ptr requires complete types, but we hide the definition in the header. - delete impl_; - // XX: Does this work if we just force the dtor definition into the .cc file with a unique_ptr? -} - -InodeResolverDependencies& InodeResolver::GetDependencies() { - return impl_->dependencies_; -} - -const InodeResolverDependencies& InodeResolver::GetDependencies() const { - return impl_->dependencies_; -} - -void InodeResolver::StartRecording() { - impl_->data_source_->StartRecording(); -} - -void InodeResolver::StopRecording() { - impl_->data_source_->StopRecording(); -} - -// TODO: refactor more code from search_directories into this file. -// XX: do we also need a DataSink class? lets see if recording gets more complicated. - -} // namespace iorap::inode2filename diff --git a/src/inode2filename/inode_resolver.h b/src/inode2filename/inode_resolver.h deleted file mode 100644 index 218640f..0000000 --- a/src/inode2filename/inode_resolver.h +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_INODE2FILENAME_INODE_RESOLVER_H_ -#define IORAP_SRC_INODE2FILENAME_INODE_RESOLVER_H_ - -#include "common/expected.h" -#include "inode2filename/data_source.h" -#include "inode2filename/inode.h" -#include "inode2filename/system_call.h" - -#include <rxcpp/rx.hpp> - -#include <memory> -#include <optional> -#include <string> -#include <vector> -namespace iorap::inode2filename { - -enum class ProcessMode { - // Test modes: - kInProcessDirect, // Execute the code directly. - kInProcessIpc, // Execute code via IPC layer using multiple threads. - // Shipping mode: - kOutOfProcessIpc, // Execute code via fork+exec with IPC. - - // Note: in-process system-wide stat(2)/readdir/etc is blocked by selinux. - // Attempting to call the test modes will fail with -EPERM. - // - // Use fork+exec mode in shipping configurations, which spawns inode2filename - // as a separate command. -}; - -enum class VerifyKind { - kNone, - kStat, -}; - -std::vector<std::string> ToArgs(VerifyKind verify_kind); - -struct InodeResolverDependencies : public DataSourceDependencies { - ProcessMode process_mode = ProcessMode::kInProcessDirect; - VerifyKind verify{VerifyKind::kStat}; // Filter out results that aren't up-to-date with stat(2) ? -}; - -std::vector<std::string> ToArgs(const InodeResolverDependencies& deps); - -// Create an rx-observable chain that allows searching for inode->filename mappings given -// a set of inode keys. -class InodeResolver : public std::enable_shared_from_this<InodeResolver> { - public: - static std::shared_ptr<InodeResolver> Create(InodeResolverDependencies dependencies, - std::shared_ptr<DataSource> data_source); // nonnull - - // Convenience function for above: Uses DataSource::Create for the data-source. - static std::shared_ptr<InodeResolver> Create(InodeResolverDependencies dependencies); - - // Search the associated data source to map each inode in 'inodes' to a file path. - // - // Observes DataSource::EmitInodes(), which is unsubscribed from early once all inodes are found. - // - // Notes: - // * Searching does not begin until all 'inodes' are observed to avoid rescanning. - // * If the observable is unsubscribed to prior to completion, searching will halt. - // - // Post-condition: All emitted results are in inodes, and all inodes are in emitted results. - rxcpp::observable<InodeResult> - FindFilenamesFromInodes(rxcpp::observable<Inode> inodes) const; - // TODO: feels like we could turn this into a general helper? - // Convenience function for above. - virtual rxcpp::observable<InodeResult> - FindFilenamesFromInodes(std::vector<Inode> inodes) const; - - // Enumerate *all* inodes available from the data source, associating it with a filepath. - // - // Depending on the data source (e.g. diskscan), it can take a very long time for this observable - // to complete. The intended use-case is for development/debugging, not for production. - // - // Observes DataSource::EmitInodes() until it reaches #on_completed. - // - // Notes: - // * If the observable is unsubscribed to prior to completion, searching will halt. - virtual rxcpp::observable<InodeResult> - EmitAll() const; - - // Notifies the DataSource to begin recording. - // Some DataSources may be continuously refreshing, but only if recording is enabled. - // To get the most up-to-date data, toggle recording before reading the inodes out. - void StartRecording(); // XX: feels like this should be BPF-specific. - - // Notifies the DataSource to stop recording. - // Some DataSources may be continuously refreshing, but only if recording is enabled. - // The snapshot of data returned by e.g. #EmitAll would then not change outside of recording. - void StopRecording(); - - virtual ~InodeResolver(); - private: - struct Impl; - Impl* impl_; - - protected: - InodeResolver(InodeResolverDependencies dependencies); - InodeResolver(InodeResolverDependencies dependencies, std::shared_ptr<DataSource> data_source); - InodeResolverDependencies& GetDependencies(); - const InodeResolverDependencies& GetDependencies() const; -}; - -} - -#endif // IORAP_SRC_INODE2FILENAME_INODE_RESOLVER_H_ diff --git a/src/inode2filename/inode_result.cc b/src/inode2filename/inode_result.cc deleted file mode 100644 index 139b038..0000000 --- a/src/inode2filename/inode_result.cc +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "inode2filename/inode_result.h" - -#include <string.h> - -namespace iorap::inode2filename { - -std::optional<std::string_view> InodeResult::ErrorMessage() const { - if (data) { - return std::nullopt; - } - - const int err_no = data.error(); - return std::string_view{ - [=]() -> const char * { - switch (err_no) { - case InodeResult::kCouldNotFindFilename: - return "Could not find filename"; - case InodeResult::kVerificationFailed: - return "Verification failed"; - default: - return strerror(err_no); - } - }() - }; -} - -std::ostream& operator<<(std::ostream& os, const InodeResult& result) { - os << "InodeResult{"; - if (result) { - os << "OK,"; - } else { - os << "ERR,"; - } - - os << result.inode << ","; - - if (result) { - os << "\"" << result.data.value() << "\""; - } else { - os << result.data.error(); - os << " (" << *result.ErrorMessage() << ")"; - } - - os << "}"; - return os; -} - -} // namespace iorap::inode2filename
\ No newline at end of file diff --git a/src/inode2filename/inode_result.h b/src/inode2filename/inode_result.h deleted file mode 100644 index df352fb..0000000 --- a/src/inode2filename/inode_result.h +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_INODE2FILENAME_INODE_RESULT_H_ -#define IORAP_SRC_INODE2FILENAME_INODE_RESULT_H_ - -#include "common/expected.h" -#include "inode2filename/inode.h" -#include "inode2filename/inode_result.h" -#include "inode2filename/system_call.h" - -#include <rxcpp/rx.hpp> - -#include <memory> -#include <optional> -#include <string> -#include <vector> -namespace iorap::inode2filename { - -// Tuple of (Inode -> (Filename|Errno)) -struct InodeResult { - // We set this error when all root directories have been searched and - // yet we still could not find a corresponding filename for the inode under search. - static constexpr int kCouldNotFindFilename = ENOKEY; - - // An initial inode->filename mapping was found, but subsequent verification for it failed. - static constexpr int kVerificationFailed = EKEYEXPIRED; - - // There is always an inode, but sometimes we may fail to resolve the filename. - Inode inode; - // Value: Contains the filename (with a root directory as a prefix). - // Error: Contains the errno, usually one of the above, otherwise some system error. - iorap::expected<std::string /*filename*/, int /*errno*/> data; - - static InodeResult makeSuccess(Inode inode, std::string filename) { - return InodeResult{inode, std::move(filename)}; - } - - static InodeResult makeFailure(Inode inode, int err_no) { - return InodeResult{inode, iorap::unexpected{err_no}}; - } - - constexpr explicit operator bool() const { - return data.has_value(); - } - - constexpr bool operator==(const InodeResult& other) const { - if (inode == other.inode) { - if (data && other.data) { - return *data == *other.data; - } else if (!data && !other.data) { - return data.error() == other.data.error(); - } - // TODO: operator== for expected - } - return false; - } - - constexpr bool operator!=(const InodeResult& other) const { - return !(*this == other); - } - - // Returns a human-readable error message, or 'nullopt' if there was no error. - std::optional<std::string_view> ErrorMessage() const; -}; - -std::ostream& operator<<(std::ostream& os, const InodeResult& result); - -} // namespace iorap::inode2filename - -#endif // IORAP_SRC_INODE2FILENAME_INODE_RESULT_H_
\ No newline at end of file diff --git a/src/inode2filename/main.cc b/src/inode2filename/main.cc deleted file mode 100644 index 986f6ac..0000000 --- a/src/inode2filename/main.cc +++ /dev/null @@ -1,455 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/debug.h" -#include "common/expected.h" -#include "inode2filename/inode_resolver.h" - -#include <android-base/strings.h> - -#include <iostream> -#include <fstream> -#include <sstream> -#include <string_view> - -#if defined(IORAP_INODE2FILENAME_MAIN) - -namespace iorap::inode2filename { - -void Usage(char** argv) { - std::cerr << "Usage: " << argv[0] << " <options> <<inode_syntax>> [inode_syntax1 inode_syntax2 ...]" << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Block until all inodes have been read in, then begin searching for filenames for those inodes." << std::endl; - std::cerr << " Results are written immediately as they are available, and once all inodes are found, " << std::endl; - std::cerr << " the program will terminate." << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Inode syntax: ('dev_t@inode' | 'major:minor:inode')" << std::endl; - std::cerr << "" << std::endl; // CLI-only flags. - std::cerr << " --help,-h Print this Usage." << std::endl; - std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; - std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; - std::cerr << "" << std::endl; // General flags. - std::cerr << " --all,-a Enumerate all inode->filename mappings in the dataset (default off)." << std::endl; - std::cerr << " All <<inode_syntaxN>> arguments are ignored." << std::endl; - std::cerr << " --data-source=, Choose a data source (default 'diskscan')." << std::endl; - std::cerr << " -ds " << std::endl; - std::cerr << " diskscan Scan disk recursively using readdir." << std::endl; - std::cerr << " textcache Use the file from the '--output-format=text'." << std::endl; - std::cerr << " bpf Query kernel BPF maps (experimental)." << std::endl; - std::cerr << " --output=,-o Choose an output file (default 'stdout')." << std::endl; - std::cerr << " --output-format=, Choose an output format (default 'log')." << std::endl; - std::cerr << " -of " << std::endl; - std::cerr << " log Log human-readable, non-parsable format to stdout+logcat." << std::endl; - std::cerr << " textcache Results are in the same format as system/extras/pagecache." << std::endl; - std::cerr << " ipc Results are in a binary inter-process communications format" << std::endl; - std::cerr << " --process-mode=, Choose a process mode (default 'in'). Test-oriented." << std::endl; - std::cerr << " -pm " << std::endl; - std::cerr << " in Use a single process to do the work in." << std::endl; - std::cerr << " out Out-of-process work (forks into a -pm=in)." << std::endl; - std::cerr << " --verify=,-vy Verification modes for the data source (default 'stat')." << std::endl; - std::cerr << " stat Use stat(2) call to validate data inodes are up-to-date. " << std::endl; - std::cerr << " none Trust that the data-source is up-to-date without checking." << std::endl; - std::cerr << "" << std::endl; // --data-source=<?> specific flags. - std::cerr << " Data-source-specific commands:" << std::endl; - std::cerr << " --data-source=diskscan" << std::endl; - std::cerr << " --root=,-r Add root directory (default '.'). Repeatable." << std::endl; - std::cerr << " --data-source=textcache" << std::endl; - std::cerr << " --textcache=,-tc Name of file that contains the textcache." << std::endl; - std::cerr << "" << std::endl; // Programmatic flags. - std::cerr << " --in-fd=# Input file descriptor. Default input is from argv." << std::endl; - std::cerr << " --out-fd=# Output file descriptor. Default stdout." << std::endl; - exit(1); -} - -static fruit::Component<SystemCall> GetSystemCallComponent() { - return fruit::createComponent().bind<SystemCall, SystemCallImpl>(); -} - -std::optional<DataSourceKind> ParseDataSourceKind(std::string_view str) { - if (str == "diskscan") { - return DataSourceKind::kDiskScan; - } else if (str == "textcache") { - return DataSourceKind::kTextCache; - } else if (str == "bpf") { - return DataSourceKind::kBpf; - } - return std::nullopt; -} - -enum class OutputFormatKind { - kLog, - kTextCache, - kIpc, -}; - -std::optional<OutputFormatKind> ParseOutputFormatKind(std::string_view str) { - if (str == "log") { - return OutputFormatKind::kLog; - } else if (str == "textcache") { - return OutputFormatKind::kTextCache; - } else if (str == "ipc") { - return OutputFormatKind::kIpc; - } - return std::nullopt; -} - -std::optional<VerifyKind> ParseVerifyKind(std::string_view str) { - if (str == "none") { - return VerifyKind::kNone; - } else if (str == "stat") { - return VerifyKind::kStat; - } - return std::nullopt; -} - -std::optional<ProcessMode> ParseProcessMode(std::string_view str) { - if (str == "in") { - return ProcessMode::kInProcessDirect; - } else if (str == "out") { - return ProcessMode::kOutOfProcessIpc; - } - return std::nullopt; -} - -bool StartsWith(std::string_view haystack, std::string_view needle) { - return haystack.size() >= needle.size() - && haystack.compare(0, needle.size(), needle) == 0; -} - -bool EndsWith(std::string_view haystack, std::string_view needle) { - return haystack.size() >= needle.size() - && haystack.compare(haystack.size() - needle.size(), haystack.npos, needle) == 0; -} - -bool StartsWithOneOf(std::string_view haystack, - std::string_view needle, - std::string_view needle2) { - return StartsWith(haystack, needle) || StartsWith(haystack, needle2); -} - -enum ParseError { - kParseSkip, - kParseFailed, -}; - -std::optional<std::string> ParseNamedArgument(std::initializer_list<std::string> names, - std::string argstr, - std::optional<std::string> arg_next, - /*inout*/ - int* arg_pos) { - for (const std::string& name : names) { - { - // Try parsing only 'argstr' for '--foo=bar' type parameters. - std::vector<std::string> split_str = ::android::base::Split(argstr, "="); - if (split_str.size() >= 2) { - /* - std::cerr << "ParseNamedArgument(name=" << name << ", argstr='" - << argstr << "')" << std::endl; - */ - - if (split_str[0] + "=" == name) { - return split_str[1]; - } - } - } - //if (EndsWith(name, "=")) { - // continue; - /*} else */ { - // Try parsing 'argstr arg_next' for '-foo bar' type parameters. - if (argstr == name) { - ++(*arg_pos); - - if (arg_next) { - return arg_next; - } else { - // Missing argument, e.g. '-foo' was the last token in the argv. - std::cerr << "Missing " << name << " flag value." << std::endl; - exit(1); - } - } - } - } - - return std::nullopt; -} - -int main(int argc, char** argv) { - android::base::InitLogging(argv); - android::base::SetLogger(android::base::StderrLogger); - - bool all = false; - bool wait_for_keystroke = false; - bool enable_verbose = false; - std::vector<std::string> root_directories; - std::vector<Inode> inode_list; - int recording_time_sec = 0; - - DataSourceKind data_source = DataSourceKind::kDiskScan; - OutputFormatKind output_format = OutputFormatKind::kLog; - VerifyKind verify = VerifyKind::kStat; - ProcessMode process_mode = ProcessMode::kInProcessDirect; - - std::optional<std::string> output_filename; - std::optional<int /*fd*/> in_fd, out_fd; // input-output file descriptors [for fork+exec]. - std::optional<std::string> text_cache_filename; - - if (argc == 1) { - Usage(argv); - } - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - std::optional<std::string> arg_next; - if ((arg + 1) < argc) { - arg_next = argv[arg+1]; - } - - if (argstr == "--help" || argstr == "-h") { - Usage(argv); - } else if (auto val = ParseNamedArgument({"--root=", "-r"}, argstr, arg_next, /*inout*/&arg); - val) { - root_directories.push_back(*val); - } else if (argstr == "--verbose" || argstr == "-v") { - enable_verbose = true; - } else if (argstr == "--wait" || argstr == "-w") { - wait_for_keystroke = true; - } - else if (argstr == "--all" || argstr == "-a") { - all = true; - } else if (auto val = ParseNamedArgument({"--data-source=", "-ds"}, - argstr, - arg_next, - /*inout*/&arg); - val) { - auto ds = ParseDataSourceKind(*val); - if (!ds) { - std::cerr << "Invalid --data-source=<value>" << std::endl; - return 1; - } - data_source = *ds; - } else if (auto val = ParseNamedArgument({"--output=", "-o"}, - argstr, - arg_next, - /*inout*/&arg); - val) { - output_filename = *val; - } else if (auto val = ParseNamedArgument({"--process-mode=", "-pm"}, - argstr, - arg_next, - /*inout*/&arg); - val) { - auto pm = ParseProcessMode(*val); - if (!pm) { - std::cerr << "Invalid --process-mode=<value>" << std::endl; - return 1; - } - process_mode = *pm; - } - else if (auto val = ParseNamedArgument({"--output-format=", "-of"}, - argstr, - arg_next, - /*inout*/&arg); - val) { - auto of = ParseOutputFormatKind(*val); - if (!of) { - std::cerr << "Missing --output-format=<value>" << std::endl; - return 1; - } - output_format = *of; - } else if (auto val = ParseNamedArgument({"--verify=", "-vy="}, - argstr, - arg_next, - /*inout*/&arg); - val) { - auto vy = ParseVerifyKind(*val); - if (!vy) { - std::cerr << "Invalid --verify=<value>" << std::endl; - return 1; - } - verify = *vy; - } else if (auto val = ParseNamedArgument({"--textcache=", "-tc"}, - argstr, - arg_next, - /*inout*/&arg); - val) { - text_cache_filename = *val; - } else { - Inode maybe_inode{}; - - std::string error_msg; - if (Inode::Parse(argstr, /*out*/&maybe_inode, /*out*/&error_msg)) { - inode_list.push_back(maybe_inode); - } else { - if (argstr.size() >= 1) { - if (argstr[0] == '-') { - std::cerr << "Unrecognized flag: " << argstr << std::endl; - return 1; - } - } - - std::cerr << "Failed to parse inode (" << argstr << ") because: " << error_msg << std::endl; - return 1; - } - } - } - - if (root_directories.size() == 0) { - root_directories.push_back("."); - } - - if (inode_list.size() == 0 && !all) { - std::cerr << "Provide at least one inode. Or use --all to dump everything." << std::endl; - return 1; - } else if (all && inode_list.size() > 0) { - std::cerr << "[WARNING]: --all flag ignores all inodes passed on command line." << std::endl; - } - - std::ofstream fout; - if (output_filename) { - fout.open(*output_filename); - if (!fout) { - std::cerr << "Failed to open output file for writing: \"" << *output_filename << "\""; - return 1; - } - } else { - fout.open("/dev/null"); // have to open *something* otherwise rdbuf fails. - fout.basic_ios<char>::rdbuf(std::cout.rdbuf()); - } - - if (enable_verbose) { - android::base::SetMinimumLogSeverity(android::base::VERBOSE); - - LOG(VERBOSE) << "Verbose check"; - LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; - - for (auto& inode_num : inode_list) { - LOG(VERBOSE) << "Searching for inode " << inode_num; - } - - LOG(VERBOSE) << "Dumping all inodes? " << all; - } - // else use - // $> ANDROID_LOG_TAGS='*:d' iorap.inode2filename <args> - // which will enable arbitrary log levels. - - // Useful to attach a debugger... - // 1) $> inode2filename -w <args> - // 2) $> gdbclient <pid> - if (wait_for_keystroke) { - LOG(INFO) << "Self pid: " << getpid(); - LOG(INFO) << "Press any key to continue..."; - std::cin >> wait_for_keystroke; - } - - fruit::Injector<SystemCall> injector(GetSystemCallComponent); - - InodeResolverDependencies ir_dependencies; - // Passed from command-line. - ir_dependencies.data_source = data_source; - ir_dependencies.process_mode = process_mode; - ir_dependencies.root_directories = root_directories; - ir_dependencies.text_cache_filename = text_cache_filename; - ir_dependencies.verify = verify; - // Hardcoded. - ir_dependencies.system_call = injector.get<SystemCall*>(); - - std::shared_ptr<InodeResolver> inode_resolver = - InodeResolver::Create(ir_dependencies); - - inode_resolver->StartRecording(); - sleep(recording_time_sec); // TODO: add cli flag for this when we add something that needs it. - inode_resolver->StopRecording(); - - auto/*observable<InodeResult>*/ inode_results = all - ? inode_resolver->EmitAll() - : inode_resolver->FindFilenamesFromInodes(std::move(inode_list)); - - int return_code = 2; - inode_results.subscribe( - /*on_next*/[&return_code, output_format, &fout](const InodeResult& result) { - if (result) { - LOG(DEBUG) << "Inode match: " << result; - if (output_format == OutputFormatKind::kLog) { - fout << "\033[1;32m[OK]\033[0m " - << result.inode - << " \"" << result.data.value() << "\"" << std::endl; - } else if (output_format == OutputFormatKind::kIpc) { - std::stringstream stream; - stream << "K " << result.inode << " " << result.data.value(); - std::string line = stream.str(); - - // Convert the size to 4 bytes. - int32_t size = line.size(); - char buf[sizeof(int32_t)]; - memcpy(buf, &size, sizeof(int32_t)); - fout.write(buf, sizeof(int32_t)); - fout.write(line.c_str(), size); - } else if (output_format == OutputFormatKind::kTextCache) { - // Same format as TextCacheDataSource (system/extras/pagecache/pagecache.py -d) - // "$device_number $inode $filesize $filename..." - const Inode& inode = result.inode; - fout << inode.GetDevice() << " " - << inode.GetInode() - << " -1 " // always -1 for filesize, since we don't track what it is. - << result.data.value() << "\n"; // don't use endl which flushes, for faster writes. - } else { - LOG(FATAL) << "Not implemented this kind of --output-format"; - } - - return_code = 0; - } else { - LOG(DEBUG) << "Failed to match inode: " << result; - if (output_format == OutputFormatKind::kLog) { - fout << "\033[1;31m[ERR]\033[0m " - << result.inode - << " '" << *result.ErrorMessage() << "'" << std::endl; - } else if (output_format == OutputFormatKind::kIpc) { - std::stringstream stream; - stream << "E " << result.inode << " " << result.data.error() << std::endl; - std::string line = stream.str(); - - // Convert the size to 4 bytes. - int32_t size = line.size(); - char buf[sizeof(int32_t)]; - memcpy(buf, &size, sizeof(int32_t)); - fout.write(buf, sizeof(int32_t)); - fout.write(line.c_str(), size); - } - else if (output_format == OutputFormatKind::kTextCache) { - // Don't add bad results to the textcache. They are dropped. - } else { - LOG(FATAL) << "Not implemented this kind of --output-format"; - } - } - }, /*on_error*/[&return_code](rxcpp::util::error_ptr error) { - // Usually occurs very early on before we see the first result. - // In this case the error is terminal so we just end up exiting out soon. - return_code = 3; - LOG(ERROR) << "Critical error: " << rxcpp::util::what(error); - }); - - // 0 -> found at least a single match, - // 1 -> bad parameters, - // 2 -> could not find any matches, - // 3 -> rxcpp on_error. - return return_code; -} - -} // namespace iorap::inode2filename - -int main(int argc, char** argv) { - return ::iorap::inode2filename::main(argc, argv); -} - -#endif diff --git a/src/inode2filename/out_of_process_inode_resolver.cc b/src/inode2filename/out_of_process_inode_resolver.cc deleted file mode 100644 index f409fd2..0000000 --- a/src/inode2filename/out_of_process_inode_resolver.cc +++ /dev/null @@ -1,402 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/cmd_utils.h" -#include "inode2filename/inode_resolver.h" -#include "inode2filename/out_of_process_inode_resolver.h" - -#include <android-base/file.h> -#include <android-base/logging.h> -#include <android-base/unique_fd.h> - -#include <sstream> -#include <stdio.h> -#include <unistd.h> - -namespace rx = rxcpp; - -namespace iorap::inode2filename { - -using ::android::base::unique_fd; -using ::android::base::borrowed_fd; - -static const char* GetCommandFileName() { - // Avoid ENOENT by execve by specifying the absolute path of inode2filename. -#ifdef __ANDROID__ - return "/system/bin/iorap.inode2filename"; -#else - static const char* file_name = nullptr; - - if (file_name == nullptr) { - char* out_dir = getenv("ANDROID_HOST_OUT"); - static std::string file_name_str = out_dir; - if (out_dir != nullptr) { - file_name_str += "/bin/"; - } else { - // Assume it's in the same directory as the binary we are in. - std::string self_path; - CHECK(::android::base::Readlink("/proc/self/exe", /*out*/&self_path)); - - std::string self_dir = ::android::base::Dirname(self_path); - file_name_str = self_dir + "/"; - } - file_name_str += "iorap.inode2filename"; - - file_name = file_name_str.c_str(); - } - - return file_name; -#endif -} - -std::error_code ErrorCodeFromErrno() { - int err = errno; - return std::error_code(err, std::system_category()); -} - -std::ios_base::failure IosBaseFailureWithErrno(const char* message) { - std::error_code ec = ErrorCodeFromErrno(); - return std::ios_base::failure(message, ec); -} - -static constexpr bool kDebugFgets = false; - -int32_t ReadLineLength(FILE* stream, bool* eof) { - char buf[sizeof(int32_t)]; - size_t count = fread(buf, 1, sizeof(int32_t), stream); - if (feof(stream)) { - // If reaching the end of the stream when trying to read the first int, just - // return. This is legitimate, because after reading the last line, the next - // iteration will reach this. - *eof = true; - return 0; - } - int32_t length; - memcpy(&length, buf, sizeof(int32_t)); - return length; -} - -// The steam is like [size1][file1][size2][file2]...[sizeN][fileN]. -std::string ReadOneLine(FILE* stream, bool* eof) { - DCHECK(stream != nullptr); - DCHECK(eof != nullptr); - - int32_t length = ReadLineLength(stream, eof); - if (length <= 0) { - PLOG(ERROR) << "unexpected 0 length line."; - *eof = true; - return ""; - } - - std::string str(length, '\0'); - size_t count = fread(&str[0], sizeof(char), length, stream); - if (feof(stream) || ferror(stream) || count != (uint32_t)length) { - // error! :( - PLOG(ERROR) << "unexpected end of the line during fread"; - *eof = true; - return ""; - } - return str; -} - -static inline void LeftTrim(/*inout*/std::string& s) { - s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { - return !std::isspace(ch); - })); -} - -// Parses an --output-format=ipc kind of line into an InodeResult. -// Returns nullopt if parsing failed. -std::optional<InodeResult> ParseFromLine(const std::string& line) { - // inode <- INT:INT:INT - // line_ok <- 'K ' inode ' ' STRING - // line_err <- 'E ' inode ' ' INT - // - // result <- line_ok | line_err - - std::stringstream ss{line}; - - bool result_ok = false; - - std::string ok_or_error; - ss >> ok_or_error; - - if (ss.fail()) { - return std::nullopt; - } - - if (ok_or_error == "K") { - result_ok = true; - } else if (ok_or_error == "E") { - result_ok = false; - } else { - return std::nullopt; - } - - std::string inode_s; - ss >> inode_s; - - if (ss.fail()) { - return std::nullopt; - } - - Inode inode; - - std::string inode_parse_error_msg; - if (!Inode::Parse(inode_s, /*out*/&inode, /*out*/&inode_parse_error_msg)) { - return std::nullopt; - } - - if (result_ok == false) { - int error_code; - ss >> error_code; - - if (ss.fail()) { - return std::nullopt; - } - - return InodeResult::makeFailure(inode, error_code); - } else if (result_ok == true) { - std::string rest_of_line; - ss >> rest_of_line; - - // parse " string with potential spaces" - // into "string with potential spaces" - LeftTrim(/*inout*/rest_of_line); - - if (ss.fail()) { - return std::nullopt; - } - - const std::string& file_name = rest_of_line; - - return InodeResult::makeSuccess(inode, file_name); - } - - return std::nullopt; -} - -struct OutOfProcessInodeResolver::Impl { - Impl() { - } - - private: - // Create the argv we will pass to the forked inode2filename, corresponds to #EmitAll. - std::vector<std::string> CreateArgvAll(const InodeResolverDependencies& deps) const { - std::vector<std::string> argv; - argv.push_back("--all"); - - return CreateArgv(deps, std::move(argv)); - } - - // Create the argv we will pass to the forked inode2filename, corresponds to - // #FindFilenamesFromInodes. - std::vector<std::string> CreateArgvFind(const InodeResolverDependencies& deps, - const std::vector<Inode>& inodes) const { - std::vector<std::string> argv; - iorap::common::AppendArgsRepeatedly(argv, inodes); - - return CreateArgv(deps, std::move(argv)); - } - - std::vector<std::string> CreateArgv(const InodeResolverDependencies& deps, - std::vector<std::string> append_argv) const { - InodeResolverDependencies deps_oop = deps; - deps_oop.process_mode = ProcessMode::kInProcessDirect; - - std::vector<std::string> argv = ToArgs(deps_oop); - - argv.push_back("--output-format=ipc"); - - if (iorap::common::GetBoolEnvOrProperty("iorap.inode2filename.log.verbose", false)) { - argv.push_back("--verbose"); - } - - iorap::common::AppendArgsRepeatedly(argv, std::move(append_argv)); - - return argv; - } - - public: - // fork+exec into inode2filename with 'inodes' as the search list. - // Each result is parsed into a dest#on_next(result). - // If a fatal error occurs, dest#on_error is called once and no other callbacks are called. - void EmitFromCommandFind(rxcpp::subscriber<InodeResult>& dest, - const InodeResolverDependencies& deps, - const std::vector<Inode>& inodes) { - // Trivial case: complete immediately. - // Executing inode2filename with empty search list will just print the --help menu. - if (inodes.empty()) { - dest.on_completed(); - } - - std::vector<std::string> argv = CreateArgvFind(deps, inodes); - EmitFromCommandWithArgv(/*inout*/dest, std::move(argv), inodes.size()); - } - - // fork+exec into inode2filename with --all (listing *all* inodes). - // Each result is parsed into a dest#on_next(result). - // If a fatal error occurs, dest#on_error is called once and no other callbacks are called. - void EmitFromCommandAll(rxcpp::subscriber<InodeResult>& dest, - const InodeResolverDependencies& deps) { - std::vector<std::string> argv = CreateArgvAll(deps); - EmitFromCommandWithArgv(/*inout*/dest, std::move(argv), /*result_count*/std::nullopt); - } - - private: - void EmitFromCommandWithArgv(rxcpp::subscriber<InodeResult>& dest, - std::vector<std::string> argv_vec, - std::optional<size_t> result_count) { - unique_fd pipe_reader, pipe_writer; - if (!::android::base::Pipe(/*out*/&pipe_reader, /*out*/&pipe_writer)) { - dest.on_error( - rxcpp::util::make_error_ptr( - IosBaseFailureWithErrno("Failed to create out-going pipe for inode2filename"))); - return; - } - - pid_t child = fork(); - if (child == -1) { - dest.on_error( - rxcpp::util::make_error_ptr( - IosBaseFailureWithErrno("Failed to fork process for inode2filename"))); - return; - } else if (child > 0) { // we are the caller of this function - LOG(DEBUG) << "forked into a process for inode2filename , pid = " << child; - } else { - // we are the child that was forked - - const char* kCommandFileName = GetCommandFileName(); - - std::stringstream argv; // for debugging. - for (std::string arg : argv_vec) { - argv << arg << ' '; - } - LOG(DEBUG) << "fork+exec: " << kCommandFileName << " " << argv.str(); - - // Redirect only stdout. stdin is unused, stderr is same as parent. - if (dup2(pipe_writer.get(), STDOUT_FILENO) == -1) { - // Trying to call #on_error does not make sense here because we are in a forked process, - // the only thing we can do is crash definitively. - PLOG(FATAL) << "Failed to dup2 for inode2filename"; - } - - std::unique_ptr<const char *[]> argv_ptr = - common::VecToArgv(kCommandFileName, argv_vec); - - if (execve(kCommandFileName, - (char **)argv_ptr.get(), - /*envp*/nullptr) == -1) { - // Trying to call #on_error does not make sense here because we are in a forked process, - // the only thing we can do is crash definitively. - PLOG(FATAL) << "Failed to execve process for inode2filename"; - } - // This should never return. - } - - // Immediately close the writer end of the pipe because we never use it. - pipe_writer.reset(); - - // Convert pipe(reader) file descriptor into FILE*. - std::unique_ptr<FILE, int(*)(FILE*)> file_reader{ - ::android::base::Fdopen(std::move(pipe_reader), /*mode*/"r"), fclose}; - if (!file_reader) { - dest.on_error( - rxcpp::util::make_error_ptr( - IosBaseFailureWithErrno("Failed to fdopen for inode2filename"))); - return; - } - - size_t actual_result_count = 0; - - bool file_eof = false; - while (!file_eof) { - std::string inode2filename_line = ReadOneLine(file_reader.get(), /*out*/&file_eof); - - if (inode2filename_line.empty()) { - if (!file_eof) { - // Ignore empty lines. - LOG(WARNING) << "inode2filename: got empty line"; - } - continue; - } - - LOG(DEBUG) << "inode2filename output-line: " << inode2filename_line; - - std::optional<InodeResult> res = ParseFromLine(inode2filename_line); - if (!res) { - std::string error_msg = "Invalid output: "; - error_msg += inode2filename_line; - dest.on_error( - rxcpp::util::make_error_ptr(std::ios_base::failure(error_msg))); - return; - } - dest.on_next(*res); - - ++actual_result_count; - } - - LOG(DEBUG) << "inode2filename output-eof"; - - // Ensure that the # of inputs into the rx stream match the # of outputs. - // This is validating the post-condition of FindFilenamesFromInodes. - if (result_count && actual_result_count != *result_count) { - std::stringstream ss; - ss << "Invalid number of results, expected: " << *result_count; - ss << ", actual: " << actual_result_count; - - dest.on_error( - rxcpp::util::make_error_ptr(std::ios_base::failure(ss.str()))); - return; - } - - CHECK(child > 0); // we are in the parent process, parse the IPC output of inode2filename - dest.on_completed(); - } -}; - -rxcpp::observable<InodeResult> - OutOfProcessInodeResolver::FindFilenamesFromInodes(std::vector<Inode> inodes) const { - return rxcpp::observable<>::create<InodeResult>( - [self=std::static_pointer_cast<const OutOfProcessInodeResolver>(shared_from_this()), - inodes=std::move(inodes)]( - rxcpp::subscriber<InodeResult> s) { - self->impl_->EmitFromCommandFind(s, self->GetDependencies(), inodes); - }); -} - -rxcpp::observable<InodeResult> - OutOfProcessInodeResolver::EmitAll() const { - auto self = std::static_pointer_cast<const OutOfProcessInodeResolver>(shared_from_this()); - CHECK(self != nullptr); - CHECK(self->impl_ != nullptr); - - return rxcpp::observable<>::create<InodeResult>( - [self](rxcpp::subscriber<InodeResult> s) { - CHECK(self != nullptr); - CHECK(self->impl_ != nullptr); - self->impl_->EmitFromCommandAll(s, self->GetDependencies()); - }); -} - -OutOfProcessInodeResolver::OutOfProcessInodeResolver(InodeResolverDependencies dependencies) - : InodeResolver{std::move(dependencies)}, impl_{new Impl{}} { -} - -OutOfProcessInodeResolver::~OutOfProcessInodeResolver() { - // std::unique_ptr requires complete types, but we hide the definition in the header. - delete impl_; -} - -} // namespace iorap::inode2filename diff --git a/src/inode2filename/out_of_process_inode_resolver.h b/src/inode2filename/out_of_process_inode_resolver.h deleted file mode 100644 index d41e93f..0000000 --- a/src/inode2filename/out_of_process_inode_resolver.h +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_INODE2FILENAME_OUT_OF_PROCESS_INDOE_RESOLVER_H_ -#define IORAP_SRC_INODE2FILENAME_OUT_OF_PROCESS_INDOE_RESOLVER_H_ - -#include "common/expected.h" -#include "inode2filename/inode_resolver.h" - -namespace iorap::inode2filename { - -// Create an InodeResolver that fork+exec+pipes into the binary 'iorap.inode2filename' -// and transmits the results back via an IPC mechanism. -// -// This is instantiated by InodeResolver::Create + ProcessMode::kOutOfProcessIpc -class OutOfProcessInodeResolver : public InodeResolver { - public: - virtual rxcpp::observable<InodeResult> - FindFilenamesFromInodes(std::vector<Inode> inodes) const override; - - virtual rxcpp::observable<InodeResult> - EmitAll() const override; - - OutOfProcessInodeResolver(InodeResolverDependencies dependencies); - ~OutOfProcessInodeResolver(); - private: - struct Impl; - Impl* impl_; -}; - -// Reads one line data from the stream. -// Each line is in the format of "<4 bytes line length><state> <inode info> <file path>" -// The <4 bytes line length> is the length rest data of "<state> <inode info> <file path>". -// The return string is "<state> <inode info> <file path>". -// For example: for "<size>K 253:9:6 ./test", the return value is -// "K 253:9:6 ./test". The <size> is encoded in the first 4 bytes. -// Note: there is no newline in the end of each line and the line shouldn't be -// empty unless there is some error. -std::string ReadOneLine(FILE* stream, bool* eof); -} - -#endif // IORAP_SRC_INODE2FILENAME_OUT_OF_PROCESS_INDOE_RESOLVER_H_ diff --git a/src/inode2filename/search_directories.cc b/src/inode2filename/search_directories.cc deleted file mode 100644 index 1d31671..0000000 --- a/src/inode2filename/search_directories.cc +++ /dev/null @@ -1,1366 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/debug.h" -#include "inode2filename/search_directories.h" -#include "inode2filename/system_call.h" - -#include <android-base/file.h> -#include <android-base/logging.h> -#include <android-base/scopeguard.h> -#include <android-base/stringprintf.h> -#include <android-base/unique_fd.h> - -#include "rxcpp/rx.hpp" - -#include <iostream> -#include <stdio.h> -#include <fstream> -#include <vector> -#include <optional> - -#include <signal.h> -#include <stdlib.h> -#include <unistd.h> - -#include <sys/types.h> - -#ifdef __ANDROID__ -#include <sys/sysmacros.h> -#endif - -#include <sys/stat.h> -#include <fcntl.h> -#include <poll.h> -#include <dirent.h> - -#include <unordered_map> - -namespace rx = rxcpp; -using android::base::unique_fd; // NOLINT -using android::base::StringPrintf; // NOLINT - -namespace iorap::inode2filename { - -#define DEBUG_INODE_SET 0 - -// A multimap of 'ino_t -> List[Inode]' (where the value Inodes have the same ino_t as the key). -// -// A flat list of Inodes is turned into the above map, then keys can be removed one at a time -// until the InodeSet eventually becomes empty. -struct InodeSet { - - InodeSet() = default; -#if DEBUG_INODE_SET - InodeSet(const InodeSet& other) { - LOG(INFO) << "InodeSet-copyctor"; - set_ = other.set_; - } - - InodeSet(InodeSet&& other) { - LOG(INFO) << "InodeSet-movector"; - set_ = std::move(other.set_); - } - - InodeSet& operator=(const InodeSet& other) { - LOG(INFO) << "InodeSet-opassign-copy"; - set_ = other.set_; - return *this; - } - - InodeSet& operator=(InodeSet&& other) { - LOG(INFO) << "InodeSet-opassign-move"; - set_ = std::move(other.set_); - return *this; - } -#else - InodeSet(InodeSet&& other) = default; - InodeSet& operator=(InodeSet&& other) = default; - // Copying InodeSet can be very expensive, refuse to even allow compiling such code. - InodeSet(const InodeSet& other) = delete; - InodeSet& operator=(const InodeSet& other) = delete; -#endif - - struct ValueRange { - auto/*Iterable<Inode>*/ begin() { - return begin_; - } - - auto/*Iterable<Inode>*/ end() { - return end_; - } - - bool empty() const { - return begin_ == end_; - } - - explicit operator bool() const { - return !empty(); - } - - std::unordered_multimap<ino_t, Inode>::iterator begin_, end_; - - friend std::ostream& operator<<(std::ostream& os, const ValueRange& s); - }; - - // Create an observable that emits the remaining inodes in the map. - // - // Mutation functions must not be called until this observable - // has been finished emitting all values (e.g. with on_completed) since that - // would cause the underlying iterators to go into an undefined state. - auto/*observable<Inode>*/ IterateValues() const { - return rxcpp::observable<>::iterate(set_).map( // XX: should we use identity_immediate here? - [](const std::pair<const ino_t, Inode>& pair) { - return pair.second; - } - ); - // TODO: this would be more efficient as a range-v3 view. - } - - constexpr bool Empty() const { - return set_.empty(); - } - - static InodeSet OfList(const std::vector<Inode>& list) { - InodeSet new_inode_set; - std::unordered_multimap<ino_t, Inode>* map = &new_inode_set.set_; - - for (const Inode& inode : list) { - map->insert({inode.inode, inode}); - } - - return new_inode_set; - } - - // Return an optional list of 'Inode' structs whose 'inode' field matches the 'inode' parameter. - // Returns an empty range if there was nothing found. - ValueRange FindInodeList(ino_t inode) { - auto range = set_.equal_range(inode); - return ValueRange{range.first, range.second}; - } - - // Match all fields of an Inode against a 'struct stat' stat_buf. - // - // The returned Inode (if any) is removed from the InodeSet; it will not be returned by - // FindInodeList in future calls. - std::optional<Inode> FindAndRemoveInodeInList(ValueRange inode_list, - const struct stat& stat_buf) { - LOG(VERBOSE) << "FindAndRemoveInodeInList " << inode_list << ", " - << "stat_buf{st_dev=" << stat_buf.st_dev << ",st_ino=" << stat_buf.st_ino << "}"; - - auto /*iterator*/ found = std::find_if(inode_list.begin(), - inode_list.end(), - [&](const std::pair<ino_t, Inode>& pair) { - const Inode& inode = pair.second; - if (inode.inode != stat_buf.st_ino) { - return false; - } - - dev_t inode_dev = - makedev(static_cast<int>(inode.device_major), static_cast<int>(inode.device_minor)); - - // Inodes could be the same across different devices. - // Also match the device id. - if (inode_dev != stat_buf.st_dev) { - LOG(VERBOSE) << "InodeSet:FindAndRemoveInodeInList matched ino: " << inode.inode - << " but not device" - << ", expected dev: " << stat_buf.st_dev - << ", actual dev: " << inode_dev; - return false; - } - return true; - }); - - if (found != inode_list.end()) { - Inode inode = found->second; - LOG(VERBOSE) << "InodeSet:FindAndRemoveInodeInList *success* inode+device " << inode; - DCHECK(found->second.inode == stat_buf.st_ino); - // Erase the inode from the list. This is important. - set_.erase(found); - return inode; - } - - return std::nullopt; - } - - // Match all fields of an Inode against another Inode. - // - // The returned Inode (if any) is removed from the InodeSet; it will not be returned by - // FindInodeList in future calls. - std::optional<Inode> FindAndRemoveInodeInList(ValueRange inode_list, - const Inode& inode) { - LOG(VERBOSE) << "FindAndRemoveInodeInList " << inode_list << ", " - << inode << "}"; - - auto /*iterator*/ found = std::find_if(inode_list.begin(), - inode_list.end(), - [&](const std::pair<ino_t, Inode>& pair) { - return inode == pair.second; - }); - - if (found != inode_list.end()) { - Inode inode = found->second; - LOG(VERBOSE) << "InodeSet:FindAndRemoveInodeInList *success* inode+device " << inode; - DCHECK_EQ(found->second, inode); - // Erase the inode from the list. This is important. - set_.erase(found); - return inode; - } - - return std::nullopt; - } - - // TODO: equality and string operators for testing/logging. - private: - // Explanation: readdir returns a 'file' -> 'ino_t inode' mapping. - // - // However inodes can be reused on different partitions (but they have a different device number). - // To handle this edge case, and to avoid calling stat whenever the inode definitely doesn't match - // store the inodes into a single-key,multi-value container. - // - // This enables fast scanning of readdir results by matching just the 'inode' portion, - // then calling stat only when the inode portion definitely matches to confirm the device. - - // There are no single-key multi-value containers in standard C++, so pretend - // we have one by writing this simple facade around an unordered set. - // - // We expect that the vector size is usually size=1 (or 2 or 3) since the # of devices - // is fixed by however many partitions there are on the system, AND the same inode # - // would have to be reused across a different file. - std::unordered_multimap<ino_t, Inode> set_; // TODO: Rename to map_. - - friend std::ostream& operator<<(std::ostream& os, const InodeSet& s); -}; - -std::ostream& operator<<(std::ostream& os, const InodeSet& s) { - os << "InodeSet{"; - for (const auto& kv : s.set_) { - // e.g. "123=>(1:2:123)" ... its expected for the 'ino_t' portion to be repeated. - os << "" << kv.first << "=>(" << kv.second << "),"; - } - os << "}"; - return os; -} - -std::ostream& operator<<(std::ostream& os, const InodeSet::ValueRange& v) { - // Don't want to make a const and non const version of ValueRange. - InodeSet::ValueRange& s = const_cast<InodeSet::ValueRange&>(v); - - os << "InodeSet::ValueRange{"; - for (const auto& kv : s) { - // e.g. "123=>(1:2:123)" ... its expected for the 'ino_t' portion to be repeated. - os << "" << kv.first << "=>(" << kv.second << "),"; - } - os << "}"; - return os; -} - -void search_for_inodes_in(std::vector<Inode>& inode_list, const std::string& dirpath); - -enum DirectoryEntryErrorCode { - kInvalid, // not a real error code. to detect bad initialization. - kOpenDir, // opendir failed. - kReadDir, // readdir failed. - kDtUnknown, // d_type was DT_UNKNOWN error. -}; - -struct DirectoryEntryError { - DirectoryEntryErrorCode code; - int err_no; - std::string filename; -}; - -std::ostream& operator<<(std::ostream& os, const DirectoryEntryError& e) { - os << "DirectoryEntryError{" - << static_cast<int>(e.code) << "," << e.err_no << "," << e.filename << "}"; - return os; - // TODO: pretty-print code and err-no -} - -static common::DebugCounter gDebugDirectoryEntryCounter{}; -static constexpr bool kDebugDirectoryEntry = false; - -#define DIRECTORY_ENTRY_MOVE_DCHECK() \ - DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__ << "CNT:" << other.debug_counter_; -#define DIRECTORY_ENTRY_TRACE_CTOR() \ - if (kDebugDirectoryEntry) LOG(VERBOSE) << __PRETTY_FUNCTION__ << "@CNT:" << debug_counter_ - -struct DirectoryEntry { - using ResultT = iorap::expected<DirectoryEntry, DirectoryEntryError>; - using ObservableT = rx::observable<ResultT>; - - static constexpr ino_t kInvalidIno = std::numeric_limits<ino_t>::max(); - static constexpr auto kInvalidFileName = ""; - - // Path to file, the prefix is one of the root directories. - std::string filename{kInvalidFileName}; - // Inode number of the file. Not unique across different devices. - ino_t d_ino{kInvalidIno}; - // File type (DT_LNK, DT_REG, DT_DIR, or DT_UNKNOWN) - unsigned char d_type{DT_UNKNOWN}; // Note: not seen outside of sentinel roots. - // TODO: Consider invariant checks for valid combinations of above fields? - - // Debug-only flags. - bool moved_from_{false}; - size_t debug_counter_{0}; - - private: - // TODO: remove default constructor? - // - // SEEMS TO BE USED by std::vector etc. FIX DAT. - DirectoryEntry() noexcept { - debug_counter_ = gDebugDirectoryEntryCounter++; - DIRECTORY_ENTRY_TRACE_CTOR(); - } - public: - DirectoryEntry(std::string filename, ino_t d_ino, unsigned char d_type) noexcept - : filename{std::move(filename)}, - d_ino{d_ino}, - d_type{d_type} { - debug_counter_ = gDebugDirectoryEntryCounter++; - DIRECTORY_ENTRY_TRACE_CTOR(); - } - - DirectoryEntry(const DirectoryEntry& other) noexcept { - // Do not use member-initialization syntax so that this DCHECK can execute first. - DIRECTORY_ENTRY_MOVE_DCHECK(); - - filename = other.filename; - d_ino = other.d_ino; - d_type = other.d_type; - children_paths_ = other.children_paths_; - children_initialized_ = other.children_initialized_; - debug_counter_ = other.debug_counter_; - DIRECTORY_ENTRY_TRACE_CTOR(); - } - - DirectoryEntry& operator=(const DirectoryEntry& other) noexcept { - if (this == &other) { - return *this; - } - - DIRECTORY_ENTRY_MOVE_DCHECK(); - - filename = other.filename; - d_ino = other.d_ino; - d_type = other.d_type; - children_paths_ = other.children_paths_; - children_initialized_ = other.children_initialized_; - debug_counter_ = other.debug_counter_; - DIRECTORY_ENTRY_TRACE_CTOR(); - - return *this; - } - - DirectoryEntry& operator=(DirectoryEntry&& other) noexcept { - if (this == &other) { - return *this; - } - - DIRECTORY_ENTRY_MOVE_DCHECK(); - - filename = std::move(other.filename); - d_ino = other.d_ino; - d_type = other.d_type; - children_paths_ = std::move(other.children_paths_); - children_initialized_ = other.children_initialized_; - debug_counter_ = other.debug_counter_; - DIRECTORY_ENTRY_TRACE_CTOR(); - - return *this; - } - - DirectoryEntry(DirectoryEntry&& other) noexcept { - DIRECTORY_ENTRY_MOVE_DCHECK(); - other.moved_from_ = true; - - filename = std::move(other.filename); - d_ino = other.d_ino; - d_type = other.d_type; - children_paths_ = std::move(other.children_paths_); - children_initialized_ = other.children_initialized_; - debug_counter_ = other.debug_counter_; - DIRECTORY_ENTRY_TRACE_CTOR(); - } - - // Create a sentinel (root of roots) whose children entries are those specified by - // children_paths. - static DirectoryEntry CreateSentinel(std::vector<std::string> children_paths) { - DirectoryEntry e; - e.d_type = DT_DIR; - ++gDebugDirectoryEntryCounter; - - for (std::string& child_path : children_paths) { - // TODO: Should we call Stat on the child path here to reconstitute the ino_t for a root dir? - // Otherwise it can look a little strange (i.e. the root dir itself will never match - // the searched inode). - // - // Probably not too big of a problem in practice. - DirectoryEntry child_entry{std::move(child_path), kInvalidIno, DT_DIR}; - ResultT child_entry_as_result{std::move(child_entry)}; - e.children_paths_.push_back(std::move(child_entry_as_result)); - } - - e.children_initialized_ = true; - - return e; - } - - // Return an observable which emits the direct children only. - // The children entries are now read from disk (with readdir) if they weren't read previously. - std::vector<ResultT> GetChildrenEntries(borrowed<SystemCall*> system_call) const& { - BuildChildrenPaths(system_call); - return children_paths_; - } - - // Return an observable which emits the direct children only. - // The children entries are now read from disk (with readdir) if they weren't read previously. - // Movable overload. - std::vector<ResultT> GetChildrenEntries(borrowed<SystemCall*> system_call) && { - BuildChildrenPaths(system_call); - return std::move(children_paths_); - } - - // Returns a (lazy) observable that emits every single node, in pre-order, - // rooted at this tree. - // - // New entries are only read from disk (with e.g. readdir) when more values are pulled - // from the observable. Only the direct children of any entry are read at any time. - // - // The emission can be stopped prematurely by unsubscribing from the observable. - // This means the maximum amount of 'redundant' IO reads is bounded by the children count - // of all entries emitted thus far minus entries actually emitted. - ObservableT GetSubTreePreOrderEntries(borrowed<SystemCall*> system_call) const; - - private: - // Out-of-line definition to avoid circular type dependency. - void BuildChildrenPaths(borrowed<SystemCall*> system_call) const; - - // We need to lazily initialize children_paths_ only when we try to read them. - // - // Assuming the underlying file system doesn't change (which isn't strictly true), - // the directory children are referentially transparent. - // - // In practice we do not need to distinguish between the file contents changing out - // from under us in this code, so we don't need the more strict requirements. - mutable std::vector<ResultT> children_paths_; - mutable bool children_initialized_{false}; - - friend std::ostream& operator<<(std::ostream& os, const DirectoryEntry& d); -}; - -std::ostream& operator<<(std::ostream& os, const DirectoryEntry& d) { - os << "DirectoryEntry{" << d.filename << ",ino:" << d.d_ino << ",type:" << d.d_type << "}"; - return os; -} - -using DirectoryEntryResult = DirectoryEntry::ResultT; - -// Read all directory entries and return it as a vector. This must be an eager operation, -// as readdir is not re-entrant. -// -// This could be considered as a limitation from the 'observable' perspective since -// one can end up reading unnecessary extra directory entries that are then never consumed. -// -// The following entries are skipped: -// - '.' self -// - ".." parent -// -// All DT types except the following are removed: -// * DT_LNK - symbolic link (empty children) -// * DT_REG - regular file (empty children) -// * DT_DIR - directory (has children) -static std::vector<DirectoryEntryResult> - ReadDirectoryEntriesFromDirectoryPath(std::string dirpath, borrowed<SystemCall*> system_call) { - DIR *dirp; - struct dirent *dp; - - LOG(VERBOSE) << "ReadDirectoryEntriesFromDirectoryPath(" << dirpath << ")"; - - if ((dirp = system_call->opendir(dirpath.c_str())) == nullptr) { - PLOG(ERROR) << "Couldn't open directory: " << dirpath; - return {DirectoryEntryError{kOpenDir, errno, dirpath}}; - } - - // Read all the results up front because readdir is not re-entrant. - std::vector<DirectoryEntryResult> results; - - // Get full path + the directory entry path. - auto child_path = [&] { return dirpath + "/" + dp->d_name; }; - - do { - errno = 0; - if ((dp = system_call->readdir(dirp)) != nullptr) { - if (dp->d_type == DT_DIR) { - if (strcmp(".", dp->d_name) == 0 || strcmp("..", dp->d_name) == 0) { - LOG(VERBOSE) << "Skip self/parent: " << dp->d_name; - continue; - } - - LOG(VERBOSE) << "Find entry " << child_path() - << ", ino: " << dp->d_ino << ", type: " << dp->d_type; - results.push_back(DirectoryEntry{child_path(), - static_cast<ino_t>(dp->d_ino), - dp->d_type}); - } else if (dp->d_type == DT_UNKNOWN) { - // This seems bad if it happens. We should probably do something about this. - LOG(WARNING) << "Found unknown DT entry: " << child_path(); - - results.push_back(DirectoryEntryError{kDtUnknown, /*errno*/0, child_path()}); - } else if (dp->d_type == DT_LNK || dp->d_type == DT_REG) { - // Regular non-directory file entry. - results.push_back(DirectoryEntry{child_path(), - static_cast<ino_t>(dp->d_ino), - dp->d_type}); - } else { - // Block device, character device, socket, etc... - LOG(VERBOSE) << "Skip DT entry of type: " << dp->d_type << " " << child_path(); - } - } else if (errno != 0) { - PLOG(ERROR) << "Error reading directory entry in " << dirpath; - - results.push_back(DirectoryEntryError{kReadDir, errno, dirpath}); - } - } while (dp != nullptr); - - if (system_call->closedir(dirp) < 0) { - PLOG(ERROR) << "Failed to close directory " << dirpath; - } - - return results; -} - -void DirectoryEntry::BuildChildrenPaths(borrowed<SystemCall*> system_call) const { - if (children_initialized_) { - return; - } - - if (d_type == DT_DIR) { - children_paths_ = ReadDirectoryEntriesFromDirectoryPath(filename, system_call); - // TODO: consider using dependency injection here to substitute this function during testing? - } -} - -struct InodeSearchParameters { - std::vector<Inode> inode_list; - std::vector<std::string> root_dirs; -}; - -// [IN] -// observable: expected<Value, Error>, ... -// [OUT] -// observable: Value, ... -// -// Any encountered 'Error' items are dropped after logging. -template <typename T> -auto MapExpectedOrLog(T&& observable, - ::android::base::LogSeverity log_level) { - return observable.filter([log_level](const auto& result) { - if (result) { - return true; - } else { - LOG(log_level) << result.error(); - return false; - } - }).map([](auto&& result) { - return IORAP_FORWARD_LAMBDA(result).value(); - }); -} - -template <typename T> -auto MapExpectedOrLogError(T&& observable) { - return MapExpectedOrLog(std::forward<T>(observable), ::android::base::ERROR); -} - -template <typename T> -auto MapOptionalOrDrop(T&& observable) { - return observable.filter([](const auto& result) { - return result.has_value(); - }).map([](auto&& result) { - return IORAP_FORWARD_LAMBDA(result).value(); - }); - // TODO: static_assert this isn't used with an unexpected. -} - -template <typename T, typename F> -auto VisitValueOrLogError(T&& expected, F&& visit_func, const char* error_prefix = "") { - if (!expected) { - LOG(ERROR) << error_prefix << " " << expected.error(); - } else { - visit_func(std::forward<T>(expected).value()); - } - // TODO: Could be good to make this more monadic by returning an optional. -} - -template <typename TSimple, typename T, typename F> -void TreeTraversalPreOrderObservableImpl(rx::subscriber<TSimple> dest, T&& node, F&& fn) { - LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (begin) " << __PRETTY_FUNCTION__; - - if (!dest.is_subscribed()) { - LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (unsubscribed)"; - return; - } else { - LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (on_next node)"; - - // Copy the node here. This is less bad than it seems since we haven't yet - // calculated its children (except in the root), so its just doing a shallow memcpy (sizeof(T)). - // - // This assumes the children are calculated lazily, otherwise we'd need to have a separate - // NodeBody class which only holds the non-children elements. - - TSimple copy = std::forward<T>(node); - dest.on_next(std::move(copy)); - - if (!node.has_value()) { - return; - } - - // Whenever we call 'on_next' also check if we end up unsubscribing. - // This avoids the expensive call into the children. - if (!dest.is_subscribed()) { - LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (post-self unsubscribe)"; - return; - } - - // Eagerly get the childrem, moving them instead of copying them. - auto&& children = fn(std::forward<T>(node)); - for (auto&& child : children) { - TreeTraversalPreOrderObservableImpl(dest, IORAP_FORWARD_LAMBDA(child), fn); - // TODO: double check this is doing the std::move properly for rvalues. - - if (!dest.is_subscribed()) { - LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (unsubscribed in children)"; - break; - } - }; - } -} - -// Creates an observable over all the nodes in the tree rooted at node. -// fn is a function that returns the children of that node. -// -// The items are emitted left-to-right pre-order, and stop early if the -// observable is unsubscribed from. -// -// Implementation requirement: -// typeof(node) -> expected<V, E> or optional<V> or similar. -// fn(node) -> iterable<typeof(node)> -// -// preorder(self): -// visit(self) -// for child in fn(self): -// preorder(child) -template <typename T, typename F> -auto/*observable<T>*/ TreeTraversalPreOrderObservable(T&& node, F&& fn) { - LOG(VERBOSE) << "TreeTraversalPreOrderObservable: " << __PRETTY_FUNCTION__; - - using T_simple = std::decay_t<T>; - return rx::observable<>::create<T_simple>( - // Copy node to avoid lifetime issues. - [node=node,fn=std::forward<F>(fn)](rx::subscriber<T_simple> dest) { - LOG(VERBOSE) << "TreeTraversalPreOrderObservable (lambda)"; - TreeTraversalPreOrderObservableImpl<T_simple>(dest, - std::move(node), - std::move(fn)); - dest.on_completed(); - } - ); -} - -DirectoryEntry::ObservableT - DirectoryEntry::GetSubTreePreOrderEntries(borrowed<SystemCall*> system_call) const { - return TreeTraversalPreOrderObservable( - DirectoryEntryResult{*this}, - [system_call=system_call](auto/*DirectoryEntryResult*/&& result) - -> std::vector<DirectoryEntryResult> { - if (!result) { - LOG(VERBOSE) << "GetSubTreePreOrderEntries (no value return)"; - // Cannot have children when it was an error. - return {}; - } - return - IORAP_FORWARD_LAMBDA(result) - .value() - .GetChildrenEntries(system_call); - }); -} - -struct StatError { - int err_no; - std::string path_name; -}; - -std::ostream& operator<<(std::ostream& os, const StatError& e) { - os << "StatError{" << e.err_no << "," << e.path_name - << ": " << strerror(e.err_no) << "}"; - return os; -} - -template <typename U = void> // suppress unused warning. -static iorap::expected<struct stat, StatError> Stat(const std::string& path_name, - borrowed<SystemCall*> system_call) { - struct stat statbuf{}; - - // Call stat(2) in live code. Overridden in test code. - if (system_call->stat(path_name.c_str(), /*out*/&statbuf) == 0) { - return statbuf; - } else { - return iorap::unexpected(StatError{errno, path_name}); - } -} - -using StatResult = iorap::expected<struct stat, StatError>; - -// An inode's corresponding filename on the system. -struct SearchMatch { - Inode inode; - // Relative path joined with a root directory. - // - // Use absolute path root dirs to get back absolute path filenames. - // If relative, this is relative to the current working directory. - std::string filename; -}; - -std::ostream& operator<<(std::ostream& os, const SearchMatch& s) { - os << "SearchMatch{" << s.inode << ", " << s.filename << "}"; - return os; -} - -struct SearchState { - // Emit 'match' Inodes corresponding to the ones here. - InodeSet inode_set; - - // An inode matching one of the ones in inode_set was discovered in the most-recently - // emitted SearchState. - // - // The InodeSet removes any matching 'Inode'. - std::optional<SearchMatch> match; - - SearchState() = default; - SearchState(SearchState&& other) = default; - - // Do not copy this because copying InodeSet is excruciatingly slow. - SearchState(const SearchState& other) = delete; - - // TODO: make sure this doesn't copy [inodes], as that would be unnecessarily expensive. -}; - -std::ostream& operator<<(std::ostream& os, const SearchState& s) { - os << "SearchState{match:"; - // Print the 'match' first. The InodeSet could be very large so it could be truncated in logs. - if (s.match) { - os << s.match.value(); - } else { - os << "(none)"; - } - os << ", inode_set:" << s.inode_set << "}"; - return os; -} - -// TODO: write operator<< etc. - -// Return a lazy observable that will search for all filenames whose inodes -// match the inodes in inode_search_list. -// -// Every unmatched inode will be emitted as an unexpected at the end of the stream. -auto/*[observable<InodeResult>, connectable]*/ SearchDirectoriesForMatchingInodes( - std::vector<std::string> root_dirs, - std::vector<Inode> inode_search_list, - borrowed<SystemCall*> system_call) { - - // Create a (lazy) observable that will emit each DirectoryEntry that is a recursive subchild - // of root_dirs. Emission will be stopped when its unsubscribed from. - // - // This is done by calling readdir(3) lazily. - auto/*obs<DirectoryEntry>*/ find_all_subdir_entries = ([&]() { - DirectoryEntry sentinel = DirectoryEntry::CreateSentinel(std::move(root_dirs)); - auto/*obs<DirectoryEntryResult*/ results = sentinel.GetSubTreePreOrderEntries(system_call); - - // Drop any errors by logging them to logcat. "Unwrap" the expected into the underlying data. - auto/*obs<DirectoryEntry*>*/ expected_drop_errors = MapExpectedOrLogError(std::move(results)); - return expected_drop_errors; - })(); - - // DirectoryEntry is missing the dev_t portion, so we may need to call scan(2) again - // to confirm the dev_t. We skip calling scan(2) when the ino_t does not match. - // InodeSet lets us optimally avoid calling scan(2). - std::shared_ptr<SearchState> initial = std::make_shared<SearchState>(); - initial->inode_set = InodeSet::OfList(inode_search_list); - - auto/*[observable<SearchState>,Connectable]*/ search_state_results = find_all_subdir_entries.scan( - std::move(initial), - [system_call=system_call](std::shared_ptr<SearchState> search_state, - const DirectoryEntry& dir_entry) { - LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#Scan " - << dir_entry << ", state: " << *search_state; - - search_state->match = std::nullopt; - - InodeSet* inodes = &search_state->inode_set; - - // Find all the possible inodes across different devices. - InodeSet::ValueRange inode_list = inodes->FindInodeList(dir_entry.d_ino); - - // This directory doesn't correspond to any inodes we are searching for. - if (!inode_list) { - return search_state; - } - - StatResult maybe_stat = Stat(dir_entry.filename, system_call); - VisitValueOrLogError(maybe_stat, [&](const struct stat& stat_buf) { - // Try to match the specific inode. Usually this will not result in a match (nullopt). - std::optional<Inode> inode = inodes->FindAndRemoveInodeInList(inode_list, stat_buf); - - if (inode) { - search_state->match = SearchMatch{inode.value(), dir_entry.filename}; - } - }); - - return search_state; - } - // Avoid exhausting a potentially 'infinite' stream of files by terminating as soon - // as we find every single inode we care about. - ).take_while([](std::shared_ptr<SearchState> state) { - // Also emit the last item that caused the search set to go empty. - bool cond = !state->inode_set.Empty() || state->match; - - if (WOULD_LOG(VERBOSE)) { - static int kCounter = 0; - LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#take_while (" << kCounter++ << - ",is_empty:" - << state->inode_set.Empty() << ", match:" << state->match.has_value(); - } - // Minor O(1) implementation inefficiency: - // (Too minor to fix but it can be strange if looking at the logs or readdir traces). - // - // Note, because we return 'true' after the search set went empty, - // the overall stream graph still pulls from search_state_results exactly once more: - // - // This means that for cond to go to false, we would've read one extra item and then discarded - // it. If that item was the first child of a directory, that means we essentially did - // one redundant pass of doing a readdir. - // In other words if the search set goes to empty while the current item is a directory, - // - // it will definitely readdir on it at least once as we try to get the first child in - // OnTreeTraversal. - // - // This could be fixed with a 'take_until(Predicate)' operator which doesn't discard - // the last item when the condition becomes false. However rxcpp seems to lack this operator, - // whereas RxJava has it. - - if (!cond) { - LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#take_while " - << "should now terminate for " << *state; - } - - return cond; - }).publish(); - // The publish here is mandatory. The stream is consumed twice (once by matched and once by - // unmatched streams). Without the publish, once all items from 'matched' were consumed it would - // start another instance of 'search_state_results' (i.e. it appears as if the search - // is restarted). - // - // By using 'publish', the search_state_results is effectively shared by both downstream nodes. - // Note that this also requires the subscriber to additionally call #connect on the above stream, - // otherwise no work will happen. - - // Lifetime notes: - // - // The the 'SearchState' is emitted into both below streams simultaneously. - // The 'unmatched_inode_values' only touches the inode_set. - // The 'matched_inode_values' only touches the match. - // Either stream can 'std::move' from those fields because they don't move each other's fields. - auto/*observable<InodeResult>*/ matched_inode_values = search_state_results - .filter([](std::shared_ptr<SearchState> search_state) { - return search_state->match.has_value(); }) - .map([](std::shared_ptr<SearchState> search_state) { - return std::move(search_state->match.value()); }) - // observable<SearchMatch> - .map([](SearchMatch search_match) { - return InodeResult::makeSuccess(search_match.inode, std::move(search_match.filename)); - }); // observable<InodeResult> - - auto/*observable<?>*/ unmatched_inode_values = search_state_results - // The 'last' SearchState is the one that contains all the remaining inodes. - .take_last(1) // observable<SearchState> - .flat_map([](std::shared_ptr<SearchState> search_state) { - LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#unmatched -- flat_map"; - // Aside: Could've used a move here if the inodes weren't so lightweight already. - return search_state->inode_set.IterateValues(); }) - // observable<Inode> - .map([](const Inode& inode) { - LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#unmatched -- map"; - return InodeResult::makeFailure(inode, InodeResult::kCouldNotFindFilename); - }); - // observable<InodeResult> - - // The matched and unmatched InodeResults are emitted together. - // Use merge, not concat, because we need both observables to be subscribed to simultaneously. - - auto/*observable<InodeResult*/ all_inode_results = - matched_inode_values.merge(unmatched_inode_values); - - // Now that all mid-stream observables have been connected, turn the Connectable observable - // into a regular observable. - - // The caller has to call 'connect' on the search_state_results after subscribing - // and before any work can actually start. - return std::make_pair(all_inode_results, search_state_results); -} - -rxcpp::observable<InodeResult> SearchDirectories::FindFilenamesFromInodes( - std::vector<std::string> root_directories, - std::vector<Inode> inode_list, - SearchMode mode) const { - DCHECK(mode == SearchMode::kInProcessDirect) << " other modes not implemented yet"; - - auto/*observable[2]*/ [inode_results, connectable] = SearchDirectoriesForMatchingInodes( - std::move(root_directories), - std::move(inode_list), - system_call_); - - return inode_results.ref_count(connectable); -} - -// I think we could avoid this with auto_connect, which rxcpp doesn't seem to have. -// -// I can't figure out any other way to avoid this, or at least to allow connecting -// on the primary observable (instead of a secondary side-observable). -// -// If using the obvious publish+ref_count then the unmerged stream gets no items emitted into it. -// If tried to ref_count later, everything turns into no-op. -// If trying to call connect too early, the subscribe is missed. -template <typename T> -struct RxAnyConnectableFromObservable : public SearchDirectories::RxAnyConnectable { - virtual void connect() override { - observable.connect(); - } - - virtual ~RxAnyConnectableFromObservable() {} - - RxAnyConnectableFromObservable(rxcpp::connectable_observable<T> observable) - : observable(observable) { - } - - rxcpp::connectable_observable<T> observable; -}; - -// Type deduction helper. -template <typename T> -std::unique_ptr<SearchDirectories::RxAnyConnectable> - MakeRxAnyConnectableFromObservable(rxcpp::connectable_observable<T> observable) { - SearchDirectories::RxAnyConnectable* ptr = new RxAnyConnectableFromObservable<T>{observable}; - return std::unique_ptr<SearchDirectories::RxAnyConnectable>{ptr}; -} - -std::pair<rxcpp::observable<InodeResult>, std::unique_ptr<SearchDirectories::RxAnyConnectable>> - SearchDirectories::FindFilenamesFromInodesPair( - std::vector<std::string> root_directories, - std::vector<Inode> inode_list, - SearchMode mode) const { - DCHECK(mode == SearchMode::kInProcessDirect) << " other modes not implemented yet"; - - auto/*observable[2]*/ [inode_results, connectable] = SearchDirectoriesForMatchingInodes( - std::move(root_directories), - std::move(inode_list), - system_call_); - - std::unique_ptr<SearchDirectories::RxAnyConnectable> connectable_ptr = - MakeRxAnyConnectableFromObservable(connectable.as_dynamic()); - - return {inode_results, std::move(connectable_ptr)}; -} - -rxcpp::observable<InodeResult> - SearchDirectories::FindFilenamesFromInodes(std::vector<std::string> root_directories, - rxcpp::observable<Inode> inodes, - SearchMode mode) const { - - // It's inefficient to search for inodes until the full search list is available, - // so first reduce to a vector so we can access all the inodes simultaneously. - return inodes.reduce(std::vector<Inode>{}, - [](std::vector<Inode> vec, Inode inode) { - vec.push_back(inode); - return vec; - }, - [](std::vector<Inode> v){ - return v; // TODO: use an identity function - }) - .flat_map([root_directories=std::move(root_directories), mode, self=*this] - (std::vector<Inode> vec) { - // All borrowed values (e.g. SystemCall) must outlive the observable. - return self.FindFilenamesFromInodes(root_directories, vec, mode); - } - ); -} - -auto/*[observable<InodeResult>]*/ EmitAllInodesFromDirectories( - std::vector<std::string> root_dirs, - borrowed<SystemCall*> system_call) { - - // Create a (lazy) observable that will emit each DirectoryEntry that is a recursive subchild - // of root_dirs. Emission will be stopped when its unsubscribed from. - // - // This is done by calling readdir(3) lazily. - auto/*obs<DirectoryEntry>*/ find_all_subdir_entries = ([&]() { - DirectoryEntry sentinel = DirectoryEntry::CreateSentinel(std::move(root_dirs)); - auto/*obs<DirectoryEntryResult*/ results = sentinel.GetSubTreePreOrderEntries(system_call); - - // Drop any errors by logging them to logcat. "Unwrap" the expected into the underlying data. - auto/*obs<DirectoryEntry*>*/ expected_drop_errors = MapExpectedOrLogError(std::move(results)); - return expected_drop_errors; - })(); - - // Fill in -1 for the dev_t since readdir only returns the ino_t. - // The caller of this function is expected to call stat(2) later on to fill in - // the full data. - return find_all_subdir_entries.map([](DirectoryEntry e) { - return InodeResult::makeSuccess(Inode::FromDeviceAndInode(-1, e.d_ino), std::move(e.filename)); - }); -} - -rxcpp::observable<InodeResult> - SearchDirectories::ListAllFilenames(std::vector<std::string> root_directories) const { - // TODO: refactor implementation into DiskScanDataSource. - return EmitAllInodesFromDirectories(std::move(root_directories), - /*borrowed*/system_call_); -} - -struct FilterState { - // Emit 'match' Inodes corresponding to the ones here. - InodeSet inode_set; - - // An inode matching one of the ones in inode_set was discovered in the most-recently - // emitted SearchState. - // - // The InodeSet removes any matching 'Inode'. - std::optional<InodeResult> match; - - FilterState() = default; - FilterState(FilterState&& other) = default; - - // Copying the InodeSet is expensive, so forbid any copies. - FilterState(const FilterState& other) = delete; -}; - -std::ostream& operator<<(std::ostream& os, const FilterState& s) { - os << "FilterState{match:"; - // Print the 'match' first. The InodeSet could be very large so it could be truncated in logs. - if (s.match) { - os << s.match.value(); - } else { - os << "(none)"; - } - os << ", inode_set:" << s.inode_set << "}"; - return os; -} - -rxcpp::observable<InodeResult> SearchDirectories::FilterFilenamesForSpecificInodes( - rxcpp::observable<InodeResult> all_inodes, - std::vector<Inode> inode_list, - bool missing_device_number, // missing dev_t portion? - bool needs_verification) const { - // TODO: refactor into InodeResolver - - borrowed<SystemCall*> system_call = system_call_; - - // InodeResult may be missing the dev_t portion, so we may need to call scan(2) again - // to confirm the dev_t. We skip calling scan(2) when the ino_t does not match. - // InodeSet lets us optimally avoid calling scan(2). - std::shared_ptr<FilterState> initial = std::make_shared<FilterState>(); - initial->inode_set = InodeSet::OfList(inode_list); - - auto/*[observable<FilterState>,Connectable]*/ filter_state_results = all_inodes.scan( - std::move(initial), - [system_call, missing_device_number] - (std::shared_ptr<FilterState> filter_state, InodeResult inode_result) { - LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#Scan " - << inode_result << ", state: " << *filter_state; - - filter_state->match = std::nullopt; - - InodeSet* inodes = &filter_state->inode_set; - - // Find all the possible (dev_t, ino_t) potential needles given an ino_t in the haystack. - InodeSet::ValueRange inode_list = inodes->FindInodeList(inode_result.inode.inode); - - // This inode result doesn't correspond to any inodes we are searching for. - if (!inode_list) { - // Drop the result and keep going. - return filter_state; - } - - if (missing_device_number) { - // Need to fill in dev_t by calling stat(2). - VisitValueOrLogError(std::move(inode_result.data), [&](std::string filename) { - StatResult maybe_stat = Stat(filename, system_call); - VisitValueOrLogError(maybe_stat, [&](const struct stat& stat_buf) { - // Try to match the specific inode. Usually this will not result in a match (nullopt). - std::optional<Inode> inode = inodes->FindAndRemoveInodeInList(inode_list, stat_buf); - - if (inode) { - filter_state->match = InodeResult::makeSuccess(inode.value(), std::move(filename)); - } - }); - - // Note: stat errors are logged here to make the error closer to the occurrence. - // In theory, we could just return it as an InodeResult but then the error would - // just get logged elsewhere. - }); - } else { - // Trust the dev_t in InodeResult is valid. Later passes can verify it. - - // Try to match the specific inode. Usually this will not result in a match (nullopt). - std::optional<Inode> inode = - inodes->FindAndRemoveInodeInList(inode_list, inode_result.inode); - - if (inode) { - filter_state->match = inode_result; - } - - // Note that the InodeResult doesn't necessarily need to have a valid filename here. - // If the earlier pass returned an error-ed result, this will forward the error code. - } - - return filter_state; - } - // Avoid exhausting a potentially 'infinite' stream of files by terminating as soon - // as we find every single inode we care about. - ).take_while([](std::shared_ptr<FilterState> state) { - // Also emit the last item that caused the search set to go empty. - bool cond = !state->inode_set.Empty() || state->match; - - if (WOULD_LOG(VERBOSE)) { - static int kCounter = 0; - LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#take_while (" << kCounter++ << - ",is_empty:" - << state->inode_set.Empty() << ", match:" << state->match.has_value(); - } - // Minor O(1) implementation inefficiency: - // (Too minor to fix but it can be strange if looking at the logs or readdir traces). - // - // Note, because we return 'true' after the search set went empty, - // the overall stream graph still pulls from filter_state_results exactly once more: - // - // This means that for cond to go to false, we would've read one extra item and then discarded - // it. If that item was the first child of a directory, that means we essentially did - // one redundant pass of doing a readdir. - // In other words if the search set goes to empty while the current item is a directory, - // - // it will definitely readdir on it at least once as we try to get the first child in - // OnTreeTraversal. - // - // This could be fixed with a 'take_until(Predicate)' operator which doesn't discard - // the last item when the condition becomes false. However rxcpp seems to lack this operator, - // whereas RxJava has it. - - if (!cond) { - LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#take_while " - << "should now terminate for " << *state; - } - - return cond; - }).publish(); - // The publish here is mandatory. The stream is consumed twice (once by matched and once by - // unmatched streams). Without the publish, once all items from 'matched' were consumed it would - // start another instance of 'filter_state_results' (i.e. it appears as if the search - // is restarted). - // - // By using 'publish', the filter_state_results is effectively shared by both downstream nodes. - // Note that this also requires the subscriber to additionally call #connect on the above stream, - // otherwise no work will happen. - - // Lifetime notes: - // - // The the 'FilterState' is emitted into both below streams simultaneously. - // The 'unmatched_inode_values' only touches the inode_set. - // The 'matched_inode_values' only touches the match. - // Either stream can 'std::move' from those fields because they don't move each other's fields. - auto/*observable<InodeResult>*/ matched_inode_values = filter_state_results - .filter([](std::shared_ptr<FilterState> filter_state) { - return filter_state->match.has_value(); }) - .map([](std::shared_ptr<FilterState> filter_state) { - return std::move(filter_state->match.value()); }); - // observable<InodeResult> - - auto/*observable<?>*/ unmatched_inode_values = filter_state_results - // The 'last' FilterState is the one that contains all the remaining inodes. - .take_last(1) // observable<FilterState> - .flat_map([](std::shared_ptr<FilterState> filter_state) { - LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#unmatched -- flat_map"; - // Aside: Could've used a move here if the inodes weren't so lightweight already. - return filter_state->inode_set.IterateValues(); }) - // observable<Inode> - .map([](const Inode& inode) { - LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#unmatched -- map"; - return InodeResult::makeFailure(inode, InodeResult::kCouldNotFindFilename); - }); - // observable<InodeResult> - - // The matched and unmatched InodeResults are emitted together. - // Use merge, not concat, because we need both observables to be subscribed to simultaneously. - - auto/*observable<InodeResult*/ all_inode_results = - matched_inode_values.merge(unmatched_inode_values); - - // Verify the inode results by calling stat(2). - // Unverified results are turned into an error. - - auto/*observable<InodeResult>*/ verified_inode_results = - all_inode_results.map([needs_verification, system_call](InodeResult result) { - if (!needs_verification || !result) { - // Skip verification if requested, or if the result didn't have a filename. - return result; - } - - const std::string& filename = result.data.value(); - StatResult maybe_stat = Stat(filename, system_call); - - if (maybe_stat) - { - if (result.inode == Inode::FromDeviceAndInode(maybe_stat->st_dev, maybe_stat->st_ino)) { - return result; - } else { - LOG(WARNING) - << "FilterFilenamesForSpecificInodes#verified fail out-of-date inode: " << result; - return InodeResult::makeFailure(result.inode, InodeResult::kVerificationFailed); - } - } else { - // Forward stat errors directly, as it could be a missing security rule, - // but turn -ENOENT into casual verification errors. - const StatError& err = maybe_stat.error(); - int error_code = err.err_no; - if (err.err_no == ENOENT) { - error_code = InodeResult::kVerificationFailed; - - // TODO: Don't LOG(WARNING) here because this could be very common if we - // access the data much much later after the initial results were read in. - LOG(WARNING) - << "FilterFilenamesForSpecificInodes#verified fail out-of-date filename: " << result; - } else { - LOG(ERROR) - << "FilterFilenamesForSpecificInodes#verified stat(2) failure: " << err; - } - - return InodeResult::makeFailure(result.inode, error_code); - } - }); - - // Now that all mid-stream observables have been connected, turn the Connectable observable - // into a regular observable. - return verified_inode_results.ref_count(filter_state_results); -} - -rxcpp::observable<InodeResult> SearchDirectories::EmitAllFilenames( - rxcpp::observable<InodeResult> all_inodes, - bool missing_device_number, // missing dev_t portion? - bool needs_verification) const { - // TODO: refactor into InodeResolver - - borrowed<SystemCall*> system_call = system_call_; - - // InodeResult may be missing the dev_t portion, so we may need to call scan(2) again - // to confirm the dev_t. - - using EmitAllState = std::optional<InodeResult>; - - auto/*[observable<FilterState>,Connectable]*/ all_inode_results = all_inodes.map( - [system_call, missing_device_number](InodeResult inode_result) { - LOG(VERBOSE) << "EmitAllFilenames#map " - << inode_result; - - // Could fail if the device number is missing _and_ stat(2) fails. - EmitAllState match = std::nullopt; - - if (missing_device_number) { - // Need to fill in dev_t by calling stat(2). - VisitValueOrLogError(std::move(inode_result.data), [&](std::string filename) { - StatResult maybe_stat = Stat(filename, system_call); - VisitValueOrLogError(maybe_stat, [&](const struct stat& stat_buf) { - Inode inode = Inode::FromDeviceAndInode(stat_buf.st_dev, stat_buf.st_ino); - match = InodeResult::makeSuccess(inode, std::move(filename)); - }); - - // Note: stat errors are logged here to make the error closer to the occurrence. - // In theory, we could just return it as an InodeResult but then the error would - // just get logged elsewhere. - }); - } else { - // Trust the dev_t in InodeResult is valid. Later passes can verify it. - match = std::move(inode_result); - - // Note that the InodeResult doesn't necessarily need to have a valid filename here. - // If the earlier pass returned an error-ed result, this will forward the error code. - } - - return match; // implicit move. - } - ); - - auto/*observable<InodeResult>*/ matched_inode_values = all_inode_results - .filter([](const EmitAllState& filter_state) { return filter_state.has_value(); }) - .map([](EmitAllState& filter_state) { return std::move(filter_state.value()); }); - // observable<InodeResult> - - // Verify the inode results by calling stat(2). - // Unverified results are turned into an error. - - auto/*observable<InodeResult>*/ verified_inode_results = - matched_inode_values.map([needs_verification, system_call](InodeResult result) { - if (!needs_verification || !result) { - // Skip verification if requested, or if the result didn't have a filename. - return result; - } - - const std::string& filename = result.data.value(); - StatResult maybe_stat = Stat(filename, system_call); - - if (maybe_stat) - { - if (result.inode == Inode::FromDeviceAndInode(maybe_stat->st_dev, maybe_stat->st_ino)) { - return result; - } else { - LOG(WARNING) - << "EmitAllFilenames#verified fail out-of-date inode: " << result; - return InodeResult::makeFailure(result.inode, InodeResult::kVerificationFailed); - } - } else { - // Forward stat errors directly, as it could be a missing security rule, - // but turn -ENOENT into casual verification errors. - const StatError& err = maybe_stat.error(); - int error_code = err.err_no; - if (err.err_no == ENOENT) { - error_code = InodeResult::kVerificationFailed; - - // TODO: Don't LOG(WARNING) here because this could be very common if we - // access the data much much later after the initial results were read in. - LOG(WARNING) - << "EmitAllFilenames#verified fail out-of-date filename: " << result; - } else { - LOG(ERROR) - << "EmitAllFilenames#verified stat(2) failure: " << err; - } - - return InodeResult::makeFailure(result.inode, error_code); - } - }); - - // TODO: refactor this function some more with the Find(inode_set) equivalent. - - // Now that all mid-stream observables have been connected, turn the Connectable observable - // into a regular observable. - return verified_inode_results; -} - -} // namespace iorap::inode2filename diff --git a/src/inode2filename/search_directories.h b/src/inode2filename/search_directories.h deleted file mode 100644 index 0b4af78..0000000 --- a/src/inode2filename/search_directories.h +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_INODE2FILENAME_SEARCH_DIRECTORIES_H_ -#define IORAP_SRC_INODE2FILENAME_SEARCH_DIRECTORIES_H_ - -#include "inode2filename/inode.h" -#include "inode2filename/system_call.h" -#include "inode2filename/inode_resolver.h" - -#include <fruit/fruit.h> - -#include <rxcpp/rx.hpp> -namespace iorap::inode2filename { - -// TODO: rename. -using SearchMode = ProcessMode; - -struct SearchDirectories { - // Type-erased subset of rxcpp::connectable_observable<?> - struct RxAnyConnectable { - // Connects to the underlying observable. - // - // This kicks off the graph, streams begin emitting items. - // This method will block until all items have been fully emitted - // and processed by any subscribers. - virtual void connect() = 0; - - virtual ~RxAnyConnectable(){} - }; - - - // Create a cold observable of inode results (a lazy stream) corresponding - // to the inode search list. - // - // A depth-first search is done on each of the root directories (in order), - // until all inodes have been found (or until all directories have been exhausted). - // - // Some internal errors may occur during emission that aren't part of an InodeResult; - // these will be sent to the error logcat and dropped. - // - // Calling this function does not begin the search. - // The returned observable will begin the search after subscribing to it. - // - // The emitted InodeResult stream has these guarantees: - // - All inodes in inode_list will eventually be emitted exactly once in an InodeResult - // - When all inodes are found, directory traversal is halted. - // - The order of emission can be considered arbitrary. - // - // Lifetime rules: - // - The observable must be fully consumed before deleting any of the SearchDirectory's - // borrowed constructor parameters (e.g. the SystemCall). - // - SearchDirectory itself can be deleted at any time after creating an observable. - rxcpp::observable<InodeResult> - FindFilenamesFromInodes(std::vector<std::string> root_directories, - std::vector<Inode> inode_list, - SearchMode mode) const; - - // Create a cold observable of inode results (a lazy stream) corresponding - // to the inode search list. - // - // A depth-first search is done on each of the root directories (in order), - // until all inodes have been found (or until all directories have been exhausted). - // - // Some internal errors may occur during emission that aren't part of an InodeResult; - // these will be sent to the error logcat and dropped. - // - // Calling this function does not begin the search. - // The returned observable will begin the search after subscribing to it. - // - // The emitted InodeResult stream has these guarantees: - // - All inodes in inode_list will eventually be emitted exactly once in an InodeResult - // - When all inodes are found, directory traversal is halted. - // - The order of emission can be considered arbitrary. - // - // Lifetime rules: - // - The observable must be fully consumed before deleting any of the SearchDirectory's - // borrowed constructor parameters (e.g. the SystemCall). - // - SearchDirectory itself can be deleted at any time after creating an observable. - std::pair<rxcpp::observable<InodeResult>, std::unique_ptr<RxAnyConnectable>> - FindFilenamesFromInodesPair(std::vector<std::string> root_directories, - std::vector<Inode> inode_list, - SearchMode mode) const; - - // No items on the output stream will be emitted until 'inodes' completes. - // - // The current algorithm is a naive DFS, so if it began too early it would either - // miss the search items or require traversal restarts. - // - // See above for more details. - rxcpp::observable<InodeResult> - FindFilenamesFromInodes(std::vector<std::string> root_directories, - rxcpp::observable<Inode> inodes, - SearchMode mode) const; - - rxcpp::observable<InodeResult> - ListAllFilenames(std::vector<std::string> root_directories) const; - - rxcpp::observable<InodeResult> FilterFilenamesForSpecificInodes( - // haystack that will be subscribed to until all in inode_list are found. - rxcpp::observable<InodeResult> all_inodes, - // key list: traverse all_inodes until we emit all results from inode_list. - std::vector<Inode> inode_list, - // all_inodes have a missing device number: use stat(2) to fill it in. - bool missing_device_number, - bool needs_verification) const; - - rxcpp::observable<InodeResult> EmitAllFilenames( - // haystack that will be subscribed to until all in inode_list are found. - rxcpp::observable<InodeResult> all_inodes, - // all_inodes have a missing device number: use stat(2) to fill it in. - bool missing_device_number, - bool needs_verification) const; - - // Any borrowed parameters here can also be borrowed by the observables returned by the above - // member functions. - // - // The observables must be fully consumed within the lifetime of the borrowed parameters. - INJECT(SearchDirectories(borrowed<SystemCall*> system_call)) - : system_call_(system_call) {} - - // TODO: is there a way to get rid of this second RxAnyConnectable parameter? - private: - // This gets passed around to lazy lambdas, so we must finish consuming any observables - // before the injected system call is deleted. - borrowed<SystemCall*> system_call_; -}; - -} // namespace iorap::inode2filename - -#endif // IORAP_SRC_INODE2FILENAME_SEARCH_DIRECTORIES_H_ diff --git a/src/inode2filename/system_call.h b/src/inode2filename/system_call.h deleted file mode 100644 index 43c371f..0000000 --- a/src/inode2filename/system_call.h +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_INODE2FILENAME_SYSTEM_CALL_H_ -#define IORAP_SRC_INODE2FILENAME_SYSTEM_CALL_H_ - -#include <fruit/fruit.h> - -#include <dirent.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <unistd.h> - -// Abstract out the system calls behind a virtual interface: -// This enables us to use dependency injection to provide mock implementations -// during tests. -struct SystemCall { - // stat(2) - virtual int stat(const char *pathname, struct stat *statbuf) = 0; - - // opendir(3) - virtual DIR *opendir(const char *name) = 0; - - // readdir(3) - virtual struct dirent *readdir(DIR *dirp) = 0; - - // closedir(3) - virtual int closedir(DIR *dirp) = 0; - - virtual ~SystemCall() {} -}; - -// "Live" implementation that calls down to libc. -struct SystemCallImpl : public SystemCall { - // Marks this constructor as the one to use for injection. - INJECT(SystemCallImpl()) = default; - - // stat(2) - virtual int stat(const char *pathname, struct stat *statbuf) override { - return ::stat(pathname, statbuf); - } - - // opendir(3) - virtual DIR *opendir(const char *name) override { - return ::opendir(name); - } - - // readdir(3) - virtual struct dirent *readdir(DIR *dirp) override { - return ::readdir(dirp); - } - - // closedir(3) - virtual int closedir(DIR *dirp) override { - return ::closedir(dirp); - } - - virtual ~SystemCallImpl() {} -}; - -#endif // IORAP_SRC_INODE2FILENAME_SYSTEM_CALL_H_ - diff --git a/src/iorapd/main.cc b/src/iorapd/main.cc deleted file mode 100644 index a3a63f2..0000000 --- a/src/iorapd/main.cc +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "binder/iiorap_impl.h" -#include "common/debug.h" -#include "common/loggers.h" -#include "common/property.h" -#include "db/models.h" -#include "manager/event_manager.h" - -#include <android-base/logging.h> -#include <android-base/properties.h> -#include <binder/IPCThreadState.h> -#include <server_configurable_flags/get_flags.h> -#include <utils/Trace.h> - -#include <stdio.h> - -static constexpr const char* kServiceName = iorap::binder::IIorapImpl::getServiceName(); - -int main(int /*argc*/, char** argv) { - bool tracing_allowed = iorap::common::IsTracingEnabled(/*default_value=*/"false"); - bool readahead_allowed = iorap::common::IsReadAheadEnabled(/*default_value*/"false"); - if (!tracing_allowed && !readahead_allowed) { - LOG(INFO) << "Turn off IORap because both tracing and prefetching are off."; - return 0; - } - - if (android::base::GetBoolProperty("iorapd.log.verbose", iorap::kIsDebugBuild)) { - // Show verbose logs if the property is enabled or if we are a debug build. - setenv("ANDROID_LOG_TAGS", "*:v", /*overwrite*/ 1); - } - - // Logs go to system logcat. - android::base::InitLogging(argv, iorap::common::StderrAndLogdLogger{android::base::SYSTEM}); - - LOG(INFO) << kServiceName << " (the prefetchening) firing up"; - { - android::ScopedTrace trace_db_init{ATRACE_TAG_ACTIVITY_MANAGER, "IorapNativeService::db_init"}; - iorap::db::SchemaModel db_schema = - iorap::db::SchemaModel::GetOrCreate( - android::base::GetProperty("iorapd.db.location", - "/data/misc/iorapd/sqlite.db")); - db_schema.MarkSingleton(); - } - - std::shared_ptr<iorap::manager::EventManager> event_manager; - { - android::ScopedTrace trace_start{ATRACE_TAG_ACTIVITY_MANAGER, "IorapNativeService::start"}; - - // TODO: use fruit for this DI. - event_manager = - iorap::manager::EventManager::Create(); - if (!iorap::binder::IIorapImpl::Start(event_manager)) { - LOG(ERROR) << "Unable to start IorapNativeService"; - exit(1); - } - } - - // This must be logged after all other initialization has finished. - LOG(INFO) << kServiceName << " (the prefetchening) readied up"; - - event_manager->Join(); // TODO: shutdown somewhere? - // Block until something else shuts down the binder service. - android::IPCThreadState::self()->joinThreadPool(); - LOG(INFO) << kServiceName << " shutting down"; - - return 0; -} diff --git a/src/maintenance/controller.cc b/src/maintenance/controller.cc deleted file mode 100644 index 8929057..0000000 --- a/src/maintenance/controller.cc +++ /dev/null @@ -1,633 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "compiler/compiler.h" -#include "maintenance/controller.h" - -#include "common/cmd_utils.h" -#include "common/debug.h" -#include "common/expected.h" -#include "common/trace.h" - -#include "db/models.h" -#include "inode2filename/inode.h" -#include "inode2filename/search_directories.h" -#include "prefetcher/read_ahead.h" - -#include <android-base/file.h> -#include <utils/Printer.h> - -#include <chrono> -#include <ctime> -#include <iostream> -#include <filesystem> -#include <fstream> -#include <limits> -#include <mutex> -#include <optional> -#include <vector> -#include <string> -#include <sys/wait.h> - -namespace iorap::maintenance { - -const constexpr int64_t kCompilerCheckIntervalMs = 10; -static constexpr size_t kMinTracesForCompilation = 1; -const constexpr char* kDenyListFilterDexFiles = "[.](art|oat|odex|vdex|dex)$"; - -struct LastJobInfo { - time_t last_run_ns_{0}; - size_t activities_last_compiled_{0}; -}; - -LastJobInfo last_job_info_; -std::mutex last_job_info_mutex_; - -// Gets the path of output compiled trace. -db::CompiledTraceFileModel CalculateNewestFilePath( - const std::string& package_name, - const std::string& activity_name, - int version) { - db::VersionedComponentName versioned_component_name{ - package_name, activity_name, version}; - - db::CompiledTraceFileModel output_file = - db::CompiledTraceFileModel::CalculateNewestFilePath(versioned_component_name); - - return output_file; -} - -using ArgString = const char*; - -static constexpr const char kCommandFileName[] = "/system/bin/iorap.cmd.compiler"; - -int Exec::Execve(const std::string& pathname, - std::vector<std::string>& argv_vec, - char *const envp[]) { - std::unique_ptr<ArgString[]> argv_ptr = - common::VecToArgv(kCommandFileName, argv_vec); - - return execve(pathname.c_str(), (char**)argv_ptr.get(), envp); -} - -pid_t Exec::Fork() { - return fork(); -} - -// Represents the parameters used when fork+exec compiler. -struct CompilerForkParameters { - std::vector<std::string> input_pbs; - std::vector<uint64_t> timestamp_limit_ns; - std::string output_proto; - std::vector<int32_t> pids; - ControllerParameters controller_params; - - CompilerForkParameters(const std::vector<compiler::CompilationInput>& perfetto_traces, - const std::string& output_proto, - ControllerParameters controller_params) : - output_proto(output_proto), controller_params(controller_params) { - for (compiler::CompilationInput perfetto_trace : perfetto_traces) { - input_pbs.push_back(perfetto_trace.filename); - timestamp_limit_ns.push_back(perfetto_trace.timestamp_limit_ns); - pids.push_back(perfetto_trace.pid); - } - } -}; - -std::vector<std::string> MakeCompilerParams(const CompilerForkParameters& params) { - std::vector<std::string> argv; - ControllerParameters controller_params = params.controller_params; - - common::AppendArgsRepeatedly(argv, params.input_pbs); - common::AppendArgsRepeatedly(argv, "--timestamp_limit_ns", params.timestamp_limit_ns); - common::AppendArgsRepeatedly(argv, "--pid", params.pids); - - if (controller_params.output_text) { - argv.push_back("--output-text"); - } - - common::AppendArgs(argv, "--output-proto", params.output_proto); - - if (controller_params.inode_textcache) { - common::AppendArgs(argv, "--inode-textcache", *controller_params.inode_textcache); - } - - if (controller_params.verbose) { - argv.push_back("--verbose"); - } - - if (controller_params.exclude_dex_files) { - common::AppendArgs(argv, "--denylist-filter", kDenyListFilterDexFiles); - } - - return argv; -} - -// Sets a watch dog for the given pid and kill it if timeout. -std::thread SetTimeoutWatchDog(pid_t pid, int64_t timeout_ms, std::atomic<bool>& cancel_watchdog) { - std::thread watchdog_thread{[pid, timeout_ms, &cancel_watchdog]() { - std::chrono::time_point start = std::chrono::system_clock::now(); - std::chrono::milliseconds timeout(timeout_ms); - while (!cancel_watchdog) { - int status = kill(pid, 0); - if (status != 0) { - LOG(DEBUG) << "Process (" << pid << ") doesn't exist now."; - break; - } - std::chrono::time_point cur = std::chrono::system_clock::now(); - if (cur - start > timeout) { - LOG(INFO) << "Process (" << pid << ") is timeout!"; - LOG(INFO) << "start time: " - << std::chrono::system_clock::to_time_t(start) - << " end time: " - << std::chrono::system_clock::to_time_t(cur) - << " timeout: " - << timeout_ms; - kill(pid, SIGKILL); - break; - } - usleep(kCompilerCheckIntervalMs * 1000); - } - }}; - - return watchdog_thread; -} - -bool StartViaFork(const CompilerForkParameters& params) { - const ControllerParameters& controller_params = params.controller_params; - pid_t child = controller_params.exec->Fork(); - - if (child == -1) { - LOG(FATAL) << "Failed to fork a process for compilation"; - } else if (child > 0) { // we are the caller of this function - LOG(DEBUG) << "forked into a process for compilation , pid = " << child; - - int64_t compiler_timeout_ms = - android::base::GetIntProperty("iorapd.maintenance.compiler_timeout_ms", - /*default*/ 10 * 60 * 1000); // 10 min - std::atomic<bool> cancel_watchdog(false); - std::thread watchdog_thread = SetTimeoutWatchDog(child, compiler_timeout_ms, cancel_watchdog); - int wstatus; - waitpid(child, /*out*/&wstatus, /*options*/0); - - // Terminate the thread after the compiler process is killed or done. - LOG(DEBUG) << "Terminate the watch dog thread."; - cancel_watchdog = true; - watchdog_thread.join(); - - if (!WIFEXITED(wstatus)) { - LOG(ERROR) << "Child terminated abnormally, status: " << WEXITSTATUS(wstatus); - return false; - } - - int status = WEXITSTATUS(wstatus); - LOG(DEBUG) << "Child terminated, status: " << status; - if (status == 0) { - LOG(DEBUG) << "Iorap compilation succeeded"; - return true; - } else { - LOG(ERROR) << "Iorap compilation failed"; - return false; - } - } else { - // we are the child that was forked. - std::vector<std::string> argv_vec = MakeCompilerParams(params); - std::unique_ptr<ArgString[]> argv_ptr = - common::VecToArgv(kCommandFileName, argv_vec); - - std::stringstream argv; // for debugging. - for (std::string arg : argv_vec) { - argv << arg << ' '; - } - LOG(DEBUG) << "fork+exec: " << kCommandFileName << " " << argv.str(); - - controller_params.exec->Execve(kCommandFileName, - argv_vec, - /*envp*/nullptr); - // This should never return. - } - return false; -} - -// Gets the perfetto trace infos in the histories. -std::vector<compiler::CompilationInput> GetPerfettoTraceInfo( - const db::DbHandle& db, - const std::vector<db::AppLaunchHistoryModel>& histories) { - std::vector<compiler::CompilationInput> perfetto_traces; - - for(db::AppLaunchHistoryModel history : histories) { - // Get perfetto trace. - std::optional<db::RawTraceModel> raw_trace = - db::RawTraceModel::SelectByHistoryId(db, history.id); - if (!raw_trace) { - // This is normal: non-cold launches do not have traces. - continue; - } - - if (!history.pid) { - LOG(DEBUG) << "Missing pid for history " << history.id; - continue; - } - - uint64_t timestamp_limit = std::numeric_limits<uint64_t>::max(); - // Get corresponding timestamp limit. - if (history.report_fully_drawn_ns) { - timestamp_limit = *history.report_fully_drawn_ns; - } else if (history.total_time_ns) { - timestamp_limit = *history.total_time_ns; - } else { - LOG(DEBUG) << " No timestamp exists. Using the max value."; - } - perfetto_traces.push_back({raw_trace->file_path, timestamp_limit, history.pid}); - } - return perfetto_traces; -} - -// Helper struct for printing vector. -template <class T> -struct VectorPrinter { - std::vector<T>& values; -}; - -std::ostream& operator<<(std::ostream& os, - const struct compiler::CompilationInput& perfetto_trace) { - os << "file_path: " << perfetto_trace.filename << " " - << "timestamp_limit: " << perfetto_trace.timestamp_limit_ns; - return os; -} - -template <class T> -std::ostream& operator<<(std::ostream& os, const struct VectorPrinter<T>& printer) { - os << "[\n"; - for (T i : printer.values) { - os << i << ",\n"; - } - os << "]\n"; - return os; -} - -// Compiled the perfetto traces for an activity. -bool CompileActivity(const db::DbHandle& db, - int package_id, - const std::string& package_name, - const std::string& activity_name, - int version, - const ControllerParameters& params) { - ScopedFormatTrace atrace_compile_package(ATRACE_TAG_PACKAGE_MANAGER, - "Compile activity %s", - activity_name.c_str()); - - LOG(DEBUG) << "CompileActivity: " << package_name << "/" << activity_name << "@" << version; - - db::CompiledTraceFileModel output_file = - CalculateNewestFilePath(package_name, activity_name, version); - - std::string file_path = output_file.FilePath(); - - if (!params.recompile) { - if (std::filesystem::exists(file_path)) { - LOG(DEBUG) << "compiled trace exists in " << file_path; - - db::VersionedComponentName vcn{package_name, activity_name, version}; - std::optional<db::PrefetchFileModel> prefetch_file = - db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn); - if (prefetch_file) { - return true; - } else { - LOG(WARNING) << "Missing corresponding prefetch_file db row for " << vcn; - // let it go and compile again. we'll insert the prefetch_file at the bottom. - } - } - } - - std::optional<db::ActivityModel> activity = - db::ActivityModel::SelectByNameAndPackageId(db, activity_name.c_str(), package_id); - if (!activity) { - LOG(ERROR) << "Cannot find activity for package_id: " << package_id - <<" activity_name: " <<activity_name; - return false; - } - - int activity_id = activity->id; - - std::vector<db::AppLaunchHistoryModel> histories = - db::AppLaunchHistoryModel::SelectActivityHistoryForCompile(db, activity_id); - - { - std::vector<compiler::CompilationInput> perfetto_traces = - GetPerfettoTraceInfo(db, histories); - - if (perfetto_traces.size() < params.min_traces) { - LOG(DEBUG) << "The number of perfetto traces is " << perfetto_traces.size() - <<", which is less than " << params.min_traces; - return false; - } - - { - std::lock_guard<std::mutex> last_job_info_guard{last_job_info_mutex_}; - last_job_info_.activities_last_compiled_++; - } - - // Show the compilation config. - LOG(DEBUG) << "Try to compiled package_id: " << package_id - << " package_name: " << package_name - << " activity_name: " << activity_name - << " version: " << version - << " file_path: " << file_path - << " verbose: " << params.verbose - << " perfetto_traces: " - << VectorPrinter<compiler::CompilationInput>{perfetto_traces}; - if (params.inode_textcache) { - LOG(DEBUG) << "inode_textcache: " << *params.inode_textcache; - } - - CompilerForkParameters compiler_params{perfetto_traces, file_path, params}; - - if (!output_file.MkdirWithParents()) { - LOG(ERROR) << "Compile activity failed. Failed to mkdirs " << file_path; - return false; - } - - ScopedFormatTrace atrace_compile_fork(ATRACE_TAG_PACKAGE_MANAGER, - "Fork+exec iorap.cmd.compiler", - activity_name.c_str()); - if (!StartViaFork(compiler_params)) { - LOG(ERROR) << "Compilation failed for package_id:" << package_id - << " activity_name: " << activity_name; - return false; - } - } - - std::optional<db::PrefetchFileModel> compiled_trace = - db::PrefetchFileModel::Insert(db, activity_id, file_path); - if (!compiled_trace) { - LOG(ERROR) << "Cannot insert compiled trace activity_id: " << activity_id - << " file_path: " << file_path; - return false; - } - return true; -} - -// Compiled the perfetto traces for activities in an package. -bool CompilePackage(const db::DbHandle& db, - const std::string& package_name, - int version, - const ControllerParameters& params) { - ScopedFormatTrace atrace_compile_package(ATRACE_TAG_PACKAGE_MANAGER, - "Compile package %s", - package_name.c_str()); - - std::optional<db::PackageModel> package = - db::PackageModel::SelectByNameAndVersion(db, package_name.c_str(), version); - - if (!package) { - LOG(ERROR) << "Cannot find package for package_name: " - << package_name - << " and version " - << version; - return false; - } - - std::vector<db::ActivityModel> activities = - db::ActivityModel::SelectByPackageId(db, package->id); - - bool ret = true; - for (db::ActivityModel activity : activities) { - if (!CompileActivity(db, package->id, package->name, activity.name, version, params)) { - ret = false; - } - } - return ret; -} - -// Compiled the perfetto traces for packages in a device. -bool CompileAppsOnDevice(const db::DbHandle& db, const ControllerParameters& params) { - { - std::lock_guard<std::mutex> last_job_info_guard{last_job_info_mutex_}; - last_job_info_.activities_last_compiled_ = 0; - } - - std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db); - bool ret = true; - for (db::PackageModel package : packages) { - if (!CompilePackage(db, package.name, package.version, params)) { - ret = false; - } - } - - { - std::lock_guard<std::mutex> last_job_info_guard{last_job_info_mutex_}; - last_job_info_.last_run_ns_ = time(nullptr); - } - - return ret; -} - -// Compiled the perfetto traces for a single package in a device. -bool CompileSingleAppOnDevice(const db::DbHandle& db, - const ControllerParameters& params, - const std::string& package_name) { - std::vector<db::PackageModel> packages = db::PackageModel::SelectByName(db, package_name.c_str()); - bool ret = true; - for (db::PackageModel package : packages) { - if (!CompilePackage(db, package.name, package.version, params)) { - ret = false; - } - } - - return ret; -} - -bool Compile(const std::string& db_path, const ControllerParameters& params) { - iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path); - db::DbHandle db{db_schema.db()}; - return CompileAppsOnDevice(db, params); -} - -bool Compile(const std::string& db_path, - const std::string& package_name, - int version, - const ControllerParameters& params) { - iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path); - db::DbHandle db{db_schema.db()}; - return CompilePackage(db, package_name, version, params); -} - -bool Compile(const std::string& db_path, - const std::string& package_name, - const std::string& activity_name, - int version, - const ControllerParameters& params) { - iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path); - db::DbHandle db{db_schema.db()}; - - std::optional<db::PackageModel> package = - db::PackageModel::SelectByNameAndVersion(db, package_name.c_str(), version); - - if (!package) { - LOG(ERROR) << "Cannot find package with name " - << package_name - << " and version " - << version; - return false; - } - return CompileActivity(db, package->id, package_name, activity_name, version, params); -} - -static std::string TimeToString(time_t the_time) { - tm tm_buf{}; - tm* tm_ptr = localtime_r(&the_time, &tm_buf); - - if (tm_ptr != nullptr) { - char time_buffer[256]; - strftime(time_buffer, sizeof(time_buffer), "%a %b %d %H:%M:%S %Y", tm_ptr); - return std::string{time_buffer}; - } else { - return std::string{"(nullptr)"}; - } -} - -static std::string GetTimestampForPrefetchFile(const db::PrefetchFileModel& prefetch_file) { - std::filesystem::path path{prefetch_file.file_path}; - - std::error_code ec{}; - auto last_write_time = std::filesystem::last_write_time(path, /*out*/ec); - if (ec) { - return std::string("Failed to get last write time: ") + ec.message(); - } - - time_t time = decltype(last_write_time)::clock::to_time_t(last_write_time); - - std::string time_str = TimeToString(time); - return time_str; -} - -void DumpPackageActivity(const db::DbHandle& db, - ::android::Printer& printer, - const db::PackageModel& package, - const db::ActivityModel& activity) { - int package_id = package.id; - const std::string& package_name = package.name; - int package_version = package.version; - const std::string& activity_name = activity.name; - db::VersionedComponentName vcn{package_name, activity_name, package_version}; - - // com.google.Settings/com.google.Settings.ActivityMain@1234567890 - printer.printFormatLine(" %s/%s@%d", - package_name.c_str(), - activity_name.c_str(), - package_version); - - std::optional<db::PrefetchFileModel> prefetch_file = - db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn); - - std::vector<db::AppLaunchHistoryModel> histories = - db::AppLaunchHistoryModel::SelectActivityHistoryForCompile(db, activity.id); - std::vector<compiler::CompilationInput> perfetto_traces = - GetPerfettoTraceInfo(db, histories); - - if (prefetch_file) { - bool exists_on_disk = std::filesystem::exists(prefetch_file->file_path); - - std::optional<size_t> prefetch_byte_sum = - prefetcher::ReadAhead::PrefetchSizeInBytes(prefetch_file->file_path); - - if (exists_on_disk) { - printer.printFormatLine(" Compiled Status: Usable compiled trace"); - } else { - printer.printFormatLine(" Compiled Status: Prefetch file deleted from disk."); - } - - if (prefetch_byte_sum) { - printer.printFormatLine(" Bytes to be prefetched: %zu", *prefetch_byte_sum); - } else { - printer.printFormatLine(" Bytes to be prefetched: (bad file path)" ); - } - - printer.printFormatLine(" Time compiled: %s", - GetTimestampForPrefetchFile(*prefetch_file).c_str()); - printer.printFormatLine(" %s", prefetch_file->file_path.c_str()); - } else { - size_t size = perfetto_traces.size(); - - if (size >= kMinTracesForCompilation) { - printer.printFormatLine(" Compiled Status: Raw traces pending compilation (%zu)", - perfetto_traces.size()); - } else { - size_t remaining = kMinTracesForCompilation - size; - printer.printFormatLine(" Compiled Status: Need %zu more traces for compilation", - remaining); - } - } - - printer.printFormatLine(" Raw traces:"); - printer.printFormatLine(" Trace count: %zu", perfetto_traces.size()); - - for (compiler::CompilationInput& compilation_input : perfetto_traces) { - std::string& raw_trace_file_name = compilation_input.filename; - - printer.printFormatLine(" %s", raw_trace_file_name.c_str()); - } -} - -void DumpPackage(const db::DbHandle& db, - ::android::Printer& printer, - db::PackageModel package) { - std::vector<db::ActivityModel> activities = - db::ActivityModel::SelectByPackageId(db, package.id); - - for (db::ActivityModel& activity : activities) { - DumpPackageActivity(db, printer, package, activity); - } -} - -void DumpAllPackages(const db::DbHandle& db, ::android::Printer& printer) { - printer.printLine("Package history in database:"); - - std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db); - for (db::PackageModel package : packages) { - DumpPackage(db, printer, package); - } - - printer.printLine(""); -} - -void Dump(const db::DbHandle& db, ::android::Printer& printer) { - bool locked = last_job_info_mutex_.try_lock(); - - LastJobInfo info = last_job_info_; - - printer.printFormatLine("Background job:"); - if (!locked) { - printer.printLine(""""" (possible deadlock)"); - } - if (info.last_run_ns_ != time_t{0}) { - std::string time_str = TimeToString(info.last_run_ns_); - - printer.printFormatLine(" Last run at: %s", time_str.c_str()); - } else { - printer.printFormatLine(" Last run at: (None)"); - } - printer.printFormatLine(" Activities last compiled: %zu", info.activities_last_compiled_); - - printer.printLine(""); - - if (locked) { - last_job_info_mutex_.unlock(); - } - - DumpAllPackages(db, printer); -} - -} // namespace iorap::maintenance diff --git a/src/maintenance/controller.h b/src/maintenance/controller.h deleted file mode 100644 index 750d0b4..0000000 --- a/src/maintenance/controller.h +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_MAINTENANCE_COMPILER_CONTROLLER_H_ -#define IORAP_SRC_MAINTENANCE_COMPILER_CONTROLLER_H_ - -#include "db/file_models.h" -#include "inode2filename/inode_resolver.h" - -#include <string> -#include <vector> - -namespace android { -class Printer; -} // namespace android - -namespace iorap::maintenance { - -// Enabling mock for testing purpose. -class IExec { - public: - virtual int Execve(const std::string& pathname, - std::vector<std::string>& argv_vec, - char *const envp[]) = 0; - virtual int Fork() = 0; - virtual ~IExec() = default; -}; - -class Exec : public IExec { - public: - virtual int Execve(const std::string& pathname, - std::vector<std::string>& argv_vec, - char *const envp[]); - virtual int Fork(); -}; - -// Represents the parameters used for compilation controller. -struct ControllerParameters { - bool output_text; - // The path of inode2filepath file. - std::optional<std::string> inode_textcache; - bool verbose; - bool recompile; - uint64_t min_traces; - std::shared_ptr<IExec> exec; - bool exclude_dex_files; - - ControllerParameters(bool output_text, - std::optional<std::string> inode_textcache, - bool verbose, - bool recompile, - uint64_t min_traces, - std::shared_ptr<IExec> exec, - bool exclude_dex_files) : - output_text(output_text), - inode_textcache(inode_textcache), - verbose(verbose), - recompile(recompile), - min_traces(min_traces), - exec(exec), - exclude_dex_files(exclude_dex_files) { - } -}; - -// Control the compilation of perfetto traces in the sqlite db. -// -// The strategy now is to compile all the existing perfetto traces for an activity -// and skip ones if the number of perfetto traces is less than the min_traces. -// -// By default, the program doesn't replace the existing compiled trace, it just -// return true. To force replace the existing compiled trace, set `force` to true. -// -// The timestamp limit of the each perfetto trace is determined by `report_fully_drawn_ns` -// timestamp. If it doesn't exists, use `total_time_ns`. If neither of them exists, -// use the max. - -// Compile all activities of all packages in the database. -bool Compile(const std::string& db_path, const ControllerParameters& params); - -// Compile all activities in the package. -// If the version is not given, an arbitrary package that has the same name is used. -bool Compile(const std::string& db_path, - const std::string& package_name, - int version, - const ControllerParameters& params); - -// Compile trace for the activity. -// If the version is not given, an arbitrary package has the same name is used. -bool Compile(const std::string& db_path, - const std::string& package_name, - const std::string& activity_name, - int version, - const ControllerParameters& params); -// Visible for testing. -bool CompileAppsOnDevice(const db::DbHandle& db, const ControllerParameters& params); - -bool CompileSingleAppOnDevice(const db::DbHandle& db, - const ControllerParameters& params, - const std::string& package_name); - -void Dump(const db::DbHandle& db, ::android::Printer& printer); - -} // iorap::maintenance - -#endif // IORAP_SRC_MAINTENANCE_COMPILER_CONTROLLER_H_ diff --git a/src/maintenance/db_cleaner.cc b/src/maintenance/db_cleaner.cc deleted file mode 100644 index 56eff70..0000000 --- a/src/maintenance/db_cleaner.cc +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "maintenance/db_cleaner.h" - -#include <android-base/file.h> - -#include <cstdio> -#include <filesystem> -#include <fstream> -#include <iostream> -#include <limits> -#include <optional> -#include <string> -#include <vector> - -#include "db/clean_up.h" -#include "db/file_models.h" -#include "db/models.h" - -namespace iorap::maintenance { - -// Enable foreign key restriction. -const constexpr char* kForeignKeyOnSql = "PRAGMA foreign_keys = ON;"; - -void CleanUpDatabase(const db::DbHandle& db, - std::shared_ptr<binder::PackageVersionMap> version_map) { - std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db); - // Enable cascade deletion. - if (!db::DbQueryBuilder::ExecuteOnce(db, kForeignKeyOnSql)) { - LOG(ERROR) << "Fail to turn on foreign key restraint!"; - } - - for (db::PackageModel package : packages) { - std::optional<int64_t> version = version_map->Find(package.name); - if (!version) { - LOG(DEBUG) << "Fail to find version for package " << package.name - << " with version " << package.version - << ". The package manager may be down."; - continue; - } - // Package is cleanup if it - // * is not in the version map, it may be uninstalled - // * has an different version with the latest one - if (*version != package.version) { - db::CleanUpFilesForPackage(db, package.id, package.name, package.version); - if (!package.Delete()) { - LOG(ERROR) << "Fail to delete package " << package.name - << " with version " << package.version; - } - } - } -} - -} // namespace iorap::maintenance diff --git a/src/maintenance/db_cleaner.h b/src/maintenance/db_cleaner.h deleted file mode 100644 index 1168bd5..0000000 --- a/src/maintenance/db_cleaner.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_MAINTENANCE_VERSION_UPDATE_H_ -#define IORAP_SRC_MAINTENANCE_VERSION_UPDATE_H_ - -#include <android/content/pm/IPackageManagerNative.h> - -#include <string> -#include <vector> - -#include "binder/package_version_map.h" -#include "db/file_models.h" - -namespace iorap::maintenance { - -// Clean up the database. -// Remove all relevant data for old-version packages. -void CleanUpDatabase(const db::DbHandle& db, - std::shared_ptr<binder::PackageVersionMap> version_map); -} // namespace iorap::maintenance - -#endif // IORAP_SRC_MAINTENANCE_VERSION_UPDATE_H_ diff --git a/src/maintenance/main.cc b/src/maintenance/main.cc deleted file mode 100644 index 8b66069..0000000 --- a/src/maintenance/main.cc +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/debug.h" -#include "compiler/compiler.h" -#include "maintenance/controller.h" -#include "db/clean_up.h" - -#include <android-base/parseint.h> -#include <android-base/properties.h> -#include <android-base/logging.h> - -#include <iostream> -#include <optional> - -#if defined(IORAP_MAINTENANCE_MAIN) - -namespace iorap::maintenance { - -void Usage(char** argv) { - std::cerr << "Usage: " << argv[0] << " <path of sqlite db>" << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Compile the perfetto trace for an package and activity." << std::endl; - std::cerr << " The info of perfetto trace is stored in the sqlite db." << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Optional flags:" << std::endl; - std::cerr << " --package $,-p $ Package name." << std::endl; - std::cerr << " --version $,-ve $ Package version." << std::endl; - std::cerr << " --activity $,-a $ Activity name." << std::endl; - std::cerr << " --inode-textcache $,-it $ Resolve inode->filename from textcache." << std::endl; - std::cerr << " --help,-h Print this Usage." << std::endl; - std::cerr << " --recompile,-r Force re-compilation, which replace the existing compiled trace ." << std::endl; - std::cerr << " --purge-package,-pp Purge all files associated with a package." << std::endl; - std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; - std::cerr << " --output-text,-ot Output ascii text instead of protobuf (default off)." << std::endl; - std::cerr << " --min_traces,-mt The min number of perfetto traces needed " - << "for compilation (default 1)." << std::endl; - std::cerr << " --exclude-dex-files,-edf Set of exclude dex files" << std::endl; - exit(1); -} - - -int Main(int argc, char** argv){ - android::base::InitLogging(argv); - android::base::SetLogger(android::base::StderrLogger); - - if (argc == 1) { - // Need at least 1 input file to do anything. - Usage(argv); - } - - std::vector<std::string> arg_input_filenames; - std::optional<std::string> arg_package; - std::optional<std::string> arg_purge_package; - int arg_version = -1; - std::optional<std::string> arg_activity; - std::optional<std::string> arg_inode_textcache; - bool recompile = false; - bool enable_verbose = false; - bool arg_output_text = false; - uint64_t arg_min_traces = 1; - bool exclude_dex_files = false; - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - bool has_arg_next = (arg+1)<argc; - std::string arg_next = has_arg_next ? argv[arg+1] : ""; - - if (argstr == "--help" || argstr == "-h") { - Usage(argv); - } else if (argstr == "--package" || argstr == "-p") { - if (!has_arg_next) { - std::cerr << "Missing --package <value>" << std::endl; - return 1; - } - arg_package = arg_next; - ++arg; - } else if (argstr == "--version" || argstr == "-ve") { - if (!has_arg_next) { - std::cerr << "Missing --version <value>" << std::endl; - return 1; - } - int version; - if (!android::base::ParseInt<int>(arg_next, &version)) { - std::cerr << "Invalid --version " << arg_next << std::endl; - return 1; - } - arg_version = version; - ++arg; - } else if (argstr == "--activity" || argstr == "-a") { - if (!has_arg_next) { - std::cerr << "Missing --activity <value>" << std::endl; - return 1; - } - arg_activity = arg_next; - ++arg; - } else if (argstr == "--inode-textcache" || argstr == "-it") { - if (!has_arg_next) { - std::cerr << "Missing --inode-textcache <value>" << std::endl; - return 1; - } - arg_inode_textcache = arg_next; - ++arg; - } else if (argstr == "--purge-package" || argstr == "-pp") { - if (!has_arg_next) { - std::cerr << "Missing --purge-package <value>" << std::endl; - return 1; - } - arg_purge_package = arg_next; - ++arg; - } else if (argstr == "--verbose" || argstr == "-v") { - enable_verbose = true; - } else if (argstr == "--recompile" || argstr == "-r") { - recompile = true; - } else if (argstr == "--output-text" || argstr == "-ot") { - arg_output_text = true; - } else if (argstr == "--min_traces" || argstr == "-mt") { - if (!has_arg_next) { - std::cerr << "Missing --min_traces <value>" << std::endl; - return 1; - } - arg_min_traces = std::stoul(arg_next); - ++arg; - } else if (argstr == "--exclude-dex-files" || argstr == "-edf") { - exclude_dex_files = true; - } else { - arg_input_filenames.push_back(argstr); - } - } - - if (arg_input_filenames.empty()) { - LOG(ERROR) << "Missing filename to a sqlite database."; - Usage(argv); - } else if (arg_input_filenames.size() > 1) { - LOG(ERROR) << "More than one filename to a sqlite database."; - Usage(argv); - } - - std::string db_path = arg_input_filenames[0]; - - if (enable_verbose) { - android::base::SetMinimumLogSeverity(android::base::VERBOSE); - - LOG(VERBOSE) << "Verbose check"; - LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; - } else { - android::base::SetMinimumLogSeverity(android::base::DEBUG); - } - - if (arg_purge_package) { - db::CleanUpFilesForPackage(db_path, *arg_purge_package); - return 0; - // Don't do any more work because SchemaModel can only be created once. - } - - maintenance::ControllerParameters params{ - arg_output_text, - arg_inode_textcache, - enable_verbose, - recompile, - arg_min_traces, - std::make_shared<Exec>(), - exclude_dex_files}; - - int ret_code = 0; - if (arg_package && arg_activity) { - ret_code = !Compile(std::move(db_path), - std::move(*arg_package), - std::move(*arg_activity), - arg_version, - params); - } else if (arg_package) { - ret_code = !Compile(std::move(db_path), std::move(*arg_package), arg_version, params); - } else { - ret_code = !Compile(std::move(db_path), params); - } - return ret_code; -} - -} // iorap::maintenance - -int main(int argc, char** argv) { - return ::iorap::maintenance::Main(argc, argv); -} - - -#endif // IORAP_MAINTENANCE_MAIN diff --git a/src/manager/event_manager.cc b/src/manager/event_manager.cc deleted file mode 100644 index 0242c10..0000000 --- a/src/manager/event_manager.cc +++ /dev/null @@ -1,1401 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "binder/package_version_map.h" -#include "common/debug.h" -#include "common/expected.h" -#include "common/printer.h" -#include "common/rx_async.h" -#include "common/property.h" -#include "common/trace.h" -#include "db/app_component_name.h" -#include "db/file_models.h" -#include "db/models.h" -#include "maintenance/controller.h" -#include "maintenance/db_cleaner.h" -#include "manager/event_manager.h" -#include "perfetto/rx_producer.h" -#include "prefetcher/read_ahead.h" -#include "prefetcher/task_id.h" - -#include <android-base/chrono_utils.h> -#include <android-base/strings.h> -#include <android-base/properties.h> -#include <rxcpp/rx.hpp> -#include <server_configurable_flags/get_flags.h> -#include <utils/misc.h> -#include <utils/Trace.h> - -#include <atomic> -#include <filesystem> -#include <functional> -#include <type_traits> -#include <unordered_map> - -using rxcpp::observe_on_one_worker; - -namespace iorap::manager { - -using binder::AppLaunchEvent; -using binder::DexOptEvent; -using binder::JobScheduledEvent; -using binder::RequestId; -using binder::TaskResult; - -using common::AsyncPool; -using common::RxAsync; - -using perfetto::PerfettoStreamCommand; -using perfetto::PerfettoTraceProto; - -using db::AppComponentName; - -const constexpr bool kExcludeDexFilesDefault = true; - -static std::atomic<bool> s_tracing_allowed{false}; -static std::atomic<bool> s_readahead_allowed{false}; -static std::atomic<uint64_t> s_min_traces{3}; - -struct PackageBlacklister { - // "x.y.z;foo.bar.baz" colon-separated list of substrings - PackageBlacklister(std::string blacklist_string) { - LOG(VERBOSE) << "Configuring package blacklister with string: " << blacklist_string; - - std::vector<std::string> split = ::android::base::Split(blacklist_string, ";"); - - // Ignore any l/r whitespace or empty strings. - for (const std::string& s : split) { - std::string t = ::android::base::Trim(s); - if (!t.empty()) { - LOG(INFO) << "Blacklisted package: " << t << "; will not optimize."; - packages_.push_back(t); - } - } - } - - PackageBlacklister() = default; - - bool IsBlacklisted(const std::string& package_name) const { - return std::find(packages_.begin(), packages_.end(), package_name) != packages_.end(); - } - - bool IsBlacklisted(const AppComponentName& component_name) const { - return IsBlacklisted(component_name.package); - } - - bool IsBlacklisted(const std::optional<AppComponentName>& component_name) const { - return component_name.has_value() && IsBlacklisted(component_name->package); - } - - private: - std::vector<std::string> packages_; -}; - -using PackageVersionMap = std::unordered_map<std::string, int64_t>; - -// Main logic of the #OnAppLaunchEvent scan method. -// -// All functions are called from the same thread as the event manager -// functions. -// -// This is a data type, it's moved (std::move) around from one iteration -// of #scan to another. -struct AppLaunchEventState { - std::optional<AppComponentName> component_name_; - // Sequence ID is shared amongst the same app launch sequence, - // but changes whenever a new app launch sequence begins. - size_t sequence_id_ = static_cast<size_t>(-1); - std::optional<AppLaunchEvent::Temperature> temperature_; - - // Push data to perfetto rx chain for associating - // the raw_trace with the history_id. - std::optional<rxcpp::subscriber<int>> history_id_subscriber_; - rxcpp::observable<int> history_id_observable_; - - std::optional<uint64_t> intent_started_ns_; - std::optional<uint64_t> total_time_ns_; - - // Used by kReportFullyDrawn to find the right history_id. - // We assume no interleaving between different sequences. - // This assumption is checked in the Java service code. - std::optional<uint64_t> recent_history_id_; - - // labeled as 'shared' due to rx not being able to handle move-only objects. - // lifetime: in practice equivalent to unique_ptr. - std::shared_ptr<prefetcher::ReadAhead> read_ahead_; - bool allowed_readahead_{true}; - bool is_read_ahead_{false}; - std::optional<prefetcher::TaskId> read_ahead_task_; - - bool allowed_tracing_{true}; - bool is_tracing_{false}; - std::optional<rxcpp::composite_subscription> rx_lifetime_; - std::vector<rxcpp::composite_subscription> rx_in_flight_; - - PackageBlacklister package_blacklister_{}; - - borrowed<perfetto::RxProducerFactory*> perfetto_factory_; // not null - borrowed<observe_on_one_worker*> thread_; // not null - borrowed<observe_on_one_worker*> io_thread_; // not null - borrowed<AsyncPool*> async_pool_; // not null - - std::shared_ptr<binder::PackageVersionMap> version_map_; - - explicit AppLaunchEventState(borrowed<perfetto::RxProducerFactory*> perfetto_factory, - bool allowed_readahead, - bool allowed_tracing, - PackageBlacklister package_blacklister, - borrowed<observe_on_one_worker*> thread, - borrowed<observe_on_one_worker*> io_thread, - borrowed<AsyncPool*> async_pool, - std::shared_ptr<binder::PackageVersionMap> version_map) - : read_ahead_{std::make_shared<prefetcher::ReadAhead>()} - { - perfetto_factory_ = perfetto_factory; - DCHECK(perfetto_factory_ != nullptr); - - allowed_readahead_ = allowed_readahead; - allowed_tracing_ = allowed_tracing; - - package_blacklister_ = package_blacklister; - - thread_ = thread; - DCHECK(thread_ != nullptr); - - io_thread_ = io_thread; - DCHECK(io_thread_ != nullptr); - - async_pool_ = async_pool; - DCHECK(async_pool_ != nullptr); - - version_map_ = version_map; - DCHECK(version_map_ != nullptr); - } - - // Updates the values in this struct only as a side effect. - // - // May create and fire a new rx chain on the same threads as passed - // in by the constructors. - void OnNewEvent(const AppLaunchEvent& event) { - LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent: " << event; - - android::ScopedTrace trace_db_init{ATRACE_TAG_ACTIVITY_MANAGER, - "IorapNativeService::OnAppLaunchEvent"}; - - using Type = AppLaunchEvent::Type; - - DCHECK_GE(event.sequence_id, 0); - sequence_id_ = static_cast<size_t>(event.sequence_id); - allowed_readahead_ = s_readahead_allowed; - allowed_tracing_ = s_tracing_allowed; - - switch (event.type) { - case Type::kIntentStarted: { - const std::string& package_name = event.intent_proto->component().package_name(); - const std::string& class_name = event.intent_proto->component().class_name(); - AppComponentName component_name{package_name, class_name}; - component_name = component_name.Canonicalize(); - component_name_ = component_name; - - if (package_blacklister_.IsBlacklisted(component_name)) { - LOG(DEBUG) << "kIntentStarted: package " << component_name.package - << " ignored due to blacklisting."; - break; - } - - // Create a new history ID chain for each new app start-up sequence. - auto history_id_observable = rxcpp::observable<>::create<int>( - [&](rxcpp::subscriber<int> subscriber) { - history_id_subscriber_ = std::move(subscriber); - LOG(VERBOSE) << " set up the history id subscriber "; - }) - .tap([](int history_id) { LOG(VERBOSE) << " tap rx history id = " << history_id; }) - .replay(1); // Remember the history id in case we subscribe too late. - - history_id_observable_ = history_id_observable; - - // Immediately turn observable hot, creating the subscriber. - history_id_observable.connect(); - - DCHECK(!IsTracing()); - - // The time should be set before perfetto tracing. - // Record the timestamp even no perfetto tracing is triggered, - // because the tracing may start in the following ActivityLaunched - // event. Otherwise, there will be no starting timestamp and - // trace without starting timestamp is not considered for compilation. - if (event.timestamp_nanos >= 0) { - intent_started_ns_ = event.timestamp_nanos; - } else { - LOG(WARNING) << "Negative event timestamp: " << event.timestamp_nanos; - } - break; - } - case Type::kIntentFailed: - if (package_blacklister_.IsBlacklisted(component_name_)) { - LOG(VERBOSE) << "kIntentFailed: package " << component_name_->package - << " ignored due to blacklisting."; - break; - } - - if (history_id_subscriber_) { - history_id_subscriber_->on_error(rxcpp::util::make_error_ptr( - std::ios_base::failure("Aborting due to intent failed"))); - history_id_subscriber_ = std::nullopt; - } - - break; - case Type::kActivityLaunched: { - // TODO add test in Android framework to verify this. - const std::string& title = - event.activity_record_proto->window_token().window_container().identifier().title(); - if (!AppComponentName::HasAppComponentName(title)) { - // Proto comment claim this is sometimes a window title. - // We need the actual 'package/component' here, so just ignore it if it's a title. - LOG(WARNING) << "App launched without a component name: " << event; - break; - } - - AppComponentName component_name = AppComponentName::FromString(title); - component_name = component_name.Canonicalize(); - component_name_ = component_name; - - if (package_blacklister_.IsBlacklisted(component_name_)) { - LOG(VERBOSE) << "kActivityLaunched: package " << component_name_->package - << " ignored due to blacklisting."; - break; - } - - // Cancel tracing for warm/hot. - // Restart tracing if the activity was unexpected. - - AppLaunchEvent::Temperature temperature = event.temperature; - temperature_ = temperature; - if (temperature != AppLaunchEvent::Temperature::kCold) { - LOG(DEBUG) << "AppLaunchEventState#OnNewEvent don't trace due to non-cold temperature"; - } else if (!IsTracing() && !IsReadAhead()) { // and the temperature is Cold. - // Start late trace when intent didn't have a component name - LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent need to start new trace"; - - if (allowed_readahead_ && !IsReadAhead()) { - StartReadAhead(sequence_id_, component_name); - } - if (allowed_tracing_ && !IsTracing() && !IsReadAhead()) { - rx_lifetime_ = StartTracing(std::move(component_name)); - } - } else { - // FIXME: match actual component name against intent component name. - // abort traces if they don't match. - - if (allowed_tracing_) { - LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent already tracing"; - } - LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent already doing readahead"; - } - break; - } - case Type::kActivityLaunchFinished: - if (package_blacklister_.IsBlacklisted(component_name_)) { - LOG(VERBOSE) << "kActivityLaunchFinished: package " << component_name_->package - << " ignored due to blacklisting."; - break; - } - - if (event.timestamp_nanos >= 0) { - total_time_ns_ = event.timestamp_nanos; - } - RecordDbLaunchHistory(event.activity_record_proto->proc_id()); - // Finish tracing and collect trace buffer. - // - // TODO: this happens automatically when perfetto finishes its - // trace duration. - if (IsTracing()) { - MarkPendingTrace(); - } - FinishReadAhead(); - break; - case Type::kActivityLaunchCancelled: - if (package_blacklister_.IsBlacklisted(component_name_)) { - LOG(VERBOSE) << "kActivityLaunchCancelled: package " << component_name_->package - << " ignored due to blacklisting."; - break; - } - - // Abort tracing. - AbortTrace(); - AbortReadAhead(); - break; - case Type::kReportFullyDrawn: { - if (package_blacklister_.IsBlacklisted(component_name_)) { - LOG(VERBOSE) << "kReportFullyDrawn: package " << component_name_->package - << " ignored due to blacklisting."; - break; - } - - if (!recent_history_id_) { - LOG(WARNING) << "Dangling kReportFullyDrawn event"; - return; - } - UpdateReportFullyDrawn(*recent_history_id_, event.timestamp_nanos); - recent_history_id_ = std::nullopt; - break; - } - default: - DCHECK(false) << "invalid type: " << event; // binder layer should've rejected this. - LOG(ERROR) << "invalid type: " << event; // binder layer should've rejected this. - } - } - - // Is there an in-flight readahead task currently? - bool IsReadAhead() const { - return read_ahead_task_.has_value(); - } - - // Gets the compiled trace. - // If a compiled trace exists in sqlite, use that one. Otherwise, try - // to find a prebuilt one. - std::optional<std::string> GetCompiledTrace(const AppComponentName& component_name) { - ScopedFormatTrace atrace_get_compiled_trace(ATRACE_TAG_ACTIVITY_MANAGER, "GetCompiledTrace"); - // Firstly, try to find the compiled trace from sqlite. - android::base::Timer timer{}; - db::DbHandle db{db::SchemaModel::GetSingleton()}; - std::optional<int> version = - version_map_->GetOrQueryPackageVersion(component_name.package); - if (!version) { - LOG(DEBUG) << "The version is NULL, maybe package manager is down."; - return std::nullopt; - } - db::VersionedComponentName vcn{component_name.package, - component_name.activity_name, - *version}; - - std::optional<db::PrefetchFileModel> compiled_trace = - db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn); - - std::chrono::milliseconds duration_ms = timer.duration(); - LOG(DEBUG) << "EventManager: Looking up compiled trace done in " - << duration_ms.count() // the count of ticks. - << "ms."; - - if (compiled_trace) { - if (std::filesystem::exists(compiled_trace->file_path)) { - return compiled_trace->file_path; - } else { - LOG(DEBUG) << "Compiled trace in sqlite doesn't exists. file_path: " - << compiled_trace->file_path; - } - } - - LOG(DEBUG) << "Cannot find compiled trace in sqlite for package_name: " - << component_name.package - << " activity_name: " - << component_name.activity_name; - - // If sqlite doesn't have the compiled trace, try the prebuilt path. - std::string file_path = "/product/iorap-trace/"; - file_path += component_name.ToMakeFileSafeEncodedPkgString(); - file_path += ".compiled_trace.pb"; - - if (std::filesystem::exists(file_path)) { - return file_path; - } - - LOG(DEBUG) << "Prebuilt compiled trace doesn't exists. file_path: " - << file_path; - - return std::nullopt; - } - - void StartReadAhead(size_t id, const AppComponentName& component_name) { - DCHECK(allowed_readahead_); - DCHECK(!IsReadAhead()); - - std::optional<std::string> file_path = GetCompiledTrace(component_name); - if (!file_path) { - LOG(VERBOSE) << "Cannot find a compiled trace."; - return; - } - - prefetcher::TaskId task{id, *file_path}; - read_ahead_->BeginTask(task); - // TODO: non-void return signature? - - read_ahead_task_ = std::move(task); - } - - void FinishReadAhead() { - // if no readahead task exist, do nothing. - if (!IsReadAhead()){ - return; - } - - read_ahead_->FinishTask(*read_ahead_task_); - read_ahead_task_ = std::nullopt; - } - - void AbortReadAhead() { - FinishReadAhead(); - } - - bool IsTracing() const { - return is_tracing_; - } - - std::optional<rxcpp::composite_subscription> StartTracing( - AppComponentName component_name) { - DCHECK(allowed_tracing_); - DCHECK(!IsTracing()); - - std::optional<int> version = - version_map_->GetOrQueryPackageVersion(component_name_->package); - if (!version) { - LOG(DEBUG) << "The version is NULL, maybe package manager is down."; - return std::nullopt; - } - db::VersionedComponentName versioned_component_name{component_name.package, - component_name.activity_name, - *version}; - db::DbHandle db{db::SchemaModel::GetSingleton()}; - { - ScopedFormatTrace atrace_traces_number_check( - ATRACE_TAG_ACTIVITY_MANAGER, "IorapNativeService::CheckPerfettoTracesNnumber"); - // Just return if we have enough perfetto traces. - if (!db::PerfettoTraceFileModel::NeedMorePerfettoTraces( - db, versioned_component_name)) { - return std::nullopt; - } - } - - auto /*observable<PerfettoStreamCommand>*/ perfetto_commands = - rxcpp::observable<>::just(PerfettoStreamCommand::kStartTracing) - // wait 1x - .concat( - // Pick a value longer than the perfetto config delay_ms, so that we send - // 'kShutdown' after tracing has already finished. - rxcpp::observable<>::interval(std::chrono::milliseconds(10000)) - .take(2) // kStopTracing, kShutdown. - .map([](int value) { - // value is 1,2,3,... - return static_cast<PerfettoStreamCommand>(value); // 1,2, ... - }) - ); - - auto /*observable<PerfettoTraceProto>*/ trace_proto_stream = - perfetto_factory_->CreateTraceStream(perfetto_commands); - // This immediately connects to perfetto asynchronously. - // - // TODO: create a perfetto handle earlier, to minimize perfetto startup latency. - - rxcpp::composite_subscription lifetime; - - auto stream_via_threads = trace_proto_stream - .tap([](const PerfettoTraceProto& trace_proto) { - LOG(VERBOSE) << "StartTracing -- PerfettoTraceProto received (1)"; - }) - .combine_latest(history_id_observable_) - .observe_on(*thread_) // All work prior to 'observe_on' is handled on thread_. - .subscribe_on(*thread_) // All work prior to 'observe_on' is handled on thread_. - .observe_on(*io_thread_) // Write data on an idle-class-priority thread. - .tap([](std::tuple<PerfettoTraceProto, int> trace_tuple) { - LOG(VERBOSE) << "StartTracing -- PerfettoTraceProto received (2)"; - }); - - lifetime = RxAsync::SubscribeAsync(*async_pool_, - std::move(stream_via_threads), - /*on_next*/[versioned_component_name] - (std::tuple<PerfettoTraceProto, int> trace_tuple) { - PerfettoTraceProto& trace_proto = std::get<0>(trace_tuple); - int history_id = std::get<1>(trace_tuple); - - db::PerfettoTraceFileModel file_model = - db::PerfettoTraceFileModel::CalculateNewestFilePath(versioned_component_name); - - std::string file_path = file_model.FilePath(); - - ScopedFormatTrace atrace_write_to_file(ATRACE_TAG_ACTIVITY_MANAGER, - "Perfetto Write Trace To File %s", - file_path.c_str()); - - if (!file_model.MkdirWithParents()) { - LOG(ERROR) << "Cannot save TraceBuffer; failed to mkdirs " << file_path; - return; - } - - if (!trace_proto.WriteFullyToFile(file_path)) { - LOG(ERROR) << "Failed to save TraceBuffer to " << file_path; - } else { - LOG(INFO) << "Perfetto TraceBuffer saved to file: " << file_path; - - ScopedFormatTrace atrace_update_raw_traces_table( - ATRACE_TAG_ACTIVITY_MANAGER, - "update raw_traces table history_id = %d", - history_id); - db::DbHandle db{db::SchemaModel::GetSingleton()}; - std::optional<db::RawTraceModel> raw_trace = - db::RawTraceModel::Insert(db, history_id, file_path); - - if (!raw_trace) { - LOG(ERROR) << "Failed to insert raw_traces for " << file_path; - } else { - LOG(VERBOSE) << "Inserted into db: " << *raw_trace; - } - } - }, - /*on_error*/[](rxcpp::util::error_ptr err) { - LOG(ERROR) << "Perfetto trace proto collection error: " << rxcpp::util::what(err); - }); - - is_tracing_ = true; - - return lifetime; - } - - void AbortTrace() { - LOG(VERBOSE) << "AppLaunchEventState - AbortTrace"; - - // if the tracing is not running, do nothing. - if (!IsTracing()){ - return; - } - - is_tracing_ = false; - if (rx_lifetime_) { - // TODO: it would be good to call perfetto Destroy. - - rx_in_flight_.erase(std::remove(rx_in_flight_.begin(), - rx_in_flight_.end(), *rx_lifetime_), - rx_in_flight_.end()); - - LOG(VERBOSE) << "AppLaunchEventState - AbortTrace - Unsubscribe"; - rx_lifetime_->unsubscribe(); - - rx_lifetime_.reset(); - } - } - - void MarkPendingTrace() { - LOG(VERBOSE) << "AppLaunchEventState - MarkPendingTrace"; - DCHECK(is_tracing_); - DCHECK(rx_lifetime_.has_value()); - - if (rx_lifetime_) { - LOG(VERBOSE) << "AppLaunchEventState - MarkPendingTrace - lifetime moved"; - // Don't unsubscribe because that would cause the perfetto TraceBuffer - // to get dropped on the floor. - // - // Instead, we want to let it finish and write it out to a file. - rx_in_flight_.push_back(*std::move(rx_lifetime_)); - rx_lifetime_.reset(); - } else { - LOG(VERBOSE) << "AppLaunchEventState - MarkPendingTrace - lifetime was empty"; - } - - is_tracing_ = false; - // FIXME: how do we clear this vector? - } - - void RecordDbLaunchHistory(int32_t pid) { - std::optional<db::AppLaunchHistoryModel> history = InsertDbLaunchHistory(pid); - - // RecordDbLaunchHistory happens-after kIntentStarted - if (!history_id_subscriber_.has_value()) { - LOG(WARNING) << "Logic error? Should always have a subscriber here."; - return; - } - - // Ensure that the history id rx chain is terminated either with an error or with - // the newly inserted app_launch_histories.id - if (!history) { - history_id_subscriber_->on_error(rxcpp::util::make_error_ptr( - std::ios_base::failure("Failed to insert history id"))); - recent_history_id_ = std::nullopt; - } else { - // Note: we must have already subscribed, or this value will disappear. - LOG(VERBOSE) << "history_id_subscriber on_next history_id=" << history->id; - history_id_subscriber_->on_next(history->id); - history_id_subscriber_->on_completed(); - - recent_history_id_ = history->id; - } - history_id_subscriber_ = std::nullopt; - } - - std::optional<db::AppLaunchHistoryModel> InsertDbLaunchHistory(int32_t pid) { - // TODO: deferred queue into a different lower priority thread. - if (!component_name_ || !temperature_) { - LOG(VERBOSE) << "Skip RecordDbLaunchHistory, no component name available."; - - return std::nullopt; - } - - android::ScopedTrace trace{ATRACE_TAG_ACTIVITY_MANAGER, - "IorapNativeService::RecordDbLaunchHistory"}; - db::DbHandle db{db::SchemaModel::GetSingleton()}; - - using namespace iorap::db; - - std::optional<int> version = - version_map_->GetOrQueryPackageVersion(component_name_->package); - if (!version) { - LOG(DEBUG) << "The version is NULL, maybe package manager is down."; - return std::nullopt; - } - std::optional<ActivityModel> activity = - ActivityModel::SelectOrInsert(db, - component_name_->package, - *version, - component_name_->activity_name); - - if (!activity) { - LOG(WARNING) << "Failed to query activity row for : " << *component_name_; - return std::nullopt; - } - - auto temp = static_cast<db::AppLaunchHistoryModel::Temperature>(*temperature_); - - std::optional<AppLaunchHistoryModel> alh = - AppLaunchHistoryModel::Insert(db, - activity->id, - temp, - IsTracing(), - IsReadAhead(), - intent_started_ns_, - total_time_ns_, - // ReportFullyDrawn event normally occurs after this. Need update later. - /* report_fully_drawn_ns= */ std::nullopt, - pid); - //Repo - if (!alh) { - LOG(WARNING) << "Failed to insert app_launch_histories row"; - return std::nullopt; - } - - LOG(VERBOSE) << "RecordDbLaunchHistory: " << *alh; - return alh; - } - - void UpdateReportFullyDrawn(int history_id, uint64_t timestamp_ns) { - LOG(DEBUG) << "Update kReportFullyDrawn for history_id:" - << history_id - << " timestamp_ns: " - << timestamp_ns; - - android::ScopedTrace trace{ATRACE_TAG_ACTIVITY_MANAGER, - "IorapNativeService::UpdateReportFullyDrawn"}; - db::DbHandle db{db::SchemaModel::GetSingleton()}; - - bool result = - db::AppLaunchHistoryModel::UpdateReportFullyDrawn(db, - history_id, - timestamp_ns); - - if (!result) { - LOG(WARNING) << "Failed to update app_launch_histories row"; - } - } -}; - -struct AppLaunchEventDefender { - binder::AppLaunchEvent::Type last_event_type_{binder::AppLaunchEvent::Type::kUninitialized}; - - enum class Result { - kAccept, // Pass-through the new event. - kOverwrite, // Overwrite the new event with a different event. - kReject // Completely reject the new event, it will not be delivered. - }; - - Result OnAppLaunchEvent(binder::RequestId request_id, - const binder::AppLaunchEvent& event, - binder::AppLaunchEvent* overwrite) { - using Type = binder::AppLaunchEvent::Type; - CHECK(overwrite != nullptr); - - // Ensure only legal transitions are allowed. - switch (last_event_type_) { - case Type::kUninitialized: - case Type::kIntentFailed: - case Type::kActivityLaunchCancelled: - case Type::kReportFullyDrawn: { // From a terminal state, only go to kIntentStarted - if (event.type != Type::kIntentStarted) { - LOG(DEBUG) << "Rejecting transition from " << last_event_type_ << " to " << event.type; - last_event_type_ = Type::kUninitialized; - return Result::kReject; - } else { - LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type; - last_event_type_ = event.type; - return Result::kAccept; - } - } - case Type::kIntentStarted: { - if (event.type == Type::kIntentFailed || - event.type == Type::kActivityLaunched) { - LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type; - last_event_type_ = event.type; - return Result::kAccept; - } else { - LOG(DEBUG) << "Overwriting transition from kIntentStarted to " - << event.type << " into kIntentFailed"; - last_event_type_ = Type::kIntentFailed; - - *overwrite = event; - overwrite->type = Type::kIntentFailed; - return Result::kOverwrite; - } - } - case Type::kActivityLaunched: { - if (event.type == Type::kActivityLaunchFinished || - event.type == Type::kActivityLaunchCancelled) { - LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type; - last_event_type_ = event.type; - return Result::kAccept; - } else { - LOG(DEBUG) << "Overwriting transition from kActivityLaunched to " - << event.type << " into kActivityLaunchCancelled"; - last_event_type_ = Type::kActivityLaunchCancelled; - - *overwrite = event; - overwrite->type = Type::kActivityLaunchCancelled; - return Result::kOverwrite; - } - } - case Type::kActivityLaunchFinished: { - if (event.type == Type::kIntentStarted || - event.type == Type::kReportFullyDrawn) { - LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type; - last_event_type_ = event.type; - return Result::kAccept; - } else { - LOG(DEBUG) << "Rejecting transition from " << last_event_type_ << " to " << event.type; - last_event_type_ = Type::kUninitialized; - return Result::kReject; - } - } - } - } -}; - -// Convert callback pattern into reactive pattern. -struct AppLaunchEventSubject { - using RefWrapper = - std::reference_wrapper<const AppLaunchEvent>; - - AppLaunchEventSubject() {} - - void Subscribe(rxcpp::subscriber<RefWrapper> subscriber) { - DCHECK(ready_ != true) << "Cannot Subscribe twice"; - - subscriber_ = std::move(subscriber); - - // Release edge of synchronizes-with AcquireIsReady. - ready_.store(true); - } - - void OnNext(const AppLaunchEvent& e) { - if (!AcquireIsReady()) { - return; - } - - if (!subscriber_->is_subscribed()) { - return; - } - - /* - * TODO: fix upstream. - * - * Rx workaround: this fails to compile when - * the observable is a reference type: - * - * external/Reactive-Extensions/RxCpp/Rx/v2/src/rxcpp/rx-observer.hpp:354:18: error: multiple overloads of 'on_next' instantiate to the same signature 'void (const iorap::binder::AppLaunchEvent &) const' - * virtual void on_next(T&&) const {}; - * - * external/Reactive-Extensions/RxCpp/Rx/v2/src/rxcpp/rx-observer.hpp:353:18: note: previous declaration is here - * virtual void on_next(T&) const {}; - * - * (The workaround is to use reference_wrapper instead - * of const AppLaunchEvent&) - */ - subscriber_->on_next(std::cref(e)); - - } - - void OnCompleted() { - if (!AcquireIsReady()) { - return; - } - - subscriber_->on_completed(); - } - - private: - bool AcquireIsReady() { - // Synchronizes-with the release-edge in Subscribe. - // This can happen much later, only once the subscription actually happens. - - // However, as far as I know, 'rxcpp::subscriber' is not thread safe, - // (but the observable chain itself can be made thread-safe via #observe_on, etc). - // so we must avoid reading it until it has been fully synchronized. - // - // TODO: investigate rxcpp subscribers and see if we can get rid of this atomics, - // to make it simpler. - return ready_.load(); - } - - // TODO: also track the RequestId ? - - std::atomic<bool> ready_{false}; - - - std::optional<rxcpp::subscriber<RefWrapper>> subscriber_; -}; - -// Convert callback pattern into reactive pattern. -struct JobScheduledEventSubject { - JobScheduledEventSubject() {} - - void Subscribe(rxcpp::subscriber<std::pair<RequestId, JobScheduledEvent>> subscriber) { - DCHECK(ready_ != true) << "Cannot Subscribe twice"; - - subscriber_ = std::move(subscriber); - - // Release edge of synchronizes-with AcquireIsReady. - ready_.store(true); - } - - void OnNext(RequestId request_id, JobScheduledEvent e) { - if (!AcquireIsReady()) { - return; - } - - if (!subscriber_->is_subscribed()) { - return; - } - - subscriber_->on_next(std::pair<RequestId, JobScheduledEvent>{std::move(request_id), std::move(e)}); - - } - - void OnCompleted() { - if (!AcquireIsReady()) { - return; - } - - subscriber_->on_completed(); - } - - private: - bool AcquireIsReady() { - // Synchronizes-with the release-edge in Subscribe. - // This can happen much later, only once the subscription actually happens. - - // However, as far as I know, 'rxcpp::subscriber' is not thread safe, - // (but the observable chain itself can be made thread-safe via #observe_on, etc). - // so we must avoid reading it until it has been fully synchronized. - // - // TODO: investigate rxcpp subscribers and see if we can get rid of this atomics, - // to make it simpler. - return ready_.load(); - } - - // TODO: also track the RequestId ? - - std::atomic<bool> ready_{false}; - - std::optional<rxcpp::subscriber<std::pair<RequestId, JobScheduledEvent>>> subscriber_; -}; - -std::ostream& operator<<(std::ostream& os, const android::content::pm::PackageChangeEvent& event) { - os << "PackageChangeEvent{"; - os << "packageName=" << event.packageName << ","; - os << "version=" << event.version << ","; - os << "lastUpdateTimeMillis=" << event.lastUpdateTimeMillis; - os << "}"; - return os; -} - -class EventManager::Impl { - public: - Impl(/*borrow*/perfetto::RxProducerFactory& perfetto_factory) - : perfetto_factory_(perfetto_factory), - worker_thread_(rxcpp::observe_on_new_thread()), - worker_thread2_(rxcpp::observe_on_new_thread()), - io_thread_(perfetto::ObserveOnNewIoThread()) { - // Try to create version map - RetryCreateVersionMap(); - - iorap::common::StderrLogPrinter printer{"iorapd"}; - RefreshSystemProperties(printer); - - rx_lifetime_ = InitializeRxGraph(); - rx_lifetime_jobs_ = InitializeRxGraphForJobScheduledEvents(); - - android::add_sysprop_change_callback(&Impl::OnSyspropChanged, /*priority*/-10000); - } - - void RetryCreateVersionMap() { - android::base::Timer timer{}; - version_map_ = binder::PackageVersionMap::Create(); - std::chrono::milliseconds duration_ms = timer.duration(); - LOG(DEBUG) << "Got versions for " - << version_map_->Size() - << " packages in " - << duration_ms.count() - << "ms"; - } - - void SetTaskResultCallbacks(std::shared_ptr<TaskResultCallbacks> callbacks) { - DCHECK(callbacks_.expired()); - callbacks_ = callbacks; - } - - void Join() { - async_pool_.Join(); - } - - bool OnAppLaunchEvent(RequestId request_id, - const AppLaunchEvent& event) { - LOG(VERBOSE) << "EventManager::OnAppLaunchEvent(" - << "request_id=" << request_id.request_id << "," - << event; - - // Filter any incoming events through a defender that enforces - // that all state transitions are as contractually documented in - // ActivityMetricsLaunchObserver's javadoc. - AppLaunchEvent overwrite_event{}; - AppLaunchEventDefender::Result result = - app_launch_event_defender_.OnAppLaunchEvent(request_id, event, /*out*/&overwrite_event); - - switch (result) { - case AppLaunchEventDefender::Result::kAccept: - app_launch_event_subject_.OnNext(event); - return true; - case AppLaunchEventDefender::Result::kOverwrite: - app_launch_event_subject_.OnNext(overwrite_event); - return false; - case AppLaunchEventDefender::Result::kReject: - // Intentionally left-empty: we drop the event completely. - return false; - } - - // In theory returns BAD_VALUE to the other side of this binder connection. - // In practice we use 'oneway' flags so this doesn't matter on a regular build. - return false; - } - - bool OnDexOptEvent(RequestId request_id, - const DexOptEvent& event) { - LOG(VERBOSE) << "EventManager::OnDexOptEvent(" - << "request_id=" << request_id.request_id << "," - << event.package_name - << ")"; - - if (common::ExcludeDexFiles(kExcludeDexFilesDefault)) { - LOG(VERBOSE) << "Dex files are excluded. Skip the purging."; - return true; - } - return PurgePackage(event.package_name); - } - - bool OnJobScheduledEvent(RequestId request_id, - const JobScheduledEvent& event) { - LOG(VERBOSE) << "EventManager::OnJobScheduledEvent(" - << "request_id=" << request_id.request_id << ",event=TODO)."; - - job_scheduled_event_subject_.OnNext(std::move(request_id), event); - - return true; // No errors. - } - - bool OnPackageChanged(const android::content::pm::PackageChangeEvent& event) { - LOG(DEBUG) << "Received " << event; - if (event.isDeleted) { - // Do nothing if the package is deleted rignt now. - // The package will be removed from db during maintenance. - return true; - } - // Update the version map. - if (version_map_->Update(event.packageName, event.version)) { - return true; - } - - // Sometimes a package is updated without any version change. - // Clean it up in this case. - db::DbHandle db{db::SchemaModel::GetSingleton()}; - db::CleanUpFilesForPackage(db, event.packageName, event.version); - return true; - } - - void Dump(/*borrow*/::android::Printer& printer) { - ::iorap::prefetcher::ReadAhead::Dump(printer); - ::iorap::perfetto::PerfettoConsumerImpl::Dump(/*borrow*/printer); - ::iorap::maintenance::Dump(db::SchemaModel::GetSingleton(), printer); - } - - rxcpp::composite_subscription InitializeRxGraph() { - LOG(VERBOSE) << "EventManager::InitializeRxGraph"; - - app_launch_events_ = rxcpp::observable<>::create<AppLaunchEventRefWrapper>( - [&](rxcpp::subscriber<AppLaunchEventRefWrapper> subscriber) { - app_launch_event_subject_.Subscribe(std::move(subscriber)); - }); - - rxcpp::composite_subscription lifetime; - - if (!tracing_allowed_) { - LOG(WARNING) << "Tracing disabled by system property"; - } - if (!readahead_allowed_) { - LOG(WARNING) << "Readahead disabled by system property"; - } - - AppLaunchEventState initial_state{&perfetto_factory_, - readahead_allowed_, - tracing_allowed_, - package_blacklister_, - &worker_thread2_, - &io_thread_, - &async_pool_, - version_map_}; - app_launch_events_ - .subscribe_on(worker_thread_) - .scan(std::move(initial_state), - [](AppLaunchEventState state, AppLaunchEventRefWrapper event) { - state.OnNewEvent(event.get()); - return state; - }) - .subscribe(/*out*/lifetime, [](const AppLaunchEventState& state) { - // Intentionally left blank. - (void)state; - }); - - return lifetime; - } - - // Runs the maintenance code to compile perfetto traces to compiled - // trace for a package. - void StartMaintenance(bool output_text, - std::optional<std::string> inode_textcache, - bool verbose, - bool recompile, - uint64_t min_traces, - std::string package_name, - bool should_update_versions) { - ScopedFormatTrace atrace_bg_scope(ATRACE_TAG_PACKAGE_MANAGER, - "Background Job Scope"); - - db::DbHandle db{db::SchemaModel::GetSingleton()}; - if (should_update_versions) { - { - ScopedFormatTrace atrace_update_versions(ATRACE_TAG_PACKAGE_MANAGER, - "Update package versions map cache"); - // Update the version map. - version_map_->UpdateAll(); - } - - { - ScopedFormatTrace atrace_cleanup_db(ATRACE_TAG_PACKAGE_MANAGER, - "Clean up obsolete data in database"); - // Cleanup the obsolete data in the database. - maintenance::CleanUpDatabase(db, version_map_); - } - } - - { - ScopedFormatTrace atrace_compile_apps(ATRACE_TAG_PACKAGE_MANAGER, - "Compile apps on device"); - // Compilation - maintenance::ControllerParameters params{ - output_text, - inode_textcache, - verbose, - recompile, - min_traces, - std::make_shared<maintenance::Exec>(), - common::ExcludeDexFiles(kExcludeDexFilesDefault)}; - - LOG(DEBUG) << "StartMaintenance: min_traces=" << min_traces; - maintenance::CompileSingleAppOnDevice(db, params, package_name); - } - } - - rxcpp::composite_subscription InitializeRxGraphForJobScheduledEvents() { - LOG(VERBOSE) << "EventManager::InitializeRxGraphForJobScheduledEvents"; - - using RequestAndJobEvent = std::pair<RequestId, JobScheduledEvent>; - - job_scheduled_events_ = rxcpp::observable<>::create<RequestAndJobEvent>( - [&](rxcpp::subscriber<RequestAndJobEvent> subscriber) { - job_scheduled_event_subject_.Subscribe(std::move(subscriber)); - }); - - rxcpp::composite_subscription lifetime; - - job_scheduled_events_ - .observe_on(worker_thread_) // async handling. - .tap([this](const RequestAndJobEvent& e) { - LOG(VERBOSE) << "EventManager#JobScheduledEvent#tap(1) - job begins"; - this->NotifyProgress(e.first, TaskResult{TaskResult::State::kBegan}); - - LOG(VERBOSE) << "Compile " << std::get<1>(e).package_name; - StartMaintenance(/*output_text=*/false, - /*inode_textcache=*/std::nullopt, - /*verbose=*/false, - /*recompile=*/false, - s_min_traces, - std::get<1>(e).package_name, - std::get<1>(e).should_update_versions); - - // TODO: probably this shouldn't be emitted until most of the usual DCHECKs - // (for example, validate a job isn't already started, the request is not reused, etc). - // In this way we could block from the client until it sees 'kBegan' and Log.wtf otherwise. - }) - .tap([](const RequestAndJobEvent& e) { - // TODO. Actual work. - LOG(VERBOSE) << "EventManager#JobScheduledEvent#tap(2) - job is being processed"; - - // TODO: abort functionality for in-flight jobs. - // - // maybe something like scan that returns an observable<Job> + flat map to that job. - // then we could unsubscribe from the scan to do a partial abort? need to try it and see if it works. - // - // other option is to create a new outer subscription for each job id which seems less ideal. - }) - .subscribe(/*out*/lifetime, - /*on_next*/ - [this](const RequestAndJobEvent& e) { - LOG(VERBOSE) << "EventManager#JobScheduledEvent#subscribe - job completed"; - this->NotifyComplete(e.first, TaskResult{TaskResult::State::kCompleted}); - } -#if 0 - , - /*on_error*/ - [](rxcpp::util::error_ptr err) { - LOG(ERROR) << "Scheduled job event failed: " << rxcpp::util::what(err); - - //std::shared_ptr<TaskResultCallbacks> callbacks = callbacks_.lock(); - //if (callbacks != nullptr) { - // FIXME: How do we get the request ID back out of the error? Seems like a problem. - // callbacks->OnComplete(, TaskResult{TaskResult::kError}); - // We may have to wrap with an iorap::expected instead of using on_error. - //} - - // FIXME: need to add a 'OnErrorResumeNext' operator? - DCHECK(false) << "forgot to implement OnErrorResumeNext"; - } -#endif - ); - - // TODO: error output should happen via an observable. - - return lifetime; - } - - void NotifyComplete(RequestId request_id, TaskResult result) { - std::shared_ptr<TaskResultCallbacks> callbacks = callbacks_.lock(); - if (callbacks != nullptr) { - callbacks->OnComplete(std::move(request_id), std::move(result)); - } else { - LOG(WARNING) << "EventManager: TaskResultCallbacks may have been released early"; - } - } - - void NotifyProgress(RequestId request_id, TaskResult result) { - std::shared_ptr<TaskResultCallbacks> callbacks = callbacks_.lock(); - if (callbacks != nullptr) { - callbacks->OnProgress(std::move(request_id), std::move(result)); - } else { - LOG(WARNING) << "EventManager: TaskResultCallbacks may have been released early"; - } - } - - static void OnSyspropChanged() { - LOG(DEBUG) << "OnSyspropChanged"; - } - - void RefreshSystemProperties(::android::Printer& printer) { - // TODO: read all properties from one config class. - // PH properties do not work if they contain ".". "_" was instead used here. - tracing_allowed_ = common::IsTracingEnabled(/*default_value=*/"false"); - s_tracing_allowed = tracing_allowed_; - printer.printFormatLine("iorapd.perfetto.enable = %s", tracing_allowed_ ? "true" : "false"); - - readahead_allowed_ = common::IsReadAheadEnabled(/*default_value=*/"false"); - s_readahead_allowed = readahead_allowed_; - printer.printFormatLine("iorapd.readahead.enable = %s", s_readahead_allowed ? "true" : "false"); - - s_min_traces = - ::android::base::GetUintProperty<uint64_t>("iorapd.maintenance.min_traces", /*default*/1); - uint64_t min_traces = s_min_traces; - printer.printFormatLine("iorapd.maintenance.min_traces = %" PRIu64, min_traces); - - printer.printFormatLine("iorapd.exclude_dex_files = %s", - common::ExcludeDexFiles(kExcludeDexFilesDefault) ? "true" : "false"); - - package_blacklister_ = PackageBlacklister{ - /* Colon-separated string list of blacklisted packages, e.g. - * "foo.bar.baz;com.fake.name" would blacklist {"foo.bar.baz", "com.fake.name"} packages. - * - * Blacklisted packages are ignored by iorapd. - */ - server_configurable_flags::GetServerConfigurableFlag( - common::ph_namespace, - "iorap_blacklisted_packages", - ::android::base::GetProperty("iorapd.blacklist_packages", - /*default*/"")) - }; - - LOG(DEBUG) << "RefreshSystemProperties"; - } - - bool PurgePackage(::android::Printer& printer, const std::string& package_name) { - (void)printer; - return PurgePackage(package_name); - } - - bool PurgePackage(const std::string& package_name) { - db::DbHandle db{db::SchemaModel::GetSingleton()}; - db::CleanUpFilesForPackage(db, package_name); - LOG(DEBUG) << "PurgePackage: " << package_name; - return true; - } - - bool CompilePackage(::android::Printer& printer, const std::string& package_name) { - (void)printer; - - ScopedFormatTrace atrace_compile_app(ATRACE_TAG_PACKAGE_MANAGER, - "Compile one app on device"); - - maintenance::ControllerParameters params{ - /*output_text*/false, - /*inode_textcache*/std::nullopt, - WOULD_LOG(VERBOSE), - /*recompile*/false, - s_min_traces, - std::make_shared<maintenance::Exec>(), - common::ExcludeDexFiles(kExcludeDexFilesDefault)}; - - db::DbHandle db{db::SchemaModel::GetSingleton()}; - bool res = maintenance::CompileSingleAppOnDevice(db, std::move(params), package_name); - LOG(DEBUG) << "CompilePackage: " << package_name; - - return res; - } - - bool readahead_allowed_{true}; - - perfetto::RxProducerFactory& perfetto_factory_; - bool tracing_allowed_{true}; - - PackageBlacklister package_blacklister_{}; - - std::weak_ptr<TaskResultCallbacks> callbacks_; // avoid cycles with weakptr. - - using AppLaunchEventRefWrapper = AppLaunchEventSubject::RefWrapper; - rxcpp::observable<AppLaunchEventRefWrapper> app_launch_events_; - AppLaunchEventSubject app_launch_event_subject_; - AppLaunchEventDefender app_launch_event_defender_; - - rxcpp::observable<std::pair<RequestId, JobScheduledEvent>> job_scheduled_events_; - JobScheduledEventSubject job_scheduled_event_subject_; - - rxcpp::observable<RequestId> completed_requests_; - - // regular-priority thread to handle binder callbacks. - observe_on_one_worker worker_thread_; - observe_on_one_worker worker_thread2_; - // low priority idle-class thread for IO operations. - observe_on_one_worker io_thread_; - // async futures pool for async rx operations. - AsyncPool async_pool_; - - rxcpp::composite_subscription rx_lifetime_; // app launch events - rxcpp::composite_subscription rx_lifetime_jobs_; // job scheduled events - - // package version map - std::shared_ptr<binder::PackageVersionMap> version_map_; - -//INTENTIONAL_COMPILER_ERROR_HERE: - // FIXME: - // ok so we want to expose a 'BlockingSubscribe' or a 'Subscribe' or some kind of function - // that the main thread can call. This would subscribe on all the observables we internally - // have here (probably on an event-manager-dedicated thread for simplicity). - // - // ideally we'd just reuse the binder thread to handle the events but I'm not super sure, - // maybe this already works with the identity_current_thread coordination? -}; -using Impl = EventManager::Impl; - -EventManager::EventManager(perfetto::RxProducerFactory& perfetto_factory) - : impl_(new Impl(perfetto_factory)) {} - -std::shared_ptr<EventManager> EventManager::Create() { - static perfetto::PerfettoDependencies::Injector injector{ - perfetto::PerfettoDependencies::CreateComponent - }; - static perfetto::RxProducerFactory producer_factory{ - /*borrow*/injector - }; - return EventManager::Create(/*borrow*/producer_factory); -} - -std::shared_ptr<EventManager> EventManager::Create(perfetto::RxProducerFactory& perfetto_factory) { - std::shared_ptr<EventManager> p{new EventManager{/*borrow*/perfetto_factory}}; - return p; -} - -void EventManager::SetTaskResultCallbacks(std::shared_ptr<TaskResultCallbacks> callbacks) { - return impl_->SetTaskResultCallbacks(std::move(callbacks)); -} - -void EventManager::Join() { - return impl_->Join(); -} - -bool EventManager::OnAppLaunchEvent(RequestId request_id, - const AppLaunchEvent& event) { - return impl_->OnAppLaunchEvent(request_id, event); -} - -bool EventManager::OnDexOptEvent(RequestId request_id, - const DexOptEvent& event) { - return impl_->OnDexOptEvent(request_id, event); -} - -bool EventManager::OnJobScheduledEvent(RequestId request_id, - const JobScheduledEvent& event) { - return impl_->OnJobScheduledEvent(request_id, event); -} - -bool EventManager::OnPackageChanged(const android::content::pm::PackageChangeEvent& event) { - return impl_->OnPackageChanged(event); -} - -void EventManager::Dump(/*borrow*/::android::Printer& printer) { - return impl_->Dump(printer); -} - -void EventManager::RefreshSystemProperties(::android::Printer& printer) { - return impl_->RefreshSystemProperties(printer); -} - -bool EventManager::PurgePackage(::android::Printer& printer, const std::string& package_name) { - return impl_->PurgePackage(printer, package_name); -} - -bool EventManager::CompilePackage(::android::Printer& printer, const std::string& package_name) { - return impl_->CompilePackage(printer, package_name); -} - -} // namespace iorap::manager diff --git a/src/manager/event_manager.h b/src/manager/event_manager.h deleted file mode 100644 index fa09eec..0000000 --- a/src/manager/event_manager.h +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef IORAP_MANAGER_EVENT_MANAGER_H_ -#define IORAP_MANAGER_EVENT_MANAGER_H_ - -#include "binder/app_launch_event.h" -#include "binder/dexopt_event.h" -#include "binder/job_scheduled_event.h" -#include "binder/request_id.h" -#include "binder/task_result.h" - -#include <android/content/pm/PackageChangeEvent.h> - -#include <memory> - -namespace android { -class Printer; -} // namespace android - -namespace iorap::perfetto { -struct RxProducerFactory; -} // namespace iorap::perfetto - -namespace iorap::manager { - -// These callbacks are invoked by the EventManager to provide asynchronous notification for the status -// of an event handler. -// -// Calling 'On_Event' in EventManager should be considered merely to start the task. -// Calling 'OnComplete' here is considered to terminate the request (either with a success or error). -// OnProgress is optional, but if it is called it must be called prior to 'OnComplete'. -// -// All callbacks for the same request-id are sequentially consistent. -class TaskResultCallbacks { - public: - virtual void OnProgress(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) {} - virtual void OnComplete(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) {} - - virtual ~TaskResultCallbacks() {} -}; - -class EventManager { - public: - static std::shared_ptr<EventManager> Create(); - static std::shared_ptr<EventManager> Create( - /*borrow*/perfetto::RxProducerFactory& perfetto_factory); - void SetTaskResultCallbacks(std::shared_ptr<TaskResultCallbacks> callbacks); - - // Joins any background threads created by EventManager. - void Join(); - - // Handles an AppLaunchEvent: - // - // * Intent starts and app launch starts are treated critically - // and will be handled immediately. This means the caller - // (e.g. the binder pool thread) could be starved in the name - // of low latency. - // - // * Other types are handled in a separate thread. - bool OnAppLaunchEvent(binder::RequestId request_id, - const binder::AppLaunchEvent& event); - - // Handles a DexOptEvent: - // - // Clean up the invalidate traces after package is updated by dexopt. - bool OnDexOptEvent(binder::RequestId request_id, - const binder::DexOptEvent& event); - - - // Handles a JobScheduledEvent: - // - // * Start/stop background jobs (typically for idle maintenance). - // * For example, this could kick off a background compiler. - bool OnJobScheduledEvent(binder::RequestId request_id, - const binder::JobScheduledEvent& event); - - // Handles a PackageChangeEvent: - // - // * The package manager service send this event for package install - // update or delete. - bool OnPackageChanged(const android::content::pm::PackageChangeEvent& event); - - // Print to adb shell dumpsys (for bugreport info). - void Dump(/*borrow*/::android::Printer& printer); - - // A dumpsys --refresh-properties command signaling that we should - // refresh our system properties. - void RefreshSystemProperties(::android::Printer& printer); - - // A dumpsys --purge-package <name> command signaling - // that all db rows and files associated with a package should be deleted. - bool PurgePackage(::android::Printer& printer, const std::string& package_name); - - // A dumpsys --compile-package <name> command signaling - // that a package should be recompiled. - bool CompilePackage(::android::Printer& printer, const std::string& package_name); - - class Impl; - private: - std::unique_ptr<Impl> impl_; - - EventManager(perfetto::RxProducerFactory& perfetto_factory); -}; - -} // namespace iorap::manager - -#endif // IORAP_MANAGER_EVENT_MANAGER_H_ diff --git a/src/perfetto/main.cc b/src/perfetto/main.cc deleted file mode 100644 index 0b1056e..0000000 --- a/src/perfetto/main.cc +++ /dev/null @@ -1,244 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//#undef NDEBUG // get DCHECK etc. - - -#include "common/debug.h" -#include "common/expected.h" -#include "perfetto/rx_producer.h" - -#include <android-base/unique_fd.h> -#include <android-base/parseint.h> -#include <android-base/file.h> - -#include "rxcpp/rx.hpp" -#include <iostream> -#include <optional> - -#include <sched.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <syscall.h> -#include <fcntl.h> -#include <unistd.h> - -using namespace iorap::perfetto; // NOLINT - -#if defined(IORAP_PERFETTO_MAIN) - -void Usage(char** argv) { - std::cerr << "Usage: " << argv[0] << " [--config-proto=config.pb] [--duration-ms=5000] [--output-proto=output.pb]" << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Request a perfetto trace, blocking until it's complete. The resulting trace proto" << std::endl; - std::cerr << " is output to stdout as text, or to --output-proto as a binary." << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Optional flags:" << std::endl; - std::cerr << " --help,-h Print this Usage." << std::endl; - std::cerr << " --output-proto $,-op $ Perfetto tracebuffer output file (default stdout)." << std::endl; - std::cerr << " --config-proto $,-cp $ Path to binary protobuf config." << std::endl; - std::cerr << " --duration-ms $,-dm $ How long to run trace for in milliseconds." << std::endl; - std::cerr << " --simple Simplest possible perfetto state transitions (default off)." << std::endl; - std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; - std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; - exit(1); -} - -PerfettoDependencies::Component CreateCommandLinePerfettoDependenciesComponent( - uint32_t duration_ms) { - // TODO: read from command line. - static const uint32_t kBufferSize = 4096; - - // TODO: remove this hack. - static const uint32_t kTraceDurationMs = duration_ms; - - // fruit: using 'bindInstance' causes a segfault every time. -#if 0 - - // fruit: Can't use a stateful lambda, so use bindInstance instead of registerProvider. - auto config = PerfettoDependencies::CreateConfig(duration_ms, - /*deferred_start*/true, - kBufferSize); - - .... bindInstance(config); -#endif - - return fruit::createComponent() - .bind<PerfettoConsumer, PerfettoConsumerImpl>() - .registerProvider([]() /* -> TraceConfig */ { - return PerfettoDependencies::CreateConfig(kTraceDurationMs, - /*deferred_start*/true, - kBufferSize); - }); -} - -static void CollectPerfettoTraceBufferViaAbstractions( - RxProducerFactory& producer_factory, - const std::string& arg_output_proto, - const int arg_duration_ms) { - LOG(VERBOSE) << "CollectPerfettoTraceBufferViaAbstractions"; - - // Don't create a subscriber to emit the PerfettoStreamCommand. - // RxCpp is "greedy" and consumes every possible item emitted (it doesn't support 'pull'). We want - // to operate on a (command,state) iteration every time, just like in a real scenario. - // Adding the 'interval' turns into a non-greedy version (i.e. push). - - // Immediately emit 'kStartTracing', wait and emit kStopTracing, wait and emit kShutdown. - // In reality, there would be a delay between all these events. - auto /*observable<PerfettoStreamCommand>*/ commands = - rxcpp::observable<>::just(PerfettoStreamCommand::kStartTracing) - // wait 1x - .concat( - // Pick a value longer than the perfetto config delay_ms, so that we send - // 'kShutdown' after tracing has already finished. - rxcpp::observable<>::interval(std::chrono::milliseconds(arg_duration_ms * 2)) - .take(2) // kStopTracing, kShutdown. - .map([](int value) { - // value is 1,2,3,... - return static_cast<PerfettoStreamCommand>(value); // 1,2, ... - }) - ); - - auto /*observable<PerfettoTraceProto>*/ trace_proto_stream = - producer_factory.CreateTraceStream(commands); - - trace_proto_stream - .observe_on(ObserveOnNewIoThread()) // Write data on an idle-class-priority thread. - .as_blocking() // Wait for observable to terminate with on_completed or on_error. - .subscribe(/*on_next*/[arg_output_proto] - (PerfettoTraceProto trace_proto) { - if (!trace_proto.WriteFullyToFile(arg_output_proto)) { - LOG(ERROR) << "Failed to save TraceBuffer to " << arg_output_proto; - } else { - LOG(INFO) << "TraceBuffer saved to file: " << arg_output_proto; - LOG(INFO); - LOG(INFO) << "To print this in a human readable form, execute these commands:"; - LOG(INFO) << "$> adb pull '" << arg_output_proto << "'"; - LOG(INFO) << "$> trace_to_text systrace <filename.pb>"; - } - }, - /*on_error*/[](rxcpp::util::error_ptr err) { - LOG(ERROR) << "Perfetto trace proto collection error: " << rxcpp::util::what(err); - }); -} - -namespace iorap::perfetto { -// Reach inside rx_producer.cc -// Not part of any headers because it's internal. -void CollectPerfettoTraceBufferImmediately( - RxProducerFactory& producer_factory, - const std::string& arg_output_proto); -} - -int main(int argc, char** argv) { - android::base::InitLogging(argv); - android::base::SetLogger(android::base::StderrLogger); - - bool wait_for_keystroke = false; - bool enable_verbose = false; - - std::string arg_output_proto; - std::string arg_config_proto; - uint32_t arg_duration_ms = 1000; - bool arg_simple = false; - - if (argc == 1) { - Usage(argv); - } - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - bool has_arg_next = (arg+1)<argc; - std::string arg_next = has_arg_next ? argv[arg+1] : ""; - - if (argstr == "--help" || argstr == "-h") { - Usage(argv); - } else if (argstr == "--output-proto" || argstr == "-op") { - if (!has_arg_next) { - std::cerr << "Missing --output-proto <value>" << std::endl; - return 1; - } - arg_output_proto = arg_next; - ++arg; - } else if (argstr == "--config-proto" || argstr == "-cp") { - if (!has_arg_next) { - std::cerr << "Missing --config-proto <value>" << std::endl; - return 1; - } - arg_config_proto = arg_next; - LOG(WARNING) << "TODO: parse configs from a file, not implemented yet."; - ++arg; - } else if (argstr == "--duration-ms" || argstr == "-dm") { - if (!has_arg_next) { - std::cerr << "Missing --duration-ms <value>" << std::endl; - return 1; - } - if (!android::base::ParseUint(arg_next.c_str(), /*out*/&arg_duration_ms)) { - std::cerr << "Invalid --duration-ms " << arg_next << ", reason: " << strerror(errno); - return 1; - } - ++arg; - } else if (argstr == "--simple") { - arg_simple = true; - } else if (argstr == "--verbose" || argstr == "-v") { - enable_verbose = true; - } else if (argstr == "--wait" || argstr == "-w") { - wait_for_keystroke = true; - } - } - - if (enable_verbose) { - android::base::SetMinimumLogSeverity(android::base::VERBOSE); - - LOG(VERBOSE) << "Verbose check"; - LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; - } - - // Useful to attach a debugger... - // 1) $> iorap-cmd-perfetto -w <args> - // 2) $> gdbclient <pid> - if (wait_for_keystroke) { - LOG(INFO) << "Self pid: " << getpid(); - LOG(INFO) << "Press any key to continue..."; - std::cin >> wait_for_keystroke; - } - - int return_code = 0; - // TODO: convert #on-error into a non-0 return code. - - PerfettoDependencies::Injector injector{ - CreateCommandLinePerfettoDependenciesComponent, - arg_duration_ms - }; - RxProducerFactory rx_producer_factory{/*borrow*/injector}; - - if (arg_simple) { - // To debug any kind of low-level perfetto issues. - CollectPerfettoTraceBufferImmediately(/*inout*/rx_producer_factory, arg_output_proto); - } else { - // To debug our own iorap internal abstractions. - CollectPerfettoTraceBufferViaAbstractions(/*inout*/rx_producer_factory, - arg_output_proto, - arg_duration_ms); - } - - // Uncomment this if we want to leave the process around to inspect it from adb shell. - // sleep(100000); - - // 0 -> successfully wrote the TraceProto out to file. - // 1 -> failed along the way (#on_error and also see the error logs). - return return_code; -} - -#endif diff --git a/src/perfetto/perfetto_consumer.cc b/src/perfetto/perfetto_consumer.cc deleted file mode 100644 index 4fe677b..0000000 --- a/src/perfetto/perfetto_consumer.cc +++ /dev/null @@ -1,608 +0,0 @@ -// Copyright (C) 2020 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "perfetto/perfetto_consumer.h" - -#include "common/trace.h" - -#include <android-base/logging.h> -#include <android-base/properties.h> -#include <utils/Looper.h> -#include <utils/Printer.h> - -#include <limits> -#include <map> -#include <memory> -#include <mutex> -#include <sstream> -#include <vector> - -#include <inttypes.h> -#include <time.h> - -namespace iorap::perfetto { - -using State = PerfettoConsumer::State; -using Handle = PerfettoConsumer::Handle; -static constexpr Handle kInvalidHandle = PerfettoConsumer::kInvalidHandle; -using OnStateChangedCb = PerfettoConsumer::OnStateChangedCb; -using TraceBuffer = PerfettoConsumer::TraceBuffer; - -enum class StateKind { - kUncreated, - kCreated, - kStartedTracing, - kReadTracing, - kTimedOutDestroyed, // same as kDestroyed but timed out. - kDestroyed, // calling kDestroyed before timing out. -}; - -std::ostream& operator<<(std::ostream& os, StateKind kind) { - switch (kind) { - case StateKind::kUncreated: - os << "kUncreated"; - break; - case StateKind::kCreated: - os << "kCreated"; - break; - case StateKind::kStartedTracing: - os << "kStartedTracing"; - break; - case StateKind::kReadTracing: - os << "kReadTracing"; - break; - case StateKind::kTimedOutDestroyed: - os << "kTimedOutDestroyed"; - break; - case StateKind::kDestroyed: - os << "kDestroyed"; - break; - default: - os << "(invalid)"; - break; - } - return os; -} - -std::string ToString(StateKind kind) { - std::stringstream ss; - ss << kind; - return ss.str(); -} - -static constexpr uint64_t kSecToNano = 1000000000LL; - -static uint64_t GetTimeNanoseconds() { - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - - uint64_t now_ns = (now.tv_sec * kSecToNano + now.tv_nsec); - return now_ns; -} - -// Describe the state of our handle in detail for debugging/logging. -struct HandleDescription { - Handle handle_; - StateKind kind_{StateKind::kUncreated}; // Our state. required for correctness. - OnStateChangedCb callback_{nullptr}; // Required for Destroy callbacks. - void* callback_arg_{nullptr}; - - // For dumping to logs: - State state_{State::kSessionNotFound}; // perfetto state - std::optional<uint64_t> started_tracing_ns_; // when StartedTracing last called. - std::optional<uint64_t> read_trace_ns_; // When ReadTrace last called. - std::uint64_t last_transition_ns_{0}; - std::optional<uint64_t> trace_cookie_; // atrace beginning at StartTracing. - bool trace_ended_{false}; // atrace ending at ReadTrace or Destroy. - - HandleDescription(Handle handle): handle_(handle) {} -}; - -// pimpl idiom to hide the implementation details from header -// -// Track and verify that our perfetto usage is sane. -struct PerfettoConsumerImpl::Impl { - Impl() : raw_{new PerfettoConsumerRawImpl{}}, - message_handler_{new TraceMessageHandler{this}} { - std::thread watchdog_thread{ [this]() { - ::android::sp<::android::Looper> looper; - { - std::lock_guard<std::mutex> guard{looper_mutex_}; - looper = ::android::Looper::prepare(/*opts*/0); - looper_ = looper; - } - - static constexpr int kTimeoutMillis = std::numeric_limits<int>::max(); - - while (true) { - // Execute any pending callbacks, otherwise just block forever. - int result = looper->pollAll(kTimeoutMillis); - - if (result == ::android::Looper::POLL_ERROR) { - LOG(ERROR) << "PerfettoConsumerImpl::Looper got a POLL_ERROR"; - } else { - LOG(DEBUG) << "PerfettoConsumerImpl::Looper result was " << result; - } - } - }}; - - // Let thread run freely on its own. - watchdog_thread.detach(); - - // Block until looper_ is prepared. - while (true) { - std::lock_guard<std::mutex> guard{looper_mutex_}; - if (looper_ != nullptr) { - break; - } - } - } - - private: - std::unique_ptr<PerfettoConsumerRawImpl> raw_; - std::map<Handle, HandleDescription> states_; - - // We need this to be a counter to avoid memory leaks. - Handle last_created_{0}; - Handle last_destroyed_{0}; - uint64_t trace_cookie_{0}; - - std::mutex mutex_; // Guard above values. - - ::android::sp<::android::Looper> looper_; - std::mutex looper_mutex_; // Guard looper_. - - struct TraceMessageHandler : public ::android::MessageHandler { - TraceMessageHandler(Impl* impl) : impl_{impl} { - CHECK(impl != nullptr); - } - - Impl* impl_; - - virtual void handleMessage(const ::android::Message& message) override { - impl_->OnTraceMessage(static_cast<Handle>(message.what)); - } - }; - - ::android::sp<TraceMessageHandler> message_handler_; - - public: - Handle Create(const void* config_proto, - size_t config_len, - OnStateChangedCb callback, - void* callback_arg) { - LOG(VERBOSE) << "PerfettoConsumer::Create(" - << "config_len=" << config_len << ")"; - Handle handle = raw_->Create(config_proto, config_len, callback, callback_arg); - - std::lock_guard<std::mutex> guard{mutex_}; - - // Assume every Handle starts at 0 and then increments by 1 every Create. - ++last_created_; - CHECK_EQ(last_created_, handle) << "perfetto handle had unexpected behavior."; - // Without this^ increment-by-1 behavior our detection of untracked state values is broken. - // If we have to, we can go with Untracked=Uncreated|Destroyed but it's better to distinguish - // the two if possible. - - HandleDescription handle_desc{handle}; - handle_desc.handle_ = handle; - handle_desc.callback_ = callback; - handle_desc.callback_arg_ = callback_arg; - UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kCreated); - - // assume we never wrap around due to using int64 - bool inserted = states_.insert({handle, handle_desc}).second; - CHECK(inserted) << "perfetto handle was re-used: " << handle; - - return handle; - } - - void StartTracing(Handle handle) { - LOG(DEBUG) << "PerfettoConsumer::StartTracing(handle=" << handle << ")"; - - uint64_t trace_cookie; - { - std::lock_guard<std::mutex> guard{mutex_}; - - auto it = states_.find(handle); - if (it == states_.end()) { - LOG(ERROR) << "Cannot StartTracing(" << handle << "), untracked handle"; - return; - } - HandleDescription& handle_desc = it->second; - - raw_->StartTracing(handle); - UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kStartedTracing); - } - - // Use a looper here to add a timeout and immediately destroy the trace buffer. - CHECK_LE(static_cast<int64_t>(handle), static_cast<int64_t>(std::numeric_limits<int>::max())); - int message_code = static_cast<int>(handle); - ::android::Message message{message_code}; - - std::lock_guard<std::mutex> looper_guard{looper_mutex_}; - looper_->sendMessageDelayed(static_cast<nsecs_t>(GetPropertyTraceTimeoutNs()), - message_handler_, - message); - } - - TraceBuffer ReadTrace(Handle handle) { - LOG(DEBUG) << "PerfettoConsumer::ReadTrace(handle=" << handle << ")"; - - std::lock_guard<std::mutex> guard{mutex_}; - - auto it = states_.find(handle); - if (it == states_.end()) { - LOG(ERROR) << "Cannot ReadTrace(" << handle << "), untracked handle"; - return TraceBuffer{}; - } - - HandleDescription& handle_desc = it->second; - - TraceBuffer trace_buffer = raw_->ReadTrace(handle); - UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kReadTracing); - - return trace_buffer; - } - - void Destroy(Handle handle) { - HandleDescription handle_desc{handle}; - TryDestroy(handle, /*do_destroy*/true, /*out*/&handle_desc);; - } - - bool TryDestroy(Handle handle, bool do_destroy, /*out*/HandleDescription* handle_desc_out) { - CHECK(handle_desc_out != nullptr); - - LOG(VERBOSE) << "PerfettoConsumer::Destroy(handle=" << handle << ")"; - - std::lock_guard<std::mutex> guard{mutex_}; - - auto it = states_.find(handle); - if (it == states_.end()) { - // Leniency for calling Destroy multiple times. It's not a mistake. - LOG(ERROR) << "Cannot Destroy(" << handle << "), untracked handle"; - return false; - } - - HandleDescription& handle_desc = it->second; - - if (do_destroy) { - raw_->Destroy(handle); - } - UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kDestroyed); - - *handle_desc_out = handle_desc; - - // No longer track this handle to avoid memory leaks. - last_destroyed_ = handle; - states_.erase(it); - - return true; - } - - State PollState(Handle handle) { - // Just pass-through the call, we never use it directly anyway. - return raw_->PollState(handle); - } - - // Either fetch or infer the current handle state from a handle. - // Meant for debugging/logging only. - HandleDescription GetOrInferHandleDescription(Handle handle) { - std::lock_guard<std::mutex> guard{mutex_}; - - auto it = states_.find(handle); - if (it == states_.end()) { - HandleDescription state{handle}; - // If it's untracked it hasn't been created yet, or it was already destroyed. - if (IsDestroyed(handle)) { - UpdateHandleDescription(/*inout*/&state, StateKind::kDestroyed); - } else { - if (!IsUncreated(handle)) { - LOG(WARNING) << "bad state detection"; - } - UpdateHandleDescription(/*inout*/&state, StateKind::kUncreated); - } - return state; - } - return it->second; - } - - void OnTraceMessage(Handle handle) { - LOG(VERBOSE) << "OnTraceMessage(" << static_cast<int64_t>(handle) << ")"; - HandleDescription handle_desc{handle}; - { - std::lock_guard<std::mutex> guard{mutex_}; - - auto it = states_.find(handle); - if (it == states_.end()) { - // Handle values are never re-used, so we can simply ignore the message here - // instead of having to remove it from the message queue. - LOG(VERBOSE) << "OnTraceMessage(" << static_cast<int64_t>(handle) - << ") no longer tracked handle"; - return; - } - handle_desc = it->second; - } - - // First check. Has this trace been active for too long? - uint64_t now_ns = GetTimeNanoseconds(); - if (handle_desc.kind_ == StateKind::kStartedTracing) { - // Ignore other kinds of traces because they don't exhaust perfetto resources. - CHECK(handle_desc.started_tracing_ns_.has_value()) << static_cast<int64_t>(handle); - - uint64_t started_tracing_ns = *handle_desc.started_tracing_ns_; - - if ((now_ns - started_tracing_ns) > GetPropertyTraceTimeoutNs()) { - LOG(WARNING) << "Perfetto Handle timed out after " << (now_ns - started_tracing_ns) << "ns" - << ", forcibly destroying"; - - // Let the callback handler call Destroy. - handle_desc.callback_(handle, State::kTraceFailed, handle_desc.callback_arg_); - } - } - - // Second check. Are there too many traces now? Cull the old traces. - std::vector<HandleDescription> handle_list; - do { - std::lock_guard<std::mutex> guard{mutex_}; - - size_t max_trace_count = GetPropertyMaxTraceCount(); - if (states_.size() > max_trace_count) { - size_t overflow_count = states_.size() - max_trace_count; - LOG(WARNING) << "Too many perfetto handles, overflowed by " << overflow_count - << ", pruning down to " << max_trace_count; - } else { - break; - } - - size_t prune_count = states_.size() - max_trace_count; - auto it = states_.begin(); - for (size_t i = 0; i < prune_count; ++i) { - // Simply prune by handle 1,2,3,4... - // We could do better with a timestamp if we wanted to. - ++it; - handle_list.push_back(it->second); - } - } while (false); - - for (HandleDescription& handle_desc : handle_list) { - LOG(DEBUG) << "Perfetto handle pruned: " << static_cast<int64_t>(handle); - - // Let the callback handler call Destroy. - handle_desc.callback_(handle, State::kTraceFailed, handle_desc.callback_arg_); - } - } - - private: - static uint64_t GetPropertyTraceTimeoutNs() { - static uint64_t value = // property is timeout in seconds - ::android::base::GetUintProperty<uint64_t>("iorapd.perfetto.timeout", /*default*/10); - return value * kSecToNano; - } - - static size_t GetPropertyMaxTraceCount() { - static size_t value = - ::android::base::GetUintProperty<size_t>("iorapd.perfetto.max_traces", /*default*/5); - return value; - } - - void UpdateHandleDescription(/*inout*/HandleDescription* handle_desc, StateKind kind) { - CHECK(handle_desc != nullptr); - handle_desc->kind_ = kind; - handle_desc->state_ = raw_->PollState(handle_desc->handle_); - - handle_desc->last_transition_ns_ = GetTimeNanoseconds(); - if (kind == StateKind::kStartedTracing) { - if (!handle_desc->started_tracing_ns_) { - handle_desc->started_tracing_ns_ = handle_desc->last_transition_ns_; - - handle_desc->trace_cookie_ = ++trace_cookie_; - - atrace_async_begin(ATRACE_TAG_ACTIVITY_MANAGER, - "Perfetto Scoped Trace", - *handle_desc->trace_cookie_); - atrace_int(ATRACE_TAG_ACTIVITY_MANAGER, - "Perfetto::Trace Handle", - static_cast<int32_t>(handle_desc->handle_)); - } - } - - if (kind == StateKind::kReadTracing) { - if (!handle_desc->read_trace_ns_) { - handle_desc->read_trace_ns_ = handle_desc->last_transition_ns_; - - if (handle_desc->trace_cookie_.has_value() && !handle_desc->trace_ended_) { - atrace_async_end(ATRACE_TAG_ACTIVITY_MANAGER, - "Perfetto Scoped Trace", - handle_desc->trace_cookie_.value()); - - handle_desc->trace_ended_ = true; - } - } - } - - // If Destroy is called prior to ReadTrace, mark the atrace as finished. - if (kind == StateKind::kDestroyed && handle_desc->trace_cookie_ && !handle_desc->trace_ended_) { - atrace_async_end(ATRACE_TAG_ACTIVITY_MANAGER, - "Perfetto Scoped Trace", - *handle_desc->trace_cookie_); - handle_desc->trace_ended_ = true; - } - } - - // The following state detection is for debugging only. - // We figure out if something is destroyed, uncreated, or live. - - // Does not distinguish between kTimedOutDestroyed and kDestroyed. - bool IsDestroyed(Handle handle) const { - auto it = states_.find(handle); - if (it != states_.end()) { - // Tracked values are not destroyed yet. - return false; - } - - if (handle == kInvalidHandle) { - return false; - } - - // The following assumes handles are incrementally generated: - if (it == states_.end()) { - // value is in range of [0, last_destroyed] => destroyed. - return handle <= last_destroyed_; - } - - auto min_it = states_.begin(); - if (handle < min_it->first) { - // value smaller than anything tracked: it was destroyed and we stopped tracking it. - return true; - } - - auto max_it = states_.rbegin(); - if (handle > max_it->first) { - // value too big: it's uncreated; - return false; - } - - // else it was a value that was previously tracked within [min,max] but no longer - return true; - } - - bool IsUncreated(Handle handle) const { - auto it = states_.find(handle); - if (it != states_.end()) { - // Tracked values are not uncreated. - return false; - } - - if (handle == kInvalidHandle) { - // Strangely enough, an invalid handle can never be created. - return true; - } - - // The following assumes handles are incrementally generated: - if (it == states_.end()) { - // value is in range of (last_destroyed, inf) => uncreated. - return handle > last_destroyed_; - } - - auto min_it = states_.begin(); - if (handle < min_it->first) { - // value smaller than anything tracked: it was destroyed and we stopped tracking it. - return false; - } - - auto max_it = states_.rbegin(); - if (handle > max_it->first) { - // value too big: it's uncreated; - return true; - } - - // else it was a value that was previously tracked within [min,max] but no longer - return false; - } - - public: - void Dump(::android::Printer& printer) { - // Locking can fail if we dump during a deadlock, so just do a best-effort lock here. - bool is_it_locked = mutex_.try_lock(); - - printer.printFormatLine("Perfetto consumer state:"); - if (!is_it_locked) { - printer.printLine(""""" (possible deadlock)"); - } - printer.printFormatLine(" Last destroyed handle: %" PRId64, last_destroyed_); - printer.printFormatLine(" Last created handle: %" PRId64, last_created_); - printer.printFormatLine(""); - printer.printFormatLine(" In-flight handles:"); - - for (auto it = states_.begin(); it != states_.end(); ++it) { - HandleDescription& handle_desc = it->second; - uint64_t started_tracing = - handle_desc.started_tracing_ns_ ? *handle_desc.started_tracing_ns_ : 0; - printer.printFormatLine(" Handle %" PRId64, handle_desc.handle_); - printer.printFormatLine(" Kind: %s", ToString(handle_desc.kind_).c_str()); - printer.printFormatLine(" Perfetto State: %d", static_cast<int>(handle_desc.state_)); - printer.printFormatLine(" Started tracing at: %" PRIu64, started_tracing); - printer.printFormatLine(" Last transition at: %" PRIu64, - handle_desc.last_transition_ns_); - } - if (states_.empty()) { - printer.printFormatLine(" (None)"); - } - - printer.printFormatLine(""); - - if (is_it_locked) { // u.b. if calling unlock on an unlocked mutex. - mutex_.unlock(); - } - } - - static PerfettoConsumerImpl::Impl* GetImplSingleton() { - static PerfettoConsumerImpl::Impl impl; - return &impl; - } -}; - -// Use a singleton because fruit instantiates a new PerfettoConsumer object for every -// new rx chain in RxProducerFactory. However, we want to track all perfetto transitions globally -// through 1 impl object. -// -// TODO: Avoiding a singleton would mean a more significant refactoring to remove the fruit/perfetto -// usage. - - -// -// Forward all calls to PerfettoConsumerImpl::Impl -// - -PerfettoConsumerImpl::~PerfettoConsumerImpl() { - // delete impl_; // TODO: no singleton -} - -void PerfettoConsumerImpl::Initialize() { - // impl_ = new PerfettoConsumerImpl::Impl(); // TODO: no singleton - impl_ = PerfettoConsumerImpl::Impl::GetImplSingleton(); -} - -void PerfettoConsumerImpl::Dump(::android::Printer& printer) { - PerfettoConsumerImpl::Impl::GetImplSingleton()->Dump(/*borrow*/printer); -} - -PerfettoConsumer::Handle PerfettoConsumerImpl::Create(const void* config_proto, - size_t config_len, - PerfettoConsumer::OnStateChangedCb callback, - void* callback_arg) { - return impl_->Create(config_proto, - config_len, - callback, - callback_arg); -} - -void PerfettoConsumerImpl::StartTracing(PerfettoConsumer::Handle handle) { - impl_->StartTracing(handle); -} - -PerfettoConsumer::TraceBuffer PerfettoConsumerImpl::ReadTrace(PerfettoConsumer::Handle handle) { - return impl_->ReadTrace(handle); -} - -void PerfettoConsumerImpl::Destroy(PerfettoConsumer::Handle handle) { - impl_->Destroy(handle); -} - -PerfettoConsumer::State PerfettoConsumerImpl::PollState(PerfettoConsumer::Handle handle) { - return impl_->PollState(handle); -} - -} // namespace iorap::perfetto diff --git a/src/perfetto/perfetto_consumer.h b/src/perfetto/perfetto_consumer.h deleted file mode 100644 index 2d828c0..0000000 --- a/src/perfetto/perfetto_consumer.h +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright (C) 2018 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_PERFETTO_PERFETTO_CONSUMER_H_ -#define IORAP_SRC_PERFETTO_PERFETTO_CONSUMER_H_ - -#include <fruit/fruit.h> -#include <perfetto/public/consumer_api.h> // libperfetto - -namespace android { -class Printer; -} // namespace android - -namespace iorap::perfetto { - -// Abstract out the Perfetto C API behind a virtual interface: -// This enables us to use dependency injection to provide mock implementations -// during tests. -struct PerfettoConsumer { - // 1:1 aliasing of type definitions and constants in perfetto/public/consumer_api.h - // Refer to the documentation there. - using State = ::perfetto::consumer::State; - using Handle = ::perfetto::consumer::Handle; - static constexpr Handle kInvalidHandle = ::perfetto::consumer::kInvalidHandle; - using OnStateChangedCb = ::perfetto::consumer::OnStateChangedCb; - using TraceBuffer = ::perfetto::consumer::TraceBuffer; - - // 1:1 forwarding of C-style functions in perfetto/public/consumer_api.h - // Refer to the documentation there. - - virtual Handle Create(const void* config_proto, - size_t config_len, - OnStateChangedCb callback, - void* callback_arg) = 0; - virtual void StartTracing(Handle) = 0; - virtual TraceBuffer ReadTrace(Handle) = 0; - virtual void Destroy(Handle) = 0; - virtual State PollState(Handle) = 0; - - virtual ~PerfettoConsumer() {} -}; - -// "Live" implementation that calls down to libperfetto. -struct PerfettoConsumerRawImpl : public PerfettoConsumer { - // Marks this constructor as the one to use for injection. - INJECT(PerfettoConsumerRawImpl()) = default; - - virtual Handle Create(const void* config_proto, - size_t config_len, - OnStateChangedCb callback, - void* callback_arg) override { - return ::perfetto::consumer::Create(config_proto, - config_len, - callback, - callback_arg); - } - - virtual void StartTracing(Handle handle) override { - ::perfetto::consumer::StartTracing(handle); - } - - virtual TraceBuffer ReadTrace(Handle handle) override { - return ::perfetto::consumer::ReadTrace(handle); - } - - virtual void Destroy(Handle handle) override { - ::perfetto::consumer::Destroy(handle); - } - virtual State PollState(Handle handle) override { - return ::perfetto::consumer::PollState(handle); - } - - virtual ~PerfettoConsumerRawImpl() {} -}; - -// "Safe" implementation that has extra checking around it. -class PerfettoConsumerImpl : public PerfettoConsumer { - public: - // Marks this constructor as the one to use for injection. - INJECT(PerfettoConsumerImpl()) { Initialize(); } - - virtual Handle Create(const void* config_proto, - size_t config_len, - OnStateChangedCb callback, - void* callback_arg) override; - virtual void StartTracing(Handle handle) override; - virtual TraceBuffer ReadTrace(Handle handle) override; - virtual void Destroy(Handle handle) override; - virtual State PollState(Handle handle) override; - - virtual ~PerfettoConsumerImpl(); - - static void Dump(/*borrow*/::android::Printer& printer); - - private: - void Initialize(); - struct Impl; - PerfettoConsumerImpl::Impl* impl_; -}; - -} // namespace iorap::perfetto - -#endif // IORAP_SRC_PERFETTO_PERFETTO_CONSUMER_H_ - diff --git a/src/perfetto/rx_producer.cc b/src/perfetto/rx_producer.cc deleted file mode 100644 index 824600c..0000000 --- a/src/perfetto/rx_producer.cc +++ /dev/null @@ -1,939 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/debug.h" -#include "common/expected.h" -#include "perfetto/rx_producer.h" - -#include <android-base/file.h> -#include <android-base/properties.h> -#include <android-base/unique_fd.h> - -#include <iostream> - -#include <sched.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <syscall.h> -#include <fcntl.h> -#include <unistd.h> - -// TODO: move to perfetto code -namespace perfetto { -namespace consumer { - -std::ostream& operator<<(std::ostream& os, State state) { - switch (state) { - case State::kTraceFailed: - os << "kTraceFailed"; - break; - case State::kConnectionError: - os << "kConnectionError"; - break; - case State::kSessionNotFound: - os << "kSessionNotFound"; - break; - case State::kIdle: - os << "kIdle"; - break; - case State::kConnecting: - os << "kConnecting"; - break; - case State::kConfigured: - os << "kConfigured"; - break; - case State::kTracing: - os << "kTracing"; - break; - case State::kTraceEnded: - os << "kTraceEnded"; - break; - default: - os << "(unknown)"; // did someone forget to update this code? - break; - } - return os; -} - -} // namespace consumer -} // namespace perfetto - -namespace iorap::perfetto { - -PerfettoDependencies::Component PerfettoDependencies::CreateComponent() { - // TODO: read from config. - static const uint32_t kTraceDurationMs = - ::android::base::GetUintProperty("iorapd.perfetto.trace_duration_ms", /*default*/5000U); - - static const uint32_t kBufferSize = - ::android::base::GetUintProperty("iorapd.perfetto.buffer_size", /*default*/4096U); - - return fruit::createComponent() - .bind<PerfettoConsumer, PerfettoConsumerImpl>() - .registerProvider([]() /* -> TraceConfig */ { - return CreateConfig(kTraceDurationMs, - /*deferred_start*/false, - kBufferSize); - }); -} - -::perfetto::protos::TraceConfig PerfettoDependencies::CreateConfig(uint32_t duration_ms, - bool deferred_start, - uint32_t buffer_size) { - ::perfetto::protos::TraceConfig trace_config; - - trace_config.set_duration_ms(duration_ms); - trace_config.add_buffers()->set_size_kb(buffer_size); - trace_config.set_deferred_start(deferred_start); - - auto* ds_config = trace_config.add_data_sources()->mutable_config(); - ds_config->set_name("linux.ftrace"); - ds_config->mutable_ftrace_config()->add_ftrace_events( - "mm_filemap_add_to_page_cache"); - ds_config->mutable_ftrace_config()->add_ftrace_events( - "mm_filemap_delete_from_page_cache"); - ds_config->set_target_buffer(0); - - return trace_config; -} - -// RAII-style wrapper around a perfetto handle that calls Destroy -// in a thread-safe manner. -struct PerfettoConsumerHandle { - private: - std::shared_ptr<PerfettoConsumer> consumer_; - PerfettoConsumer::Handle handle_; - - public: - // Takes over ownership of the 'handle'. - // - // Consumer must not be null. - PerfettoConsumerHandle(std::shared_ptr<PerfettoConsumer> consumer, - PerfettoConsumer::Handle handle) - : consumer_{std::move(consumer)}, - handle_{std::move(handle)} { - DCHECK(consumer_ != nullptr); - } - - std::shared_ptr<PerfettoConsumer> GetConsumer() const { - return consumer_; - } - - PerfettoConsumer::Handle GetHandle() const { - return handle_; - } - - ~PerfettoConsumerHandle() { - LOG(VERBOSE) << "PerfettoConsumerHandle::Destroy(" << handle_ << ")"; - consumer_->Destroy(handle_); - } - - bool operator==(const PerfettoConsumerHandle& other) const { - return handle_ == other.handle_ && consumer_ == other.consumer_; - } - - bool operator!=(const PerfettoConsumerHandle& other) const { - return !(*this == other); - } -}; - - -// Snapshot of a single perfetto OnStateChanged callback. -// -// Operate on the PerfettoConsumer to further change the state. -// -// The Handle is kept 'valid' until all references to the PerfettoConsumerHandle -// are dropped to 0. This ensures the Handle is not destroyed too early. All -// direct usages of 'Handle' must be scoped by the PerfettoConsumerHandle. -struct PerfettoStateChange { - public: - using State = ::perfetto::consumer::State; - using Handle = ::perfetto::consumer::Handle; - - State state; // Never invalid. - std::shared_ptr<PerfettoConsumerHandle> perfetto_consumer_and_handle; // Never null. - - // Safety: Use only within scope of the PerfettoStateChange. - Handle GetHandle() const { - // TODO: it would be even safer to wrap all the calls to the handle inside a class, - // instead of exposing this raw Handle. - return perfetto_consumer_and_handle->GetHandle(); - } - - std::shared_ptr<PerfettoConsumer> GetConsumer() const { - return perfetto_consumer_and_handle->GetConsumer(); - } -}; - -std::ostream& operator<<(std::ostream& os, const PerfettoStateChange& state_change) { - os << "PerfettoStateChange{" << state_change.state << "," - << state_change.GetHandle() << "," - << state_change.GetConsumer().get() << "}"; - return os; -} - -// Once created, this acts as a hot observable, emitting 'PerfettoStateChange' transition items. -// Only the 'state' will vary, the handle and perfetto_consumer are always the same value. -// -// Clients only need to handle the success states in #on_next, all failure states will go to -// #on_error. -// -// Upon reaching the appropriate terminal states, either #on_completed or #on_error is called. -// No future callbacks will then occur, so this object should be subsequently deleted. -// -// The Handle is destroyed automatically after the last item is emitted, so it must only be -// manipulated from the #on_next callbacks. Do not save the Handle and use it at other times. -class StateChangedSubject { - public: - using State = ::perfetto::consumer::State; - using Handle = ::perfetto::consumer::Handle; - - // Static members to solve use-after-free bug. - // The object is accessed from not only perfetto thread, but also iorap - // thread. Use this global map to manage it. - static std::mutex state_subject_mutex_; - static std::unordered_map<Handle, StateChangedSubject*> state_subject_map_; - - StateChangedSubject(const ::perfetto::protos::TraceConfig& trace_config, - rxcpp::subscriber<PerfettoStateChange> destination, - std::shared_ptr<PerfettoConsumer> perfetto_consumer) - : deferred_start(trace_config.deferred_start()), - dest(std::move(destination)), - perfetto_consumer_(std::move(perfetto_consumer)) { - DCHECK(perfetto_consumer_ != nullptr); - } - - private: - struct StateChangedError : public std::runtime_error { - explicit StateChangedError(const std::string& what_arg) : std::runtime_error(what_arg) {} - }; - - std::shared_ptr<PerfettoConsumerHandle> handle_; // non-null after bound_ == true. - std::atomic<bool> bound_{false}; // synchronize-with for BindHandle -> OnStateChanged. - - State last_state{State::kIdle}; - bool deferred_start{false}; - - rxcpp::subscriber<PerfettoStateChange> dest; - std::shared_ptr<PerfettoConsumer> perfetto_consumer_; // This is never null. - - void DcheckBadStateTransition(State state, bool fail_unless = false) const { - DCHECK(fail_unless) << "Invalid state transition to " << state << " from " << last_state; - } - - void DcheckValidStateTransition(State state) { - // State must not be out of range. - DCHECK_GE(state, State::kTraceFailed); - DCHECK_LE(state, State::kTraceEnded); - - // Internal state that should never leak out into public perfetto API: - DCHECK_NE(state, State::kIdle); - // These can only be returned by PollState: - DCHECK_NE(state, State::kSessionNotFound); - - // Validate state transitions as per the perfetto API contract. - // See the 'state diagram' in consumer_api.h - switch (last_state) { - case State::kTraceFailed: // Final and unrecoverable. - // b/122548195: this can transition to 'kConnectionError' if selinux is disabled. - if (state == State::kConnectionError) { - LOG(WARNING) << "b/122548195: kTraceFailed is non-terminal, ignoring."; - // This is a bit awkward: rxcpp will drop the #on_error calls if its more than once. - break; - } - DcheckBadStateTransition(state); - break; - case State::kConnectionError: // Final and unrecoverable. - DcheckBadStateTransition(state); - break; - case State::kSessionNotFound: - DcheckBadStateTransition(state); - break; - case State::kIdle: - // OK: we initialized our own state to idle prior to the first callback. - break; - case State::kConnecting: - switch (state) { - case State::kConfigured: - // kConfigured, if |deferred_start| == true in the trace config. - DcheckBadStateTransition(state, deferred_start); - break; - case State::kTracing: - // kTracing, if |deferred_start| == false. - DcheckBadStateTransition(state, !deferred_start); - break; - case State::kConnectionError: - // An error state, e.g. if cannot reach the traced daemon. - break; - default: - // Unconditionally invalid state transitions from kConnecting to anything else. - DcheckBadStateTransition(state); - } - break; - case State::kConfigured: - DCHECK(deferred_start); - if (state != State::kTracing // OK: this is documented. - && state != State::kTraceFailed) { // Undocumented selinux failure. - // Undocumented, but it appears to go directly from Configured->TraceEnded - // it can also go to kTraceFailed if e.g. there's an selinux violation - // however this appears to be underdocumented. - // b/122607276 #2 - - if (state != State::kTraceEnded) { // b/122607276 #1 - DcheckBadStateTransition(state); - } - } - break; - case State::kTracing: - switch (state) { - case State::kTraceEnded: - break; - case State::kTraceFailed: - break; - default: - DcheckBadStateTransition(state); - } - break; - case State::kTraceEnded: - // Cannot transition from terminal state to another state. - DcheckBadStateTransition(state); - break; - - // default: This list is exhaustive - } - } - - constexpr bool IsTerminalState() const { - switch (last_state) { - case State::kTraceFailed: - case State::kConnectionError: - case State::kTraceEnded: - return true; - default: - return false; - } - } - - // Returns true for non-terminal states (i.e. this callback will be invoked again). - // Returns false otherwise. - bool OnStateChanged(Handle handle, State state) { - using namespace ::perfetto::consumer; - - // Block until 'BoundHandle' is called by the other thread. - while (!bound_.load()) {} // seq_cst acquire. - - std::shared_ptr<PerfettoConsumerHandle> handle_ptr = handle_; - DCHECK(handle_ptr != nullptr); - - DCHECK_EQ(handle_ptr->GetHandle(), handle); - DcheckValidStateTransition(state); - - switch (state) { - // Error states (terminal). - case State::kTraceFailed: - EmitError("kTraceFailed"); - break; - case State::kConnectionError: - EmitError("kConnectionError"); - break; - - // Regular transitions (non-terminal). - case State::kConnecting: - case State::kConfigured: - case State::kTracing: - EmitNext(state); - break; - // Regular transitions (terminal). - case State::kTraceEnded: // XX: do we even need to emit the 'TraceEnded' state? - EmitNext(state); - dest.on_completed(); - break; - default: - DcheckBadStateTransition(state); - } - - bool force_non_terminal = false; - - if (last_state == State::kConfigured && state == State::kConnectionError) { - // b/122548195: this can transition to 'kConnectionError' if selinux is disabled. - force_non_terminal = true; - // This function must 'return true' in this buggy case, otherwise we will - // call the destructor too early and subsequent callbacks will crash. - } - - // Remember the state to validate prior state transitions. - last_state = state; - - // The owner of this class should avoid leaking memory once we reach a terminal state. - return !IsTerminalState() || force_non_terminal; - } - - public: - // Thread safety: Called by main thread, terminates the rx stream. - // When this function is invoked, no calls to this class from other threads can occur. - void OnCreateFailed() { - // returned when an invalid handle is passed to PollState(). - last_state = State::kSessionNotFound; - EmitError("Create returned kInvalidHandle"); - } - - // Thread safety: Called by main thread, this could be concurrent to - // 'CallbackOnStateChanged'. - void BindHandle(const std::shared_ptr<PerfettoConsumerHandle>& handle) { - handle_ = handle; - - // Unblock OnStateChanged. - bound_.store(true); // seq_cst release. - } - - - // Called by libperfetto background thread (same one every time) and iorap - // thread. - static void CallbackOnStateChanged(Handle handle, State state, void* callback_arg) { - LOG(VERBOSE) << "CallbackOnStateChanged(handle=" << handle << ",state=" << state - << ",callback_arg=" << callback_arg << ")"; - - // Validate OnStateChanged callback invariants, guaranteed by libperfetto. - DCHECK_NE(handle, ::perfetto::consumer::kInvalidHandle); - - // TODO: the memory ordering guarantees should be explicitly specified in consumer_api.h: - // This isn't specific enough: - // "The callback will be invoked on an internal thread and must not block." - // However looking at the implementation it posts onto a single-thread task runner, - // so this must be the case. - - // This current thread owns 'StateChangedSubject', no other threads must access it. - // Explicit synchronization is not necessary. - - { - std::lock_guard<std::mutex> guard(StateChangedSubject::state_subject_mutex_); - auto it = StateChangedSubject::state_subject_map_.find(handle); - // If the object is already deleted, do nothing. - if (it == StateChangedSubject::state_subject_map_.end()) { - return; - } - - StateChangedSubject* state_subject = it->second; - if (!state_subject->OnStateChanged(handle, state)) { - // Clean up the state tracker when we reach a terminal state. - // This means that no future callbacks will occur anymore. - StateChangedSubject::state_subject_map_.erase(it); - delete state_subject; - } - } - } - - private: - void EmitError(const std::string& msg) { - // Sidenote: Exact error class does not matter, rxcpp only lets us access the error - // as a string (rxcpp::util::what). - // - // Either way, the recovery strategy is identical (log then try and restart). - dest.on_error(rxcpp::util::make_error_ptr(StateChangedError{msg})); - } - - void EmitNext(State state) { - if (WOULD_LOG(VERBOSE) && !dest.is_subscribed()) { - // This is purely for logging: #on_next already filters out items after unsubscription. - LOG(VERBOSE) << "StateChangedSubject#EmitNext(" << state << ") - drop due to unsubscribe"; - } - - auto handle_ptr = handle_; - DCHECK(handle_ptr != nullptr); - - // Non-null guarantee for the items emitted into this stream. - PerfettoStateChange state_change{state, handle_ptr}; - dest.on_next(std::move(state_change)); - } - - // TODO: inherit from rx subject and handle #unsubscribe explicitly, instead - // of just being subject-like? -}; - -std::mutex StateChangedSubject::state_subject_mutex_; -std::unordered_map<::perfetto::consumer::Handle, - StateChangedSubject*> StateChangedSubject::state_subject_map_; - -// Note: The states will be emitted on a separate thread, so e.g. #as_blocking() -// needs to be used to avoid dropping everything on the floor. -// -// Important: The #on_error case must be handled explicitly by the observable, -// because the default behavior is to 'throw' which will cause an std::terminate with -fno-except. -static auto /*[observable<State>, shared_ptr<PerfettoConsumerHandle>]*/ - CreatePerfettoStateStream(::perfetto::protos::TraceConfig perfetto_config, - std::shared_ptr<PerfettoConsumer> perfetto_consumer) { - auto obs = rxcpp::observable<>::create<PerfettoStateChange>( - [perfetto_config = std::move(perfetto_config), perfetto_consumer = std::move(perfetto_consumer)] - (rxcpp::subscriber<PerfettoStateChange> subscriber) { - std::unique_ptr<StateChangedSubject> state_subject{ - new StateChangedSubject{perfetto_config, subscriber, perfetto_consumer}}; - - // Perfetto API requires a pointer to a serialized protobuf, it doesn't accept - // the code-generated object. - std::string perfetto_config_str = perfetto_config.SerializeAsString(); - - ::perfetto::consumer::Handle handle = - perfetto_consumer->Create(perfetto_config_str.data(), - perfetto_config_str.size(), - // executes on the same background thread repeatedly. - &StateChangedSubject::CallbackOnStateChanged, - // inter-thread-move - reinterpret_cast<void*>(state_subject.get())); - // perfetto::consumer::Create synchronizes-with OnStateChanged callback, this means - // we don't need to explicitly synchronize state_subject here so long as we don't access - // it on this thread again. - LOG(DEBUG) << "Create Perfetto handle " << handle; - - if (handle == ::perfetto::consumer::kInvalidHandle) { - LOG(ERROR) << "Failed to create Perfetto handle"; - // No callbacks will occur, so our thread still owns the state subject. - state_subject->OnCreateFailed(); - return; - } - - { - std::lock_guard<std::mutex> guard(StateChangedSubject::state_subject_mutex_); - StateChangedSubject::state_subject_map_[handle] = state_subject.get(); - } - - std::shared_ptr<PerfettoConsumerHandle> safe_handle{ - new PerfettoConsumerHandle{perfetto_consumer, handle}}; - - // Share ownership of the Handle with the StateSubject. - // This way we defer calling 'Destroy' until the callback reaches a terminal state - // *and* all users of the stream are done with the handle. - state_subject->BindHandle(safe_handle); - - // state_subject ownership is taken over by OnStateChanged. - // It will also be touched in a separate thread, so we must never access it here again. - state_subject.release(); - - // 'subscriber#add' is actually a call to register an on_unsubscribe listener. - subscriber.add([safe_handle]() { - LOG(VERBOSE) << "PerfettoStateChange#unsubscribe"; - - // Release our ref-count to the handle. - // safe_handle.reset(); // This happens implicitly. - - // TODO: I think this won't handle the case where we need to shut down early. - // Need to use the explicit kShutdown for that? - }); - - // TODO: this would be an excellent place to shuffle the perfetto config protobuf - // into a global debug state for dumpsys. - }); - - return obs; -} - -template <typename T> -bool BinaryWireProtobuf<T>::WriteFullyToFile(const std::string& path, - bool follow_symlinks) const { - // TODO: it would be great if android::base had a string_view overload to avoid copying - // data into an std::string. - - // u g o - // rw-rw---- - // - // Protobufs can be read/written but not executed. - static constexpr const mode_t kMode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; - - int flags = - O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_BINARY | (follow_symlinks ? 0 : O_NOFOLLOW); - android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), flags, kMode))); - - if (fd == -1) { - PLOG(ERROR) << "BinaryWireProtobuf::WriteFullyToFile open failed"; - return false; - } - - if (!::android::base::WriteFully(fd, data_.data(), size())) { - PLOG(ERROR) << "BinaryWireProtobuf::WriteFullyToFile write failed"; - return CleanUpAfterFailedWrite(path); - } - - return true; -} - -template <typename T> -bool BinaryWireProtobuf<T>::CleanUpAfterFailedWrite(const std::string& path) { - // Something went wrong. Let's not leave a corrupt file lying around. - int saved_errno = errno; - unlink(path.c_str()); - errno = saved_errno; - return false; -} - -template <typename T> -bool BinaryWireProtobuf<T>::WriteStringToFd(int fd) const { - const char* p = reinterpret_cast<const char*>(data_.data()); - size_t left = size(); - while (left > 0) { - ssize_t n = TEMP_FAILURE_RETRY(write(fd, p, left)); - if (n == -1) { - return false; - } - p += n; - left -= n; - } - return true; -} - -template <typename T> -std::optional<BinaryWireProtobuf<T>> BinaryWireProtobuf<T>::ReadFullyFromFile( - const std::string& path, - bool follow_symlinks) { - std::vector<std::byte> data; - - int flags = O_RDONLY | O_CLOEXEC | O_BINARY | (follow_symlinks ? 0 : O_NOFOLLOW); - android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), flags))); - if (fd == -1) { - return std::nullopt; - } - - if (ReadFdToString(fd.get(), /*out*/&data)) { - return BinaryWireProtobuf<T>{std::move(data)}; - } else { - return std::nullopt; - } -} - -template <typename T> -bool BinaryWireProtobuf<T>::operator==(const BinaryWireProtobuf<T>& other) const { - if (data_.size() != other.data_.size()) { - return false; - } - return std::equal(data_.begin(), data_.end(), other.data_.begin()); -} - -template <typename T> -bool BinaryWireProtobuf<T>::ReadFdToString(int fd, /*out*/std::vector<std::byte>* content) { - DCHECK(content != nullptr); - - content->clear(); - - struct stat sb; - if (fstat(fd, /*out*/&sb) != -1 && sb.st_size > 0) { - content->reserve(sb.st_size); - } - - char buf[BUFSIZ]; - auto it = content->begin(); - ssize_t n; - while ((n = TEMP_FAILURE_RETRY(read(fd, &buf[0], sizeof(buf)))) > 0) { - content->insert(it, - reinterpret_cast<std::byte*>(&buf[0]), - reinterpret_cast<std::byte*>(&buf[n])); - - std::advance(/*inout*/it, static_cast<size_t>(n)); - - static_assert(sizeof(char) == sizeof(std::byte), "sanity check for reinterpret cast"); - } - return (n == 0) ? true : false; -} - -// explicit template instantiation. -template struct BinaryWireProtobuf<::google::protobuf::MessageLite>; -// TODO: refactor this not to need the template instantiation. - -// Copy of the 2.6.18 kernel header (linux/ioprio.h) - -#define IOPRIO_WHO_PROCESS (1) -#define IOPRIO_CLASS_IDLE (3) - -#define IOPRIO_BITS (16) -#define IOPRIO_CLASS_SHIFT (13) -#define IOPRIO_PRIO_MASK ((1UL << IOPRIO_CLASS_SHIFT) - 1) - -#define IOPRIO_PRIO_CLASS(mask) ((mask) >> IOPRIO_CLASS_SHIFT) -#define IOPRIO_PRIO_DATA(mask) ((mask) & IOPRIO_PRIO_MASK) -#define IOPRIO_PRIO_VALUE(class, data) (((class) << IOPRIO_CLASS_SHIFT) | data) - -static int ioprio_get(int which, int who) { - return syscall(SYS_ioprio_get, which, who); -} - -static int ioprio_set(int which, int who, int ioprio) { - return syscall(SYS_ioprio_set, which, who, ioprio); -} - -// An rx Coordination, which will cause a new thread to spawn for each new Worker. -// -// Idle-class priority is set for the CPU and IO priorities on the new thread. -rxcpp::observe_on_one_worker ObserveOnNewIoThread() { - // IO thread factory for idle-priority threads. - // Both the CPU scheduler and the IO scheduler are set to idle. - // - // Use this when needing to schedule disk access from a normal-priority thread onto a - // very low priority thread, but not so low that we need to use a BackgroundJobScheduler. - struct io_thread_factory { - std::thread operator()(std::function<void()> start) const { - return std::thread{ - [start=std::move(start)]() { - // Set IO priority to idle. - do { - int value = ioprio_get(IOPRIO_WHO_PROCESS, /*pid*/0); - if (value == -1) { - PLOG(ERROR) << "io_thread_factory failed ioprio_get"; - break; // Can't set the ioprio, we don't know what data to use. - } - - int data = IOPRIO_PRIO_DATA(value); // priority level - // This appears to be '4' in practice. We may want to raise to - // be the highest-priority within the idle class. - - // idle scheduling class. only access disk when nobody else needs disk. - int res = ioprio_set(IOPRIO_WHO_PROCESS, - /*pid*/0, - IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, data)); - if (res < 0) { - PLOG(ERROR) << "io_thread_factory failed ioprio_set"; - break; - } - - // Changing the IO priority only has any effect with cfq scheduler: - // $> cat /sys/block/sda/queue/scheduler - LOG(VERBOSE) << "ioprio_set(WHO_PROCESS, class=IDLE, data=" << data << ")"; - } while (false); - - // Set CPU priority to idle. - do { - struct sched_param param{}; - param.sched_priority = 0; // Required to be statically 0 when used with SCHED_IDLE. - - if (sched_setscheduler(/*pid*/0, // current thread, - SCHED_IDLE, - /*in*/¶m) != 0) { - PLOG(ERROR) << "io_thread_factory failed sched_setscheduler"; - break; - } - - LOG(VERBOSE) << "sched_setscheduler(self, IDLE)"; - } while (false); - - // XX: if changing the scheduling is too aggressive (i.e. it causes starvation), - // we may want to stick with the default class and change the nice (priority) levels - // to the minimum. - - // TODO: future work, maybe use cgroups configuration file instead? - - // Call the rxcpp-supplied code. - start(); - } - }; - } - }; - - static rxcpp::schedulers::scheduler thread_scheduler = - rxcpp::schedulers::make_new_thread(io_thread_factory{}); - - static rxcpp::observe_on_one_worker observe_on_io_thread{thread_scheduler}; - - return observe_on_io_thread; -} - -static auto/*observable<PerfettoTraceProto>*/ - CreatePerfettoStream(rxcpp::observable<PerfettoStreamCommand> input, - std::shared_ptr<PerfettoConsumer> perfetto_consumer, - const ::perfetto::protos::TraceConfig& trace_config) { - // XX: should I also take a scheduler for input here??? - - auto /*observable<PerfettoStateChange>*/ perfetto_states = - CreatePerfettoStateStream(trace_config, perfetto_consumer); - - using State = ::perfetto::consumer::State; - - auto/*coordinator*/ serialize_coordinator = rxcpp::observe_on_new_thread(); - // Rx note: - // The optimal thing to do would be to have a lock/unlock for an entire subset of a chain. - // This would avoid creating new threads, and could also be used to intentionally block - // the regular C-callback perfetto thread. - // - // It seems possible to create a coordinator to lock a single operator in a chain, but this - // appears to be unsound. In particular, it doesn't even make life any simpler below because - // it would only apply the synchronization to 'zip' but not 'flat_map' which is unsound. - // - // There is also the built-in 'serialize_new_thread' which seems to create a new thread but - // then never actually uses it, that seems unfortunate and wasteful. - // - // Instead, do the simple thing which is create a new thread and always queue on there. - // Execution an action on that worker is itself unsynchronized, but this doesn't matter since - // the worker is only backed by 1 thread (no 2 schedulables can be executed concurrently - // on the 'observe_new_thread' worker). - return input - .tap([](PerfettoStreamCommand command) { - LOG(VERBOSE) << "CreatePerfettoStreamCommand#tap(command=" << command << ")"; - }) - // Input A, thread tA. Input B, thread tB. Continue execution with (A,B) on thread tC. - .zip(serialize_coordinator, // rest of chain is also executed on the same thread. - perfetto_states) - // Note: zip terminates when either of the streams complete. - .flat_map( - [](std::tuple<PerfettoStreamCommand, PerfettoStateChange> p) { - auto& [command, state_change] = p; - LOG(VERBOSE) << "CreatePerfettoStream#combine(" - << command << "," << state_change << ")"; - if (command == PerfettoStreamCommand::kShutdown) { - // Perfetto: Always safe to call ::perfetto::consumer::Destroy - // at any time. - // - // XX: How do we clean up the StateChangedSubject without racing - // against the callback? It strikes me that we may need a 'kDestroyed' - // state that perfetto can transition to from kConfigured. - LOG(VERBOSE) << "Call Perfetto_Consumer->Destroy"; - state_change.GetConsumer()->Destroy(state_change.GetHandle()); - - // XX: Do we even have any guarantees about not getting more callbacks? - // We could just say 'there can still be spurious output after Shutdown' - // and just ignore it (e.g. Shutdown and immediately unsubscribe). - } else if (command == PerfettoStreamCommand::kStartTracing - && state_change.state == State::kConfigured) { - LOG(VERBOSE) << "Call Perfetto_Consumer->StartTracing"; - state_change.GetConsumer()->StartTracing(state_change.GetHandle()); - } else if (command == PerfettoStreamCommand::kStopTracing && - state_change.state == State::kTraceEnded) { - // TODO: if perfetto actually had a 'StopTracing' we could call that here. - // right now we just pretend it exists, but rely on the config timer instead. - ::perfetto::consumer::TraceBuffer trace_buffer = - state_change.GetConsumer()->ReadTrace(state_change.GetHandle()); - - LOG(VERBOSE) << "Perfetto Trace ended" - << ", addr=" << reinterpret_cast<void*>(trace_buffer.begin) - << ",size= " << trace_buffer.size; - - PerfettoTraceProto wire_proto{trace_buffer.begin, trace_buffer.size}; - return rxcpp::observable<>::just(std::move(wire_proto)).as_dynamic(); - } - return rxcpp::observable<>::empty<PerfettoTraceProto>().as_dynamic(); - } - ); -} - -std::ostream& operator<<(std::ostream& os, PerfettoStreamCommand c) { - switch (c) { - case PerfettoStreamCommand::kStartTracing: - os << "kStartTracing"; - break; - case PerfettoStreamCommand::kStopTracing: - os << "kStopTracing"; - break; - case PerfettoStreamCommand::kShutdown: - os << "kShutdown"; - break; - default: - os << "(unknown)"; - break; - } - return os; -} - -RxProducerFactory::RxProducerFactory(PerfettoDependencies::Injector& injector) - : injector_(injector) { -} - -// TODO: (fruit) maybe this could be streamlined further by avoiding this boilerplate? -rxcpp::observable<PerfettoTraceProto> RxProducerFactory::CreateTraceStream( - rxcpp::observable<PerfettoStreamCommand> commands) { - std::shared_ptr<PerfettoConsumer> perfetto_consumer = - injector_.get<std::shared_ptr<PerfettoConsumer>>(); - const ::perfetto::protos::TraceConfig& trace_config = - injector_.get<::perfetto::protos::TraceConfig>(); - - DCHECK(perfetto_consumer != nullptr); - DCHECK(reinterpret_cast<volatile const void*>(&trace_config) != nullptr); - - return CreatePerfettoStream(commands, - perfetto_consumer, - trace_config); -} - -// For testing/debugging only. -// -// Saves protobuf results in file name specified by 'arg_output_proto'. -void CollectPerfettoTraceBufferImmediately( - RxProducerFactory& producer_factory, - const std::string& arg_output_proto) { - LOG(VERBOSE) << "CollectPerfettoTraceBufferImmediately"; - - std::shared_ptr<PerfettoConsumer> perfetto_consumer = - producer_factory.injector_.get<std::shared_ptr<PerfettoConsumer>>(); - const ::perfetto::protos::TraceConfig& trace_config = - producer_factory.injector_.get<const ::perfetto::protos::TraceConfig&>(); - - auto /*observable<PerfettoStateChange>*/ perfetto_states = - CreatePerfettoStateStream(trace_config, perfetto_consumer); - - perfetto_states - .as_blocking() // Wait for observable to terminate with on_completed or on_error. - .subscribe(/*on_next*/[&](auto state_change) { - LOG(VERBOSE) << "Perfetto post-processed State change: " << state_change; - - using State = ::perfetto::consumer::State; - switch (state_change.state) { - case State::kConnecting: - LOG(VERBOSE) << "Perfetto Tracing is Connecting"; - // Transitional state. No-op. - break; - case State::kConfigured: - state_change.GetConsumer()->StartTracing(state_change.GetHandle()); - break; - case State::kTracing: - LOG(VERBOSE) << "Perfetto Tracing started"; - // Transitional state. No-op. - break; - case State::kTraceEnded: { - ::perfetto::consumer::TraceBuffer trace_buffer = - state_change.GetConsumer()->ReadTrace(state_change.GetHandle()); - - LOG(VERBOSE) << "Perfetto Trace ended" - << ", addr=" << reinterpret_cast<void*>(trace_buffer.begin) - << ",size= " << trace_buffer.size; - - if (!arg_output_proto.empty()) { - std::string trace_buffer_str; - trace_buffer_str.resize(trace_buffer.size); - std::copy(trace_buffer.begin, - trace_buffer.begin + trace_buffer.size, - trace_buffer_str.data()); - if (!android::base::WriteStringToFile(trace_buffer_str, arg_output_proto)) { - LOG(ERROR) << "Failed to save TraceBuffer to " << arg_output_proto; - } else { - LOG(INFO) << "TraceBuffer saved to file: " << arg_output_proto; - LOG(INFO); - LOG(INFO) << "To print this in a human readable form, execute these commands:"; - LOG(INFO) << "$> adb pull '" << arg_output_proto << "'"; - LOG(INFO) << "$> trace_to_text systrace <filename.pb>"; - } - } - - // TODO: something more useful with this TraceBuffer, such as saving it to a file - // and printing the output. - break; - } - default: - // No other states are possible, because they go to #on_error or cause a dcheck. - DCHECK(false) << "Invalid state: " << state_change; - } - - //INTENTIONAL_COMPILER_ERROR_HERE // lets make sure this code actually does a trace. - - }, /*on_error*/[](rxcpp::util::error_ptr err) { - LOG(ERROR) << "Perfetto post-processed state change failed: " << rxcpp::util::what(err); - }, /*on_completed*/[]() { - LOG(VERBOSE) << "Perfetto post-processed State #on_completed"; - }); -} - - -} // namespace iorap::perfetto diff --git a/src/perfetto/rx_producer.h b/src/perfetto/rx_producer.h deleted file mode 100644 index 4e93f0d..0000000 --- a/src/perfetto/rx_producer.h +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef IORAP_SRC_PERFETTO_RX_PRODUCER_H_ -#define IORAP_SRC_PERFETTO_RX_PRODUCER_H_ - -#include "perfetto/perfetto_consumer.h" // libiorap - -#include <perfetto/config/trace_config.pb.h> // libperfetto -#include <rxcpp/rx.hpp> - -#include <iosfwd> -#include <functional> -#include <optional> -#include <vector> - -namespace iorap::perfetto { - -struct PerfettoDependencies { - using Component = - fruit::Component<PerfettoConsumer, ::perfetto::protos::TraceConfig>; - using Injector = - fruit::Injector<PerfettoConsumer, ::perfetto::protos::TraceConfig>; - using NormalizedComponent = - fruit::NormalizedComponent<PerfettoConsumer, ::perfetto::protos::TraceConfig>; - - // Create a 'live' component that will talk to perfetto via traced. - static Component CreateComponent(/*TODO: config params*/); - - // Create perfetto.protos.TraceConfig , serialized as a (machine-readable) string. - // - // The following ftrace events are enabled: - // * mm_filemap_add_to_page_cache - // * mm_filemap_delete_from_page_cache - // - // If deferred starting is also enabled, no tracing will begin until - // ::perfetto::consumer::StartTracing is invoked. - static ::perfetto::protos::TraceConfig CreateConfig(uint32_t duration_ms, - bool deferred_start = true, - uint32_t buffer_size = 4096); -}; - -namespace detail { - template <typename T> - struct concept_message_lite_base { - static_assert(std::is_base_of_v<::google::protobuf::MessageLite, T>, - "T must inherit from MessageLite"); - using type = T; - }; - - template <typename T> - using concept_message_lite_base_t = typename concept_message_lite_base<T>::type; -} // namespace detail - -/* - * In Android's version of libprotobuf, move-constructors are not generated. - * This results in a legitimate (~10sec per TracePacket being compiled) slowdown, - * so we need to avoid it everywhere. - * - * 1) Don't copy the protos, move them instead. - * 2) Use 'shared_ptr' because rxcpp won't compile with unique_ptr. - */ -template <typename T> -using ProtobufPtr = std::shared_ptr<detail::concept_message_lite_base_t<const T>>; - -template <typename T> -using ProtobufMutablePtr = std::shared_ptr<detail::concept_message_lite_base_t<T>>; - -// This acts as a lightweight type marker so that we know what data has actually -// encoded under the hood. -template <typename T> -struct BinaryWireProtobuf { - static_assert(std::is_base_of_v<::google::protobuf::MessageLite, T>, - "T should be a base class of MessageLite"); - - std::vector<std::byte>& data() { - return data_; - } - - const std::vector<std::byte>& data() const { - return data_; - } - - size_t size() const { - return data_.size(); - } - - explicit BinaryWireProtobuf(char* data, size_t size) - : BinaryWireProtobuf(reinterpret_cast<std::byte*>(data), size) { - } - - explicit BinaryWireProtobuf(std::byte* data, size_t size) { - data_.resize(size); - std::copy(data, - data + size, - data_.data()); - } - - explicit BinaryWireProtobuf(std::vector<std::byte> data) : data_{std::move(data)} { - } - - // You wouldn't want to accidentally copy a giant multi-megabyte chunk would you? - // BinaryWireProtobuf(const BinaryWireProtobuf& other) = delete; // FIXME: rx likes to copy. - BinaryWireProtobuf(const BinaryWireProtobuf& other) = default; - BinaryWireProtobuf(BinaryWireProtobuf&& other) = default; - - // Important: Deserialization could fail, for example data is truncated or - // some minor disc corruption occurred. - template <typename U> - std::optional<ProtobufPtr<U>> MaybeUnserialize() { - ProtobufMutablePtr<U> unencoded{new U{}}; - - if (!unencoded->ParseFromArray(data_.data(), data_.size())) { - return std::nullopt; - } - - return {std::move(unencoded)}; - } - - bool WriteFullyToFile(const std::string& path, - bool follow_symlinks = false) const; - - static std::optional<BinaryWireProtobuf<T>> ReadFullyFromFile(const std::string& path, - bool follow_symlinks = false); - - bool operator==(const BinaryWireProtobuf<T>& other) const; - bool operator!=(const BinaryWireProtobuf<T>& other) const { - return !(*this == other); - } - - private: - static bool CleanUpAfterFailedWrite(const std::string& path); - bool WriteStringToFd(int fd) const; - - static bool ReadFdToString(int fd, /*out*/std::vector<std::byte>* data); - - std::vector<std::byte> data_; -}; - -//using PerfettoTraceProto = BinaryWireProtobuf<::perfetto::protos::Trace>; -using PerfettoTraceProto = BinaryWireProtobuf<::google::protobuf::MessageLite>; - -enum class PerfettoStreamCommand { - kStartTracing, // -> () | on_error - kStopTracing, // -> on_next(PerfettoTraceProto) | on_error - kShutdown, // -> on_completed | on_error - // XX: should shutdown be converted to use the rx suscriber#unsubscribe instead? -}; - -std::ostream& operator<<(std::ostream& os, PerfettoStreamCommand c); - -struct RxProducerFactory { - // Passing anything by value leads to a lot of pain and headache. - // Pass in the injector by reference because nothing else seems to work. - explicit RxProducerFactory(PerfettoDependencies::Injector& injector); - - // Create a one-shot perfetto observable that will begin - // asynchronously producing a PerfettoTraceProto after the 'kStartTracing' - // command is observed. - // - // libperfetto is immediately primed (i.e. connected in a deferred state) - // upon calling this function, to reduce the latency of 'kStartTracing'. - // - // To finish the trace, push 'kStopTracing'. To cancel or tear down at any - // time, push 'kShutdown'. - // - // The TraceProto may come out at any time after 'kStartTracing', - // this is controlled by duration_ms in the TraceConfig. - // - // TODO: libperfetto should actually stop tracing when we ask it to, - // instead of using a hardcoded time. - // - // The observable may go into #on_error at any time, if the underlying - // libperfetto states transition to a failing state. - // This usually means the OS is not configured correctly. - rxcpp::observable<PerfettoTraceProto> CreateTraceStream( - rxcpp::observable<PerfettoStreamCommand> commands); - - // TODO: is this refactor-able into a subscriber factory that takes - // the commands-observable as a parameter? - - // TODO: infinite perfetto stream. - - private: - // XX: why doesn't this just let me pass in a regular Component? - PerfettoDependencies::Injector& injector_; - - friend void CollectPerfettoTraceBufferImmediately( - RxProducerFactory& producer_factory, - const std::string& arg_output_proto); -}; - -// An rx Coordination, which will cause a new thread to spawn for each new Worker. -// -// Idle-class priority is set for the CPU and IO priorities on the new thread. -// -// TODO: move to separate file -rxcpp::observe_on_one_worker ObserveOnNewIoThread(); - -} // namespace iorap::perfetto -#endif // IORAP_SRC_PERFETTO_RX_PRODUCER_H_ diff --git a/src/prefetcher/main.cc b/src/prefetcher/main.cc deleted file mode 100644 index 94ba141..0000000 --- a/src/prefetcher/main.cc +++ /dev/null @@ -1,190 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/debug.h" -#include "common/loggers.h" -#include "prefetcher/prefetcher_daemon.h" - -#include <android-base/parseint.h> -#include <android-base/logging.h> - -#include <iostream> -#include <optional> -#include <string_view> -#include <string> -#include <vector> - -#include <signal.h> - -#if defined(IORAP_PREFETCHER_MAIN) - -namespace iorap::prefetcher { - -void Usage(char** argv) { - std::cerr << "Usage: " << argv[0] << " [--input-fd=#] [--output-fd=#]" << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Run the readahead daemon which can prefetch files given a command." << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Optional flags:" << std::endl; - std::cerr << " --help,-h Print this Usage." << std::endl; - std::cerr << " --input-fd,-if Input FD (default stdin)." << std::endl; - std::cerr << " --output-fd,-of Output FD (default stdout)." << std::endl; - std::cerr << " --use-sockets,-us Use AF_UNIX sockets (default off)." << std::endl; - std::cerr << " --command-format=[text|binary],-cf (default text)." << std::endl; - std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; - std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; - exit(1); -} - -int Main(int argc, char** argv) { - // Go to system logcat + stderr when running from command line. - android::base::InitLogging(argv, iorap::common::StderrAndLogdLogger{android::base::SYSTEM}); - - bool wait_for_keystroke = false; - bool enable_verbose = false; - - bool command_format_text = false; // false = binary. - - int arg_input_fd = -1; - int arg_output_fd = -1; - - std::vector<std::string> arg_input_filenames; - bool arg_use_sockets = false; - - LOG(VERBOSE) << "argparse: argc=" << argc; - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - bool has_arg_next = (arg+1)<argc; - std::string arg_next = has_arg_next ? argv[arg+1] : ""; - - LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; - - if (argstr == "--help" || argstr == "-h") { - Usage(argv); - } else if (argstr == "--input-fd" || argstr == "-if") { - if (!has_arg_next) { - LOG(ERROR) << "--input-fd=<numeric-value>"; - Usage(argv); - } - if (!::android::base::ParseInt(arg_next, /*out*/&arg_input_fd)) { - LOG(ERROR) << "--input-fd value must be numeric"; - Usage(argv); - } - } else if (argstr == "--output-fd" || argstr == "-of") { - if (!has_arg_next) { - LOG(ERROR) << "--output-fd=<numeric-value>"; - Usage(argv); - } - if (!::android::base::ParseInt(arg_next, /*out*/&arg_output_fd)) { - LOG(ERROR) << "--output-fd value must be numeric"; - Usage(argv); - } - } else if (argstr == "--command-format=" || argstr == "-cf") { - if (!has_arg_next) { - LOG(ERROR) << "--command-format=text|binary"; - Usage(argv); - } - if (arg_next == "text") { - command_format_text = true; - } else if (arg_next == "binary") { - command_format_text = false; - } else { - LOG(ERROR) << "--command-format must be one of {text,binary}"; - Usage(argv); - } - } else if (argstr == "--use-sockets" || argstr == "-us") { - arg_use_sockets = true; - } else if (argstr == "--verbose" || argstr == "-v") { - enable_verbose = true; - } else if (argstr == "--wait" || argstr == "-w") { - wait_for_keystroke = true; - } else { - arg_input_filenames.push_back(argstr); - } - } - - if (enable_verbose) { - android::base::SetMinimumLogSeverity(android::base::VERBOSE); - - LOG(VERBOSE) << "Verbose check"; - LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; - } else { - android::base::SetMinimumLogSeverity(android::base::DEBUG); - } - - LOG(VERBOSE) << "argparse: argc=" << argc; - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - - LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; - } - - // Useful to attach a debugger... - // 1) $> iorap.cmd.readahead -w <args> - // 2) $> gdbclient <pid> - if (wait_for_keystroke) { - LOG(INFO) << "Self pid: " << getpid(); - - raise(SIGSTOP); - // LOG(INFO) << "Press any key to continue..."; - // std::cin >> wait_for_keystroke; - } - - // auto system_call = std::make_unique<SystemCallImpl>(); - // TODO: mock readahead calls? - // - // Uncomment this if we want to leave the process around to inspect it from adb shell. - // sleep(100000); - - int return_code = 0; - - LOG(VERBOSE) << "Hello world"; - - if (arg_input_fd == -1) { - arg_input_fd = STDIN_FILENO; - } - if (arg_output_fd == -1) { - arg_output_fd = STDOUT_FILENO; - } - - PrefetcherForkParameters params{}; - params.input_fd = arg_input_fd; - params.output_fd = arg_output_fd; - params.format_text = command_format_text; - params.use_sockets = arg_use_sockets; - - LOG(VERBOSE) << "main: Starting PrefetcherDaemon: " - << "input_fd=" << params.input_fd - << ",output_fd=" << params.output_fd; - { - PrefetcherDaemon daemon; - // Blocks until receiving an exit command. - daemon.Main(std::move(params)); - } - LOG(VERBOSE) << "main: Terminating"; - - // 0 -> successfully executed all commands. - // 1 -> failed along the way (#on_error and also see the error logs). - return return_code; -} - -} // namespace iorap::prefetcher - -int main(int argc, char** argv) { - return ::iorap::prefetcher::Main(argc, argv); -} - -#endif // IORAP_PREFETCHER_MAIN diff --git a/src/prefetcher/main_client.cc b/src/prefetcher/main_client.cc deleted file mode 100644 index ff7ee6d..0000000 --- a/src/prefetcher/main_client.cc +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "common/debug.h" -#include "prefetcher/read_ahead.h" -#include "prefetcher/task_id.h" - -#include <android-base/parseint.h> -#include <android-base/logging.h> - -#include <iostream> -#include <optional> -#include <string_view> -#include <string> -#include <vector> - -#include <signal.h> -#include <unistd.h> - - -namespace iorap::prefetcher { - -static void UsageClient(char** argv) { - std::cerr << "UsageClient: " << argv[0] << " <path-to-compiled-trace.pb> [... pathN]" << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Run the readahead daemon which can prefetch files given a command." << std::endl; - std::cerr << "" << std::endl; - std::cerr << " Optional flags:" << std::endl; - std::cerr << " --help,-h Print this UsageClient." << std::endl; - std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; - std::cerr << " --task-duration-ms,-tdm Set task duration (default: 0ms)." << std::endl; - std::cerr << " --use-sockets,-us Use AF_UNIX sockets (default: off)" << std::endl; - std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; - exit(1); -} - -int MainClient(int argc, char** argv) { - android::base::InitLogging(argv); - android::base::SetLogger(android::base::StderrLogger); - - bool wait_for_keystroke = false; - bool enable_verbose = false; - - bool command_format_text = false; // false = binary. - - unsigned int arg_task_duration_ms = 10000; - std::vector<std::string> arg_input_filenames; - bool arg_use_sockets = false; - - LOG(VERBOSE) << "argparse: argc=" << argc; - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - bool has_arg_next = (arg+1)<argc; - std::string arg_next = has_arg_next ? argv[arg+1] : ""; - - LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; - - if (argstr == "--help" || argstr == "-h") { - UsageClient(argv); - } else if (argstr == "--use-sockets" || argstr == "-us") { - arg_use_sockets = true; - } else if (argstr == "--verbose" || argstr == "-v") { - enable_verbose = true; - } else if (argstr == "--wait" || argstr == "-w") { - wait_for_keystroke = true; - } else if (argstr == "--task-duration-ms" || argstr == "-tdm") { - if (!has_arg_next) { - LOG(ERROR) << "--task-duration-ms: requires uint parameter"; - UsageClient(argv); - } else if (!::android::base::ParseUint(arg_next, &arg_task_duration_ms)) { - LOG(ERROR) << "--task-duration-ms: requires non-negative parameter"; - UsageClient(argv); - } - } else { - arg_input_filenames.push_back(argstr); - } - } - - if (enable_verbose) { - android::base::SetMinimumLogSeverity(android::base::VERBOSE); - - LOG(VERBOSE) << "Verbose check"; - LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; - } else { - android::base::SetMinimumLogSeverity(android::base::DEBUG); - } - - LOG(VERBOSE) << "argparse: argc=" << argc; - - for (int arg = 1; arg < argc; ++arg) { - std::string argstr = argv[arg]; - - LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; - } - - // Useful to attach a debugger... - // 1) $> iorap.cmd.readahead -w <args> - // 2) $> gdbclient <pid> - if (wait_for_keystroke) { - LOG(INFO) << "Self pid: " << getpid(); - - raise(SIGSTOP); - // LOG(INFO) << "Press any key to continue..."; - // std::cin >> wait_for_keystroke; - } - - // auto system_call = std::make_unique<SystemCallImpl>(); - // TODO: mock readahead calls? - // - // Uncomment this if we want to leave the process around to inspect it from adb shell. - // sleep(100000); - - int return_code = 0; - - LOG(VERBOSE) << "Hello world"; - - ReadAhead read_ahead{arg_use_sockets}; // Don't count the time it takes to fork+exec. - - size_t task_id_counter = 0; - for (const std::string& compiled_trace_path : arg_input_filenames) { - TaskId task_id{task_id_counter++, compiled_trace_path}; - - LOG(DEBUG) << "main: ReadAhead BeginTask: " - << "task_duration_ms=" << arg_task_duration_ms << "," - << task_id; - - read_ahead.BeginTask(task_id); - usleep(arg_task_duration_ms*1000); - - LOG(DEBUG) << "main: ReadAhead FinishTask: " << task_id; - - read_ahead.FinishTask(task_id); - } - LOG(VERBOSE) << "main: Terminating"; - - // 0 -> successfully executed all commands. - // 1 -> failed along the way (#on_error and also see the error logs). - return return_code; -} - -} // namespace iorap::prefetcher - -#if defined(IORAP_PREFETCHER_MAIN_CLIENT) -int main(int argc, char** argv) { - return ::iorap::prefetcher::MainClient(argc, argv); -} - -#endif // IORAP_PREFETCHER_MAIN_CLIENT diff --git a/src/prefetcher/minijail.cc b/src/prefetcher/minijail.cc deleted file mode 100644 index df4b11b..0000000 --- a/src/prefetcher/minijail.cc +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "prefetcher/minijail.h" - -#include <android-base/logging.h> -#include <libminijail.h> - -namespace iorap::prefetcher { - -static const char kSeccompFilePath[] = "/system/etc/seccomp_policy/iorap.prefetcherd.policy"; - -bool MiniJail() { - /* no seccomp policy for this architecture */ - if (access(kSeccompFilePath, R_OK) == -1) { - LOG(WARNING) << "No seccomp filter defined for this architecture."; - return true; - } - - struct minijail* jail = minijail_new(); - if (jail == NULL) { - LOG(WARNING) << "Failed to create minijail."; - return false; - } - - minijail_no_new_privs(jail); - minijail_log_seccomp_filter_failures(jail); - minijail_use_seccomp_filter(jail); - minijail_parse_seccomp_filters(jail, kSeccompFilePath); - minijail_enter(jail); - minijail_destroy(jail); - - LOG(DEBUG) << "minijail installed."; - - return true; -} - -} diff --git a/src/prefetcher/minijail.h b/src/prefetcher/minijail.h deleted file mode 100644 index 9caefe6..0000000 --- a/src/prefetcher/minijail.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef SRC_PREFETCHER_MINIJAIL_H -#define SRC_PREFETCHER_MINIJAIL_H - -namespace iorap::prefetcher { -// Install a minijail using seccomp_policy/prefetcherd.{ARCH}.policy file. -bool MiniJail(); -} - -#endif // SRC_PREFETCHER_MINIJAIL_H diff --git a/src/prefetcher/prefetcher_daemon.cc b/src/prefetcher/prefetcher_daemon.cc deleted file mode 100644 index f4b9087..0000000 --- a/src/prefetcher/prefetcher_daemon.cc +++ /dev/null @@ -1,1367 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "prefetcher/minijail.h" -#include "common/cmd_utils.h" -#include "prefetcher/prefetcher_daemon.h" -#include "prefetcher/session_manager.h" -#include "prefetcher/session.h" - -#include <android-base/logging.h> -#include <android-base/properties.h> - -#include <deque> -#include <iomanip> -#include <string> -#include <sstream> -#include <vector> - -#include <fcntl.h> -#include <string.h> -#include <sys/socket.h> -#include <sys/types.h> -#include <sys/wait.h> -#include <sys/un.h> -#include <unistd.h> - -namespace iorap::prefetcher { - -// Gate super-spammy IPC logging behind a property. -// This is beyond merely annoying, enabling this logging causes prefetching to be about 1000x slower. -static bool LogVerboseIpc() { - static bool initialized = false; - static bool verbose_ipc; - - if (initialized == false) { - initialized = true; - - verbose_ipc = - ::android::base::GetBoolProperty("iorapd.readahead.verbose_ipc", /*default*/false); - } - - return verbose_ipc; -} - -static const bool kInstallMiniJail = - ::android::base::GetBoolProperty("iorapd.readahead.minijail", /*default*/true); - -static constexpr const char kCommandFileName[] = "/system/bin/iorap.prefetcherd"; - -static constexpr size_t kPipeBufferSize = 1024 * 1024; // matches /proc/sys/fs/pipe-max-size - -using ArgString = const char*; - -std::ostream& operator<<(std::ostream& os, ReadAheadKind ps) { - switch (ps) { - case ReadAheadKind::kFadvise: - os << "fadvise"; - break; - case ReadAheadKind::kMmapLocked: - os << "mmap"; - break; - case ReadAheadKind::kMlock: - os << "mlock"; - break; - default: - os << "<invalid>"; - } - return os; -} - -std::ostream& operator<<(std::ostream& os, CommandChoice choice) { - switch (choice) { - case CommandChoice::kRegisterFilePath: - os << "kRegisterFilePath"; - break; - case CommandChoice::kUnregisterFilePath: - os << "kUnregisterFilePath"; - break; - case CommandChoice::kReadAhead: - os << "kReadAhead"; - break; - case CommandChoice::kExit: - os << "kExit"; - break; - case CommandChoice::kCreateSession: - os << "kCreateSession"; - break; - case CommandChoice::kDestroySession: - os << "kDestroySession"; - break; - case CommandChoice::kDumpSession: - os << "kDumpSession"; - break; - case CommandChoice::kDumpEverything: - os << "kDumpEverything"; - break; - case CommandChoice::kCreateFdSession: - os << "kCreateFdSession"; - break; - default: - CHECK(false) << "forgot to handle this choice"; - break; - } - return os; -} - -std::ostream& operator<<(std::ostream& os, const Command& command) { - os << "Command{"; - os << "choice=" << command.choice << ","; - - bool has_session_id = true; - bool has_id = true; - switch (command.choice) { - case CommandChoice::kDumpEverything: - case CommandChoice::kExit: - has_session_id = false; - FALLTHROUGH_INTENDED; - case CommandChoice::kCreateFdSession: - case CommandChoice::kCreateSession: - case CommandChoice::kDestroySession: - case CommandChoice::kDumpSession: - has_id = false; - break; - default: - break; - } - - if (has_session_id) { - os << "sid=" << command.session_id << ","; - } - - if (has_id) { - os << "id=" << command.id << ","; - } - - switch (command.choice) { - case CommandChoice::kRegisterFilePath: - os << "file_path="; - - if (command.file_path) { - os << *(command.file_path); - } else { - os << "(nullopt)"; - } - break; - case CommandChoice::kUnregisterFilePath: - break; - case CommandChoice::kReadAhead: - os << "read_ahead_kind=" << command.read_ahead_kind << ","; - os << "length=" << command.length << ","; - os << "offset=" << command.offset << ","; - break; - case CommandChoice::kExit: - break; - case CommandChoice::kCreateFdSession: - os << "fd="; - if (command.fd.has_value()) { - os << command.fd.value(); - } else { - os << "(nullopt)"; - } - os << ","; - FALLTHROUGH_INTENDED; - case CommandChoice::kCreateSession: - os << "description="; - if (command.file_path) { - os << "'" << *(command.file_path) << "'"; - } else { - os << "(nullopt)"; - } - break; - case CommandChoice::kDestroySession: - break; - case CommandChoice::kDumpSession: - break; - case CommandChoice::kDumpEverything: - break; - default: - CHECK(false) << "forgot to handle this choice"; - break; - } - - os << "}"; - - return os; -} - -template <typename T> -struct ParseResult { - T value; - char* next_token; - size_t stream_size; - - ParseResult() : value{}, next_token{nullptr}, stream_size{} { - } - - constexpr operator bool() const { - return next_token != nullptr; - } -}; - -// Very spammy: Keep it off by default. Set to true if changing this code. -static constexpr bool kDebugParsingRead = false; - -#define DEBUG_PREAD if (kDebugParsingRead) LOG(VERBOSE) << "ParsingRead " - - - -// Parse a strong type T from a buffer stream. -// If there's insufficient space left to parse the value, an empty ParseResult is returned. -template <typename T> -ParseResult<T> ParsingRead(char* stream, size_t stream_size) { - if (stream == nullptr) { - DEBUG_PREAD << "stream was null"; - return {}; - } - - if constexpr (std::is_same_v<T, std::string>) { - ParseResult<uint32_t> length = ParsingRead<uint32_t>(stream, stream_size); - - if (!length) { - DEBUG_PREAD << "could not find length"; - // Not enough bytes left? - return {}; - } - - ParseResult<std::string> string_result; - string_result.value.reserve(length); - - stream = length.next_token; - stream_size = length.stream_size; - - for (size_t i = 0; i < length.value; ++i) { - ParseResult<char> char_result = ParsingRead<char>(stream, stream_size); - - stream = char_result.next_token; - stream_size = char_result.stream_size; - - if (!char_result) { - DEBUG_PREAD << "too few chars in stream, expected length: " << length.value; - // Not enough bytes left? - return {}; - } - - string_result.value += char_result.value; - - DEBUG_PREAD << "string preliminary is : " << string_result.value; - } - - DEBUG_PREAD << "parsed string to: " << string_result.value; - string_result.next_token = stream; - return string_result; - } else { - if (sizeof(T) > stream_size) { - return {}; - } - - ParseResult<T> result; - result.next_token = stream + sizeof(T); - result.stream_size = stream_size - sizeof(T); - - memcpy(&result.value, stream, sizeof(T)); - - return result; - } -} - -// Convenience overload to chain multiple ParsingRead together. -template <typename T, typename U> -ParseResult<T> ParsingRead(ParseResult<U> result) { - return ParsingRead<T>(result.next_token, result.stream_size); -} - -class CommandParser { - public: - CommandParser(PrefetcherForkParameters params) { - params_ = params; - } - - std::vector<Command> ParseSocketCommands(bool& eof) { - eof = false; - - std::vector<Command> commands_vec; - - std::vector<char> buf_vector; - buf_vector.resize(1024*1024); // 1MB. - char* buf = &buf_vector[0]; - - // Binary only parsing. The higher level code can parse text - // with ifstream if it really wants to. - char* stream = &buf[0]; - size_t stream_size = buf_vector.size(); - - while (true) { - if (stream_size == 0) { - // TODO: reply with an overflow command. - LOG(WARNING) << "prefetcher_daemon command overflow, dropping all commands."; - stream = &buf[0]; - stream_size = buf_vector.size(); - memset(&buf[0], /*c*/0, buf_vector.size()); - } - - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon block recvmsg for commands (fd=" << params_.input_fd << ")"; - } - - ssize_t count; - struct msghdr hdr; - memset(&hdr, 0, sizeof(hdr)); - - { - union { - struct cmsghdr cmh; - char control[CMSG_SPACE(sizeof(int))]; - } control_un; - memset(&control_un, 0, sizeof(control_un)); - - /* Set 'control_un' to describe ancillary data that we want to receive */ - control_un.cmh.cmsg_len = CMSG_LEN(sizeof(int)); /* fd is sizeof(int) */ - control_un.cmh.cmsg_level = SOL_SOCKET; - control_un.cmh.cmsg_type = SCM_CREDENTIALS; - - // the regular message data will be read into stream - struct iovec iov; - memset(&iov, 0, sizeof(iov)); - iov.iov_base = stream; - iov.iov_len = stream_size; - - /* Set hdr fields to describe 'control_un' */ - hdr.msg_control = control_un.control; - hdr.msg_controllen = sizeof(control_un.control); - hdr.msg_iov = &iov; - hdr.msg_iovlen = 1; - hdr.msg_name = nullptr; /* no peer address */ - hdr.msg_namelen = 0; - - count = TEMP_FAILURE_RETRY(recvmsg(params_.input_fd, &hdr, /*flags*/0)); - } - - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon recvmsg " << count << " for stream size:" << stream_size; - } - - if (count < 0) { - PLOG(ERROR) << "failed to recvmsg from input fd"; - break; - // TODO: let the daemon be restarted by higher level code? - } else if (count == 0) { - LOG(WARNING) << "prefetcher_daemon input_fd end-of-file; terminating"; - eof = true; - break; - // TODO: let the daemon be restarted by higher level code? - } - - { - /* Extract fd from ancillary data if present */ - struct cmsghdr* hp; - hp = CMSG_FIRSTHDR(&hdr); - if (hp && - // FIXME: hp->cmsg_len returns an absurdly large value. is it overflowing? - // (hp->cmsg_len == CMSG_LEN(sizeof(int))) && - (hp->cmsg_level == SOL_SOCKET) && - (hp->cmsg_type == SCM_RIGHTS)) { - - int passed_fd = *(int*) CMSG_DATA(hp); - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon received FD " << passed_fd; - } - - // tack the FD into our dequeue. - // we assume the FDs are sent in-order same as the regular iov are sent in-order. - longbuf_fds_.insert(longbuf_fds_.end(), passed_fd); - } else if (hp != nullptr) { - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon::read got CMSG but it wasn't matching SCM_RIGHTS," - << "cmsg_len=" << hp->cmsg_len << "," - << "cmsg_level=" << hp->cmsg_level << "," - << "cmsg_type=" << hp->cmsg_type; - } - } - } - - longbuf_.insert(longbuf_.end(), stream, stream + count); - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon updated longbuf size: " << longbuf_.size(); - } - - // reconstruct a stream of [iov_Command chdr_fd?]* back into [Command]* - { - if (longbuf_.size() == 0) { - break; - } - - std::vector<char> v(longbuf_.begin(), - longbuf_.end()); - - std::vector<int> v_fds{longbuf_fds_.begin(), longbuf_fds_.end()}; - - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon longbuf_ size: " << v.size(); - if (WOULD_LOG(VERBOSE)) { - std::stringstream dump; - dump << std::hex << std::setfill('0'); - for (size_t i = 0; i < v.size(); ++i) { - dump << std::setw(2) << static_cast<unsigned>(v[i]); - } - - LOG(VERBOSE) << "PrefetcherDaemon longbuf_ dump: " << dump.str(); - } - LOG(VERBOSE) << "PrefetcherDaemon longbuf_fds_ size: " << v_fds.size(); - if (WOULD_LOG(VERBOSE)) { - std::stringstream dump; - for (size_t i = 0; i < v_fds.size(); ++i) { - dump << v_fds[i] << ", "; - } - - LOG(VERBOSE) << "PrefetcherDaemon longbuf_fds_ dump: " << dump.str(); - } - - } - - size_t v_fds_off = 0; - size_t consumed_fds_total = 0; - - size_t v_off = 0; - size_t consumed_bytes = std::numeric_limits<size_t>::max(); - size_t consumed_total = 0; - - while (true) { - std::optional<Command> maybe_command; - maybe_command = Command::Read(&v[v_off], v.size() - v_off, &consumed_bytes); - consumed_total += consumed_bytes; - // Normal every time we get to the end of a buffer. - if (!maybe_command) { - if (LogVerboseIpc()) { - LOG(VERBOSE) << "failed to read command, v_off=" << v_off << ",v_size:" << v.size(); - } - break; - } - - if (maybe_command->RequiresFd()) { - if (v_fds_off < v_fds.size()) { - maybe_command->fd = v_fds[v_fds_off++]; - consumed_fds_total++; - if (LogVerboseIpc()) { - LOG(VERBOSE) << "Append the FD to " << *maybe_command; - } - } else { - LOG(WARNING) << "Failed to acquire FD for " << *maybe_command; - } - } - - // in the next pass ignore what we already consumed. - v_off += consumed_bytes; - - // true as long we don't hit the 'break' above. - DCHECK_EQ(v_off, consumed_total); - if (LogVerboseIpc()) { - LOG(VERBOSE) << "success to read command, v_off=" << v_off << ",v_size:" << v.size() - << "," << *maybe_command; - - // Pretty-print a single command for debugging/testing. - LOG(VERBOSE) << *maybe_command; - } - - // add to the commands we parsed. - commands_vec.push_back(*maybe_command); - } - - // erase however many were consumed - longbuf_.erase(longbuf_.begin(), longbuf_.begin() + consumed_total); - - // erase however many FDs were consumed. - longbuf_fds_.erase(longbuf_fds_.begin(), longbuf_fds_.begin() + consumed_fds_total); - } - break; - } - - return commands_vec; - } - - std::vector<Command> ParseCommands(bool& eof) { - eof = false; - - std::vector<Command> commands_vec; - - std::vector<char> buf_vector; - buf_vector.resize(kPipeBufferSize); - char* buf = &buf_vector[0]; - - // Binary only parsing. The higher level code can parse text - // with ifstream if it really wants to. - char* stream = &buf[0]; - size_t stream_size = buf_vector.size(); - - while (true) { - if (stream_size == 0) { - // TODO: reply with an overflow command. - LOG(WARNING) << "prefetcher_daemon command overflow, dropping all commands."; - stream = &buf[0]; - stream_size = buf_vector.size(); - memset(&buf[0], /*c*/0, buf_vector.size()); - } - - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon block read for commands (fd=" << params_.input_fd << ")"; - } - ssize_t count = TEMP_FAILURE_RETRY(read(params_.input_fd, stream, stream_size)); - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon::read " << count << " for stream size:" << stream_size; - } - - if (count < 0) { - PLOG(ERROR) << "failed to read from input fd"; - break; - // TODO: let the daemon be restarted by higher level code? - } else if (count == 0) { - LOG(WARNING) << "prefetcher_daemon input_fd end-of-file; terminating"; - eof = true; - break; - // TODO: let the daemon be restarted by higher level code? - } - - longbuf_.insert(longbuf_.end(), stream, stream + count); - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon updated longbuf size: " << longbuf_.size(); - } - - std::optional<Command> maybe_command; - { - if (longbuf_.size() == 0) { - break; - } - - std::vector<char> v(longbuf_.begin(), - longbuf_.end()); - - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon longbuf_ size: " << v.size(); - if (WOULD_LOG(VERBOSE)) { - std::stringstream dump; - dump << std::hex << std::setfill('0'); - for (size_t i = 0; i < v.size(); ++i) { - dump << std::setw(2) << static_cast<unsigned>(v[i]); - } - - LOG(VERBOSE) << "PrefetcherDaemon longbuf_ dump: " << dump.str(); - } - } - - size_t v_off = 0; - size_t consumed_bytes = std::numeric_limits<size_t>::max(); - size_t consumed_total = 0; - - while (true) { - maybe_command = Command::Read(&v[v_off], v.size() - v_off, &consumed_bytes); - consumed_total += consumed_bytes; - // Normal every time we get to the end of a buffer. - if (!maybe_command) { - if (LogVerboseIpc()) { - LOG(VERBOSE) << "failed to read command, v_off=" << v_off << ",v_size:" << v.size(); - } - break; - } - - // in the next pass ignore what we already consumed. - v_off += consumed_bytes; - - // true as long we don't hit the 'break' above. - DCHECK_EQ(v_off, consumed_total); - if (LogVerboseIpc()) { - LOG(VERBOSE) << "success to read command, v_off=" << v_off << ",v_size:" << v.size() - << "," << *maybe_command; - - // Pretty-print a single command for debugging/testing. - LOG(VERBOSE) << *maybe_command; - } - - // add to the commands we parsed. - commands_vec.push_back(*maybe_command); - } - - // erase however many were consumed - longbuf_.erase(longbuf_.begin(), longbuf_.begin() + consumed_total); - } - break; - } - - return commands_vec; - } - - private: - bool IsTextMode() const { - return params_.format_text; - } - - PrefetcherForkParameters params_; - - // A buffer long enough to contain a lot of buffers. - // This handles reads that only contain a partial command. - std::deque<char> longbuf_; - - // File descriptor buffers. - std::deque<int> longbuf_fds_; -}; - -static constexpr bool kDebugCommandRead = true; - -#define DEBUG_READ if (kDebugCommandRead) LOG(VERBOSE) << "Command::Read " - -std::optional<Command> Command::Read(char* buf, size_t buf_size, /*out*/size_t* consumed_bytes) { - *consumed_bytes = 0; - if (buf == nullptr) { - return std::nullopt; - } - - Command cmd{}; // zero-initialize any unused fields - ParseResult<CommandChoice> parsed_choice = ParsingRead<CommandChoice>(buf, buf_size); - cmd.choice = parsed_choice.value; - - if (!parsed_choice) { - DEBUG_READ << "no choice"; - return std::nullopt; - } - - switch (parsed_choice.value) { - case CommandChoice::kRegisterFilePath: { - ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); - if (!parsed_session_id) { - DEBUG_READ << "no parsed session id"; - return std::nullopt; - } - - ParseResult<uint32_t> parsed_id = ParsingRead<uint32_t>(parsed_session_id); - if (!parsed_id) { - DEBUG_READ << "no parsed id"; - return std::nullopt; - } - - ParseResult<std::string> parsed_file_path = ParsingRead<std::string>(parsed_id); - - if (!parsed_file_path) { - DEBUG_READ << "no file path"; - return std::nullopt; - } - *consumed_bytes = parsed_file_path.next_token - buf; - - cmd.session_id = parsed_session_id.value; - cmd.id = parsed_id.value; - cmd.file_path = parsed_file_path.value; - - break; - } - case CommandChoice::kUnregisterFilePath: { - ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); - if (!parsed_session_id) { - DEBUG_READ << "no parsed session id"; - return std::nullopt; - } - - ParseResult<uint32_t> parsed_id = ParsingRead<uint32_t>(parsed_session_id); - if (!parsed_id) { - DEBUG_READ << "no parsed id"; - return std::nullopt; - } - *consumed_bytes = parsed_id.next_token - buf; - - cmd.session_id = parsed_session_id.value; - cmd.id = parsed_id.value; - - break; - } - case CommandChoice::kReadAhead: { - ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); - if (!parsed_session_id) { - DEBUG_READ << "no parsed session id"; - return std::nullopt; - } - - ParseResult<uint32_t> parsed_id = ParsingRead<uint32_t>(parsed_session_id); - if (!parsed_id) { - DEBUG_READ << "no parsed id"; - return std::nullopt; - } - - ParseResult<ReadAheadKind> parsed_kind = ParsingRead<ReadAheadKind>(parsed_id); - if (!parsed_kind) { - DEBUG_READ << "no parsed kind"; - return std::nullopt; - } - ParseResult<uint64_t> parsed_length = ParsingRead<uint64_t>(parsed_kind); - if (!parsed_length) { - DEBUG_READ << "no parsed length"; - return std::nullopt; - } - ParseResult<uint64_t> parsed_offset = ParsingRead<uint64_t>(parsed_length); - if (!parsed_offset) { - DEBUG_READ << "no parsed offset"; - return std::nullopt; - } - *consumed_bytes = parsed_offset.next_token - buf; - - cmd.session_id = parsed_session_id.value; - cmd.id = parsed_id.value; - cmd.read_ahead_kind = parsed_kind.value; - cmd.length = parsed_length.value; - cmd.offset = parsed_offset.value; - - break; - } - case CommandChoice::kCreateSession: - case CommandChoice::kCreateFdSession: { - ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); - if (!parsed_session_id) { - DEBUG_READ << "no parsed session id"; - return std::nullopt; - } - - ParseResult<std::string> parsed_description = ParsingRead<std::string>(parsed_session_id); - - if (!parsed_description) { - DEBUG_READ << "no description"; - return std::nullopt; - } - *consumed_bytes = parsed_description.next_token - buf; - - cmd.session_id = parsed_session_id.value; - cmd.file_path = parsed_description.value; - - break; - } - case CommandChoice::kDestroySession: - case CommandChoice::kDumpSession: { - ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); - if (!parsed_session_id) { - DEBUG_READ << "no parsed session id"; - return std::nullopt; - } - - *consumed_bytes = parsed_session_id.next_token - buf; - - cmd.session_id = parsed_session_id.value; - - break; - } - case CommandChoice::kExit: - case CommandChoice::kDumpEverything: - *consumed_bytes = parsed_choice.next_token - buf; - // Only need to parse the choice. - break; - default: - LOG(FATAL) << "unrecognized command number " << static_cast<uint32_t>(parsed_choice.value); - break; - } - - return cmd; -} - -bool Command::Write(char* buf, size_t buf_size, /*out*/size_t* produced_bytes) const { - *produced_bytes = 0; - if (buf == nullptr) { - LOG(WARNING) << "null buf, is this expected?"; - return false; - } - - bool has_enough_space = false; - size_t space_requirement = std::numeric_limits<size_t>::max(); - - space_requirement = sizeof(choice); - - switch (choice) { - case CommandChoice::kRegisterFilePath: - space_requirement += sizeof(session_id); - space_requirement += sizeof(id); - space_requirement += sizeof(uint32_t); // string length - - if (!file_path) { - LOG(WARNING) << "Missing file path for kRegisterFilePath"; - return false; - } - - space_requirement += file_path->size(); // string contents - break; - case CommandChoice::kUnregisterFilePath: - space_requirement += sizeof(session_id); - space_requirement += sizeof(id); - break; - case CommandChoice::kReadAhead: - space_requirement += sizeof(session_id); - space_requirement += sizeof(id); - space_requirement += sizeof(read_ahead_kind); - space_requirement += sizeof(length); - space_requirement += sizeof(offset); - break; - case CommandChoice::kCreateSession: - case CommandChoice::kCreateFdSession: - space_requirement += sizeof(session_id); - space_requirement += sizeof(uint32_t); // string length - - if (!file_path) { - LOG(WARNING) << "Missing file path for kCreateSession"; - return false; - } - - space_requirement += file_path->size(); // string contents - break; - case CommandChoice::kDestroySession: - case CommandChoice::kDumpSession: - space_requirement += sizeof(session_id); - break; - case CommandChoice::kExit: - case CommandChoice::kDumpEverything: - // Only need space for the choice. - break; - default: - LOG(FATAL) << "unrecognized command number " << static_cast<uint32_t>(choice); - break; - } - - if (buf_size < space_requirement) { - return false; - } - - *produced_bytes = space_requirement; - - // Always write out the choice. - size_t buf_offset = 0; - - memcpy(&buf[buf_offset], &choice, sizeof(choice)); - buf_offset += sizeof(choice); - - switch (choice) { - case CommandChoice::kRegisterFilePath: - memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); - buf_offset += sizeof(session_id); - memcpy(&buf[buf_offset], &id, sizeof(id)); - buf_offset += sizeof(id); - - { - uint32_t string_length = static_cast<uint32_t>(file_path->size()); - memcpy(&buf[buf_offset], &string_length, sizeof(string_length)); - buf_offset += sizeof(string_length); - } - - DCHECK(file_path.has_value()); - - memcpy(&buf[buf_offset], file_path->c_str(), file_path->size()); - buf_offset += file_path->size(); - break; - case CommandChoice::kUnregisterFilePath: - memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); - buf_offset += sizeof(session_id); - memcpy(&buf[buf_offset], &id, sizeof(id)); - buf_offset += sizeof(id); - break; - case CommandChoice::kReadAhead: - memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); - buf_offset += sizeof(session_id); - memcpy(&buf[buf_offset], &id, sizeof(id)); - buf_offset += sizeof(id); - memcpy(&buf[buf_offset], &read_ahead_kind, sizeof(read_ahead_kind)); - buf_offset += sizeof(read_ahead_kind); - memcpy(&buf[buf_offset], &length, sizeof(length)); - buf_offset += sizeof(length); - memcpy(&buf[buf_offset], &offset, sizeof(offset)); - buf_offset += sizeof(offset); - break; - case CommandChoice::kCreateSession: - case CommandChoice::kCreateFdSession: - memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); - buf_offset += sizeof(session_id); - - { - uint32_t string_length = static_cast<uint32_t>(file_path->size()); - memcpy(&buf[buf_offset], &string_length, sizeof(string_length)); - buf_offset += sizeof(string_length); - } - - DCHECK(file_path.has_value()); - - memcpy(&buf[buf_offset], file_path->c_str(), file_path->size()); - buf_offset += file_path->size(); - - DCHECK_EQ(buf_offset, space_requirement) << *this << ",file_path_size:" << file_path->size(); - DCHECK_EQ(buf_offset, *produced_bytes) << *this; - - break; - case CommandChoice::kDestroySession: - case CommandChoice::kDumpSession: - memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); - buf_offset += sizeof(session_id); - break; - case CommandChoice::kExit: - case CommandChoice::kDumpEverything: - // Only need to write out the choice. - break; - default: - LOG(FATAL) << "should have fallen out in the above switch" - << static_cast<uint32_t>(choice); - break; - } - - DCHECK_EQ(buf_offset, space_requirement) << *this; - DCHECK_EQ(buf_offset, *produced_bytes) << *this; - - return true; -} - -class PrefetcherDaemon::Impl { - public: - std::optional<PrefetcherForkParameters> StartPipesViaFork() { - int pipefds[2]; - if (pipe(&pipefds[0]) != 0) { - PLOG(FATAL) << "Failed to create read/write pipes"; - } - - if (WOULD_LOG(VERBOSE)) { - long pipe_size = static_cast<long>(fcntl(pipefds[0], F_GETPIPE_SZ)); - if (pipe_size < 0) { - PLOG(ERROR) << "Failed to F_GETPIPE_SZ:"; - } - LOG(VERBOSE) << "StartPipesViaFork: default pipe size: " << pipe_size; - } - - for (int i = 0; i < 2; ++i) { - // Default pipe size is usually 64KB. - // Increase to 1MB so that iorapd has to rarely run during prefetching. - if (fcntl(pipefds[i], F_SETPIPE_SZ, kPipeBufferSize) < 0) { - PLOG(FATAL) << "Failed to increase pipe size to max"; - } - } - - pipefd_read_ = pipefds[0]; - pipefd_write_ = pipefds[1]; - - PrefetcherForkParameters params; - params.input_fd = pipefd_read_; - params.output_fd = pipefd_write_; - params.format_text = false; - params.use_sockets = false; - - bool res = StartViaFork(params); - if (res) { - return params; - } else { - return std::nullopt; - } - } - -std::optional<PrefetcherForkParameters> StartSocketViaFork() { - int socket_fds[2]; - if (socketpair(AF_UNIX, SOCK_STREAM, /*protocol*/0, &socket_fds[0]) != 0) { - PLOG(FATAL) << "Failed to create read/write socketpair"; - } - - pipefd_read_ = socket_fds[0]; // iorapd writer, iorap.prefetcherd reader - pipefd_write_ = socket_fds[1]; // iorapd reader, iorap.prefetcherd writer - - PrefetcherForkParameters params; - params.input_fd = pipefd_read_; - params.output_fd = pipefd_write_; - params.format_text = false; - params.use_sockets = true; - - bool res = StartViaFork(params); - if (res) { - return params; - } else { - return std::nullopt; - } - } - - bool StartViaFork(PrefetcherForkParameters params) { - params_ = params; - - forked_ = true; - child_ = fork(); - - if (child_ == -1) { - LOG(FATAL) << "Failed to fork PrefetcherDaemon"; - } else if (child_ > 0) { // we are the caller of this function - LOG(DEBUG) << "forked into iorap.prefetcherd, pid = " << child_; - - return true; - } else { - // we are the child that was forked. - std::stringstream argv; // for logging - std::vector<std::string> argv_vec; - - { - std::stringstream s; - s << "--input-fd"; - argv_vec.push_back(s.str()); - - std::stringstream s2; - s2 << params.input_fd; - argv_vec.push_back(s2.str()); - - argv << " --input-fd" << " " << params.input_fd; - } - - { - std::stringstream s; - s << "--output-fd"; - argv_vec.push_back(s.str()); - - std::stringstream s2; - s2 << params.output_fd; - argv_vec.push_back(s2.str()); - - argv << " --output-fd" << " " << params.output_fd; - } - - - if (params.use_sockets) { - std::stringstream s; - s << "--use-sockets"; - argv_vec.push_back(s.str()); - - argv << " --use-sockets"; - } - - if (WOULD_LOG(VERBOSE)) { - std::stringstream s; - s << "--verbose"; - argv_vec.push_back(s.str()); - - argv << " --verbose"; - } - - std::unique_ptr<ArgString[]> argv_ptr = common::VecToArgv(kCommandFileName, argv_vec); - - LOG(DEBUG) << "fork+exec: " << kCommandFileName << " " - << argv.str(); - execve(kCommandFileName, (char **)argv_ptr.get(), /*envp*/nullptr); - // This should never return. - _exit(EXIT_FAILURE); - } - - DCHECK(false); - return false; - } - - // TODO: Not very useful since this can never return 'true' - // -> in the child we would've already execd which loses all this code. - bool IsDaemon() { - // In the child the pid is always 0. - return child_ > 0; - } - - bool Main(PrefetcherForkParameters params) { - LOG(VERBOSE) << "PrefetcherDaemon::Main " << params; - - CommandParser command_parser{params}; - - Command next_command{}; - - std::vector<Command> many_commands; - - // Ensure alogd is pre-initialized before installing minijail. - LOG(DEBUG) << "Installing minijail"; - - // Install seccomp filter using libminijail. - if (kInstallMiniJail) { - MiniJail(); - } - - while (true) { - bool eof = false; - - if (params.use_sockets) { - // use recvmsg(2). supports receiving FDs. - many_commands = command_parser.ParseSocketCommands(/*out*/eof); - } else { - // use read(2). does not support receiving FDs. - many_commands = command_parser.ParseCommands(/*out*/eof); - } - - if (eof) { - LOG(WARNING) << "PrefetcherDaemon got EOF, terminating"; - return true; - } - - for (auto& command : many_commands) { - if (LogVerboseIpc()) { - LOG(VERBOSE) << "PrefetcherDaemon got command: " << command; - } - - if (command.choice == CommandChoice::kExit) { - LOG(DEBUG) << "PrefetcherDaemon got kExit command, terminating"; - return true; - } - - if (!ReceiveCommand(command)) { - // LOG(WARNING) << "PrefetcherDaemon command processing failure: " << command; - } - - // ReceiveCommand should dup to keep the FD. Avoid leaks. - if (command.fd.has_value()) { - close(*command.fd); - } - } - } - - LOG(VERBOSE) << "PrefetcherDaemon::Main got exit, terminating"; - - return true; - // Terminate. - } - - Impl(PrefetcherDaemon* daemon) { - session_manager_ = SessionManager::CreateManager(SessionKind::kInProcessDirect); - DCHECK(session_manager_ != nullptr); - }; - - ~Impl() { - // Don't do anything if we never called 'StartViaFork' - if (forked_) { - if (!IsDaemon()) { - int status; - waitpid(child_, /*out*/&status, /*options*/0); - } else { - LOG(WARNING) << "execve should have avoided this path"; - // DCHECK(false) << "not possible because the execve would avoid this path"; - } - } - } - - bool SendCommand(const Command& command) { - // Only parent is the sender. - DCHECK(forked_); - //DCHECK(!IsDaemon()); - - char buf[1024]; - size_t stream_size; - if (!command.Write(buf, sizeof(buf), /*out*/&stream_size)) { - PLOG(ERROR) << "Failed to serialize command: " << command; - return false; - } - - if (LogVerboseIpc()) { - LOG(VERBOSE) << "pre-write(fd=" << pipefd_write_ << ", buf=" << buf - << ", size=" << stream_size<< ")"; - } - - if (params_.use_sockets) { - /* iov contains the normal message (Command) */ - struct iovec iov; - memset(&iov, 0, sizeof(iov)); - iov.iov_base = &buf[0]; - iov.iov_len = stream_size; - - struct msghdr msg; - memset(&msg, 0, sizeof(msg)); - - /* point to iov to transmit */ - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - - /* no dest address; socket is connected */ - msg.msg_name = nullptr; - msg.msg_namelen = 0; - - // append a CMSG with SCM_RIGHTS if we have an FD. - if (command.fd.has_value()) { - union { - struct cmsghdr cmh; - char control[CMSG_SPACE(sizeof(int))]; /* sized to hold an fd (int) */ - } control_un; - memset(&control_un, 0, sizeof(control_un)); - - msg.msg_control = &control_un.control[0]; - msg.msg_controllen = sizeof(control_un.control); - - struct cmsghdr *hp; - hp = CMSG_FIRSTHDR(&msg); - hp->cmsg_len = CMSG_LEN(sizeof(int)); - hp->cmsg_level = SOL_SOCKET; - hp->cmsg_type = SCM_RIGHTS; - *((int *) CMSG_DATA(hp)) = *(command.fd); - - DCHECK(command.RequiresFd()) << command; - - if (LogVerboseIpc()) { - LOG(VERBOSE) << "append FD to sendmsg: " << *(command.fd); - } - } - - // TODO: add CMSG for the FD passage. - - if (TEMP_FAILURE_RETRY(sendmsg(pipefd_write_, &msg, /*flags*/0)) < 0) { - PLOG(ERROR) << "Failed to sendmsg command: " << command; - return false; - } - } else { - if (TEMP_FAILURE_RETRY(write(pipefd_write_, buf, stream_size)) < 0) { - PLOG(ERROR) << "Failed to write command: " << command; - return false; - } - } - - if (LogVerboseIpc()) { - LOG(VERBOSE) << "write(fd=" << pipefd_write_ << ", buf=" << buf - << ", size=" << stream_size<< ")"; - } - - // TODO: also read the reply? - return true; - } - - bool ReceiveCommand(const Command& command) { - // Only child is the command receiver. - // DCHECK(IsDaemon()); - - switch (command.choice) { - case CommandChoice::kRegisterFilePath: { - std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); - - if (!session) { - LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command; - return false; - } - - CHECK(command.file_path.has_value()) << command; - return session->RegisterFilePath(command.id, *command.file_path); - } - case CommandChoice::kUnregisterFilePath: { - std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); - - if (!session) { - LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command; - return false; - } - - return session->UnregisterFilePath(command.id); - } - case CommandChoice::kReadAhead: { - std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); - - if (!session) { - LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command; - return false; - } - - return session->ReadAhead(command.id, command.read_ahead_kind, command.length, command.offset); - } - // TODO: unreadahead - case CommandChoice::kExit: { - LOG(WARNING) << "kExit should be handled earlier."; - return true; - } - case CommandChoice::kCreateSession: { - std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); - if (session != nullptr) { - LOG(ERROR) << "ReceiveCommand: session for ID already exists: " << command; - return false; - } - CHECK(command.file_path.has_value()) << command; - if (session_manager_->CreateSession(command.session_id, /*description*/*command.file_path) - == nullptr) { - LOG(ERROR) << "ReceiveCommand: Failure to kCreateSession: " << command; - return false; - } - return true; - } - case CommandChoice::kDestroySession: { - if (!session_manager_->DestroySession(command.session_id)) { - LOG(ERROR) << "ReceiveCommand: Failure to kDestroySession: " << command; - return false; - } - return true; - } - case CommandChoice::kDumpSession: { - std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); - - if (!session) { - LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command; - return false; - } - - // TODO: Consider doing dumpsys support somehow? - session->Dump(LOG_STREAM(DEBUG), /*multiline*/true); - return true; - } - case CommandChoice::kDumpEverything: { - session_manager_->Dump(LOG_STREAM(DEBUG), /*multiline*/true); - break; - } - case CommandChoice::kCreateFdSession: { - std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); - if (session != nullptr) { - LOG(ERROR) << "ReceiveCommand: session for ID already exists: " << command; - return false; - } - CHECK(command.file_path.has_value()) << command; - CHECK(command.fd.has_value()) << command; - - LOG(VERBOSE) << "ReceiveCommand: kCreateFdSession fd=" << *(command.fd); - - // TODO: Maybe use CreateFdSession instead? - session = - session_manager_->CreateSession(command.session_id, - /*description*/*command.file_path, - command.fd.value()); - if (session == nullptr) { - LOG(ERROR) << "ReceiveCommand: Failure to kCreateFdSession: " << command; - return false; - } - - return session->ProcessFd(*command.fd); - } - } - - return true; - } - - pid_t child_; - bool forked_; - int pipefd_read_; - int pipefd_write_; - PrefetcherForkParameters params_; - // do not ever use an indirect session manager here, as it would cause a lifetime cycle. - std::unique_ptr<SessionManager> session_manager_; // direct only. -}; - -PrefetcherDaemon::PrefetcherDaemon() - : impl_{new Impl{this}} { - LOG(VERBOSE) << "PrefetcherDaemon() constructor"; -} - -bool PrefetcherDaemon::StartViaFork(PrefetcherForkParameters params) { - return impl_->StartViaFork(std::move(params)); -} - - -std::optional<PrefetcherForkParameters> PrefetcherDaemon::StartPipesViaFork() { - return impl_->StartPipesViaFork(); -} - -std::optional<PrefetcherForkParameters> PrefetcherDaemon::StartSocketViaFork() { - return impl_->StartSocketViaFork(); -} - -bool PrefetcherDaemon::Main(PrefetcherForkParameters params) { - return impl_->Main(params); -} - -bool PrefetcherDaemon::SendCommand(const Command& command) { - return impl_->SendCommand(command); -} - -PrefetcherDaemon::~PrefetcherDaemon() { - // required for unique_ptr for incomplete types. -} - -} // namespace iorap::prefetcher diff --git a/src/prefetcher/prefetcher_daemon.h b/src/prefetcher/prefetcher_daemon.h deleted file mode 100644 index 693f871..0000000 --- a/src/prefetcher/prefetcher_daemon.h +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef PREFETCHER_DAEMON_H_ -#define PREFETCHER_DAEMON_H_ - -#include "prefetcher/session_manager.h" - -#include <memory> -#include <optional> -#include <ostream> - -namespace iorap { -namespace prefetcher { - -struct PrefetcherForkParameters { - int input_fd; - int output_fd; - bool use_sockets; // use the socket path instead of simpler read/write path. - bool format_text; // true=>text, false=>binary -}; - -inline std::ostream& operator<<(std::ostream& os, const PrefetcherForkParameters& p) { - os << "PrefetcherForkParameters{"; - os << "input_fd=" << p.input_fd << ","; - os << "output_fd=" << p.output_fd << ","; - os << "format_text=" << p.format_text << ","; - os << "use_sockets=" << p.use_sockets << ","; - os << "}"; - return os; -} - - -#ifndef READ_AHEAD_KIND -enum class ReadAheadKind : uint32_t { - kFadvise = 0, - kMmapLocked = 1, - kMlock = 2, -}; -#define READ_AHEAD_KIND 1 -#endif - -std::ostream& operator<<(std::ostream& os, ReadAheadKind k); - -enum class CommandChoice : uint32_t { - kRegisterFilePath, // kRegisterFilePath <sid:uint32> <id:uint32> <path:c-string> - kUnregisterFilePath, // kUnregisterFilePath <sid:uint32> <id:uint32> - kReadAhead, // kReadAhead <sid:uint32> <id:uint32> <kind:uint32_t> <length:uint64> <offset:uint64> - kExit, // kExit - kCreateSession, // kCreateSession <sid:uint32> <description:c-string> - kDestroySession, // kDestroySession <sid:uint32> - kDumpSession, // kDumpSession <sid:uint32> - kDumpEverything, // kDumpEverything - kCreateFdSession, // kCreateFdSession $CMSG{<fd:int>} <sid:uint32> <description:c-string> -}; - -struct Command { - CommandChoice choice; - uint32_t session_id; - uint32_t id; // file_path_id - std::optional<std::string> file_path; // required for choice=kRegisterFilePath. - // also serves as the description for choice=kCreateSession - - // choice=kReadAhead - ReadAheadKind read_ahead_kind; - uint64_t length; - uint64_t offset; - - std::optional<int> fd; // only valid in kCreateFdSession. - - // Deserialize from a char buffer. - // This can only fail if buf_size is too small. - static std::optional<Command> Read(char* buf, size_t buf_size, /*out*/size_t* consumed_bytes); - // Serialize to a char buffer. - // This can only fail if the buf_size is too small. - bool Write(char* buf, size_t buf_size, /*out*/size_t* produced_bytes) const; - - bool RequiresFd() const { - return choice == CommandChoice::kCreateFdSession; - } -}; - -std::ostream& operator<<(std::ostream& os, const Command& command); - -class PrefetcherDaemon { - public: - PrefetcherDaemon(); - ~PrefetcherDaemon(); - - // Asynchronously launch a new fork. - // - // The destructor will waitpid automatically on the child process. - bool StartViaFork(PrefetcherForkParameters params); - - // Launch a new fork , returning the pipes as input/output fds. - std::optional<PrefetcherForkParameters> StartPipesViaFork(); - - // Launch a new fork , returning the socket pair as input/output fds. - std::optional<PrefetcherForkParameters> StartSocketViaFork(); - - // Execute the main code in-process. - // - // Intended as the execve target. - bool Main(PrefetcherForkParameters params); - - // Send a command via IPC. - // The caller must be the parent process after using StartViaFork. - bool SendCommand(const Command& command); - - private: - class Impl; - std::unique_ptr<PrefetcherDaemon::Impl> impl_; -}; - -} // namespace prefetcher -} // namespace iorap - -#endif - diff --git a/src/prefetcher/read_ahead.cc b/src/prefetcher/read_ahead.cc deleted file mode 100644 index 8759f32..0000000 --- a/src/prefetcher/read_ahead.cc +++ /dev/null @@ -1,447 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "read_ahead.h" - -#include "common/trace.h" -#include "prefetcher/session_manager.h" -#include "prefetcher/session.h" -#include "prefetcher/task_id.h" -#include "serialize/arena_ptr.h" -#include "serialize/protobuf_io.h" - -#include <android-base/chrono_utils.h> -#include <android-base/logging.h> -#include <android-base/scopeguard.h> -#include <android-base/properties.h> -#include <android-base/unique_fd.h> -#include <cutils/trace.h> -#include <deque> -#include <fcntl.h> -#include <functional> -#include <stdint.h> -#include <sys/mman.h> -#include <sys/time.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <unordered_map> -#include <utils/Printer.h> - -namespace iorap { -namespace prefetcher { - -enum class PrefetchStrategy { - kFadvise = 0, - kMmapLocked = 1, - kMlock = 2, -}; - -std::ostream& operator<<(std::ostream& os, PrefetchStrategy ps) { - switch (ps) { - case PrefetchStrategy::kFadvise: - os << "fadvise"; - break; - case PrefetchStrategy::kMmapLocked: - os << "mmap"; - break; - case PrefetchStrategy::kMlock: - os << "mlock"; - break; - default: - os << "<invalid>"; - } - return os; -} - -static constexpr PrefetchStrategy kPrefetchStrategy = PrefetchStrategy::kFadvise; -static uint64_t kMaxPrefetchBytes = ::android::base::GetUintProperty<uint64_t>( - "iorapd.max_prefetch_bytes", /*default*/100 * 1024 * 1024); // 100MB by default - -static PrefetchStrategy GetPrefetchStrategy() { - PrefetchStrategy strat = PrefetchStrategy::kFadvise; - - std::string prefetch_env = - ::android::base::GetProperty("iorapd.readahead.strategy", /*default*/""); - - if (prefetch_env == "") { - LOG(VERBOSE) - << "ReadAhead strategy defaulted. Did you want to set iorapd.readahead.strategy ?"; - } else if (prefetch_env == "mmap") { - strat = PrefetchStrategy::kMmapLocked; - LOG(VERBOSE) << "ReadAhead strategy: kMmapLocked"; - } else if (prefetch_env == "mlock") { - strat = PrefetchStrategy::kMlock; - LOG(VERBOSE) << "ReadAhead strategy: kMlock"; - } else if (prefetch_env == "fadvise") { - strat = PrefetchStrategy::kFadvise; - LOG(VERBOSE) << "ReadAhead strategy: kFadvise"; - } else { - LOG(WARNING) << "Unknown iorapd.readahead.strategy: " << prefetch_env << ", ignoring"; - } - - return strat; -} - -struct TaskData { - TaskId task_id; // also the session ID. - - size_t SessionId() const { - if (session != nullptr) { - DCHECK_EQ(session->SessionId(), task_id.id); - } - - // good enough to be used as the session ID. Task IDs are always monotonically increasing. - return task_id.id; - } - - std::shared_ptr<Session> session; - int32_t trace_cookie; // async trace cookie in BeginTask/FinishTask. -}; - -// Remember the last 5 files being prefetched. -static constexpr size_t kRecentDataCount = 5; - -struct RecentData { - TaskId task_id; - size_t file_lengths_sum; -}; - -struct RecentDataKeeper { - std::deque<RecentData> recents_; - std::mutex mutex_; - - void RecordRecent(TaskId task_id, size_t file_lengths_sum) { - std::lock_guard<std::mutex> guard{mutex_}; - - while (recents_.size() > kRecentDataCount) { - recents_.pop_front(); - } - recents_.push_back(RecentData{std::move(task_id), file_lengths_sum}); - } - - void Dump(/*borrow*/::android::Printer& printer) { - bool locked = mutex_.try_lock(); - - printer.printFormatLine("Recent prefetches:"); - if (!locked) { - printer.printLine(""""" (possible deadlock)"); - } - - for (const RecentData& data : recents_) { - printer.printFormatLine(" %s", data.task_id.path.c_str()); - printer.printFormatLine(" Task ID: %zu", data.task_id.id); - printer.printFormatLine(" Bytes count: %zu", data.file_lengths_sum); - } - - if (recents_.empty()) { - printer.printFormatLine(" (None)"); - } - - printer.printLine(""); - - if (locked) { - mutex_.unlock(); - } - } -}; - -struct ReadAhead::Impl { - Impl(bool use_sockets) { - // Flip this property to test in-process vs out-of-process for the prefetcher code. - bool out_of_process = - ::android::base::GetBoolProperty("iorapd.readahead.out_of_process", /*default*/true); - - SessionKind session_kind = - out_of_process ? SessionKind::kOutOfProcessIpc : SessionKind::kInProcessDirect; - - if (use_sockets) { - session_kind = SessionKind::kOutOfProcessSocket; - } - - session_manager_ = SessionManager::CreateManager(session_kind); - session_kind_ = session_kind; - } - - std::unique_ptr<SessionManager> session_manager_; - SessionKind session_kind_; - std::unordered_map<size_t /*task index*/, TaskData> read_ahead_file_map_; - static RecentDataKeeper recent_data_keeper_; - int32_t trace_cookie_{0}; - - bool UseSockets() const { - return session_kind_ == SessionKind::kOutOfProcessSocket; - } -}; - -RecentDataKeeper ReadAhead::Impl::recent_data_keeper_{}; - -ReadAhead::ReadAhead() : ReadAhead(/*use_sockets*/false) { -} - -ReadAhead::ReadAhead(bool use_sockets) : impl_(new Impl(use_sockets)) { -} - -ReadAhead::~ReadAhead() {} - -static bool PerformReadAhead(std::shared_ptr<Session> session, size_t path_id, ReadAheadKind kind, size_t length, size_t offset) { - return session->ReadAhead(path_id, kind, length, offset); -} - -void ReadAhead::FinishTask(const TaskId& id) { - auto it = impl_->read_ahead_file_map_.find(id.id); - if (it == impl_->read_ahead_file_map_.end()) { - LOG(DEBUG) << "Could not find any TaskData for " << id; - return; - } - - TaskData& task_data = it->second; - atrace_async_end(ATRACE_TAG_ACTIVITY_MANAGER, - "ReadAhead Task Scope (for File Descriptors)", - task_data.trace_cookie); - - auto deleter = [&]() { - impl_->read_ahead_file_map_.erase(it); - }; - auto scope_guard = android::base::make_scope_guard(deleter); - (void)scope_guard; - - LOG(VERBOSE) << "ReadAhead (Finish)"; - - if (!impl_->session_manager_->DestroySession(task_data.SessionId())) { - LOG(WARNING) << "ReadAhead: Failed to destroy Session " << task_data.SessionId(); - } -} - -void ReadAhead::BeginTaskForSockets(const TaskId& id, int32_t trace_cookie) { - LOG(VERBOSE) << "BeginTaskForSockets: " << id; - - // TODO: atrace. - android::base::Timer timer{}; - android::base::Timer open_timer{}; - - int trace_file_fd_raw = - TEMP_FAILURE_RETRY(open(id.path.c_str(), /*flags*/O_RDONLY)); - - android::base::unique_fd trace_file_fd{trace_file_fd_raw}; - - if (!trace_file_fd.ok()) { - PLOG(ERROR) << "ReadAhead failed to open trace file: " << id.path; - return; - } - - TaskData task_data; - task_data.task_id = id; - task_data.trace_cookie = trace_cookie; - - std::shared_ptr<Session> session = - impl_->session_manager_->CreateSession(task_data.SessionId(), - /*description*/id.path, - trace_file_fd.get()); - task_data.session = session; - CHECK(session != nullptr); - - task_data.trace_cookie = ++trace_cookie; - - // TODO: maybe getprop and a single line by default? - session->Dump(LOG_STREAM(INFO), /*multiline*/true); - - impl_->read_ahead_file_map_[id.id] = std::move(task_data); - // FinishTask is identical, as it just destroys the session. -} - -void ReadAhead::BeginTask(const TaskId& id) { - { - struct timeval now; - gettimeofday(&now, nullptr); - - uint64_t now_usec = (now.tv_sec * 1000000LL + now.tv_usec); - LOG(DEBUG) << "BeginTask: beginning usec: " << now_usec; - } - - int32_t trace_cookie = ++impl_->trace_cookie_; - atrace_async_begin(ATRACE_TAG_ACTIVITY_MANAGER, - "ReadAhead Task Scope (for File Descriptors)", - trace_cookie); - - if (impl_->UseSockets()) { - BeginTaskForSockets(id, trace_cookie); - return; - } - - LOG(VERBOSE) << "BeginTask: " << id; - - // TODO: atrace. - android::base::Timer timer{}; - - // TODO: refactor this code with SessionDirect::ProcessFd ? - TaskData task_data; - task_data.task_id = id; - task_data.trace_cookie = trace_cookie; - - ScopedFormatTrace atrace_begin_task(ATRACE_TAG_ACTIVITY_MANAGER, - "ReadAhead::BeginTask %s", - id.path.c_str()); - - // Include CreateSession above the Protobuf deserialization so that we can include - // the 'total_duration' as part of the Session dump (relevant when we use IPC mode only). - std::shared_ptr<Session> session = - impl_->session_manager_->CreateSession(task_data.SessionId(), - /*description*/id.path); - - android::base::Timer open_timer{}; - - // XX: Should we rename all the 'Create' to 'Make', or rename the 'Make' to 'Create' ? - // Unfortunately make_unique, make_shared, etc is the standard C++ terminology. - serialize::ArenaPtr<serialize::proto::TraceFile> trace_file_ptr = - serialize::ProtobufIO::Open(id.path); - - if (trace_file_ptr == nullptr) { - // TODO: distinguish between missing trace (this is OK, most apps wont have one) - // and a bad error. - LOG(DEBUG) << "ReadAhead could not start, missing trace file? " << id.path; - return; - } - - task_data.session = session; - CHECK(session != nullptr); - - ReadAheadKind kind = static_cast<ReadAheadKind>(GetPrefetchStrategy()); - - // TODO: The "Task[Id]" should probably be the one owning the trace file. - // When the task is fully complete, the task can be deleted and the - // associated arenas can go with them. - - // TODO: we should probably have the file entries all be relative - // to the package path? - - // Open every file in the trace index. - const serialize::proto::TraceFileIndex& index = trace_file_ptr->index(); - - size_t count_entries = 0; - { - ScopedFormatTrace atrace_register_file_paths(ATRACE_TAG_ACTIVITY_MANAGER, - "ReadAhead::RegisterFilePaths %s", - id.path.c_str()); - for (const serialize::proto::TraceFileIndexEntry& index_entry : index.entries()) { - LOG(VERBOSE) << "ReadAhead: found file entry: " << index_entry.file_name(); - - if (index_entry.id() < 0) { - LOG(WARNING) << "ReadAhead: Skip bad TraceFileIndexEntry, negative ID not allowed: " - << index_entry.id(); - continue; - } - - size_t path_id = index_entry.id(); - const auto& path_file_name = index_entry.file_name(); - - if (!session->RegisterFilePath(path_id, path_file_name)) { - LOG(WARNING) << "ReadAhead: Failed to register file path: " << path_file_name; - } else { - ++count_entries; - } - } - } - LOG(VERBOSE) << "ReadAhead: Registered " << count_entries << " file paths"; - std::chrono::milliseconds open_duration_ms = open_timer.duration(); - - LOG(DEBUG) << "ReadAhead: Opened file&headers in " << open_duration_ms.count() << "ms"; - - size_t length_sum = 0; - size_t prefetch_bytes = 0; - size_t entry_offset = 0; - { - ScopedFormatTrace atrace_perform_read_ahead(ATRACE_TAG_ACTIVITY_MANAGER, - "ReadAhead::PerformReadAhead entries=%zu, path=%s", - count_entries, - id.path.c_str()); - - // Go through every trace entry and readahead every (file,offset,len) tuple. - const serialize::proto::TraceFileList& file_list = trace_file_ptr->list(); - for (const serialize::proto::TraceFileEntry& file_entry : file_list.entries()) { - ++entry_offset; - - if (file_entry.file_length() < 0 || file_entry.file_offset() < 0) { - LOG(WARNING) << "ReadAhead entry negative file length or offset, illegal: " - << "index_id=" << file_entry.index_id() << ", skipping"; - continue; - } - - // Attempt to perform readahead. This can generate more warnings dynamically. - if (!PerformReadAhead(session, file_entry.index_id(), kind, file_entry.file_length(), file_entry.file_offset())) { - // TODO: Do we need below at all? The always-on Dump already prints a % of failed entries. - // LOG(WARNING) << "Failed readahead, bad file length/offset in entry @ " << (entry_offset - 1); - } else { - prefetch_bytes += static_cast<size_t>(file_entry.file_length()); - if (prefetch_bytes >= kMaxPrefetchBytes) { - LOG(WARNING) << "The prefetching size is " - << prefetch_bytes - << " and it exceeds the threshold " - << kMaxPrefetchBytes; - break; - } - } - - length_sum += static_cast<size_t>(file_entry.file_length()); - } - } - - { - ScopedFormatTrace atrace_session_dump(ATRACE_TAG_ACTIVITY_MANAGER, - "ReadAhead Session Dump entries=%zu", - entry_offset); - // TODO: maybe getprop and a single line by default? - session->Dump(LOG_STREAM(INFO), /*multiline*/true); - } - - atrace_int(ATRACE_TAG_ACTIVITY_MANAGER, - "ReadAhead Bytes Length", - static_cast<int32_t>(length_sum)); - - impl_->read_ahead_file_map_[id.id] = std::move(task_data); - - ReadAhead::Impl::recent_data_keeper_.RecordRecent(id, length_sum); -} - -void ReadAhead::Dump(::android::Printer& printer) { - ReadAhead::Impl::recent_data_keeper_.Dump(printer); -} - -std::optional<size_t> ReadAhead::PrefetchSizeInBytes(const std::string& file_path) { - serialize::ArenaPtr<serialize::proto::TraceFile> trace_file_ptr = - serialize::ProtobufIO::Open(file_path); - - if (trace_file_ptr == nullptr) { - LOG(WARNING) << "PrefetchSizeInBytes: bad file at " << file_path; - return std::nullopt; - } - - size_t length_sum = 0; - const serialize::proto::TraceFileList& file_list = trace_file_ptr->list(); - for (const serialize::proto::TraceFileEntry& file_entry : file_list.entries()) { - - if (file_entry.file_length() < 0 || file_entry.file_offset() < 0) { - LOG(WARNING) << "ReadAhead entry negative file length or offset, illegal: " - << "index_id=" << file_entry.index_id() << ", skipping"; - continue; - } - - length_sum += static_cast<size_t>(file_entry.file_length()); - } - - return length_sum; -} - -} // namespace prefetcher -} // namespace iorap - diff --git a/src/prefetcher/read_ahead.h b/src/prefetcher/read_ahead.h deleted file mode 100644 index afc6ec8..0000000 --- a/src/prefetcher/read_ahead.h +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef PREFETCHER_READAHEAD_H_ -#define PREFETCHER_READAHEAD_H_ - -#include <memory> -#include <optional> - -namespace android { -class Printer; -} // namespace android - -namespace iorap { -namespace prefetcher { - -struct TaskId; -struct ReadAheadFileEntry; - -// Manage I/O readahead for a task. -class ReadAhead { - struct Impl; - public: - // Process a task *now*. Currently will block until all readaheads have been - // issued for all entries in that task. - // - // Any memory mapped or file descriptors opened as a side effect must be - // cleaned up with #FinishTask. - void BeginTask(const TaskId& id); - // Complete a task, releasing any memory/file descriptors associated with it. - void FinishTask(const TaskId& id); - - static void Dump(/*borrow*/::android::Printer& printer); - - // Calculate the sum of file_lengths. Returns nullopt if the file path does not - // point to a valid compiled TraceFile. - static std::optional<size_t> PrefetchSizeInBytes(const std::string& file_path); - - ReadAhead(bool use_sockets); - - ReadAhead(); - ~ReadAhead(); - private: - void BeginTaskForSockets(const TaskId& id, int32_t trace_cookie); - std::unique_ptr<Impl> impl_; -}; - -} // namespace prefetcher -} // namespace iorap - -#endif - diff --git a/src/prefetcher/session.cc b/src/prefetcher/session.cc deleted file mode 100644 index 8a24b24..0000000 --- a/src/prefetcher/session.cc +++ /dev/null @@ -1,724 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "session.h" - -#include "prefetcher/prefetcher_daemon.h" -#include "prefetcher/task_id.h" -#include "serialize/arena_ptr.h" -#include "serialize/protobuf_io.h" - -#include <android-base/logging.h> -#include <android-base/properties.h> -#include <android-base/unique_fd.h> -#include <fcntl.h> -#include <functional> -#include <stdint.h> -#include <sys/mman.h> -#include <sys/time.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <unordered_map> - -namespace iorap { -namespace prefetcher { - -// Print per-entry details even if successful. Default-off, too spammy. -static constexpr bool kLogVerboseReadAhead = false; - -std::ostream& operator<<(std::ostream& os, const Session& session) { - session.Dump(os, /*multiline*/false); - return os; -} - -Session::Session() { -} - -void SessionBase::Dump(std::ostream& os, bool multiline) const { - if (!multiline) { - os << "Session{"; - os << "session_id=" << SessionId(); - os << "}"; - return; - } else { - os << "Session (id=" << SessionId() << ")" << std::endl; - return; - } -} - -SessionBase::SessionBase(size_t session_id, std::string description) - : session_id_{session_id}, description_{description} { -} - -std::optional<std::string_view> SessionBase::GetFilePath(size_t path_id) const { - auto it = path_map_.find(path_id); - if (it != path_map_.end()) { - return {it->second}; - } else { - return std::nullopt; - } -} - -bool SessionBase::RemoveFilePath(size_t path_id) { - auto it = path_map_.find(path_id); - if (it != path_map_.end()) { - path_map_.erase(it); - return true; - } else { - return false; - } -} - -bool SessionBase::InsertFilePath(size_t path_id, std::string file_path) { - path_map_.insert({path_id, std::move(file_path)}); - return true; -} - -bool SessionBase::ProcessFd(int fd) { - // Only SessionDirect has an implementation of this. - // TODO: Maybe add a CommandChoice::kProcessFd ? instead of kCreateFdSession? - LOG(FATAL) << "SessionBase::ProcessFd is not implemented"; - - return false; -} - -// -// Direct -// - -std::ostream& operator<<(std::ostream& os, const SessionDirect::Entry& entry) { - os << "Entry{"; - os << "path_id=" << entry.path_id << ","; - os << "kind=" << static_cast<int>(entry.kind) << ","; - os << "length=" << entry.length << ","; - os << "offset=" << entry.offset << ","; - os << "}"; - - return os; -} - -// Just in case the failures are slowing down performance, turn them off. -// static constexpr bool kLogFailures = true; -static constexpr bool kLogFailures = false; - -bool SessionDirect::RegisterFilePath(size_t path_id, std::string_view file_path) { - std::string file_path_str{file_path}; // no c_str for string_view. - - auto fd = TEMP_FAILURE_RETRY(open(file_path_str.c_str(), O_RDONLY)); - if (fd < 0) { - if (kLogFailures) { - PLOG(ERROR) << "Failed to register file path: " << file_path << ", id=" << path_id - << ", open(2) failed: "; - } - fd = android::base::unique_fd{}; // mark as 'bad' descriptor. - } - - LOG(VERBOSE) << "RegisterFilePath path_id=" << path_id << ", file_path=" << file_path_str; - - if (!InsertFilePath(path_id, std::move(file_path_str))) { - return false; - } - - path_fd_map_.insert(std::make_pair(path_id, std::move(fd))); - DCHECK(entry_list_map_[path_id].empty()); - - return true; -} - - -bool SessionDirect::UnregisterFilePath(size_t path_id) { - if (!RemoveFilePath(path_id)) { - return false; - } - - { // Scoped FD reference lifetime. - auto maybe_fd = GetFdForPath(path_id); - - DCHECK(*maybe_fd != nullptr); - const android::base::unique_fd& entry_fd = **maybe_fd; - - auto list = entry_list_map_[path_id]; - - for (const EntryMapping& entry_mapping : list) { - ReadAheadKind kind = entry_mapping.entry.kind; - - switch (kind) { - case ReadAheadKind::kFadvise: - // Nothing to do. - break; - case ReadAheadKind::kMmapLocked: - FALLTHROUGH_INTENDED; - case ReadAheadKind::kMlock: - // Don't do any erases in the unregister file path to avoid paying O(n^2) erase cost. - UnmapWithoutErase(entry_mapping); - break; - } - } - } - - auto it = entry_list_map_.find(path_id); - auto end = entry_list_map_.end(); - DCHECK(it != end); - entry_list_map_.erase(it); - - // Close the FD for this file path. - auto fd_it = path_fd_map_.find(path_id); - DCHECK(fd_it != path_fd_map_.end()); - path_fd_map_.erase(fd_it); - - return true; -} - -// Note: return a pointer because optional doesn't hold references directly. -std::optional<android::base::unique_fd*> SessionDirect::GetFdForPath(size_t path_id) { - auto it = path_fd_map_.find(path_id); - if (it == path_fd_map_.end()) { - return std::nullopt; - } else { - return &it->second; - } -} - -bool SessionDirect::ReadAhead(size_t path_id, - ReadAheadKind kind, - size_t length, - size_t offset) { - // Take by-reference so we can mutate list at the end. - auto& list = entry_list_map_[path_id]; - - Entry entry{path_id, kind, length, offset}; - EntryMapping entry_mapping{entry, /*address*/nullptr, /*success*/false}; - - bool success = true; - - auto maybe_fd = GetFdForPath(path_id); - if (!maybe_fd) { - LOG(ERROR) << "SessionDirect: Failed to find FD for path_id=" << path_id; - return false; - } - - DCHECK(*maybe_fd != nullptr); - const android::base::unique_fd& entry_fd = **maybe_fd; - - std::optional<std::string_view> file_name_opt = GetFilePath(path_id); - DCHECK(file_name_opt.has_value()); // if one map has it, all maps have it. - std::string_view file_name = *file_name_opt; - - if (!entry_fd.ok()) { - LOG(VERBOSE) << "SessionDirect: No file descriptor for (path_id=" << path_id << ") " - << "path '" << file_name << "', failed to readahead entry."; - // Even failures get kept with success=false. - list.push_back(entry_mapping); - return false; - } - - switch (kind) { - case ReadAheadKind::kFadvise: - if (posix_fadvise(entry_fd, offset, length, POSIX_FADV_WILLNEED) != 0) { - PLOG(ERROR) << "SessionDirect: Failed to fadvise entry " << file_name - << ", offset=" << offset << ", length=" << length; - success = false; - } - break; - case ReadAheadKind::kMmapLocked: - FALLTHROUGH_INTENDED; - case ReadAheadKind::kMlock: { - const bool need_mlock = kind == ReadAheadKind::kMlock; - - int flags = MAP_SHARED; - if (!need_mlock) { - // MAP_LOCKED is a best-effort to lock the page. it could still be - // paged in later at a fault. - flags |= MAP_LOCKED; - } - - entry_mapping.address = - mmap(/*addr*/nullptr, length, PROT_READ, flags, entry_fd, offset); - - if (entry_mapping.address == nullptr) { - PLOG(ERROR) << "SessionDirect: Failed to mmap entry " << file_name - << ", offset=" << offset << ", length=" << length; - success = false; - break; - } - - // Strong guarantee that page will be locked if mlock returns successfully. - if (need_mlock && mlock(entry_mapping.address, length) < 0) { - PLOG(ERROR) << "SessionDirect: Failed to mlock entry " << file_name - << ", offset=" << offset << ", length=" << length; - // We already have a mapping address, so we should add it to the list. - // However this didn't succeed 100% because the lock failed, so return false later. - success = false; - } - } - } - - // Keep track of success so we know in Dump() what the number of failed entry mappings were. - entry_mapping.success = success; - - // Keep track of this so that we can clean it up later in UnreadAhead. - list.push_back(entry_mapping); - - if (entry_mapping.success) { - if (kLogVerboseReadAhead) { - LOG(VERBOSE) << "SessionDirect: ReadAhead for " << entry_mapping.entry; - } - } // else one of the errors above already did print. - - return success; -} - -bool SessionDirect::UnreadAhead(size_t path_id, - ReadAheadKind kind, - size_t length, - size_t offset) { - Entry entry{path_id, kind, length, offset}; - - auto list = entry_list_map_[path_id]; - if (list.empty()) { - return false; - } - - std::optional<EntryMapping> entry_mapping; - size_t idx = 0; - - for (size_t i = 0; i < list.size(); ++i) { - if (entry == list[i].entry) { - entry_mapping = list[i]; - idx = 0; - break; - } - } - - if (!entry_mapping) { - return false; - } - - switch (kind) { - case ReadAheadKind::kFadvise: - // Nothing to do. - // TODO: maybe fadvise(RANDOM)? - return true; - case ReadAheadKind::kMmapLocked: - FALLTHROUGH_INTENDED; - case ReadAheadKind::kMlock: - UnmapWithoutErase(*entry_mapping); - return true; - } - - list.erase(list.begin() + idx); - - // FDs close only with UnregisterFilePath. - return true; -} - -void SessionDirect::UnmapWithoutErase(const EntryMapping& entry_mapping) { - void* address = entry_mapping.address; - size_t length = entry_mapping.entry.length; - - // munmap also unlocks. Do not need explicit munlock. - if (munmap(address, length) < 0) { - PLOG(WARNING) << "ReadAhead (Finish): Failed to munmap address: " - << address << ", length: " << length; - } - -} - -bool SessionDirect::ProcessFd(int fd) { - // TODO: the path is advisory, but it would still be cleaner to pass it separately - const char* fd_path = SessionDescription().c_str(); - - android::base::Timer open_timer{}; - android::base::Timer total_timer{}; - - serialize::ArenaPtr<serialize::proto::TraceFile> trace_file_ptr = - serialize::ProtobufIO::Open(fd, fd_path); - - if (trace_file_ptr == nullptr) { - LOG(ERROR) << "SessionDirect::ProcessFd failed, corrupted protobuf format? " << fd_path; - return false; - } - - // TODO: maybe make it part of a kProcessFd type of command? - ReadAheadKind kind = ReadAheadKind::kFadvise; - - // TODO: The "Task[Id]" should probably be the one owning the trace file. - // When the task is fully complete, the task can be deleted and the - // associated arenas can go with them. - - // TODO: we should probably have the file entries all be relative - // to the package path? - - // Open every file in the trace index. - const serialize::proto::TraceFileIndex& index = trace_file_ptr->index(); - - size_t count_entries = 0; - for (const serialize::proto::TraceFileIndexEntry& index_entry : index.entries()) { - LOG(VERBOSE) << "ReadAhead: found file entry: " << index_entry.file_name(); - - if (index_entry.id() < 0) { - LOG(WARNING) << "ReadAhead: Skip bad TraceFileIndexEntry, negative ID not allowed: " - << index_entry.id(); - continue; - } - - size_t path_id = index_entry.id(); - const auto& path_file_name = index_entry.file_name(); - - if (!this->RegisterFilePath(path_id, path_file_name)) { - LOG(WARNING) << "ReadAhead: Failed to register file path: " << path_file_name; - ++count_entries; - } - } - LOG(VERBOSE) << "ReadAhead: Registered " << count_entries << " file paths"; - std::chrono::milliseconds open_duration_ms = open_timer.duration(); - - LOG(DEBUG) << "ProcessFd: open+parsed headers in " << open_duration_ms.count() << "ms"; - - // Go through every trace entry and readahead every (file,offset,len) tuple. - size_t entry_offset = 0; - const serialize::proto::TraceFileList& file_list = trace_file_ptr->list(); - for (const serialize::proto::TraceFileEntry& file_entry : file_list.entries()) { - ++entry_offset; - - if (file_entry.file_length() < 0 || file_entry.file_offset() < 0) { - LOG(WARNING) << "ProcessFd entry negative file length or offset, illegal: " - << "index_id=" << file_entry.index_id() << ", skipping"; - continue; - } - - // Attempt to perform readahead. This can generate more warnings dynamically. - if (!this->ReadAhead(file_entry.index_id(), - kind, - file_entry.file_length(), - file_entry.file_offset())) { - if (kLogFailures) { - LOG(WARNING) << "Failed readahead, bad file length/offset in entry @ " - << (entry_offset - 1); - } - } - } - - std::chrono::milliseconds total_duration_ms = total_timer.duration(); - LOG(DEBUG) << "ProcessFd: total duration " << total_duration_ms.count() << "ms"; - - { - struct timeval now; - gettimeofday(&now, nullptr); - - uint64_t now_usec = (now.tv_sec * 1000000LL + now.tv_usec); - LOG(DEBUG) << "ProcessFd: finishing usec: " << now_usec; - } - - return true; -} - - -static bool IsDumpEveryEntry() { - // Set to 'true' to dump every single entry for debugging (multiline). - // Otherwise it only prints per-file-path summaries. - return ::android::base::GetBoolProperty("iorapd.readahead.dump_all", /*default*/false); -} - -static bool IsDumpEveryPath() { - // Dump per-file-path (entry) stats. Defaults to on if the above property is on. - return ::android::base::GetBoolProperty("iorapd.readahead.dump_paths", /*default*/false); -} - -void SessionDirect::Dump(std::ostream& os, bool multiline) const { - { - struct timeval now; - gettimeofday(&now, nullptr); - - uint64_t now_usec = (now.tv_sec * 1000000LL + now.tv_usec); - LOG(DEBUG) << "SessionDirect::Dump: beginning usec: " << now_usec; - } - - size_t path_count = entry_list_map_.size(); - - size_t read_ahead_entries = 0; - size_t read_ahead_bytes = 0; - - size_t overall_entry_count = 0; - size_t overall_byte_count = 0; - for (auto it = entry_list_map_.begin(); it != entry_list_map_.end(); ++it) { - const auto& entry_mapping_list = it->second; - - for (size_t j = 0; j < entry_mapping_list.size(); ++j) { - const EntryMapping& entry_mapping = entry_mapping_list[j]; - const Entry& entry = entry_mapping.entry; - - ++overall_entry_count; - overall_byte_count += entry.length; - - if (entry_mapping.success) { - ++read_ahead_entries; - read_ahead_bytes += entry.length; - } - } - } - - double overall_success_entry_rate = - read_ahead_entries * 100.0 / overall_entry_count; - double overall_success_byte_rate = - read_ahead_bytes * 100.0 / overall_byte_count; - - size_t fd_count = path_fd_map_.size(); - size_t good_fd_count = 0; - for (auto it = path_fd_map_.begin(); it != path_fd_map_.end(); ++it) { - if (it->second.ok()) { - ++good_fd_count; - } - } - double good_fd_rate = good_fd_count * 100.0 / fd_count; - // double bad_fd_rate = (fd_count - good_fd_count) * 1.0 / fd_count; - - if (!multiline) { - os << "SessionDirect{"; - os << "session_id=" << SessionId() << ","; - - os << "file_paths=" << path_count << " (good: " << good_fd_rate << "),"; - os << "read_ahead_entries=" << read_ahead_entries; - os << "(" << overall_success_entry_rate << "%),"; - os << "read_ahead_bytes=" << read_ahead_bytes << ""; - os << "(" << overall_success_byte_rate << "%),"; - os << "timer=" << timer_.duration().count() << ","; - - os << "}"; - return; - } else { - // Always try to pay attention to these stats below. - // They can be signs of potential performance problems. - os << "Session Direct (id=" << SessionId() << ")" << std::endl; - - os << " Summary: " << std::endl; - os << " Description = " << SessionDescription() << std::endl; - os << " Duration = " << timer_.duration().count() << "ms" << std::endl; - os << " Total File Paths=" << path_count << " (good: " << good_fd_rate << "%)" << std::endl; - os << " Total Entries=" << overall_entry_count; - os << " (good: " << overall_success_entry_rate << "%)" << std::endl; - os << " Total Bytes=" << overall_byte_count << ""; - os << " (good: " << overall_success_byte_rate << "%)" << std::endl; - os << std::endl; - - // Probably too spammy, but they could narrow down the issue for a problem in above stats. - if (!IsDumpEveryPath() && !IsDumpEveryEntry()) { - return; - } - - for (auto it = entry_list_map_.begin(); it != entry_list_map_.end(); ++it) { - size_t path_id = it->first; - const auto& entry_mapping_list = it->second; - - std::optional<std::string_view> file_path = GetFilePath(path_id); - os << " File Path (id=" << path_id << "): "; - if (file_path.has_value()) { - os << "'" << *file_path << "'"; - } else { - os << "(nullopt)"; - } - - auto fd_it = path_fd_map_.find(path_id); - os << ", FD="; - if (fd_it != path_fd_map_.end()) { - const android::base::unique_fd& fd = fd_it->second; - os << fd.get(); // -1 for failed fd. - } else { - os << "(none)"; - } - os << std::endl; - - size_t total_entries = entry_mapping_list.size(); - size_t total_bytes = 0; - - size_t local_read_ahead_entries = 0; - size_t local_read_ahead_bytes = 0; - for (size_t j = 0; j < entry_mapping_list.size(); ++j) { - const EntryMapping& entry_mapping = entry_mapping_list[j]; - const Entry& entry = entry_mapping.entry; - - total_bytes += entry.length; - - // Sidenote: Bad FDs will have 100% failed mappings. - // Good FDs may sometimes have failed mappings. - if (entry_mapping.success) { - ++local_read_ahead_entries; - local_read_ahead_bytes += entry.length; - } - - if (IsDumpEveryEntry()) { - os << " Entry " << j << " details:" << std::endl; - os << " " << entry << std::endl; - os << " Mapping " << (entry_mapping.success ? "Succeeded" : "Failed") - << ", Address " << entry_mapping.address << std::endl; - } - } - - double entry_success_rate = local_read_ahead_entries * 100.0 / total_entries; - double bytes_success_rate = local_read_ahead_bytes * 100.0 / total_bytes; - - double entry_failure_rate = (total_entries - local_read_ahead_entries) * 100.0 / total_entries; - double bytes_failure_rate = (total_bytes - local_read_ahead_bytes) * 100.0 / total_bytes; - - os << " Successful: Entries=" << local_read_ahead_entries - << " (" << entry_success_rate << "%)" - << ", Bytes=" << local_read_ahead_bytes - << " (" << bytes_success_rate << "%)" - << std::endl; - os << " Failed: Entries=" << (total_entries - local_read_ahead_entries) - << " (" << entry_failure_rate << "%)" - << ", Bytes=" << (total_bytes - local_read_ahead_bytes) - << " (" << bytes_failure_rate << "%)" - << std::endl; - os << " Total: Entries=" << total_entries - << ", Bytes=" << total_bytes - << std::endl; - } - - return; - } -} - -SessionDirect::~SessionDirect() { - for (auto it = entry_list_map_.begin(); it != entry_list_map_.end();) { - size_t path_id = it->first; - - ++it; // the iterator is removed in the following Unregister method. - UnregisterFilePath(path_id); - } -} - -// -// Indirect -// - -SessionIndirect::SessionIndirect(size_t session_id, - std::string description, - std::shared_ptr<PrefetcherDaemon> daemon, - bool send_command) - : SessionBase{session_id, description}, - daemon_{daemon} { - // Don't do anything in e.g. subclasses. - if (!send_command) { - return; - } - - Command cmd{}; - cmd.choice = CommandChoice::kCreateSession; - cmd.session_id = session_id; - cmd.file_path = description; - - LOG(VERBOSE) << "SessionIndirect: " << cmd; - - if (!daemon_->SendCommand(cmd)) { - LOG(FATAL) << "SessionIndirect: Failure to create session " << session_id - << ", description: " << description; - } -} - -SessionIndirect::~SessionIndirect() { - Command cmd{}; - cmd.choice = CommandChoice::kDestroySession; - cmd.session_id = SessionId(); - - if (!daemon_->SendCommand(cmd)) { - LOG(WARNING) << "SessionIndirect: Failure to destroy session " << SessionId() - << ", description: " << SessionDescription(); - } -} - -void SessionIndirect::Dump(std::ostream& os, bool multiline) const { - // SessionBase::Dump(os, multiline); - // TODO: does having the local dump do anything for us? - - Command cmd{}; - cmd.choice = CommandChoice::kDumpSession; - cmd.session_id = SessionId(); - - daemon_->SendCommand(cmd); -} - -bool SessionIndirect::RegisterFilePath(size_t path_id, std::string_view file_path) { - Command cmd{}; - cmd.choice = CommandChoice::kRegisterFilePath; - cmd.session_id = SessionId(); - cmd.id = path_id; - cmd.file_path = file_path; - - return daemon_->SendCommand(cmd); -} - -bool SessionIndirect::UnregisterFilePath(size_t path_id) { - Command cmd{}; - cmd.choice = CommandChoice::kUnregisterFilePath; - cmd.session_id = SessionId(); - cmd.id = path_id; - - return daemon_->SendCommand(cmd); -} -bool SessionIndirect::ReadAhead(size_t path_id, - ReadAheadKind kind, - size_t length, - size_t offset) { - Command cmd{}; - cmd.choice = CommandChoice::kReadAhead; - cmd.session_id = SessionId(); - cmd.id = path_id; - cmd.read_ahead_kind = kind; - cmd.length = length; - cmd.offset = offset; - - return daemon_->SendCommand(cmd); -} - -bool SessionIndirect::UnreadAhead(size_t path_id, - ReadAheadKind kind, - size_t length, - size_t offset) { - LOG(WARNING) << "UnreadAhead: command not implemented yet"; - return true; -} - -// -// IndirectSocket -// - -SessionIndirectSocket::SessionIndirectSocket(size_t session_id, - int fd, - std::string description, - std::shared_ptr<PrefetcherDaemon> daemon) - : SessionIndirect{session_id, description, daemon, /*send_command*/false} { - // TODO: all of the WriteCommand etc in the daemon. - Command cmd{}; - cmd.choice = CommandChoice::kCreateFdSession; - cmd.fd = fd; - cmd.session_id = session_id; - cmd.file_path = description; - - LOG(VERBOSE) << "SessionIndirectSocket: " << cmd; - - if (!daemon_->SendCommand(cmd)) { - LOG(FATAL) << "SessionIndirectSocket: Failure to create session " << session_id - << ", description: " << description; - } - - // This goes into the SessionDirect ctor + SessionDirect::ProcessFd - // as implemented in PrefetcherDaemon::ReceiveCommand -} - -SessionIndirectSocket::~SessionIndirectSocket() { -} - -} // namespace prefetcher -} // namespace iorap diff --git a/src/prefetcher/session.h b/src/prefetcher/session.h deleted file mode 100644 index a4a9e6b..0000000 --- a/src/prefetcher/session.h +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef PREFETCHER_SESSION_H_ -#define PREFETCHER_SESSION_H_ - -#include <android-base/chrono_utils.h> -#include <android-base/unique_fd.h> - -#include <optional> -#include <memory> -#include <string> -#include <string_view> -#include <unordered_map> -#include <vector> - -namespace iorap { -namespace prefetcher { - -#ifndef READ_AHEAD_KIND -#define READ_AHEAD_KIND 1 -enum class ReadAheadKind : uint32_t { - kFadvise = 0, - kMmapLocked = 1, - kMlock = 2, -}; -#endif - -class Session { - public: - virtual bool RegisterFilePath(size_t path_id, std::string_view file_path) = 0; - virtual bool UnregisterFilePath(size_t path_id) = 0; - - // Immediately perform a readahead now. - // Fadvise: the readahead will have been queued by the kernel. - // MmapLocked/Mlock: the memory is pinned by the requested process. - virtual bool ReadAhead(size_t path_id, ReadAheadKind kind, size_t length, size_t offset) = 0; - - // Cancels a readahead previously done. - // The length/offset should match the call of ReadAhead. - virtual bool UnreadAhead(size_t path_id, ReadAheadKind kind, size_t length, size_t offset) = 0; - - // Multi-line detailed dump, e.g. for dumpsys. - // Single-line summary dump, e.g. for logcat. - virtual void Dump(std::ostream& os, bool multiline) const = 0; - - // Process the FD for kCreateFdSession. - // Assumes there's a compiled_trace.pb at the fd, calling this function - // will immediately process it and execute any read-aheads. - // - // FD is borrowed only for the duration of the function call. - virtual bool ProcessFd(int fd) = 0; - - // Get the session ID associated with this session. - // Session IDs are distinct, they are not used for new sessions. - virtual size_t SessionId() const = 0; - - // Get this session's description. - // Only useful for logging/dumping. - virtual const std::string& SessionDescription() const = 0; - - // Implicitly unregister any remaining file paths. - // All read-aheads are also cancelled. - virtual ~Session() {} - - protected: - Session(); -}; - -// Single-line summary dump of Session. -std::ostream& operator<<(std::ostream&os, const Session& session); - -class SessionBase : public Session { - public: - virtual void Dump(std::ostream& os, bool multiline) const override; - virtual ~SessionBase() {} - - virtual size_t SessionId() const override { - return session_id_; - } - - virtual const std::string& SessionDescription() const override { - return description_; - } - - virtual bool ProcessFd(int fd) override; - - protected: - SessionBase(size_t session_id, std::string description); - std::optional<std::string_view> GetFilePath(size_t path_id) const; - bool RemoveFilePath(size_t path_id); - bool InsertFilePath(size_t path_id, std::string file_path); - - android::base::Timer timer_{}; - private: - // Note: Store filename for easier debugging and for dumping. - std::unordered_map</*path_id*/size_t, std::string> path_map_; - size_t session_id_; - std::string description_; -}; - -// In-process session. -class SessionDirect : public SessionBase { - public: - virtual bool RegisterFilePath(size_t path_id, std::string_view file_path) override; - - virtual bool UnregisterFilePath(size_t path_id) override; - virtual bool ReadAhead(size_t path_id, - ReadAheadKind kind, - size_t length, - size_t offset); - - virtual bool UnreadAhead(size_t path_id, - ReadAheadKind kind, - size_t length, - size_t offset) override; - - virtual bool ProcessFd(int fd) override; - - virtual void Dump(std::ostream& os, bool multiline) const override; - - virtual ~SessionDirect(); - - SessionDirect(size_t session_id, std::string description) - : SessionBase{session_id, std::move(description)} { - } - protected: - struct Entry { - size_t path_id; - ReadAheadKind kind; - size_t length; - size_t offset; - - constexpr bool operator==(const Entry& other) const { - return path_id == other.path_id && - kind == other.kind && - length == other.length && - offset == other.offset; - } - constexpr bool operator!=(const Entry& other) const { - return !(*this == other); - } - - friend std::ostream& operator<<(std::ostream& os, const Entry& entry); - }; - - struct EntryMapping { - Entry entry; - void* address; - bool success; - }; - - // bool EntryReadAhead(const Entry& entry) const; - // bool EntryUnReadAhead(const Entry& entry) const; - - void UnmapWithoutErase(const EntryMapping& entry_mapping); - std::optional<android::base::unique_fd*> GetFdForPath(size_t path_id); - - private: - std::unordered_map</*path_id*/size_t, std::vector<EntryMapping>> entry_list_map_; - std::unordered_map</*path_id*/size_t, android::base::unique_fd> path_fd_map_; - - public: - friend std::ostream& operator<<(std::ostream& os, const SessionDirect::Entry& entry); -}; - -std::ostream& operator<<(std::ostream& os, const SessionDirect::Entry& entry); - -class PrefetcherDaemon; - -// Out-of-process session. Requires prefetcher daemon. -class SessionIndirect : public SessionBase { - public: - virtual bool RegisterFilePath(size_t path_id, std::string_view file_path) override; - - virtual bool UnregisterFilePath(size_t path_id) override; - virtual bool ReadAhead(size_t path_id, - ReadAheadKind kind, - size_t length, - size_t offset) override; - - virtual bool UnreadAhead(size_t path_id, - ReadAheadKind kind, - size_t length, - size_t offset) override; - - virtual void Dump(std::ostream& os, bool multiline) const override; - - // Creates a new session indirectly. - // Writes to daemon the new session command. - SessionIndirect(size_t session_id, - std::string description, - std::shared_ptr<PrefetcherDaemon> daemon, - bool send_command = true); - - // Destroys the current session. - // Writes to daemon that the session is to be destroyed. - virtual ~SessionIndirect(); - - protected: - std::shared_ptr<PrefetcherDaemon> daemon_; -}; - -// Out-of-process session. Requires prefetcher daemon. -class SessionIndirectSocket : public SessionIndirect { - public: - // Creates a new session indirectly. - // Writes to daemon the new session command. - SessionIndirectSocket(size_t session_id, - int fd, - std::string description, - std::shared_ptr<PrefetcherDaemon> daemon); - // Destroys the current session. - // Writes to daemon that the session is to be destroyed. - virtual ~SessionIndirectSocket(); - - private: -}; - - -} // namespace prefetcher -} // namespace iorap - -#endif - diff --git a/src/prefetcher/session_manager.cc b/src/prefetcher/session_manager.cc deleted file mode 100644 index d6fab1f..0000000 --- a/src/prefetcher/session_manager.cc +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "session_manager.h" - -#include "prefetcher/prefetcher_daemon.h" -#include "prefetcher/session.h" -#include "prefetcher/task_id.h" -#include "serialize/arena_ptr.h" -#include "serialize/protobuf_io.h" - -#include <android-base/logging.h> -#include <android-base/chrono_utils.h> -#include <android-base/unique_fd.h> -#include <fcntl.h> -#include <functional> -#include <stdint.h> -#include <sys/mman.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <unordered_map> - -namespace iorap { -namespace prefetcher { - -std::ostream& operator<<(std::ostream& os, const SessionManager& manager) { - manager.Dump(os, /*multiline*/false); - return os; -} - -SessionManager::SessionManager() { -} - -class SessionManagerBase : public SessionManager { - public: - virtual void Dump(std::ostream& os, bool multiline) const { - if (!multiline) { - os << "SessionManager{"; - - os << "sessions=["; - for (auto it = sessions_map_.begin(); - it != sessions_map_.end(); - ++it) { - os << "(" << it->second.description << ") "; - it->second.session->Dump(os, /*multiline*/false); - } - os << "]"; - return; - } - - os << "SessionManager (session count = " << sessions_map_.size() << "):" << std::endl; - os << std::endl; - - for (auto it = sessions_map_.begin(); - it != sessions_map_.end(); - ++it) { - os << "Description: " << it->second.description << std::endl; - it->second.session->Dump(os, /*multiline*/true); - } - - // TODO: indentations? Use this pseudo line break for the time being. - os << "--------------------------------" << std::endl; - } - - virtual ~SessionManagerBase() {} - - virtual std::shared_ptr<Session> FindSession(size_t session_id) const override { - auto it = sessions_map_.find(session_id); - if (it != sessions_map_.end()) { - DCHECK_EQ(session_id, it->second.SessionId()); - return it->second.session; - } else { - return nullptr; - } - } - - virtual bool DestroySession(size_t session_id) override { - auto it = sessions_map_.find(session_id); - if (it != sessions_map_.end()) { - sessions_map_.erase(it); - return true; - } else { - return false; - } - } - - protected: - void InsertNewSession(std::shared_ptr<Session> session, std::string description) { - DCHECK(!FindSession(session->SessionId())) << "session cannot already exist"; - - size_t session_id = session->SessionId(); - - SessionData data; - data.session = std::move(session); - data.description = std::move(description); - - sessions_map_.insert({session_id, std::move(data)}); - } - - private: - struct SessionData { - std::shared_ptr<Session> session; - std::string description; - - size_t SessionId() const { - return session->SessionId(); - } - }; - - std::unordered_map</*session_id*/size_t, SessionData> sessions_map_; -}; - -class SessionManagerDirect : public SessionManagerBase { - public: - virtual std::shared_ptr<Session> CreateSession(size_t session_id, - std::string description) override { - LOG(VERBOSE) << "CreateSessionDirect id=" << session_id << ", description=" << description; - - std::shared_ptr<Session> session = - std::static_pointer_cast<Session>(std::make_shared<SessionDirect>(session_id, - description)); - DCHECK(FindSession(session_id) == nullptr); - InsertNewSession(session, std::move(description)); - return session; - } - - SessionManagerDirect() { - // Intentionally left empty. - } - - private: -}; - - -class SessionManagerIndirect : public SessionManagerBase { - public: - virtual std::shared_ptr<Session> CreateSession(size_t session_id, - std::string description) override { - LOG(VERBOSE) << "CreateSessionIndirect id=" << session_id << ", description=" << description; - - std::shared_ptr<Session> session = - std::static_pointer_cast<Session>(std::make_shared<SessionIndirect>(session_id, - description, - daemon_)); - InsertNewSession(session, description); - return session; - } - - SessionManagerIndirect() : daemon_{std::make_shared<PrefetcherDaemon>()} { - //StartViaFork etc. - // TODO: also expose a 'MainLoop(...) -> daemon::Main(..)' somehow in the base interface. - auto params = daemon_->StartPipesViaFork(); - if (!params) { - LOG(FATAL) << "Failed to fork+exec iorap.prefetcherd"; - } - } - - virtual ~SessionManagerIndirect() { - Command cmd{}; - cmd.choice = CommandChoice::kExit; - - if (!daemon_->SendCommand(cmd)) { - LOG(FATAL) << "Failed to nicely exit iorap.prefetcherd"; - } - } - - virtual void Dump(std::ostream& os, bool multiline) const override { - Command cmd{}; - cmd.choice = CommandChoice::kDumpEverything; - - if (!daemon_->SendCommand(cmd)) { - LOG(ERROR) << "Failed to transmit kDumpEverything to iorap.prefetcherd"; - } - } - - - private: - // No lifetime cycle: PrefetcherDaemon only has a SessionManagerDirect in it. - std::shared_ptr<PrefetcherDaemon> daemon_; -}; - -class SessionManagerIndirectSocket : public SessionManagerBase { - public: - virtual std::shared_ptr<Session> CreateSession(size_t session_id, - std::string description) override { - DCHECK(false) << "not supposed to create a regular session for Socket"; - - LOG(VERBOSE) << "CreateSessionIndirect id=" << session_id << ", description=" << description; - - std::shared_ptr<Session> session = - std::static_pointer_cast<Session>(std::make_shared<SessionIndirect>(session_id, - description, - daemon_)); - InsertNewSession(session, description); - return session; - } - - virtual std::shared_ptr<Session> CreateSession(size_t session_id, - std::string description, - std::optional<int> fd) override { - CHECK(fd.has_value()); - LOG(VERBOSE) << "CreateSessionIndirectSocket id=" << session_id - << ", description=" << description - << ", fd=" << *fd; - - std::shared_ptr<Session> session = - std::static_pointer_cast<Session>(std::make_shared<SessionIndirectSocket>(session_id, - *fd, - description, - daemon_)); - InsertNewSession(session, description); - return session; - } - - SessionManagerIndirectSocket() : daemon_{std::make_shared<PrefetcherDaemon>()} { - auto params = daemon_->StartSocketViaFork(); - if (!params) { - LOG(FATAL) << "Failed to fork+exec iorap.prefetcherd"; - } - } - - virtual ~SessionManagerIndirectSocket() { - Command cmd{}; - cmd.choice = CommandChoice::kExit; - - if (!daemon_->SendCommand(cmd)) { - LOG(FATAL) << "Failed to nicely exit iorap.prefetcherd"; - } - } - - virtual void Dump(std::ostream& os, bool multiline) const override { - Command cmd{}; - cmd.choice = CommandChoice::kDumpEverything; - - if (!daemon_->SendCommand(cmd)) { - LOG(ERROR) << "Failed to transmit kDumpEverything to iorap.prefetcherd"; - } - } - - - private: - // No lifetime cycle: PrefetcherDaemon only has a SessionManagerDirect in it. - std::shared_ptr<PrefetcherDaemon> daemon_; -}; - -std::unique_ptr<SessionManager> SessionManager::CreateManager(SessionKind kind) { - LOG(VERBOSE) << "SessionManager::CreateManager kind=" << kind; - - switch (kind) { - case SessionKind::kInProcessDirect: { - SessionManager* ptr = new SessionManagerDirect(); - return std::unique_ptr<SessionManager>{ptr}; - } - case SessionKind::kOutOfProcessIpc: { - SessionManager* ptr = new SessionManagerIndirect(); - return std::unique_ptr<SessionManager>{ptr}; - } - case SessionKind::kOutOfProcessSocket: { - SessionManager* ptr = new SessionManagerIndirectSocket(); - return std::unique_ptr<SessionManager>{ptr}; - } - default: { - LOG(FATAL) << "Invalid session kind: " << static_cast<int>(kind); - break; - } - } -} - -} // namespace prefetcher -} // namespace iorap diff --git a/src/prefetcher/session_manager.h b/src/prefetcher/session_manager.h deleted file mode 100644 index 45120d5..0000000 --- a/src/prefetcher/session_manager.h +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef PREFETCHER_SESSION_MANAGER_H_ -#define PREFETCHER_SESSION_MANAGER_H_ - -#include <optional> -#include <ostream> -#include <memory> - -namespace iorap { -namespace prefetcher { - -class Session; - -enum class SessionKind : uint32_t { - kInProcessDirect, - kOutOfProcessIpc, - kOutOfProcessSocket, -}; - -inline std::ostream& operator<<(std::ostream& os, SessionKind kind) { - if (kind == SessionKind::kInProcessDirect) { - os << "kInProcessDirect"; - } else if (kind == SessionKind::kOutOfProcessIpc) { - os << "kOutOfProcessIpc"; - } else if (kind == SessionKind::kOutOfProcessSocket) { - os << "kOutOfProcessSocket"; - } else { - os << "(invalid)"; - } - return os; -} - -class SessionManager { - public: - static std::unique_ptr<SessionManager> CreateManager(SessionKind kind); - - // Create a new session. The description is used by Dump. - // Manager maintains a strong ref to this session, so DestroySession must also - // be called prior to all refs dropping to 0. - virtual std::shared_ptr<Session> CreateSession(size_t session_id, - std::string description) = 0; - - // Create a new session. The description is used by Dump. - // Manager maintains a strong ref to this session, so DestroySession must also - // be called prior to all refs dropping to 0. - virtual std::shared_ptr<Session> CreateSession(size_t session_id, - std::string description, - std::optional<int> fd) { - return CreateSession(session_id, description); - } - - // Look up an existing session that was already created. - // Returns null if there is no such session. - virtual std::shared_ptr<Session> FindSession(size_t session_id) const = 0; - - // Drop all manager references to an existing session. - // Returns false if the session does not exist already. - virtual bool DestroySession(size_t session_id) = 0; - - // Multi-line detailed dump, e.g. for dumpsys. - // Single-line summary dump, e.g. for logcat. - virtual void Dump(std::ostream& os, bool multiline) const = 0; - - // Note: session lifetime is tied to manager. The manager has strong pointers to sessions. - virtual ~SessionManager() {} - - protected: - SessionManager(); -}; - -// Single-line summary dump of Session. -std::ostream& operator<<(std::ostream&os, const SessionManager& session); - -} // namespace prefetcher -} // namespace iorap - -#endif - diff --git a/src/prefetcher/task_id.h b/src/prefetcher/task_id.h deleted file mode 100644 index dc26954..0000000 --- a/src/prefetcher/task_id.h +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef PREFETCHER_TASK_ID_H_ -#define PREFETCHER_TASK_ID_H_ - -#include <ostream> -#include <string> - -namespace iorap { -namespace prefetcher { - -struct TaskId { - size_t id; // Unique monotonically increasing ID. - std::string path; // File path to the trace file. - - friend std::ostream& operator<<(std::ostream& os, const TaskId& task_id) { - os << "TaskId { id: " << task_id.id << ", path: " << task_id.path << "}"; - return os; - } - -}; - -} // namespace prefetcher -} // namespace iorap - -#endif - diff --git a/src/serialize/TraceFile.proto b/src/serialize/TraceFile.proto deleted file mode 100644 index fc72d0d..0000000 --- a/src/serialize/TraceFile.proto +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -syntax = "proto2"; // use required fields, which aren't in proto3. - -package iorap.serialize.proto; // C++ namespace iorap::serialize::proto -option java_package = "com.google.android.iorap"; -option optimize_for = LITE_RUNTIME; - -// TODO: should these fields be 'packed' for "smaller encoding" ? - -message TraceFile { - required TraceFileIndex index = 1; - required TraceFileList list = 2; -} - -message TraceFileIndex { - repeated TraceFileIndexEntry entries = 1; -} - -message TraceFileIndexEntry { - required int64 id = 1; - required string file_name = 2; -} - -message TraceFileList { - repeated TraceFileEntry entries = 1; -} - -message TraceFileEntry { - required int64 index_id = 1; - required int64 file_offset = 2; - required int64 file_length = 3; -} - -// XX: use nested messages?
\ No newline at end of file diff --git a/src/serialize/arena_ptr.h b/src/serialize/arena_ptr.h deleted file mode 100644 index d8bea26..0000000 --- a/src/serialize/arena_ptr.h +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef SERIALIZE_ARENA_PTR_H_ -#define SERIALIZE_ARENA_PTR_H_ - -#include <google/protobuf/arena.h> -#include <memory> - -namespace iorap { -namespace serialize { - -/** - * @file - * - * Helpers for protobuf arena allocators. We use smart pointers - * with an arena embedded inside of them to avoid caring about the - * arena in other parts of libiorap. - */ - -// Arena-managed objects must not be deleted manually. -// When the Arena goes out of scope, it cleans everything up itself. -template <typename T> -void DoNotDelete(T*) {} - -template <typename T, typename Base = std::unique_ptr<T, decltype(&DoNotDelete<T>)>> -struct ArenaPtr : public Base { - template <typename... Args> - static ArenaPtr<T> Make(Args&& ... args) { - ArenaPtr<T> arena_ptr(nullptr); - arena_ptr.reset(google::protobuf::Arena::Create<T>(arena_ptr.arena_.get(), - std::forward<Args>(args)...)); - return arena_ptr; - } - - ArenaPtr(std::nullptr_t) : Base(nullptr, &DoNotDelete<T>) {} // NOLINT explicit. - - private: - // Use a unique_ptr because Arena doesn't support move semantics. - std::unique_ptr<google::protobuf::Arena> arena_{new google::protobuf::Arena{}}; -}; - -template <typename T, typename Base = std::shared_ptr<T>> -struct ArenaSharedPtr : public Base { - template <typename... Args> - static ArenaSharedPtr<T> Make(Args&& ... args) { - ArenaSharedPtr<T> arena_ptr(nullptr, &DoNotDelete<T>); - arena_ptr.reset(google::protobuf::Arena::Create<T>(arena_ptr.arena_.get(), - std::forward<Args>(args)...)); - return arena_ptr; - } - - ArenaSharedPtr() = default; - template <typename Deleter> - ArenaSharedPtr(std::nullptr_t, Deleter d) : Base(nullptr, d) {} // NOLINT explicit. - - private: - std::shared_ptr<google::protobuf::Arena> arena_{new google::protobuf::Arena{}}; -}; - -} // namespace serialize -} // namespace iorap - -#endif - diff --git a/src/serialize/protobuf_io.cc b/src/serialize/protobuf_io.cc deleted file mode 100644 index 1b6420f..0000000 --- a/src/serialize/protobuf_io.cc +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "protobuf_io.h" - -#include "common/trace.h" -#include "serialize/arena_ptr.h" - -#include <android-base/chrono_utils.h> -#include <android-base/logging.h> -#include <android-base/unique_fd.h> -#include <fcntl.h> -#include <sys/mman.h> -#include <sys/stat.h> -#include <sys/types.h> -#include <unistd.h> -#include <utils/Trace.h> - -#include "google/protobuf/io/zero_copy_stream_impl_lite.h" -#include "system/iorap/src/serialize/TraceFile.pb.h" - -namespace iorap { -namespace serialize { - -ArenaPtr<proto::TraceFile> ProtobufIO::Open(std::string file_path) { - // TODO: file a bug about this. - // Note: can't use {} here, clang think it's narrowing from long->int. - android::base::unique_fd fd(TEMP_FAILURE_RETRY(::open(file_path.c_str(), O_RDONLY))); - if (fd.get() < 0) { - PLOG(DEBUG) << "ProtobufIO: open failed: " << file_path; - return nullptr; - } - - return Open(fd.get(), file_path.c_str()); -} - -ArenaPtr<proto::TraceFile> ProtobufIO::Open(int fd, const char* file_path) { - - ScopedFormatTrace atrace_protobuf_io_open(ATRACE_TAG_ACTIVITY_MANAGER, - "ProtobufIO::Open %s", - file_path); - android::base::Timer timer{}; - - struct stat buf; - if (fstat(fd, /*out*/&buf) < 0) { - PLOG(ERROR) << "ProtobufIO: open error, fstat failed: " << file_path; - return nullptr; - } - // XX: off64_t for stat::st_size ? - - // Using the mmap appears to be the only way to do zero-copy with protobuf lite. - void* data = mmap(/*addr*/nullptr, - buf.st_size, - PROT_READ, MAP_SHARED | MAP_POPULATE, - fd, - /*offset*/0); - if (data == nullptr) { - PLOG(ERROR) << "ProtobufIO: open error, mmap failed: " << file_path; - return nullptr; - } - - ArenaPtr<proto::TraceFile> protobuf_trace_file = ArenaPtr<proto::TraceFile>::Make(); - if (protobuf_trace_file == nullptr) { - LOG(ERROR) << "ProtobufIO: open error, failed to create arena: " << file_path; - return nullptr; - } - - google::protobuf::io::ArrayInputStream protobuf_input_stream{data, static_cast<int>(buf.st_size)}; - if (!protobuf_trace_file->ParseFromZeroCopyStream(/*in*/&protobuf_input_stream)) { - // XX: Does protobuf on android already have the right LogHandler ? - LOG(ERROR) << "ProtobufIO: open error, protobuf parsing failed: " << file_path; - return nullptr; - } - - if (munmap(data, buf.st_size) < 0) { - PLOG(WARNING) << "ProtobufIO: open problem, munmap failed, possibly memory leak? " - << file_path; - } - - LOG(VERBOSE) << "ProtobufIO: open succeeded: " << file_path << ", duration: " << timer; - return protobuf_trace_file; -} - -iorap::expected<size_t /*bytes written*/, int /*errno*/> ProtobufIO::WriteFully( - const ::google::protobuf::MessageLite& message, - std::string_view file_path) { - - std::string str{file_path}; - android::base::unique_fd fd(TEMP_FAILURE_RETRY( - ::open(str.c_str(), - O_CREAT | O_TRUNC | O_RDWR, - S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP))); // ugo: rw-rw---- - if (fd.get() < 0) { - int err = errno; - PLOG(ERROR) << "ProtobufIO: open failed: " << file_path; - return unexpected{err}; - } - - return WriteFully(message, fd.get(), file_path); -} - -iorap::expected<size_t /*bytes written*/, int /*errno*/> ProtobufIO::WriteFully( - const ::google::protobuf::MessageLite& message, - int fd, - std::string_view file_path) { - - int byte_size = message.ByteSize(); - if (byte_size < 0) { - DCHECK(false) << "Invalid protobuf size: " << byte_size; - LOG(ERROR) << "ProtobufIO: Invalid protobuf size: " << byte_size; - return unexpected{EDOM}; - } - size_t serialized_size = static_cast<size_t>(byte_size); - - // Change the file to be exactly the length of the protobuf. - if (ftruncate(fd, static_cast<off_t>(serialized_size)) < 0) { - int err = errno; - PLOG(ERROR) << "ProtobufIO: ftruncate (size=" << serialized_size << ") failed"; - return unexpected{err}; - } - - // Using the mmap appears to be the only way to do zero-copy with protobuf lite. - void* data = mmap(/*addr*/nullptr, - serialized_size, - PROT_WRITE, - MAP_SHARED, - fd, - /*offset*/0); - if (data == nullptr) { - int err = errno; - PLOG(ERROR) << "ProtobufIO: mmap failed: " << file_path; - return unexpected{err}; - } - - // Zero-copy write from protobuf to file via memory-map. - ::google::protobuf::io::ArrayOutputStream output_stream{data, byte_size}; - if (!message.SerializeToZeroCopyStream(/*inout*/&output_stream)) { - // This should never happen since we pre-allocated the file and memory map to be large - // enough to store the full protobuf. - DCHECK(false) << "ProtobufIO:: SerializeToZeroCopyStream failed despite precalculating size"; - LOG(ERROR) << "ProtobufIO: SerializeToZeroCopyStream failed"; - return unexpected{EXFULL}; - } - - // Guarantee that changes are written back prior to munmap. - if (msync(data, static_cast<size_t>(serialized_size), MS_SYNC) < 0) { - int err = errno; - PLOG(ERROR) << "ProtobufIO: msync failed"; - return unexpected{err}; - } - - if (munmap(data, serialized_size) < 0) { - PLOG(WARNING) << "ProtobufIO: munmap failed, possibly memory leak? " - << file_path; - } - - return serialized_size; -} - -} // namespace serialize -} // namespace iorap - diff --git a/src/serialize/protobuf_io.h b/src/serialize/protobuf_io.h deleted file mode 100644 index 1092fd7..0000000 --- a/src/serialize/protobuf_io.h +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (C) 2017 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef SERIALIZE_PROTOBUF_IO_H_ -#define SERIALIZE_PROTOBUF_IO_H_ - -#include "common/expected.h" -#include "serialize/arena_ptr.h" -#include "system/iorap/src/serialize/TraceFile.pb.h" - -#include <string> -#include <string_view> - -namespace iorap { -namespace serialize { - -// XX: either the namespace should be called pb|proto[buf] -// or we should hide the protobuf-ness from the names and call this class "IO" , "Reader", etc? -// although an obvious name might be the "OpenFactory" or "ProtobufFacade" that reads too much -// like a bad joke. - -// Helpers to read a TraceFile protobuf from a file [descriptor]. -class ProtobufIO { - public: - // XX: proto::TraceFile seems annoying, maybe just serialize::TraceFile ? - - // Open the protobuf associated at the filepath. Returns null on failure. - static ArenaPtr<proto::TraceFile> Open(std::string file_path); - // Open the protobuf from the file descriptor. Returns null on failure. - static ArenaPtr<proto::TraceFile> Open(int fd, const char* file_path = "<unknown>"); - - // Save the protobuf by overwriting the file at file_path. - // The file state is indeterminate at failure. - // Returns # of bytes written out on success, otherwise the errno value. - static iorap::expected<size_t /*bytes written*/, int /*errno*/> WriteFully( - const ::google::protobuf::MessageLite& message, - std::string_view file_path); - // Save the protobuf by truncating the file already open at 'fd'. - // The file state is indeterminate at failure. - // Returns # of bytes written out on success, otherwise the errno value. - static iorap::expected<size_t /*bytes written*/, int /*errno*/> WriteFully( - const ::google::protobuf::MessageLite& message, - int fd, - std::string_view file_path = "<unknown>"); - - ProtobufIO() = delete; -}; - -} // namespace serialize -} // namespace iorap - -#endif - |