diff options
| author | Eric Jeong <ericjeong@google.com> | 2022-01-31 19:00:37 +0000 |
|---|---|---|
| committer | Eric Jeong <ericjeong@google.com> | 2022-01-31 19:00:37 +0000 |
| commit | 7618b6d59ee4b5be2e3cc85fb7e1830bb757ac18 (patch) | |
| tree | 5cca9b0e00823d6173d96bf61193fbb3044e0fe6 | |
| parent | a461ee48caade7c184b848e6bf4827f75268190f (diff) | |
| download | iorap-7618b6d59ee4b5be2e3cc85fb7e1830bb757ac18.tar.gz | |
Revert "Remove iorap daemon codes"
Revert "Remove configs relevant to iorap"
Revert submission 16528474-remove-iorap
Reason for revert: build break
Reverted Changes:
I464c9e9c4:Remove scripts related to iorap
I0b8b1b064:Remove iorap daemon codes
I848f65908:Remove iorap framework codes
I294f37265:Remove configs relevant to iorap
Change-Id: I06b7c2fe6e211617233c35eb870d57666592d6ae
110 files changed, 20787 insertions, 0 deletions
diff --git a/Android.bp b/Android.bp new file mode 100644 index 0000000..1d181bd --- /dev/null +++ b/Android.bp @@ -0,0 +1,744 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +filegroup { + name: "iorap-aidl", + srcs: [ + // note: using **/* doesn't work, so list each file one by one: + // see also b/70046217 + + // note: list only 'interface' aidl files, otherwise + // aidl generates an error "Refusing to generate code with unstructured parcelables." + "binder/com/google/android/startop/iorap/IIorap.aidl", + "binder/com/google/android/startop/iorap/ITaskListener.aidl", + ], + path: "binder", +} + +cc_defaults { + name: "iorap-default-flags", + + cflags: [ + "-Wall", + "-Werror", + "-Wextra", + "-Wno-missing-field-initializers", + "-Wno-unused-parameter", + "-Wno-unused-variable", + ], + + local_include_dirs: [ + "include", + "src", + ], + // TODO: shouldn't need this but there's a soong/cmake generator bug. + export_include_dirs: [ + "include", + "src", + ], + + /* + TODO: Header refactoring cleanup: + + Option 1): Move src/$component/file_name.h to src/$component/include/$component/file_name.h + Option 2): Symlink src/$component/include/$component to src/$component + + Set export_include_dirs to '$component/include' for that component. + + Also delete the 'include' directory unless we have code other non-iorap + targets are allowed to reference. + */ + + clang: true, + static_libs: ["libc++fs"], + shared_libs: ["libbase"], + + // build all ioraps for host. + host_supported: true, + target: { + darwin: { + enabled: false, + } + }, +} + +cc_defaults { + name: "iorap-default-dependencies", + + static_libs: [ + "libiorap-binder", + "libplatformprotos", // android framework C++ protos. + ], + shared_libs: [ + "libbinder", + "libutils", + "libcutils", // tracing. + + "libfruit", // dependency injection. + // TODO: remove these annoying dependencies by hiding them in the main library code. + + // dependency for libplatformprotos + // "libprotobuf-cpp-lite", + + // libplatformprotos has an indirect dependency on full, causing compilation/linking + // errors if we use lite + "libprotobuf-cpp-full", + + "packagemanager_aidl-cpp", + + // phenotype flags support + "server_configurable_flags", + ], + + // srcs: [":libprotobuf-internal-protos"], + // commented out because it causes compilation errors + // TODO: can we use the lite library somehow? + + header_libs: ["librxcpp"], +} + +cc_library_static { + name: "libiorap-binder", + defaults: ["iorap-default-flags"], + + srcs: [ + ":iorap-aidl", + "src/binder/iiorap_impl.cc", + "src/binder/package_change_observer.cc", + "src/binder/package_manager_remote.cc", + "src/binder/package_version_map.cc", + ], + shared_libs: [ + "libbinder", + "libutils", + "libcutils", // tracing. + "packagemanager_aidl-cpp", + ], + aidl: { + local_include_dirs: ["binder"], + include_dirs: ["frameworks/native/aidl/binder"], + export_aidl_headers: true, + }, + + static_libs: [ + "libplatformprotos", // android framework C++ protos. + ], +} + +cc_defaults { + name: "libiorap-manager-default-dependencies", + static_libs: [ + "libiorap-binder", + "libiorap-perfetto", + "libiorap-prefetcher", + "libiorap-db", + "libiorap-maintenance", + ], + defaults: [ + "libiorap-perfetto-default-dependencies", + "libiorap-prefetcher-default-dependencies", + "libiorap-db-default-dependencies", + ], + // Users of 'libiorap-manager' also need to include these defaults to avoid + // linking errors. +} + +cc_library_static { + name: "libiorap-manager", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-manager-default-dependencies", + ], + + srcs: [ + "src/manager/**/*.cc", + ], +} + +cc_binary { + name: "iorapd", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-manager-default-dependencies", + ], + srcs: [ + "src/iorapd/main.cc", + ], + static_libs: [ + "libiorap-manager", + ], + init_rc: [ + "iorapd.rc", + ], + // iorapd fork+execs into iorap.prefetcherd and iorap.cmd.compiler + // maintenance used by tests + required: [ + "iorap.cmd.compiler", + "iorap.prefetcherd", + "iorap.cmd.maintenance", + ], +} + +cc_library_static { + name: "libiorap-inode2filename", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + ], + + srcs: [ + "src/inode2filename/**/*.cc", + ], +} + +cc_binary { + name: "iorap.inode2filename", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + ], + srcs: [ + "src/inode2filename/**/*.cc", + ], + // Easier debugging. TODO: make a separate debug config. + // XX: Using -O0 seems to completely hide some errors. + cflags: ["-O2", "-UNDEBUG", "-DIORAP_INODE2FILENAME_MAIN=1"], + sanitize: { + undefined: true, + all_undefined: true, + // Pretty print when ubsan detects a problem. + // Otherwise it just calls abort(). + +/* + diag: { + undefined: true, + }, + */ // don't use the diag when you want it to crash. + }, +} + +cc_test { + name: "iorapd-tests", + test_suites: ["device-tests"], + gtest: false, // we use gtest *and* gmock. + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-compiler-default-dependencies", + ], + srcs: [ + "tests/src/binder/*.cc", + "tests/src/inode2filename/*.cc", + "tests/src/log/*.cc", + "tests/src/tmp/*.cc", + ], + data: [ + "tests/src/compiler/testdata/*", + ], + cflags: ["-O2", "-UNDEBUG"], + + // TODO: we should probably have per-component tests. + static_libs: ["libgmock_main", "libgmock", "libgtest", "libiorap-inode2filename"], + +} + + +cc_test_host { + name: "iorapd-host-tests", + gtest: false, // we use gtest *and* gmock. + target: { + darwin: { + enabled: false, + } + }, + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-compiler-default-dependencies", + "libiorap-maintenance-default-dependencies", + ], + srcs: [ + "tests/src/compiler/*.cc", + "tests/src/db/*.cc", + "tests/src/maintenance/*.cc", + ], + data: [ + "tests/src/compiler/testdata/*", + "tests/src/maintenance/testdata/*", + ], + cflags: ["-O2", "-UNDEBUG"], + + // TODO: we should probably have per-component tests. + static_libs: [ + "libgmock_main", + "libgmock", + "libgtest", + "libiorap-compiler", + "libiorap-maintenance", + ], +} + +filegroup { + name: "libiorap-perfetto-protos", + srcs: [ + ], +} + +// Static libraries cannot export their dependencies, +// the current convention is to use an extra 'defaults' rule for statics +// to bring in all the dependencies. +cc_defaults { + name: "libiorap-perfetto-default-dependencies", + + // Some of the libperfetto header typedefs leak out into iorap. + // Avoids compilation #include errors. + // TODO: clean this up, the headers should not leak out (maybe all we need is a PerfettoConsumer + // forward declaration?). + include_dirs: ["external/perfetto/include"], + // Various perfetto protos are used directly by iorap. + // + // Furthermore, we need this regardless to avoid linking errors when linking + // libiorap-perfetto.a into the main cc_binary rule. + static_libs: [ + "perfetto_trace_protos", + ], + + shared_libs: [ + // Not part of true dependencies: Users of 'libiorap-perfetto' do not link against + // libperfetto. + // We only put this to avoid linking errors when building iorapd. + // TODO: can we split iorapd into libiorapd-main that doesn't link against libperfetto? + // only the last cc_binary should need the full transitive closure of the dependency graph. + "libperfetto", + ] +} + +cc_library_static { + name: "libiorap-perfetto", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-perfetto-default-dependencies", + ], + + srcs: [ + "src/perfetto/**/*.cc", + ], +} + +cc_binary { + name: "iorap.cmd.perfetto", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + ], + shared_libs: ["libperfetto"], + include_dirs: ["external/perfetto/include"], + srcs: [ + "src/perfetto/**/*.cc", + ], + // Easier debugging. TODO: make a separate debug config. + // XX: Using -O0 seems to completely hide some errors. + cflags: ["-O2", "-UNDEBUG", "-DIORAP_PERFETTO_MAIN=1"], + sanitize: { + undefined: true, + all_undefined: true, + // Pretty print when ubsan detects a problem. + // Otherwise it just calls abort(). + +/* + diag: { + undefined: true, + }, + */ // don't use the diag when you want it to crash. + }, + + static_libs: [ + "perfetto_trace_protos", + ], +} + +// Static libraries cannot export their dependencies, +// the current convention is to use an extra 'defaults' rule for statics +// to bring in all the dependencies. +cc_defaults { + name: "libiorap-compiler-default-dependencies", + + defaults: [ + // use the perfetto namespace + "libiorap-perfetto-default-dependencies", + // use the inode2filename namespace + "libiorap-serialize-default-dependencies", // uses but does not re-export serialize. + ], + + // Some of the libperfetto header typedefs leak out into iorap. + // Avoids compilation #include errors. + // TODO: clean this up, the headers should not leak out (maybe all we need is a PerfettoConsumer + // forward declaration?). + include_dirs: [], + // Various perfetto protos are used directly by iorap. + // + // Furthermore, we need this regardless to avoid linking errors when linking + // libiorap-compiler.a into the main cc_binary rule. + static_libs: [ + "libiorap-perfetto", + // "perfetto_trace_protos", + "libiorap-inode2filename", + "libiorap-serialize", + ], + + shared_libs: [ + // Not part of true dependencies: Users of 'libiorap-compiler' do not link against + // libperfetto. + // We only put this to avoid linking errors when building iorapd. + // TODO: can we split iorapd into libiorapd-main that doesn't link against libperfetto? + // only the last cc_binary should need the full transitive closure of the dependency graph. + ] +} + +cc_library_static { + name: "libiorap-compiler", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-compiler-default-dependencies", + ], + + srcs: [ + "src/compiler/**/*.cc", + ], +} + +cc_binary { + name: "iorap.cmd.compiler", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-compiler-default-dependencies", + ], + shared_libs: [], + include_dirs: [], + srcs: [ + "src/compiler/**/*.cc", + ], + // Easier debugging. TODO: make a separate debug config. + // XX: Using -O0 seems to completely hide some errors. + cflags: ["-O2", "-UNDEBUG", "-DIORAP_COMPILER_MAIN=1"], + sanitize: { + undefined: true, + all_undefined: true, + // Pretty print when ubsan detects a problem. + // Otherwise it just calls abort(). + +/* + diag: { + undefined: true, + }, + */ // don't use the diag when you want it to crash. + }, + + static_libs: [ + ], + required: [ + "iorap.inode2filename", + ], +} + +// Static libraries cannot export their dependencies, +// the current convention is to use an extra 'defaults' rule for statics +// to bring in all the dependencies. +cc_defaults { + name: "libiorap-serialize-default-dependencies", + + defaults: [ + ], + + include_dirs: [], + static_libs: [ + ], + shared_libs: [ + ], + // Above intentionally left empty. + srcs: [ + "src/serialize/**/*.proto", + ], +} + +cc_library_static { + name: "libiorap-serialize", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-serialize-default-dependencies", + ], + + srcs: [ + "src/serialize/**/*.cc", + ], +} + + +// Static libraries cannot export their dependencies, +// the current convention is to use an extra 'defaults' rule for statics +// to bring in all the dependencies. +cc_defaults { + name: "libiorap-prefetcher-default-dependencies", + + defaults: [ + ], + + include_dirs: [], + static_libs: [ + "libiorap-serialize", + ], + shared_libs: [ + "libminijail", + ], + + // disable mac builds because libminijail doesn't work there + target: { + darwin: { + enabled: false, + }, + }, +} + +cc_library_static { + name: "libiorap-prefetcher", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-prefetcher-default-dependencies", + "libiorap-serialize-default-dependencies", + ], + + srcs: [ + "src/prefetcher/**/*.cc", + ], +} + +cc_binary { + name: "iorap.prefetcherd", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-prefetcher-default-dependencies", + "libiorap-serialize-default-dependencies", + ], + shared_libs: [], + include_dirs: [], + srcs: [ + "src/prefetcher/**/*.cc", + ], + // Easier debugging. TODO: make a separate debug config. + // XX: Using -O0 seems to completely hide some errors. + cflags: ["-O2", "-UNDEBUG", "-DIORAP_PREFETCHER_MAIN=1"], + sanitize: { + undefined: true, + all_undefined: true, + // Pretty print when ubsan detects a problem. + // Otherwise it just calls abort(). + +/* + diag: { + undefined: true, + }, + */ // don't use the diag when you want it to crash. + }, + + static_libs: [ + ], +} + +cc_binary { + name: "iorap.cmd.prefetcher.client", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-prefetcher-default-dependencies", + "libiorap-serialize-default-dependencies", + ], + shared_libs: [], + include_dirs: [], + srcs: [ + "src/prefetcher/**/*.cc", + ], + // Easier debugging. TODO: make a separate debug config. + // XX: Using -O0 seems to completely hide some errors. + cflags: ["-O2", "-UNDEBUG", "-DIORAP_PREFETCHER_MAIN_CLIENT=1"], + sanitize: { + undefined: true, + all_undefined: true, + // Pretty print when ubsan detects a problem. + // Otherwise it just calls abort(). + +/* + diag: { + undefined: true, + }, + */ // don't use the diag when you want it to crash. + }, + + static_libs: [ + ], +} + +prebuilt_etc { + name: "iorap.prefetcherd.policy", + sub_dir: "seccomp_policy", + arch: { + arm: { + src: "seccomp_policy/prefetcherd.arm.policy" + }, + arm64: { + src: "seccomp_policy/prefetcherd.arm64.policy" + }, + x86: { + src: "seccomp_policy/prefetcherd.x86.policy" + }, + x86_64: { + src: "seccomp_policy/prefetcherd.x86_64.policy" + }, + }, + required: [ + "crash_dump.policy", + ], +} + +// Static libraries cannot export their dependencies, +// the current convention is to use an extra 'defaults' rule for statics +// to bring in all the dependencies. +cc_defaults { + name: "libiorap-db-default-dependencies", + + defaults: [ + ], + + include_dirs: [], + static_libs: [ + ], + shared_libs: [ + "libsqlite", + ], +} + +cc_library_static { + name: "libiorap-db", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-db-default-dependencies", + ], + + srcs: [ + "src/db/**/*.cc", + ], +} + +cc_binary { + name: "iorap.cmd.db", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-db-default-dependencies", + ], + shared_libs: [], + include_dirs: [], + srcs: [ + "src/db/**/*.cc", + ], + // Easier debugging. TODO: make a separate debug config. + // XX: Using -O0 seems to completely hide some errors. + cflags: ["-O2", "-UNDEBUG", "-DIORAP_DB_MAIN=1"], + sanitize: { + undefined: true, + all_undefined: true, + // Pretty print when ubsan detects a problem. + // Otherwise it just calls abort(). + +/* + diag: { + undefined: true, + }, + */ // don't use the diag when you want it to crash. + }, + + static_libs: [ + ], +} + +cc_defaults { + name: "libiorap-maintenance-default-dependencies", + + defaults: [ + "libiorap-compiler-default-dependencies", + "libiorap-db-default-dependencies", + "libiorap-prefetcher-default-dependencies", + ], + + include_dirs: [], + + static_libs: [ + "libiorap-binder", + "libiorap-compiler", + "libiorap-db", + "libiorap-prefetcher", + ], + + shared_libs: [] +} + +cc_library_static { + name: "libiorap-maintenance", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-maintenance-default-dependencies", + ], + + srcs: [ + "src/maintenance/*.cc", + ], +} + +cc_binary { + name: "iorap.cmd.maintenance", + defaults: [ + "iorap-default-flags", + "iorap-default-dependencies", + "libiorap-maintenance-default-dependencies", + ], + shared_libs: [], + include_dirs: [], + srcs: [ + "src/maintenance/*.cc", + ], + cflags: ["-O2", "-UNDEBUG", "-DIORAP_MAINTENANCE_MAIN=1"], + sanitize: { + undefined: true, + all_undefined: true, + + diag: { + undefined: true, + }, + }, + + static_libs: [], +} diff --git a/Android.mk b/Android.mk new file mode 100644 index 0000000..4e9f5fd --- /dev/null +++ b/Android.mk @@ -0,0 +1,33 @@ +# +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +LOCAL_PATH := $(call my-dir) + +# Build every binary target in system/iorap +.PHONY: iorap-nall +iorap-nall: \ + iorapd iorap.inode2filename iorapd-tests iorap.cmd.perfetto \ + iorap.cmd.compiler + +# Build every binary target required for +# frameworks/base/startop/scripts/app_startup_runner +# to work with iorap. +.PHONY: iorap-app-startup-runner +iorap-app-startup-runner: \ + iorapd iorap.inode2filename \ + iorap.cmd.compiler + + @@ -0,0 +1,4 @@ +keunyoung@google.com +ericjeong@google.com +enh@google.com +yawanng@google.com diff --git a/binder/com/google/android/startop/iorap/AppIntentEvent.aidl b/binder/com/google/android/startop/iorap/AppIntentEvent.aidl new file mode 100644 index 0000000..dbf1d02 --- /dev/null +++ b/binder/com/google/android/startop/iorap/AppIntentEvent.aidl @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +/** @hide */ +parcelable AppIntentEvent cpp_header "binder/app_intent_event.h"; diff --git a/binder/com/google/android/startop/iorap/AppLaunchEvent.aidl b/binder/com/google/android/startop/iorap/AppLaunchEvent.aidl new file mode 100644 index 0000000..ea3f9f1 --- /dev/null +++ b/binder/com/google/android/startop/iorap/AppLaunchEvent.aidl @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +/** @hide */ +parcelable AppLaunchEvent cpp_header "binder/app_launch_event.h"; diff --git a/binder/com/google/android/startop/iorap/DexOptEvent.aidl b/binder/com/google/android/startop/iorap/DexOptEvent.aidl new file mode 100644 index 0000000..a4e175b --- /dev/null +++ b/binder/com/google/android/startop/iorap/DexOptEvent.aidl @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +/** @hide */ +parcelable DexOptEvent cpp_header "binder/dexopt_event.h"; diff --git a/binder/com/google/android/startop/iorap/IIorap.aidl b/binder/com/google/android/startop/iorap/IIorap.aidl new file mode 100644 index 0000000..05faefe --- /dev/null +++ b/binder/com/google/android/startop/iorap/IIorap.aidl @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +import com.google.android.startop.iorap.ITaskListener; + +import com.google.android.startop.iorap.AppIntentEvent; +import com.google.android.startop.iorap.AppLaunchEvent; +import com.google.android.startop.iorap.DexOptEvent; +import com.google.android.startop.iorap.JobScheduledEvent; +import com.google.android.startop.iorap.PackageEvent; +import com.google.android.startop.iorap.RequestId; +import com.google.android.startop.iorap.SystemServiceEvent; +import com.google.android.startop.iorap.SystemServiceUserEvent; + +/** +* IIOrap is a client interface to the input/output readahead and pin daemon (iorapd). +* +* The aim is to speed-up the cold start-up time of certain use-cases like application startup +* by utilizing trace-based pinning or readahead. +* +* Programatically, the behavior of iorapd should be treated like a black box. There is no +* "correctness", but only performance. By sending the right events at the appropriate time, +* we can squeeze more performance out of the system. +* +* If some events are not appropriately wired up, system performance may (temporarily) degrade. +* +* {@hide} */ +oneway interface IIorap { + /** + * Set an ITaskListener which will be used to deliver notifications of in-progress/completition + * for the onXEvent method calls below this.<br /><br /> + * + * iorapd does all the work asynchronously and may deliver one or more onProgress events after + * the event begins to be processed. It will always send back one onComplete that is considered + * terminal.<br /><br /> + * + * onProgress/onComplete are matched to the original event by the requestId. Once an onComplete + * occurs for any given requestId, no further callbacks with the same requestId will occur. + * It is illegal for the caller to reuse the same requestId on different invocations of IIorap. + * <br /><br /> + * + * onXEvent(id1) must be well-ordered w.r.t. onXEvent(id2), the assumption is that later + * calls happen-after earlier calls and that id2 > id1. Decreasing request IDs will + * immediately get rejected. + * <br /><br /> + * + * Sequence diagram of stereotypical successful event delivery and response notification: + * + * <pre> + * + * ┌─────────────┐ ┌──────┐ + * │system_server│ │iorapd│ + * └──────┬──────┘ └──┬───┘ + * Request [01]: onSomeEvent ┌┴┐ + * │────────────────────────>│ │ + * │ │ │ + * │ │ │ ╔════════════════════════╗ + * │ │ │ ║start processing event ░║ + * │ │ │ ╚════════════════════════╝ + * │ │ │ + * ╔═══════╤════════╪═════════════════════════╪═╪══════════════════════════════╗ + * ║ LOOP │ 1 or more times │ │ ║ + * ╟───────┘ │ │ │ ║ + * ║ │Request [01]: onProgress │ │ ║ + * ║ │<────────────────────────│ │ ║ + * ║ │ │ │ ║ + * ║ │ │ │────┐ ║ + * ║ │ │ │ │ workload in progress ║ + * ║ │ │ │<───┘ ║ + * ╚════════════════╪═════════════════════════╪═╪══════════════════════════════╝ + * │ └┬┘ + * . . + * . . + * . . + * . . + * . . + * │ ┌┴┐ ╔═════════════════════════╗ + * │ │ │ ║finish processing event ░║ + * │ │ │ ╚═════════════════════════╝ + * │Request [01]: onComplete │ │ + * │<────────────────────────│ │ + * ┌──────┴──────┐ ┌─└┬┘──┐ + * │system_server│ │iorapd│ + * └─────────────┘ └──────┘ + * + * </pre> <!-- system/iorap/docs/binder/IIorap_setTaskListener.plantuml --> + */ + void setTaskListener(ITaskListener listener); + + // All callbacks will be done via the ITaskListener. + // The RequestId passed in is the same RequestId sent back via the ITaskListener. + // See above for more details. + + // Note: For each ${Type}Event, see the ${Type}Event.java for more documentation + // in frameworks/base/startop/src/com/google/android/startop/iorap/${Type}Event.java + + // void onActivityHintEvent(in RequestId request, in ActivityHintEvent event); + void onAppLaunchEvent(in RequestId request, in AppLaunchEvent event); + void onDexOptEvent(in RequestId request, in DexOptEvent event); + void onJobScheduledEvent(in RequestId request, in JobScheduledEvent event); + void onPackageEvent(in RequestId request, in PackageEvent event); + void onAppIntentEvent(in RequestId request, in AppIntentEvent event); + void onSystemServiceEvent(in RequestId request, in SystemServiceEvent event); + void onSystemServiceUserEvent(in RequestId request, in SystemServiceUserEvent event); +} diff --git a/binder/com/google/android/startop/iorap/ITaskListener.aidl b/binder/com/google/android/startop/iorap/ITaskListener.aidl new file mode 100644 index 0000000..0b5f4a6 --- /dev/null +++ b/binder/com/google/android/startop/iorap/ITaskListener.aidl @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +import com.google.android.startop.iorap.TaskResult; +import com.google.android.startop.iorap.RequestId; + +/** +* Provide callbacks to the {@code IIorap} client in response to method invocations.<br /><br /> +* +* @see com.google.android.startop.iorap.IIorap +* +* {@hide} */ +oneway interface ITaskListener { + void onProgress(in RequestId requestId, in TaskResult result); + void onComplete(in RequestId requestId, in TaskResult result); +} + +// TODO: we can probably delete the multiple methods. one is likely sufficient? diff --git a/binder/com/google/android/startop/iorap/JobScheduledEvent.aidl b/binder/com/google/android/startop/iorap/JobScheduledEvent.aidl new file mode 100644 index 0000000..d4b5454 --- /dev/null +++ b/binder/com/google/android/startop/iorap/JobScheduledEvent.aidl @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +/** @hide */ +parcelable JobScheduledEvent cpp_header "binder/job_scheduled_event.h"; diff --git a/binder/com/google/android/startop/iorap/PackageEvent.aidl b/binder/com/google/android/startop/iorap/PackageEvent.aidl new file mode 100644 index 0000000..8f9d18e --- /dev/null +++ b/binder/com/google/android/startop/iorap/PackageEvent.aidl @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +/** @hide */ +parcelable PackageEvent cpp_header "binder/package_event.h"; diff --git a/binder/com/google/android/startop/iorap/RequestId.aidl b/binder/com/google/android/startop/iorap/RequestId.aidl new file mode 100644 index 0000000..895af87 --- /dev/null +++ b/binder/com/google/android/startop/iorap/RequestId.aidl @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +/** @hide */ +parcelable RequestId cpp_header "binder/request_id.h"; diff --git a/binder/com/google/android/startop/iorap/SystemServiceEvent.aidl b/binder/com/google/android/startop/iorap/SystemServiceEvent.aidl new file mode 100644 index 0000000..005676f --- /dev/null +++ b/binder/com/google/android/startop/iorap/SystemServiceEvent.aidl @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +/** @hide */ +parcelable SystemServiceEvent cpp_header "binder/system_service_event.h"; diff --git a/binder/com/google/android/startop/iorap/SystemServiceUserEvent.aidl b/binder/com/google/android/startop/iorap/SystemServiceUserEvent.aidl new file mode 100644 index 0000000..c165dc4 --- /dev/null +++ b/binder/com/google/android/startop/iorap/SystemServiceUserEvent.aidl @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +/** @hide */ +parcelable SystemServiceUserEvent cpp_header "binder/system_service_user_event.h"; diff --git a/binder/com/google/android/startop/iorap/TaskResult.aidl b/binder/com/google/android/startop/iorap/TaskResult.aidl new file mode 100644 index 0000000..76cce8f --- /dev/null +++ b/binder/com/google/android/startop/iorap/TaskResult.aidl @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.startop.iorap; + +/** @hide */ +parcelable TaskResult cpp_header "binder/task_result.h"; diff --git a/docs/binder/ActivityHint.dot b/docs/binder/ActivityHint.dot new file mode 100644 index 0000000..94b5a84 --- /dev/null +++ b/docs/binder/ActivityHint.dot @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Convert with `graph-easy --as=boxart` to get textual rendering. + */ +digraph finite_state_machine { +rankdir=LR; +size="8,5" +node [shape = circle ]; STARTED; +node [shape = doublecircle]; + +"" -> STARTED; +STARTED -> CANCELLED; +STARTED -> COMPLETED; +COMPLETED -> POST_COMPLETED; +COMPLETED -> CANCELLED; +} diff --git a/docs/binder/IIorap_setTaskListener.plantuml b/docs/binder/IIorap_setTaskListener.plantuml new file mode 100644 index 0000000..5cdfac7 --- /dev/null +++ b/docs/binder/IIorap_setTaskListener.plantuml @@ -0,0 +1,53 @@ +@startuml +' Copyright (C) 2018 The Android Open Source Project +' +' Licensed under the Apache License, Version 2.0 (the "License"); +' you may not use this file except in compliance with the License. +' You may obtain a copy of the License at +' +' http://www.apache.org/licenses/LICENSE-2.0 +' +' Unless required by applicable law or agreed to in writing, software +' distributed under the License is distributed on an "AS IS" BASIS, +' WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +' See the License for the specific language governing permissions and +' limitations under the License. + +' Compile with PlantUML: +' http://www.plantuml.com/plantuml/uml/ + + +'hide footbox + +participant "system_server" as ss +participant iorapd + +autonumber 1 0 "Request [00]:" + +ss -\ iorapd : onSomeEvent +activate iorapd +note right of iorapd + start processing event +end note + +loop 1 or more times +iorapd -\ ss : onProgress + +autonumber stop +iorapd -> iorapd : workload in progress +autonumber resume + +end + +' some time later... +... ... + +note right of iorapd + finish processing event +end note + + +iorapd -\ ss : onComplete + +deactivate iorapd +@enduml diff --git a/docs/binder/TaskResult.dot b/docs/binder/TaskResult.dot new file mode 100644 index 0000000..28d7819 --- /dev/null +++ b/docs/binder/TaskResult.dot @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Convert with `graph-easy --as=boxart` to get textual rendering. + */ +digraph finite_state_machine { +rankdir=LR; +size="8,5" +node [shape = circle ]; BEGAN ONGOING; +node [shape = doublecircle]; + +// graph-easy does not support multiple state syntax {}, write one-by-one + +"" -> BEGAN; +"" -> ERROR; +BEGAN -> ERROR; +ONGOING -> ERROR; + +BEGAN -> ONGOING; +BEGAN -> COMPLETED; +ONGOING -> COMPLETED; + +} diff --git a/include/binder/activity_info.h b/include/binder/activity_info.h new file mode 100644 index 0000000..0712f18 --- /dev/null +++ b/include/binder/activity_info.h @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_ACTIVITY_INFO_H_ +#define IORAP_BINDER_ACTIVITY_INFO_H_ + +#include "binder/common.h" +#include "binder/auto_parcelable.h" +#include "common/introspection.h" + +#include <string> + +namespace iorap { +namespace binder { + +struct ActivityInfo : public AutoParcelable<ActivityInfo> { + std::string package_name; + std::string activity_name; +}; + +IORAP_INTROSPECT_ADAPT_STRUCT(ActivityInfo, package_name, activity_name); + +} +} +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(ActivityInfo) + +#endif // IORAP_BINDER_ACTIVITY_INFO_H_ diff --git a/include/binder/app_intent_event.h b/include/binder/app_intent_event.h new file mode 100644 index 0000000..bbd19ee --- /dev/null +++ b/include/binder/app_intent_event.h @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_APP_INTENT_EVENT_H_ +#define IORAP_BINDER_APP_INTENT_EVENT_H_ + +#include "binder/common.h" +#include "binder/auto_parcelable.h" +#include "common/introspection.h" + +#include "binder/activity_info.h" + +namespace iorap { +namespace binder { + +struct AppIntentEvent : public AutoParcelable<AppIntentEvent> { + enum class Type : int32_t { + kDefaultIntentChanged = 0, + }; + + Type type; + ActivityInfo old_activity_info; + ActivityInfo new_activity_info; +}; + +IORAP_INTROSPECT_ADAPT_STRUCT(AppIntentEvent, type, old_activity_info, new_activity_info); + +} +} +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(AppIntentEvent) + +#endif // IORAP_BINDER_APP_INTENT_EVENT_H_ diff --git a/include/binder/app_launch_event.h b/include/binder/app_launch_event.h new file mode 100644 index 0000000..0127e09 --- /dev/null +++ b/include/binder/app_launch_event.h @@ -0,0 +1,448 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_APP_LAUNCH_EVENT_H_ +#define IORAP_BINDER_APP_LAUNCH_EVENT_H_ + +#include "binder/common.h" +#include "common/introspection.h" +#include "common/expected.h" + +#include <binder/Parcel.h> +#include <binder/Parcelable.h> +#include <frameworks/base/core/proto/android/content/intent.pb.h> // IntentProto +#include <frameworks/base/core/proto/android/server/activitymanagerservice.pb.h> // ActivityRecord + +namespace iorap { +namespace binder { + +// These protos are part of the iorapd binder ABI, alias them for easier usage. +using IntentProto = ::android::content::IntentProto; +using ActivityRecordProto = ::com::android::server::wm::ActivityRecordProto; + +struct AppLaunchEvent : public ::android::Parcelable { + // Index position matters: Keep up-to-date with AppLaunchEvent.java sTypes field. + enum class Type : int32_t { + kUninitialized = -1, + kIntentStarted = 0, + kIntentFailed = 1, + kActivityLaunched = 2, + kActivityLaunchFinished = 3, + kActivityLaunchCancelled = 4, + kReportFullyDrawn = 5, + }; + + enum class Temperature : int32_t { + kUninitialized = -1, + kCold = 1, + kWarm = 2, + kHot = 3, + }; + + Type type{Type::kUninitialized}; + int64_t sequence_id{-1}; + // kIntentStarted only. + std::unique_ptr<IntentProto> intent_proto; + // kActivityLaunched only. + Temperature temperature{Temperature::kUninitialized}; + // kActivityLaunch*. Can be null in kActivityLaunchCancelled. + std::unique_ptr<ActivityRecordProto> activity_record_proto; + // kIntentStarted, kActivityLaunchFinished and kReportFullyDrawn only. + int64_t timestamp_nanos{-1}; + + AppLaunchEvent() = default; + AppLaunchEvent(Type type, + int64_t sequence_id, + std::unique_ptr<IntentProto> intent_proto = nullptr, + Temperature temperature = Temperature::kUninitialized, + std::unique_ptr<ActivityRecordProto> activity_record_proto = nullptr, + int64_t timestamp_nanos = -1) + : type(type), + sequence_id(sequence_id), + intent_proto(std::move(intent_proto)), + temperature(temperature), + activity_record_proto(std::move(activity_record_proto)), + timestamp_nanos(timestamp_nanos) { + } + + AppLaunchEvent(const AppLaunchEvent& other) { + *this = other; + } + + AppLaunchEvent& operator=(const AppLaunchEvent& other) { + if (&other == this) { + return *this; + } + + type = other.type; + sequence_id = other.sequence_id; + if (other.intent_proto != nullptr) { + intent_proto.reset(new IntentProto(*other.intent_proto)); + } + temperature = other.temperature; + if (other.activity_record_proto != nullptr) { + activity_record_proto.reset(new ActivityRecordProto(*other.activity_record_proto)); + } + timestamp_nanos = other.timestamp_nanos; + + return *this; + } + + ::android::status_t readFromParcel(const android::Parcel* parcel) override { + +# define PARCEL_READ_OR_RETURN(function, ...) \ + if (::android::status_t res = function(__VA_ARGS__); res != ::android::NO_ERROR) { \ + LOG(ERROR) << "AppLaunchEvent::readFromParcel failed"; \ + return res; \ + } + + int32_t type_int; + PARCEL_READ_OR_RETURN(parcel->readInt32, &type_int); + type = static_cast<Type>(type_int); + + LOG(VERBOSE) << "AppLaunchEvent::readFromParcel (type=" << type_int << ")"; + + PARCEL_READ_OR_RETURN(parcel->readInt64, &sequence_id); + + switch (type) { + case Type::kIntentStarted: + PARCEL_READ_OR_RETURN(readIntent, parcel); + PARCEL_READ_OR_RETURN(parcel->readInt64, ×tamp_nanos); + break; + case Type::kIntentFailed: + // No extra arguments. + break; + case Type::kActivityLaunched: { + PARCEL_READ_OR_RETURN(readActivityRecordProto, parcel); + int32_t temperature_int; + PARCEL_READ_OR_RETURN(parcel->readInt32, &temperature_int); + temperature = static_cast<Temperature>(temperature_int); + break; + } + case Type::kActivityLaunchFinished: + PARCEL_READ_OR_RETURN(readActivityRecordProto, parcel); + PARCEL_READ_OR_RETURN(parcel->readInt64, ×tamp_nanos); + break; + case Type::kActivityLaunchCancelled: + PARCEL_READ_OR_RETURN(readActivityRecordProtoNullable, parcel); + break; + case Type::kReportFullyDrawn: + PARCEL_READ_OR_RETURN(readActivityRecordProto, parcel); + PARCEL_READ_OR_RETURN(parcel->readInt64, ×tamp_nanos); + break; + default: + return android::BAD_VALUE; + } +# undef PARCEL_READ_OR_RETURN + + return ::android::NO_ERROR; + + // TODO: std::variant + protobuf implementation in AutoParcelable. + } + +#define PARCEL_WRITE_OR_RETURN(function, ...) \ + if (::android::status_t res = function(__VA_ARGS__); res != ::android::NO_ERROR) { \ + return res; \ + } + + ::android::status_t writeToParcel(android::Parcel* parcel) const override { + PARCEL_WRITE_OR_RETURN(parcel->writeInt32, static_cast<int32_t>(type)); + PARCEL_WRITE_OR_RETURN(parcel->writeInt64, sequence_id); + + switch (type) { + case Type::kIntentStarted: + PARCEL_WRITE_OR_RETURN(writeIntent, parcel); + PARCEL_WRITE_OR_RETURN(parcel->writeInt64, timestamp_nanos); + break; + case Type::kIntentFailed: + // No extra arguments. + break; + case Type::kActivityLaunched: + PARCEL_WRITE_OR_RETURN(writeActivityRecordProto, parcel); + PARCEL_WRITE_OR_RETURN(parcel->writeInt32, static_cast<int32_t>(temperature)); + break; + case Type::kActivityLaunchFinished: + PARCEL_WRITE_OR_RETURN(writeActivityRecordProto, parcel); + PARCEL_WRITE_OR_RETURN(parcel->writeInt64, timestamp_nanos); + break; + case Type::kActivityLaunchCancelled: + PARCEL_WRITE_OR_RETURN(writeActivityRecordProtoNullable, parcel); + break; + case Type::kReportFullyDrawn: + PARCEL_WRITE_OR_RETURN(writeActivityRecordProtoNullable, parcel); + PARCEL_WRITE_OR_RETURN(parcel->writeInt64, timestamp_nanos); + break; + default: + DCHECK(false) << "attempted to write an uninitialized AppLaunchEvent to Parcel"; + return android::BAD_VALUE; + } + +#undef PARCEL_WRITE_OR_RETURN + + return android::NO_ERROR; + } + + private: + // Using 'unique_ptr' here because protobufs don't have a move constructor. Is there + // a better way that is cheap to pass them around? + template <typename T> + static expected<std::unique_ptr<T>, ::android::status_t> + ReadProto(const android::Parcel* parcel) { + DCHECK(parcel != nullptr); + + ::android::status_t res; + + std::vector<uint8_t> byte_vector; + if ((res = parcel->readByteVector(/*out*/&byte_vector)) != ::android::NO_ERROR) { + return unexpected(res); + } + // TODO: we may want to do this without an extra copy, by parsing + // the protobuf directly out of the parcel. + + const uint8_t* data = byte_vector.data(); + const size_t size = byte_vector.size(); + + std::unique_ptr<T> proto_ptr{new T{}}; + + if (!proto_ptr) { + return unexpected(::android::NO_MEMORY); + } + + if (!proto_ptr->ParseFromArray(data, size)) { + return unexpected(::android::BAD_VALUE); + } + + return proto_ptr; + } + + template <typename T> + static expected<std::unique_ptr<T>, ::android::status_t> + ReadNullableProto(const android::Parcel* parcel) { + DCHECK(parcel != nullptr); + + bool value; + + ::android::status_t res; + res = parcel->readBool(/*out*/&value); + + if (res != ::android::NO_ERROR) { + return unexpected(res); + } + + if (!value) { + return std::unique_ptr<T>{nullptr}; + } + + return ReadProto<T>(parcel); + } + + template <typename T> + static ::android::status_t + WriteProto(android::Parcel* parcel, const std::unique_ptr<T>& proto) { + DCHECK(parcel != nullptr); + DCHECK(proto != nullptr); + + std::vector<uint8_t> byte_vector; + { + const int serialized_size = proto->ByteSize(); + byte_vector.resize(serialized_size); + if (!proto->SerializeToArray(byte_vector.data(), serialized_size)) { + return ::android::BAD_VALUE; + } + } + + ::android::status_t res; + if ((res = parcel->writeByteVector(/*in*/byte_vector)) != ::android::NO_ERROR) { + return res; + } + + return ::android::NO_ERROR; + } + + template <typename T> + static ::android::status_t + WriteNullableProto(android::Parcel* parcel, const std::unique_ptr<T>& maybe_proto) { + bool value = (maybe_proto != nullptr); + + ::android::status_t res; + res = parcel->writeBool(value); + + if (res != ::android::NO_ERROR) { + return res; + } + + if (!value) { + return ::android::NO_ERROR; + } + + return WriteProto<T>(parcel, maybe_proto); + } + + android::status_t readIntent(const android::Parcel* parcel) { + expected<std::unique_ptr<IntentProto>, ::android::status_t> maybe_intent = + ReadProto<IntentProto>(parcel); + + if (maybe_intent) { + intent_proto = std::move(maybe_intent.value()); + return ::android::NO_ERROR; + } else { + return maybe_intent.error(); + } + } + + android::status_t readActivityRecordProto(const android::Parcel* parcel) { + expected<std::unique_ptr<ActivityRecordProto>, ::android::status_t> maybe_record = + ReadProto<ActivityRecordProto>(parcel); + + if (maybe_record) { + activity_record_proto = std::move(maybe_record.value()); + return ::android::NO_ERROR; + } else { + return maybe_record.error(); + } + } + + android::status_t readActivityRecordProtoNullable(const android::Parcel* parcel) { + expected<std::unique_ptr<ActivityRecordProto>, ::android::status_t> maybe_record = + ReadNullableProto<ActivityRecordProto>(parcel); + + if (maybe_record) { + activity_record_proto = std::move(maybe_record.value()); + return ::android::NO_ERROR; + } else { + return maybe_record.error(); + } + } + + android::status_t writeIntent(android::Parcel* parcel) const { + return WriteProto<IntentProto>(parcel, intent_proto); + } + + android::status_t writeActivityRecordProto(android::Parcel* parcel) const { + return WriteProto<ActivityRecordProto>(parcel, activity_record_proto); + } + + android::status_t writeActivityRecordProtoNullable(android::Parcel* parcel) const { + return WriteNullableProto<ActivityRecordProto>(parcel, activity_record_proto); + } +}; + +inline std::ostream& operator<<(std::ostream& os, const AppLaunchEvent::Type& type) { + switch (type) { + case AppLaunchEvent::Type::kUninitialized: + os << "kUninitialized"; + break; + case AppLaunchEvent::Type::kIntentStarted: + os << "kIntentStarted"; + break; + case AppLaunchEvent::Type::kIntentFailed: + os << "kIntentFailed"; + break; + case AppLaunchEvent::Type::kActivityLaunched: + os << "kActivityLaunched"; + break; + case AppLaunchEvent::Type::kActivityLaunchCancelled: + os << "kActivityLaunchCancelled"; + break; + case AppLaunchEvent::Type::kActivityLaunchFinished: + os << "kActivityLaunchFinished"; + break; + case AppLaunchEvent::Type::kReportFullyDrawn: + os << "kReportFullyDrawn"; + break; + default: + os << "(unknown)"; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const AppLaunchEvent::Temperature& type) { + switch (type) { + case AppLaunchEvent::Temperature::kUninitialized: + os << "kUninitialized"; + break; + case AppLaunchEvent::Temperature::kCold: + os << "kCold"; + break; + case AppLaunchEvent::Temperature::kWarm: + os << "kWarm"; + break; + case AppLaunchEvent::Temperature::kHot: + os << "kHot"; + break; + default: + os << "(unknown)"; + } + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const AppLaunchEvent& e) { + os << "AppLaunchEvent{"; + os << "type=" << e.type << ","; + os << "sequence_id=" << e.sequence_id << ","; + + os << "intent_proto="; + if (e.intent_proto == nullptr) { + os << "(nullptr)"; + } else { + os << "(action=" << e.intent_proto->action() << ","; + os << "component="; + if (e.intent_proto->has_component()) { + // $package/$class_name + os << e.intent_proto->component().package_name() << "/" + << e.intent_proto->component().class_name(); + } else { + os << "(no component)"; + } + os << ")"; + } + os << ","; + + os << "temperature=" << e.temperature << ","; + os << ","; + + os << "activity_record_proto="; + if (e.activity_record_proto == nullptr) { + os << "(nullptr)"; + } else { + // title or component name. + os << "'" << e.activity_record_proto->identifier().title() << "'"; + } + os << ","; + + os << "timestamp_nanos=" << e.timestamp_nanos << ","; + os << ","; + + os << "}"; + + return os; +} + +/* +IORAP_INTROSPECT_ADAPT_STRUCT(AppLaunchEvent, + type, + sequence_id, + intent_proto, + temperature, + activity_record_proto); +*/ + +} // namespace binder +} // namespace iorap + +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(AppLaunchEvent) + +#endif // IORAP_BINDER_APP_LAUNCH_EVENT_H_ diff --git a/include/binder/auto_parcelable.h b/include/binder/auto_parcelable.h new file mode 100644 index 0000000..357615c --- /dev/null +++ b/include/binder/auto_parcelable.h @@ -0,0 +1,188 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_AUTO_PARCELABLE_H_ +#define IORAP_BINDER_AUTO_PARCELABLE_H_ + +#include "binder/Parcelable.h" +#include "binder/Parcel.h" +#include "binder/Status.h" + +#include "common/introspection.h" + +namespace iorap { +namespace binder { + +// +// Implements the android::Parcelable interface (readFromParcel, writeToParcel) +// automatically by using compile-time introspection on T. +// +// Requires that 'T' implements introspection by using the IORAP_INTROSPECT_ADAPT_STRUCT macro. +// +template <typename T> +struct AutoParcelable : public ::android::Parcelable { + private: + using Status = android::binder::Status; + using Parcel = android::Parcel; + using Parcelable = android::Parcelable; + using status_t = android::status_t; + + public: + // Write every (introspected) field to the parcel by automatically inferring the correct + // write method to invoke on the parcel from the member type. + status_t writeToParcel(Parcel* parcel) const override { + if (parcel == nullptr) { + return ::android::UNEXPECTED_NULL; + } + + status_t result = android::NO_ERROR; + ::iorap::introspect::for_each_member_field_value(*Self(), [&](auto&& value) { + if (result == android::NO_ERROR) { + result = writeAnyToParcel(/*inout*/parcel, value); + } + }); + + return result; + } + + // Read every (introspected) field to the parcel by automatically inferring the correct + // read method to invoke on the parcel from the member type. + // + // Resilient to partial read failures: A return code other than NO_ERROR means that + // the current value is left unmodified. + status_t readFromParcel(const Parcel* parcel) override { + if (parcel == nullptr) { + return ::android::UNEXPECTED_NULL; + } + + T tmp{*Self()}; + + // Unpack all the parcelable data into a temporary copy. + // Parceling could fail halfway through, in which case + // the original object is unaffected. + + status_t result = android::NO_ERROR; + ::iorap::introspect::for_each_member_field_set_value(tmp, [&](auto field_type) { + // type<?> field_type + + using ValueT = typename decltype(field_type)::type; + + if (result == android::NO_ERROR) { + auto&& [res, read_value] = readAnyFromParcel<ValueT>(/*inout*/parcel); + result = res; + return ::iorap::introspect::aliasing_forward<ValueT>(read_value); + } else { + // TODO: nice-to-have fold over members to early-out on failure. + return ValueT{}; + } + }); + + if (result != android::NO_ERROR) { + return result; + } + + // Success! Now we can copy all the data in a single step. + *Self() = std::move(tmp); + + // TODO: nice-to-have some kind of invariants-checking after reading the parcel data. + + return ::android::NO_ERROR; + } + + private: +#define AUTO_PARCELABLE_BINDER_MAPPING(FN) \ +FN(Byte,int8_t)\ +FN(Int32,int32_t)\ +FN(Uint32,uint32_t)\ +FN(Int64,int64_t)\ +FN(Uint64,uint64_t)\ +FN(Float,float)\ +FN(Double,double)\ +FN(Bool,bool)\ +FN(CString,const char*)\ +FN(String16,const String16&)\ +FN(String16,const std::unique_ptr<String16>&)\ +FN(StrongBinder,const sp<IBinder>&)\ + + template <typename F> + static status_t writeAnyToParcel(Parcel* parcel, const F& value) { + using namespace android; // NOLINT + + // 'F' is the original type of the field here, so it's safe to use it undecayed. + // However, to make matching easier we almost always want to match against the decayed type. + using D = std::decay_t<F>; // [const] [volatile] X[&][&] -> X + + if constexpr (std::is_base_of_v<Parcelable, D>) { + return value.writeToParcel(parcel); + } else if constexpr (std::is_enum_v<D>) { + return writeAnyToParcel(parcel, static_cast<std::underlying_type_t<F>>(value)); +#define AUTO_PARCELABLE_WRITE_TO_PARCEL(fn_name, type_name) \ + } else if constexpr (std::is_same_v<D, std::decay_t<type_name>>) { \ + return parcel->write ## fn_name (value); +AUTO_PARCELABLE_BINDER_MAPPING(AUTO_PARCELABLE_WRITE_TO_PARCEL) + } else if constexpr (std::is_same_v<D, std::string>) { + return parcel->writeUtf8AsUtf16(value); + } else { + STATIC_FAIL(D, "Unsupported type: Add more manual type conversions above^^^"); + } + +#undef AUTO_PARCELABLE_WRITE_TO_PARCEL + } + + template <typename F> + static auto readAnyFromParcel(const Parcel* parcel) { + // returns pair(status_t, ~F~) + using namespace android; + + // Since 'F' is almost always an lvalue reference (due to F=decltype(auto&&), + // we should lose the references, and also any consts. + using D = std::decay_t<F>; + + D value; + status_t result; + + if constexpr (std::is_base_of_v<Parcelable, D>) { + status_t result = value.readFromParcel(/*in*/parcel); + } else if constexpr (std::is_enum_v<D>) { + auto&& [res, val] = readAnyFromParcel<std::underlying_type_t<D>>(parcel); + result = res; + value = static_cast<D>(val); +#define AUTO_PARCELABLE_READ_FROM_PARCEL(fn_name, type_name) \ + } else if constexpr (std::is_same_v<D, std::decay_t<type_name>>) { \ + result = parcel->read ## fn_name (/*out*/&value); +AUTO_PARCELABLE_BINDER_MAPPING(AUTO_PARCELABLE_READ_FROM_PARCEL) + } else if constexpr (std::is_same_v<D, std::string>) { + result = parcel->readUtf8FromUtf16(/*out*/&value); + } else { + STATIC_FAIL(D, "Unsupported type: Add more manual type conversions above^^^"); + } +#undef AUTO_PARCELABLE_READ_FROM_PARCEL + + return std::make_pair(result, std::move(value)); + } + + T* Self() { + return static_cast<T*>(this); + } + const T* Self() const { + return static_cast<const T*>(this); + } +}; + +} // namespace binder +} // namespace iorap + +#endif // IORAP_BINDER_AUTO_PARCELABLE_H_ diff --git a/include/binder/common.h b/include/binder/common.h new file mode 100644 index 0000000..f31e1e9 --- /dev/null +++ b/include/binder/common.h @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_COMMON_H_ +#define IORAP_BINDER_COMMON_H_ + +// AIDL has no way to specify a custom C++ namespace for parcelables. +// +// We want our parcelables to live in the ::iorap::binder namespace, +// not in com.google.android.startop.iorap +// So this macro is just a short-hand for doing an alias across namespaces. +#define IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(what) \ +namespace com { namespace google { namespace android { namespace startop { namespace iorap { using what = ::iorap::binder::what; } } } } } + +#endif // IORAP_BINDER_COMMON_H_ diff --git a/include/binder/dexopt_event.h b/include/binder/dexopt_event.h new file mode 100644 index 0000000..cb506f4 --- /dev/null +++ b/include/binder/dexopt_event.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_DEXOPT_EVENT_H_ +#define IORAP_BINDER_DEXOPT_EVENT_H_ + +#include "binder/common.h" +#include "binder/auto_parcelable.h" +#include "common/introspection.h" + +namespace iorap { +namespace binder { + +struct DexOptEvent : public AutoParcelable<DexOptEvent> { + enum class Type : int32_t { + kPackageUpdate = 0, + }; + + Type type; + std::string package_name; +}; + +IORAP_INTROSPECT_ADAPT_STRUCT(DexOptEvent, type, package_name); + +} +} +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(DexOptEvent) + +#endif // IORAP_BINDER_DEXOPT_EVENT_H_ diff --git a/include/binder/job_scheduled_event.h b/include/binder/job_scheduled_event.h new file mode 100644 index 0000000..6b067d7 --- /dev/null +++ b/include/binder/job_scheduled_event.h @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_JOB_SCHEDULED_EVENT_H_ +#define IORAP_BINDER_JOB_SCHEDULED_EVENT_H_ + +#include "binder/common.h" +#include "binder/auto_parcelable.h" +#include "common/introspection.h" + +namespace iorap { +namespace binder { + +struct JobScheduledEvent : public AutoParcelable<JobScheduledEvent> { + enum class Type : int32_t { + kStartJob = 0, + kStopJob = 1, + }; + + Type type; + int32_t job_id; + std::string package_name; + bool should_update_versions; + + enum class Sort : int32_t { + kIdleMaintenance = 0, + }; + + Sort sort; + + constexpr bool operator==(const JobScheduledEvent& other) const { + return type == other.type + && job_id == other.job_id + && sort == other.sort + && package_name == other.package_name + && should_update_versions == other.should_update_versions; + } + + constexpr bool operator!=(const JobScheduledEvent& other) const { + return !(*this == other); + } +}; + +IORAP_INTROSPECT_ADAPT_STRUCT( + JobScheduledEvent, type, job_id, sort, package_name, should_update_versions); + +} +} +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(JobScheduledEvent) + +#endif // IORAP_BINDER_JOB_SCHEDULED_EVENT_H_ diff --git a/include/binder/package_event.h b/include/binder/package_event.h new file mode 100644 index 0000000..d8d7933 --- /dev/null +++ b/include/binder/package_event.h @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_PACKAGE_EVENT_H_ +#define IORAP_BINDER_PACKAGE_EVENT_H_ + +#include "binder/common.h" +#include "binder/auto_parcelable.h" +#include "common/introspection.h" + +namespace iorap { +namespace binder { + +struct PackageEvent : public AutoParcelable<PackageEvent> { + enum class Type : int32_t { + kReplaced = 0, + }; + + Type type; + std::string package_uri; + std::string package_name; +}; + +IORAP_INTROSPECT_ADAPT_STRUCT(PackageEvent, type, package_uri, package_name); + +} +} +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(PackageEvent) + +#endif // IORAP_BINDER_PACKAGE_EVENT_H_ diff --git a/include/binder/request_id.h b/include/binder/request_id.h new file mode 100644 index 0000000..71a2edd --- /dev/null +++ b/include/binder/request_id.h @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_REQUEST_ID_H_ +#define IORAP_BINDER_REQUEST_ID_H_ + +#include "binder/common.h" +#include "binder/auto_parcelable.h" + +namespace iorap { +namespace binder { + +struct RequestId : public AutoParcelable<RequestId> { + int64_t request_id; + + constexpr bool operator==(const RequestId& other) const { + return request_id == other.request_id; + } + + constexpr bool operator!=(const RequestId& other) const { + return !(*this == other); + } +}; + +IORAP_INTROSPECT_ADAPT_STRUCT(RequestId, request_id); + +} +} +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(RequestId) + +#endif // IORAP_BINDER_REQUEST_ID_H_ diff --git a/include/binder/system_service_event.h b/include/binder/system_service_event.h new file mode 100644 index 0000000..a82c200 --- /dev/null +++ b/include/binder/system_service_event.h @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_SYSTEM_SERVICE_EVENT_H_ +#define IORAP_BINDER_SYSTEM_SERVICE_EVENT_H_ + +#include "binder/common.h" +#include "binder/auto_parcelable.h" +#include "common/introspection.h" + +namespace iorap { +namespace binder { + +struct SystemServiceEvent : public AutoParcelable<SystemServiceEvent> { + enum class Type : int32_t { + kBootPhase = 0, + kStart = 1, + }; + + Type type; +}; + +IORAP_INTROSPECT_ADAPT_STRUCT(SystemServiceEvent, type); + +} +} +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(SystemServiceEvent) + +#endif // IORAP_BINDER_SYSTEM_SERVICE_EVENT_H_ diff --git a/include/binder/system_service_user_event.h b/include/binder/system_service_user_event.h new file mode 100644 index 0000000..43f92ea --- /dev/null +++ b/include/binder/system_service_user_event.h @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_SYSTEM_SERVICE_USER_EVENT_H_ +#define IORAP_BINDER_SYSTEM_SERVICE_USER_EVENT_H_ + +#include "binder/common.h" +#include "binder/auto_parcelable.h" +#include "common/introspection.h" + +namespace iorap { +namespace binder { + +struct SystemServiceUserEvent : public AutoParcelable<SystemServiceUserEvent> { + enum class Type : int32_t { + kStartUser = 0, + kUnlockUser, + kSwitchUser, + kStopUser, + kCleanupUser, + }; + + Type type; + int32_t user_handle; +}; + +IORAP_INTROSPECT_ADAPT_STRUCT(SystemServiceUserEvent, type, user_handle); + +} +} +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(SystemServiceUserEvent) + +#endif // IORAP_BINDER_SYSTEM_SERVICE_USER_EVENT_H_ diff --git a/include/binder/task_result.h b/include/binder/task_result.h new file mode 100644 index 0000000..b060f3f --- /dev/null +++ b/include/binder/task_result.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_TASK_RESULT_H_ +#define IORAP_BINDER_TASK_RESULT_H_ + +#include "binder/common.h" +#include "binder/auto_parcelable.h" +#include "common/introspection.h" + +namespace iorap { +namespace binder { + +struct TaskResult : public AutoParcelable<TaskResult> { + enum class State : int32_t { + kBegan = 0, + kOngoing = 1, + kCompleted = 2, + kError = 3, + kMax = kError, + }; + + State state; + + TaskResult() = default; + explicit TaskResult(State state) : state(state) {} + + constexpr bool operator==(const TaskResult& other) const { + return state == other.state; + } + + constexpr bool operator!=(const TaskResult& other) const { + return !(*this == other); + } +}; + +IORAP_INTROSPECT_ADAPT_STRUCT(TaskResult, state); + +} +} +IORAP_JAVA_NAMESPACE_BINDER_TYPEDEF(TaskResult) + +#endif // IORAP_BINDER_TASK_RESULT_H_ diff --git a/iorapd.rc b/iorapd.rc new file mode 100644 index 0000000..63f6b13 --- /dev/null +++ b/iorapd.rc @@ -0,0 +1,38 @@ +# Copyright (C) 2018 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +service iorapd /system/bin/iorapd + class main + disabled + user iorapd + group iorapd + capabilities DAC_READ_SEARCH + task_profiles ServiceCapacityLow +# TODO: should this be something else like performance/tasks? +# the main 'activity hint' thread needs to be fairly high priority to kick off +# tracing or prefetching asap. + +on property:ro.iorapd.enable=false + stop iorapd + +on post-fs-data + # Create directory for iorapd (see iorapd_data_file in selinux file_contexts). + mkdir /data/misc/iorapd/ 0750 iorapd system + +# Start iorapd when either prefetching or tracing is enabled. +on property:persist.device_config.runtime_native_boot.iorap_perfetto_enable=true && property:ro.iorapd.enable=true + start iorapd + +on property:persist.device_config.runtime_native_boot.iorap_readahead_enable=true && property:ro.iorapd.enable=true + start iorapd diff --git a/seccomp_policy/prefetcherd.arm.policy b/seccomp_policy/prefetcherd.arm.policy new file mode 120000 index 0000000..a4642af --- /dev/null +++ b/seccomp_policy/prefetcherd.arm.policy @@ -0,0 +1 @@ +prefetcherd.arm64.policy
\ No newline at end of file diff --git a/seccomp_policy/prefetcherd.arm64.policy b/seccomp_policy/prefetcherd.arm64.policy new file mode 100644 index 0000000..a796ad1 --- /dev/null +++ b/seccomp_policy/prefetcherd.arm64.policy @@ -0,0 +1,21 @@ +# Copyright (C) 2019 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# main usage: fadvise to prefetch data +fadvise64: 1 + +# bionic's fdsan requires this +getrlimit: 1 + +@include /system/etc/seccomp_policy/crash_dump.arm64.policy diff --git a/seccomp_policy/prefetcherd.x86.policy b/seccomp_policy/prefetcherd.x86.policy new file mode 120000 index 0000000..a4642af --- /dev/null +++ b/seccomp_policy/prefetcherd.x86.policy @@ -0,0 +1 @@ +prefetcherd.arm64.policy
\ No newline at end of file diff --git a/seccomp_policy/prefetcherd.x86_64.policy b/seccomp_policy/prefetcherd.x86_64.policy new file mode 120000 index 0000000..a4642af --- /dev/null +++ b/seccomp_policy/prefetcherd.x86_64.policy @@ -0,0 +1 @@ +prefetcherd.arm64.policy
\ No newline at end of file diff --git a/src/binder/iiorap_def.h b/src/binder/iiorap_def.h new file mode 100644 index 0000000..5763380 --- /dev/null +++ b/src/binder/iiorap_def.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IIORAP_IFACE_DEF_H +#define IIORAP_IFACE_DEF_H + +// Provide an x-macro that defines the 'IIorap.aidl' interface through repeated +// macro invocation on the member functions and their respective parameters. +// +// Future changes in the AIDL file to add new methods or change parameters should avoid +// a boilerplate-ripple effect in the rest of the codebase. + +#define IIORAP_IFACE_DEF(FN_BEGIN, FN, FN_END) \ +FN_BEGIN(::com::google::android::startup::iorap::,IIorap) \ +/* name <see IORAP_BINDER_PARAM_JOIN> */ \ +FN(setTaskListener, (const ::android::sp<::com::google::android::startop::iorap::,ITaskListener,>&,listener)) /*NOLINT*/ \ +FN(onAppLaunchEvent,(const ::com::google::android::startop::iorap::,RequestId,&,request), \ + (const ::com::google::android::startop::iorap::,AppLaunchEvent,&,event)) \ +FN(onJobScheduledEvent, \ + (const ::com::google::android::startop::iorap::,RequestId,&,request), \ + (const ::com::google::android::startop::iorap::,JobScheduledEvent,&,event)) \ +FN(onPackageEvent, (const ::com::google::android::startop::iorap::,RequestId,&,request), \ + (const ::com::google::android::startop::iorap::,PackageEvent,&,event)) \ +FN(onAppIntentEvent,(const ::com::google::android::startop::iorap::,RequestId,&,request), \ + (const ::com::google::android::startop::iorap::,AppIntentEvent,&,event)) \ +FN(onSystemServiceEvent, \ + (const ::com::google::android::startop::iorap::,RequestId,&,request), \ + (const ::com::google::android::startop::iorap::,SystemServiceEvent,&,event)) \ +FN(onSystemServiceUserEvent, \ + (const ::com::google::android::startop::iorap::,RequestId,&,request), \ + (const ::com::google::android::startop::iorap::,SystemServiceUserEvent,&,event))\ +FN(onDexOptEvent, (const ::com::google::android::startop::iorap::,RequestId,&,request), \ + (const ::com::google::android::startop::iorap::,DexOptEvent,&,event)) \ +FN_END() \ + +// Convenience macros to unpack the 2nd parameter from IIORAP_IFACE_DEF#FN calls. + +#define IORAP_BINDER_PARAM_JOIN_ALL(arg) IORAP_BINDER_PARAM_JOIN_ALL_IMPL arg +#define IORAP_BINDER_PARAM_JOIN_ALL_IMPL(type_l, type, type_r, name) type_l type type_r name + +#define IORAP_BINDER_PARAM_JOIN_NAMES(arg) IORAP_BINDER_PARAM_JOIN_NAMES_IMPL arg +#define IORAP_BINDER_PARAM_JOIN_NAMES_IMPL(type_l, type, type_r, name) name + +#endif //IIORAP_IFACE_DEF_H diff --git a/src/binder/iiorap_impl.cc b/src/binder/iiorap_impl.cc new file mode 100644 index 0000000..13adcc0 --- /dev/null +++ b/src/binder/iiorap_impl.cc @@ -0,0 +1,491 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "binder/iiorap_impl.h" +#include "binder/iiorap_def.h" +#include "common/macros.h" +#include "manager/event_manager.h" + +#include <android-base/logging.h> +#include <android-base/properties.h> +#include <binder/BinderService.h> +#include <binder/IPCThreadState.h> +#include <include/binder/request_id.h> +#include <utils/Printer.h> + +#include <codecvt> +#include <locale> +#include <utility> + +/* + * Definitions for the IIorap binder native service implementation. + * See also IIorap.aidl. + */ + +using Status = ::android::binder::Status; +using ITaskListener = ::com::google::android::startop::iorap::ITaskListener; + +namespace iorap { +namespace binder { + +namespace { +// Forward declarations. +template<typename ... Args> +Status Send(const char* function_name, Args&& ... args); +} + +// Join all parameter declarations by splitting each parameter with a comma. +// Types are used fully. +#define IIORAP_IMPL_ARG_DECLARATIONS(...) \ + IORAP_PP_MAP_SEP(IORAP_BINDER_PARAM_JOIN_ALL, IORAP_PP_COMMA, __VA_ARGS__) +#define IIORAP_IMPL_ARG_NAMES(...) \ + IORAP_PP_MAP_SEP(IORAP_BINDER_PARAM_JOIN_NAMES, IORAP_PP_COMMA, __VA_ARGS__) +#define IIORAP_IMPL_BODY(name, ...) \ + ::android::binder::Status IIorapImpl::name(IIORAP_IMPL_ARG_DECLARATIONS(__VA_ARGS__)) { \ + return Send(#name, impl_.get(), IIORAP_IMPL_ARG_NAMES(__VA_ARGS__)); \ + } + +IIORAP_IFACE_DEF(/*begin*/IORAP_PP_NOP, IIORAP_IMPL_BODY, /*end*/IORAP_PP_NOP); + +#undef IIORAP_IMPL_BODY +#undef IIORAP_IMPL_ARG_NAMES +#undef IIORAP_IMPL_ARGS + +namespace { + +struct ServiceParams { + bool fake_{false}; + std::shared_ptr<manager::EventManager> event_manager_; +}; + +static std::atomic<bool> s_service_started_{false}; +static std::atomic<bool> s_service_params_ready_{false}; + +// TODO: BinderService constructs IIorapImpl, +// but how do I get a pointer to it afterwards? +// +// This is a workaround for that, by using a global. +static ServiceParams s_service_params_; +static std::atomic<ServiceParams*> s_service_params_atomic_; + +// Convert an android::String16 (UTF-16) to a UTF-8 std::string. +static std::string String16ToStdString(const ::android::String16& s16) { + std::u16string u16{s16.string()}; + std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> convert; + + std::string res = convert.to_bytes(u16); + return res; +} + +} // namespace anonymous + +class IIorapImpl::Impl { + // ITaskListener implementation for iorap::manager::EventManager. + struct EventManagerTaskCallbacks : public iorap::manager::TaskResultCallbacks { + explicit EventManagerTaskCallbacks(iorap::borrowed<IIorapImpl::Impl*> impl) { + CHECK(impl != nullptr); + impl_ = impl; + } + + virtual void OnProgress(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) override { + impl_->ReplyWithResult(request_id, /*completed*/false, std::move(task_result)); + } + virtual void OnComplete(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) override { + impl_->ReplyWithResult(request_id, /*completed*/true, std::move(task_result)); + } + + virtual ~EventManagerTaskCallbacks() {} + + iorap::borrowed<IIorapImpl::Impl*> impl_; + }; + + public: + ~Impl() { + package_manager_->UnregisterPackageChangeObserver(package_change_observer_); + } + + void SetTaskListener(const ::android::sp<ITaskListener>& listener) { + ::android::sp<ITaskListener> old_listener = listener_; + if (old_listener != nullptr && listener != nullptr) { + LOG(WARNING) << "IIorap::setTaskListener: already had a task listener set"; + } + listener_ = listener; + } + + void ReplyWithResult(const RequestId& request_id, TaskResult::State result_state) { + ::android::sp<ITaskListener> listener = listener_; + if (listener == nullptr) { + // No listener. Cannot send anything back to the client. + // This could be normal, e.g. client had set listener to null before disconnecting. + LOG(DEBUG) << "Drop result, no listener registered."; + // TODO: print the result with ostream operator<< + return; + } + + TaskResult result; + result.state = result_state; + + // TODO: verbose, not info. + if (result_state == TaskResult::State::kCompleted) { + LOG(VERBOSE) << "ITaskListener::onComplete (request_id=" << request_id.request_id << ")"; + listener->onComplete(request_id, result); + } else { + LOG(VERBOSE) << "ITaskListener::onProgress (request_id=" << request_id.request_id << ")"; + listener->onProgress(request_id, result); + } + } + + void ReplyWithResult(const RequestId& request_id, bool completed, TaskResult result) { + ::android::sp<ITaskListener> listener = listener_; + if (listener == nullptr) { + // No listener. Cannot send anything back to the client. + // This could be normal, e.g. client had set listener to null before disconnecting. + LOG(DEBUG) << "Drop result, no listener registered."; + // TODO: print the result with ostream operator<< + return; + } + + // TODO: verbose, not info. + if (completed) { + LOG(VERBOSE) << "ITaskListener::onComplete (request_id=" << request_id.request_id << ")"; + listener->onComplete(request_id, result); + } else { + LOG(VERBOSE) << "ITaskListener::onProgress (request_id=" << request_id.request_id << ")"; + listener->onProgress(request_id, result); + } + } + + bool OnAppLaunchEvent(const RequestId& request_id, + const AppLaunchEvent& event) { + if (MaybeHandleFakeBehavior(request_id)) { + return true; + } + + return service_params_.event_manager_->OnAppLaunchEvent(request_id, event); + } + + bool OnDexOptEvent(const RequestId& request_id, const DexOptEvent& event) { + if (MaybeHandleFakeBehavior(request_id)) { + return true; + } + + return service_params_.event_manager_->OnDexOptEvent(request_id, event); + } + + bool OnJobScheduledEvent(const RequestId& request_id, + const JobScheduledEvent& event) { + if (MaybeHandleFakeBehavior(request_id)) { + return true; + } + + return service_params_.event_manager_->OnJobScheduledEvent(request_id, event); + } + + void Dump(/*borrow*/::android::Printer& printer, + const ::android::Vector<::android::String16>& args) { + + if (args.size() == 0) { + service_params_.event_manager_->Dump(/*borrow*/printer); + return; + } + + ::android::String16 arg_prev; + for (const ::android::String16& arg : args) { + bool unknown = false; + if (arg == ::android::String16("--all") || arg == ::android::String16("-a")) { + // using 'dumpsys' or 'bugreport' passes a single '-a' flag to this function. + service_params_.event_manager_->Dump(/*borrow*/printer); + } else if (arg == ::android::String16("--refresh-properties")) { + service_params_.event_manager_->RefreshSystemProperties(/*borrow*/printer); + printer.printLine("System properties refreshed."); + } else if (arg == ::android::String16("--compile-package")) { + // Intentionally left blank. + } else if (arg_prev == ::android::String16("--compile-package")) { + std::string package_name = String16ToStdString(arg); + + if (!service_params_.event_manager_->CompilePackage(/*borrow*/printer, package_name)) { + printer.printFormatLine("Failed to compile package %s.", package_name.c_str()); + } else { + printer.printFormatLine("Package %s compiled.", package_name.c_str()); + } + } else if (arg == ::android::String16("--purge-package")) { + // Intentionally left blank. + } else if (arg_prev == ::android::String16("--purge-package")) { + std::string package_name = String16ToStdString(arg); + + if (!service_params_.event_manager_->PurgePackage(/*borrow*/printer, package_name)) { + printer.printFormatLine("Failed to purge package %s.", package_name.c_str()); + } else { + printer.printFormatLine("Package %s purged.", package_name.c_str()); + } + } else { + unknown = true; + } + + if (unknown && arg != ::android::String16("--help")) { + printer.printLine("Invalid arguments."); + printer.printLine(""); + + printer.printLine("Arguments were:"); + for (const ::android::String16& arg16 : args) { + printer.printFormatLine(" '%s'", String16ToStdString(arg16).c_str()); + } + printer.printLine(""); + } + + if (unknown || arg == ::android::String16("--help")) { + printer.printLine("Iorapd dumpsys commands:"); + printer.printLine(" (none),--all,-a: Print state information for debugging iorapd."); + printer.printLine(" --help: Display this help menu"); + printer.printLine(" --compile-package <name>: Compile single package on device"); + printer.printLine(" --purge-package <name>: Delete database entries/files for package"); + printer.printLine(" --refresh-properties: Refresh system properties"); + return; + } + + arg_prev = arg; + } + } + + void HandleFakeBehavior(const RequestId& request_id) { + DCHECK(service_params_.fake_); + + // Send these dummy callbacks for testing only. + ReplyWithResult(request_id, TaskResult::State::kBegan); + ReplyWithResult(request_id, TaskResult::State::kOngoing); + ReplyWithResult(request_id, TaskResult::State::kCompleted); + } + + // TODO: Subclass IIorap with a separate fake implementation. + bool MaybeHandleFakeBehavior(const RequestId& request_id) { + if (service_params_.fake_) { + HandleFakeBehavior(request_id); + return true; + } + + return false; + } + + ::android::sp<ITaskListener> listener_; + + Impl(ServiceParams p) : service_params_{std::move(p)}, event_manager_callbacks_{new EventManagerTaskCallbacks{this}} { + CHECK(service_params_.event_manager_ != nullptr); + + service_params_.event_manager_->SetTaskResultCallbacks( + std::static_pointer_cast<manager::TaskResultCallbacks>(event_manager_callbacks_)); + + // Init the package change observer. + package_manager_ = PackageManagerRemote::Create(); + + if (package_manager_ == nullptr) { + LOG(ERROR) << "Failed to get package manager service in IIorapImpl::Impl." + << " Is system_server down?"; + exit(1); + } + + package_change_observer_ = + new PackageChangeObserver(service_params_.event_manager_); + package_manager_death_recipient_ = + new PackageManagerDeathRecipient(package_manager_, package_change_observer_); + + package_manager_->RegisterPackageChangeObserver(package_change_observer_); + package_manager_-> + RegisterPackageManagerDeathRecipient(package_manager_death_recipient_); + + } + + ServiceParams service_params_; + std::shared_ptr<EventManagerTaskCallbacks> event_manager_callbacks_; + android::sp<PackageChangeObserver> package_change_observer_; + android::sp<PackageManagerDeathRecipient> package_manager_death_recipient_; + std::shared_ptr<PackageManagerRemote> package_manager_; +}; + +using Impl = IIorapImpl::Impl; + +IIorapImpl::IIorapImpl() { + // Acquire edge of synchronizes-with IIorapImpl::Start(). + CHECK(s_service_params_ready_.load()); + // Do not turn this into a DCHECK, the above atomic load + // must happen-before the read of s_service_params_ready_. + impl_.reset(new Impl(std::move(s_service_params_))); +} + +namespace { + static bool started_ = false; +} +bool IIorapImpl::Start(std::shared_ptr<manager::EventManager> event_manager) { + if (s_service_started_.load()) { // Acquire-edge (see bottom of function). + // Note: Not meant to be idempotent. Two threads could race, and the second + // one would likely fail the publish. + + LOG(ERROR) << "service was already started"; + return false; // Already started + } + + CHECK(event_manager != nullptr); + + { + // This block of code needs to happen-before IIorapImpl::IIorapImpl. + + // TODO: There should be a simpler way of passing down + // this data which doesn't involve globals and memory synchronization. + ServiceParams* p = &s_service_params_; + // TODO: move all property reads to a dedicated Config class. + p->fake_ = ::android::base::GetBoolProperty("iorapd.binder.fake", /*default*/false); + p->event_manager_ = std::move(event_manager); + + // Release edge of synchronizes-with IIorapImpl::IIorapImpl. + s_service_params_ready_.store(true); + } + + ::android::IPCThreadState::self()->disableBackgroundScheduling(/*disable*/true); + ::android::status_t ret = android::BinderService<IIorapImpl>::publish(); + if (ret != android::OK) { + LOG(ERROR) << "BinderService::publish failed with error code: " << ret; + return false; + } + + android::sp<android::ProcessState> ps = android::ProcessState::self(); + // Reduce thread consumption by only using 1 thread. + // We should also be able to leverage this by avoiding locks, etc. + ps->setThreadPoolMaxThreadCount(/*maxThreads*/1); + ps->startThreadPool(); + ps->giveThreadPoolName(); + + // Release edge synchronizes-with the top of this function. + s_service_started_.store(true); + + // TODO: IIRC thread-start(t1) synchronizes-with t1.main, so we should be able + // to delete the majority of atomics for any pre-thread-start initialization... + + return true; +} + +::android::status_t IIorapImpl::dump(int fd, const ::android::Vector<::android::String16>& args) { + + ::android::FdPrinter printer{fd}; + + impl_->Dump(printer, args); + + return ::android::NO_ERROR; +} + +namespace { + +#define MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id) \ + if (self->MaybeHandleFakeBehavior(request_id)) { return ::android::binder::Status::ok(); } + +template <typename ... Args> +Status SendArgs(const char* function_name, + Impl* self, + const RequestId& request_id, + Args&&... /*rest*/) { + LOG(VERBOSE) << "IIorap::" << function_name << " (request_id = " << request_id.request_id << ")"; + + MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id); + + // TODO: implementation. + LOG(ERROR) << "IIorap::" << function_name << " -- not implemented for real code"; + return Status::fromStatusT(::android::INVALID_OPERATION); +} + +template <typename ... Args> +Status SendArgs(const char* function_name, Impl* self, Args&&... rest) { + DCHECK_EQ(std::string(function_name), "setTaskListener"); + LOG(VERBOSE) << "IIorap::setTaskListener"; + self->SetTaskListener(std::forward<Args&&>(rest)...); + + return Status::ok(); +} + +template <typename ... Args> +Status SendArgs(const char* function_name, + Impl* self, + const RequestId& request_id, + const AppLaunchEvent& app_launch_event) { + DCHECK_EQ(std::string(function_name), "onAppLaunchEvent"); + LOG(VERBOSE) << "IIorap::onAppLaunchEvent"; + + MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id); + + if (self->OnAppLaunchEvent(request_id, app_launch_event)) { + return Status::ok(); + } else { + // TODO: I suppose this should write out an exception back, + // like a service-specific error or something. + // + // It depends on whether or not we even have any synchronous + // errors. + // + // Most of the work here is done async, so it should handle + // async callbacks. + return Status::fromStatusT(::android::BAD_VALUE); + } +} + +template <typename ... Args> +Status SendArgs(const char* function_name, + Impl* self, + const RequestId& request_id, + const DexOptEvent& event) { + DCHECK_EQ(std::string(function_name), "onDexOptEvent"); + LOG(VERBOSE) << "IIorap::onDexOptEvent"; + + MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id); + + if (self->OnDexOptEvent(request_id, event)) { + return Status::ok(); + } else { + return Status::fromStatusT(::android::BAD_VALUE); + } +} + +template <typename ... Args> +Status SendArgs(const char* function_name, + Impl* self, + const RequestId& request_id, + const JobScheduledEvent& event) { + DCHECK_EQ(std::string(function_name), "onJobScheduledEvent"); + LOG(VERBOSE) << "IIorap::onJobScheduledEvent"; + + MAYBE_HAVE_FAKE_BEHAVIOR(self, request_id); + + if (self->OnJobScheduledEvent(request_id, event)) { + return Status::ok(); + } else { + // TODO: I suppose this should write out an exception back, + // like a service-specific error or something. + // + // It depends on whether or not we even have any synchronous + // errors. + // + // Most of the work here is done async, so it should handle + // async callbacks. + return Status::fromStatusT(::android::BAD_VALUE); + } +} + +template <typename ... Args> +Status Send(const char* function_name, Args&&... args) { + LOG(VERBOSE) << "IIorap::Send(" << function_name << ")"; + + return SendArgs(function_name, std::forward<Args>(args)...); +} +} // namespace <anonymous> + +} // namespace binder +} // namespace iorap diff --git a/src/binder/iiorap_impl.h b/src/binder/iiorap_impl.h new file mode 100644 index 0000000..ae1422f --- /dev/null +++ b/src/binder/iiorap_impl.h @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_BINDER_IIORAP_IMPL_H +#define IORAP_BINDER_IIORAP_IMPL_H + +#include "binder/iiorap_def.h" +#include "binder/package_change_observer.h" +#include "binder/package_manager_remote.h" +#include "common/macros.h" + +#include "com/google/android/startop/iorap/BnIorap.h" + +#include <memory> + +namespace android { +template <typename Service> +class BinderService; +} + +namespace iorap::manager { + class EventManager; +}; + +namespace iorap { +namespace binder { + +// Implementation of the IIorap service. This enables remote clients +// to connect to this process via the binder service manager. +// +// See also IIorap.aidl. +class IIorapImpl : public ::com::google::android::startop::iorap::BnIorap { +public: + static bool Start(std::shared_ptr<iorap::manager::EventManager> event_manager); + static constexpr const char* getServiceName() { return "iorapd"; }; + + virtual ::android::status_t dump(int fd, + const ::android::Vector<::android::String16>& args) override; + +// Join all parameter declarations by splitting each parameter with a comma. +// Types are used fully. +#define IIORAP_IMPL_ARGS(...) \ + IORAP_PP_MAP_SEP(IORAP_BINDER_PARAM_JOIN_ALL, IORAP_PP_COMMA, __VA_ARGS__) +#define IIORAP_IMPL_BODY(name, ...) \ + ::android::binder::Status name(IIORAP_IMPL_ARGS(__VA_ARGS__)) override; + +IIORAP_IFACE_DEF(/*begin*/IORAP_PP_NOP, IIORAP_IMPL_BODY, /*end*/IORAP_PP_NOP); + +#undef IIORAP_IMPL_BODY +#undef IIORAP_IMPL_ARGS + + class Impl; +private: + IIorapImpl(); // open for BinderService::publish. + + friend class ::android::BinderService<IIorapImpl>; + + std::unique_ptr<Impl> impl_; +}; +} +} + + +#endif //IORAP_BINDER_IIORAP_IMPL_H diff --git a/src/binder/package_change_observer.cc b/src/binder/package_change_observer.cc new file mode 100644 index 0000000..1907af8 --- /dev/null +++ b/src/binder/package_change_observer.cc @@ -0,0 +1,37 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "package_change_observer.h" +#include "package_manager_remote.h" +#include "manager/event_manager.h" + +#include <android-base/logging.h> +#include <android-base/properties.h> + +namespace iorap::binder { + +PackageChangeObserver::PackageChangeObserver( + std::shared_ptr<iorap::manager::EventManager> event_manager) : + event_manager_(event_manager){} + +android::binder::Status PackageChangeObserver::onPackageChanged( + const android::content::pm::PackageChangeEvent& event) { + LOG(DEBUG) << "Received PackageChangeObserver::onPackageChanged"; + if (event_manager_->OnPackageChanged(event)) { + return android::binder::Status::ok(); + } else { + return android::binder::Status::fromStatusT(android::BAD_VALUE); + } +} +} // namespace iorap::binder diff --git a/src/binder/package_change_observer.h b/src/binder/package_change_observer.h new file mode 100644 index 0000000..ac488ac --- /dev/null +++ b/src/binder/package_change_observer.h @@ -0,0 +1,42 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_PACKAGE_CHANGE_OBSERVER_H_ +#define IORAP_SRC_PACKAGE_CHANGE_OBSERVER_H_ + +#include "binder/iiorap_def.h" + +#include <android/content/pm/BnPackageChangeObserver.h> +#include <android/content/pm/PackageChangeEvent.h> +#include <android/content/pm/IPackageManagerNative.h> + +namespace iorap::manager { + class EventManager; +}; + +namespace iorap::binder { + +class PackageChangeObserver : public android::content::pm::BnPackageChangeObserver { + public: + PackageChangeObserver(std::shared_ptr<iorap::manager::EventManager> event_manager); + + // Callback when the package is changed. + android::binder::Status onPackageChanged( + const ::android::content::pm::PackageChangeEvent& event) override; + private: + std::shared_ptr<iorap::manager::EventManager> event_manager_; +}; +} // namespace iorap::binder + +#endif // IORAP_SRC_PACKAGE_CHANGE_OBSERVER_H_ diff --git a/src/binder/package_manager_remote.cc b/src/binder/package_manager_remote.cc new file mode 100644 index 0000000..46671eb --- /dev/null +++ b/src/binder/package_manager_remote.cc @@ -0,0 +1,196 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "package_manager_remote.h" + +#include <android-base/logging.h> +#include <android-base/properties.h> + +#include <chrono> +#include <thread> + +namespace iorap::binder { + +const constexpr int64_t kTimeoutMs = 60 * 1000; // 1 min +const constexpr int64_t kIntervalMs = 1000; // 1 sec + +std::shared_ptr<PackageManagerRemote> PackageManagerRemote::Create() { + std::shared_ptr<PackageManagerRemote> package_manager = + std::make_shared<PackageManagerRemote>(); + if (package_manager->ReconnectWithTimeout(kTimeoutMs)) { + return package_manager; + } + return nullptr; +} + +android::sp<IPackageManager> PackageManagerRemote::GetPackageService() { + android::sp<android::IBinder> binder = android::defaultServiceManager()->getService( + android::String16{"package_native"}); + if (binder == nullptr) { + LOG(ERROR) << "Cannot get package manager service!"; + return nullptr; + } + + return android::interface_cast<IPackageManager>(binder); +} + +std::optional<int64_t> PackageManagerRemote::GetPackageVersion( + const std::string& package_name) { + int64_t version_code; + android::binder::Status status = InvokeRemote( + [this, &version_code, &package_name]() { + return package_service_->getVersionCodeForPackage( + android::String16(package_name.c_str()), /*out*/&version_code); + }); + if (status.isOk()) { + return version_code; + } else { + LOG(WARNING) << "Failed to get version: " + << status.toString8().c_str() + << " for " << package_name + << ". Retry to connect package manager service."; + return std::nullopt; + } +} + +std::optional<VersionMap> PackageManagerRemote::GetPackageVersionMap() { + VersionMap package_version_map; + std::optional<std::vector<std::string>> packages = GetAllPackages(); + if (!packages) { + LOG(DEBUG) << "Failed to get all packages. The package manager may be down."; + return std::nullopt; + } + LOG(DEBUG) << "PackageManagerRemote::GetPackageVersionMap: " + << packages->size() + << " packages are found."; + + for (const std::string& package : *packages) { + std::optional<int64_t> version = GetPackageVersion(package); + if (!version) { + LOG(DEBUG) << "Cannot get version for " << package + << "Package manager may be down"; + return std::nullopt; + } + package_version_map[package] = *version; + } + + return package_version_map; +} + +std::optional<std::vector<std::string>> PackageManagerRemote::GetAllPackages() { + std::vector<std::string> packages; + android::binder::Status status = InvokeRemote( + [this, &packages]() { + return package_service_->getAllPackages(/*out*/&packages); + }); + + if (status.isOk()) { + return packages; + } + + LOG(ERROR) << "Failed to get all packages: " << status.toString8().c_str(); + return std::nullopt; + +} + +bool PackageManagerRemote::ReconnectWithTimeout(int64_t timeout_ms) { + int64_t count = 0; + package_service_ = nullptr; + std::chrono::duration interval = std::chrono::milliseconds(1000); + std::chrono::duration timeout = std::chrono::milliseconds(timeout_ms); + + while (package_service_ == nullptr) { + LOG(WARNING) << "Reconnect to package manager service: " << ++count << " times"; + package_service_ = GetPackageService(); + std::this_thread::sleep_for(interval); + if (count * interval >= timeout) { + LOG(ERROR) << "Fail to create version map in " + << timeout.count() + << " milliseconds." + << " Reason: Failed to connect to package manager service." + << " Is system_server down?"; + return false; + } + } + + return true; +} + +template <typename T> +android::binder::Status PackageManagerRemote::InvokeRemote(T&& lambda) { + android::binder::Status status = + static_cast<android::binder::Status>(lambda()); + if (status.isOk()) { + return status; + } + + if (!ReconnectWithTimeout(kTimeoutMs)) { + return status; + } + + return lambda(); +} + +void PackageManagerRemote::RegisterPackageChangeObserver( + android::sp<PackageChangeObserver> observer) { + LOG(DEBUG) << "Register package change observer."; + android::binder::Status status = InvokeRemote( + [this, &observer]() { + return package_service_->registerPackageChangeObserver(observer); + }); + + if (!status.isOk()) { + LOG(ERROR) << "Cannot register package change observer. Is system_server down?"; + exit(1); + } +} + +void PackageManagerRemote::UnregisterPackageChangeObserver( + android::sp<PackageChangeObserver> observer) { + LOG(DEBUG) << "Unregister package change observer."; + android::binder::Status status = InvokeRemote( + [this, &observer]() { + return package_service_->unregisterPackageChangeObserver(observer); + }); + + if (!status.isOk()) { + LOG(WARNING) << "Cannot unregister package change observer."; + } +} + +void PackageManagerRemote::RegisterPackageManagerDeathRecipient( + android::sp<PackageManagerDeathRecipient> death_recipient) { + LOG(DEBUG) << "Register package manager death recipient."; + android::status_t status = + android::IInterface::asBinder(package_service_.get())->linkToDeath(death_recipient); + + if (status == android::OK) { + return; + } + + if (!ReconnectWithTimeout(kTimeoutMs) || + android::OK != android::IInterface::asBinder( + package_service_.get())->linkToDeath(death_recipient)) { + LOG(ERROR) << "Failed to register package manager death recipient. Is system_server down?"; + exit(1); + } +} + +void PackageManagerDeathRecipient::binderDied(const android::wp<android::IBinder>& /* who */) { + LOG(DEBUG) << "PackageManagerDeathRecipient::binderDied try to re-register"; + package_manager_->RegisterPackageChangeObserver(observer_); + package_manager_-> + RegisterPackageManagerDeathRecipient(this); +} +} // namespace iorap::package_manager diff --git a/src/binder/package_manager_remote.h b/src/binder/package_manager_remote.h new file mode 100644 index 0000000..9b0dcd9 --- /dev/null +++ b/src/binder/package_manager_remote.h @@ -0,0 +1,83 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_ +#define IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_ + +#include "binder/package_change_observer.h" + +#include <android/content/pm/IPackageManagerNative.h> +#include <binder/IServiceManager.h> + +#include <optional> +#include <unordered_map> + +namespace iorap::binder { + +using IPackageManager = android::content::pm::IPackageManagerNative; +// A map between package name and its version. +using VersionMap = std::unordered_map<std::string, int64_t>; + +class PackageManagerRemote; + +class PackageManagerDeathRecipient : public android::IBinder::DeathRecipient { +public: + PackageManagerDeathRecipient(std::shared_ptr<PackageManagerRemote> package_manager, + android::sp<PackageChangeObserver> observer) : + package_manager_(package_manager), observer_(observer) {} + // android::IBinder::DeathRecipient override: + void binderDied(const android::wp<android::IBinder>& /* who */) override; + +private: + std::shared_ptr<PackageManagerRemote> package_manager_; + + android::sp<PackageChangeObserver> observer_; +}; + +class PackageManagerRemote { + public: + static std::shared_ptr<PackageManagerRemote> Create(); + + // Gets the package version based on the package name. + std::optional<int64_t> GetPackageVersion(const std::string& package_name); + + // Gets a map of package name and its version. + std::optional<VersionMap> GetPackageVersionMap(); + + void RegisterPackageChangeObserver(android::sp<PackageChangeObserver> observer); + + void UnregisterPackageChangeObserver(android::sp<PackageChangeObserver> observer); + + void RegisterPackageManagerDeathRecipient( + android::sp<PackageManagerDeathRecipient> death_recipient); + + private: + template <typename T> + android::binder::Status InvokeRemote(T&& lambda); + + // Reconnects to the package manager service. + // Retry until timeout. + bool ReconnectWithTimeout(int64_t timeout_ms); + + // Gets the package manager service. + static android::sp<IPackageManager> GetPackageService(); + + // Gets all package names. + std::optional<std::vector<std::string>> GetAllPackages(); + + android::sp<IPackageManager> package_service_; +}; +} // namespace iorap::package_manager + +#endif // IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_ diff --git a/src/binder/package_version_map.cc b/src/binder/package_version_map.cc new file mode 100644 index 0000000..784bb1b --- /dev/null +++ b/src/binder/package_version_map.cc @@ -0,0 +1,117 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "package_version_map.h" + +#include <android-base/logging.h> +#include <android-base/properties.h> + +namespace iorap::binder { + +std::shared_ptr<PackageVersionMap> PackageVersionMap::Create() { + std::shared_ptr<PackageManagerRemote> package_manager = + PackageManagerRemote::Create(); + if (!package_manager) { + return std::make_shared<PackageVersionMap>(); + } + + std::optional<VersionMap> map = package_manager->GetPackageVersionMap(); + return std::make_shared<PackageVersionMap>(package_manager, map); +} + +void PackageVersionMap::UpdateAll() {std::lock_guard<std::mutex> lock(mutex_); + size_t old_size = version_map_->size(); + std::optional<VersionMap> new_version_map = + package_manager_->GetPackageVersionMap(); + if (!new_version_map) { + LOG(DEBUG) << "Failed to get the latest version map"; + return; + } + version_map_ = std::move(new_version_map); + LOG(DEBUG) << "Update for version is done. The size is from " << old_size + << " to " << version_map_->size(); +} + +bool PackageVersionMap::Update(std::string package_name, int64_t version) { + std::lock_guard<std::mutex> lock(mutex_); + if (!version_map_) { + LOG(DEBUG) << "The version map doesn't exist. " + << "The package manager may be down."; + return false; + } + + VersionMap::iterator it = version_map_->find(package_name); + if (it == version_map_->end()) { + LOG(DEBUG) << "New installed package " + << package_name + << " with version " + << version; + (*version_map_)[package_name] = version; + return true; + } + + if (it->second != version) { + LOG(DEBUG) << "New version package " + << package_name + << " with version " + << version; + (*version_map_)[package_name] = version; + return true; + } + + LOG(DEBUG) << "Same version package " + << package_name + << " with version " + << version; + return false; +} + +size_t PackageVersionMap::Size() { + if (!version_map_) { + LOG(DEBUG) << "The version map doesn't exist. " + << "The package manager may be down."; + return -1; + } + return version_map_->size(); +} + +std::optional<int64_t> PackageVersionMap::GetOrQueryPackageVersion( + const std::string& package_name) { + std::lock_guard<std::mutex> lock(mutex_); + if (!version_map_) { + LOG(DEBUG) << "The version map doesn't exist. " + << "The package manager may be down."; + return std::nullopt; + } + + VersionMap::iterator it = version_map_->find(package_name); + if (it == version_map_->end()) { + LOG(DEBUG) << "Cannot find version for: " << package_name + << " in the hash table"; + std::optional<int64_t> version = + package_manager_->GetPackageVersion(package_name); + if (version) { + LOG(VERBOSE) << "Find version for: " << package_name << " on the fly."; + (*version_map_)[package_name] = *version; + return *version; + } else { + LOG(ERROR) << "Cannot find version for: " << package_name + << " on the fly."; + return -1; + } + } + + return it->second; +} +} // namespace iorap::binder diff --git a/src/binder/package_version_map.h b/src/binder/package_version_map.h new file mode 100644 index 0000000..f2048fe --- /dev/null +++ b/src/binder/package_version_map.h @@ -0,0 +1,83 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_PACKAGE_VERSION_MAP_H_ +#define IORAP_SRC_PACKAGE_VERSION_MAP_H_ + +#include <android/content/pm/IPackageManagerNative.h> +#include <binder/IServiceManager.h> + +#include <optional> +#include <unordered_map> + +#include "package_manager_remote.h" + +namespace iorap::binder { + +class PackageVersionMap { + public: + static std::shared_ptr<PackageVersionMap> Create(); + + PackageVersionMap(std::shared_ptr<PackageManagerRemote> package_manager, + std::optional<VersionMap> version_map) + : package_manager_(package_manager), + version_map_(version_map) {} + + PackageVersionMap() + : package_manager_(nullptr), version_map_(std::nullopt) {} + + // Updates the version specified by 'package_name' to 'version'. + // + // Post-condition: Find(package_name) == version. + // * if the package is newly installed, insert and return true. + // * if the package version is changed, update the version to the + // given one and return true. + // * otherwise, return false. + bool Update(std::string package_name, int64_t version); + + void UpdateAll(); + + // Finds the version of the package in the hash table. + // -1 means the app is installed by unversioned. + // Empty means the app is not inside the RAM version map, maybe due to + // the app is newly installed. + std::optional<int64_t> Find(const std::string& package_name) { + VersionMap::iterator it = version_map_->find(package_name); + if (it == version_map_->end()) { + return std::nullopt; + } + return it->second; + } + + size_t Size(); + + // Gets or queries the version for the package. + // + // The method firstly access the hash map in the RAM, which is built when + // iorapd starts. If the version is not in the map, it tries the query + // the package manager via IPC, with a cost of ~0.6ms. + // + // If no version can be found for some reason, return -1, + // because when an app has no version the package manager returns -1. + std::optional<int64_t> GetOrQueryPackageVersion( + const std::string& package_name); + + private: + std::shared_ptr<PackageManagerRemote> package_manager_; + std::optional<VersionMap> version_map_; + std::mutex mutex_; +}; +} // namespace iorap::binder + +#endif // IORAP_SRC_PACKAGE_MANAGER_REMOTE_H_ diff --git a/src/common/async_pool.h b/src/common/async_pool.h new file mode 100644 index 0000000..51cd38c --- /dev/null +++ b/src/common/async_pool.h @@ -0,0 +1,91 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_COMMON_ASYNC_POOL_H_ +#define IORAP_SRC_COMMON_ASYNC_POOL_H_ + +#include <atomic> +#include <vector> +#include <deque> +#include <future> + +#include <condition_variable> +#include <future> + +namespace iorap::common { + +class AsyncPool { + std::atomic<bool> shutting_down_{false}; + std::deque<std::future<void>> futures_; + + std::mutex mutex_; + std::condition_variable cond_var_; + + public: + + // Any threads calling 'Join' should eventually unblock + // once all functors have run to completition. + void Shutdown() { + shutting_down_ = true; + + cond_var_.notify_all(); + } + + // Block forever until Shutdown is called *and* all + // functors passed to 'LaunchAsync' have run to completition. + void Join() { + std::unique_lock<std::mutex> lock(mutex_); + while (true) { + // Pop all items eagerly + while (true) { + auto it = futures_.begin(); + if (it == futures_.end()) { + break; + } + + std::future<void> future = std::move(*it); + futures_.pop_front(); + + lock.unlock(); // do not stall callers of LaunchAsync + future.get(); + lock.lock(); + } + + if (shutting_down_) { + break; + } + + // Wait until we either get more items or ask to be shutdown. + cond_var_.wait(lock); + } + } + + // Execute the functor 'u' in a new thread asynchronously. + // Using this spawns a new thread each time to immediately begin + // async execution. + template <typename T> + void LaunchAsync(T&& u) { + auto future = std::async(std::launch::async, std::forward<T>(u)); + + { + std::unique_lock<std::mutex> lock(mutex_); + futures_.push_back(std::move(future)); + } + cond_var_.notify_one(); + } +}; + +} // namespace iorap::common + +#endif // IORAP_SRC_COMMON_ASYNC_POOL_H_ diff --git a/src/common/cmd_utils.h b/src/common/cmd_utils.h new file mode 100644 index 0000000..4494119 --- /dev/null +++ b/src/common/cmd_utils.h @@ -0,0 +1,170 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_COMMON_CMD_UTILS_H_ +#define IORAP_SRC_COMMON_CMD_UTILS_H_ + +#include <android-base/parsebool.h> +#include <android-base/properties.h> + +#include <iostream> +#include <sstream> +#include <optional> +#include <vector> + +namespace iorap::common { +// Create execve-compatible argv. +// The lifetime is tied to that of vector. +inline std::unique_ptr<const char*[]> VecToArgv(const char* program_name, + const std::vector<std::string>& vector) { + // include program name in argv[0] + // include a NULL sentinel in the end. + std::unique_ptr<const char*[]> ptr{new const char*[vector.size() + 2]}; + + // program name + ptr[0] = program_name; + + // all the argv + for (size_t i = 0; i < vector.size(); ++i) { + ptr[i+1] = vector[i].c_str(); + } + + // null sentinel + ptr[vector.size() + 1] = nullptr; + + return ptr; +} + +// Appends an args to the argv. +template <class T> +void AppendArgs(std::vector<std::string>& argv, + const T& value) { + std::stringstream ss; + ss << value; + argv.push_back(ss.str()); +} + +// Appends an args to the argv. +template <class T, class T2> +void AppendArgs(std::vector<std::string>& argv, + const T& value, + const T2& value2) { + AppendArgs(argv, value); + AppendArgs(argv, value2); +} + +// Appends a named argument to the argv. +// +// For example if <name> is "--property" and <value> is int(200): +// the string "--property=200" is appended to the argv. +template <class T, class T2> +void AppendNamedArg(std::vector<std::string>& argv, + const T& name, + const T2& value) { + std::stringstream ss; + ss << name; + ss << "="; + ss << value; + + argv.push_back(ss.str()); +} + +// Appends args from a vector to the argv repeatedly to argv. +// +// For example, if <args> is "--timestamp" and <values> is [100, 200]. +// The "--timestamp 100" and "--timestamp 200" are appended. +template <class T> +void AppendArgsRepeatedly(std::vector<std::string>& argv, + std::string args, + const std::vector<T>& values) { + for (const T& v : values) { + AppendArgs(argv, args, v); + } +} + +// Appends args from a vector to the argv repeatedly to argv. +// +// For example, if values is [input1.pb, input2.pb], +// then the "input1.pb" and "input2.pb" are appended. +template <class T> +void AppendArgsRepeatedly(std::vector<std::string>& argv, + const std::vector<T>& values) { + for (const T& v : values) { + AppendArgs(argv, v); + } +} + +// Appends a named argument to the argv repeatedly with different values. +// +// For example if <name> is "--property" and <value> is [int(200), int(400)]: +// the strings "--property=200" and "--property=400" are both appended to the argv. +template <class T, class T2> +void AppendNamedArgRepeatedly(std::vector<std::string>& argv, + const T& name, + const std::vector<T2>& values) { + for (const T2& v :values) { + AppendNamedArg(argv, name, v); + } +} + +// Get the value of the property. +// Firstly, try to find the environment variable. If it does not exist, +// try to get the property. If neither, use the default value.. +// +// For example, for prop foo.bar.baz, it will first check for +// FOO_BAR_BAZ environment variable. +inline std::string GetEnvOrProperty(const std::string& prop, const std::string& default_val) { + std::string env_str = prop; + // a.b.c -> a_b_c + std::replace(env_str.begin(), env_str.end(), '.', '_'); + // a_b_c -> A_B_C + std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper); + char *env = getenv(env_str.c_str()); + if (env) { + return std::string(env); + } + return ::android::base::GetProperty(prop, default_val); +} + +// Get the boolean value of the property. +// Firstly, try to find the environment variable. If it does not exist, +// try to get the property. If neither, use the default value.. +// +// For example, for prop foo.bar.baz, it will first check for +// FOO_BAR_BAZ environment variable. +inline bool GetBoolEnvOrProperty(const std::string& prop, bool default_val) { + std::string env_str = prop; + // a.b.c -> a_b_c + std::replace(env_str.begin(), env_str.end(), '.', '_'); + // a_b_c -> A_B_C + std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper); + char *env = getenv(env_str.c_str()); + if (env) { + using ::android::base::ParseBoolResult; + + switch (::android::base::ParseBool(env)) { + case ParseBoolResult::kError: + break; + case ParseBoolResult::kFalse: + return false; + case ParseBoolResult::kTrue: + return true; + } + } + return ::android::base::GetBoolProperty(prop, default_val); +} + +} // namespace iorap::common + +#endif // IORAP_SRC_COMMON_CMD_UTILS_H_ diff --git a/src/common/debug.h b/src/common/debug.h new file mode 100644 index 0000000..bc21c74 --- /dev/null +++ b/src/common/debug.h @@ -0,0 +1,98 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <ostream> + +namespace iorap { + +// kIsDebugBuild is special. +// It gets to be in the 'iorap' namespace +// so that different modules don't need to qualify it. +#ifndef NDEBUG +static constexpr bool kIsDebugBuild = true; +#else +static constexpr bool kIsDebugBuild = false; +#endif + +namespace common { + +// TODO: move below code to helpers. +template <typename T, bool> +struct base_if_condition {}; + +template <typename T> +struct base_if_condition<T, true> : public T {}; + +template <typename T> +using base_if_debug = base_if_condition<T, kIsDebugBuild>; + +namespace detail { +// "if constexpr" doesn't allow us to exclude fields from a struct/class, +// and also "if constexpr" doesn't allow us to reference a field that does not +// exist. +// so we must move everything into a separate base class. +template <bool kIsDebug = kIsDebugBuild> +struct DebugCounterBase { + constexpr size_t value() const { + return counter; + } + + constexpr void set_value(size_t value) { + counter = value; + } + + size_t counter{1}; // Don't start with 0. +}; + +template <> +struct DebugCounterBase<false /*kIsDebug*/> { + constexpr size_t value() const { + return 0; + } + + constexpr void set_value(size_t value) { + } +}; +} // namespace detail + +// This counter does absolutely nothing, the code compiles to no-ops +// when debugging is disabled. +struct DebugCounter : detail::DebugCounterBase<> { + constexpr DebugCounter& operator++() { + set_value(value() + 1); + return *this; + } + + constexpr DebugCounter operator++(int) { + DebugCounter now = *this; + set_value(value() + 1); + return now; + } + + constexpr operator size_t() const { + return value(); + } + + friend std::ostream& operator<<(std::ostream& os, DebugCounter counter); +}; + +inline std::ostream& operator<<(std::ostream& os, DebugCounter counter) { + os << counter.value(); + return os; +} + +// TODO: refactor DebugCounter and base traits into their own files? + +} // namespace common +} // namespace iorap diff --git a/src/common/expected.h b/src/common/expected.h new file mode 100644 index 0000000..c58e63f --- /dev/null +++ b/src/common/expected.h @@ -0,0 +1,410 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_COMMON_EXPECTED_H_ +#define IORAP_SRC_COMMON_EXPECTED_H_ + +#include <type_traits> +#include <utility> + +#include <android-base/logging.h> // CHECK/DCHECK. + +// Ignore the tautological-undefined-compare warning. +// We obviously want to do this to protect against undefined behavior +// that sets a reference to a null value. +#define DCHECK_UB_NOT_NULL(x) \ + DCHECK(reinterpret_cast<volatile decltype(x)>(x) != nullptr) + +/** + * Result<Value, Error>-like interface. + * + * Subset of the experimental standard C++ proposal (p0323r3) + * + * Example: + * + * expected<std::string, status_t> x = function_which_might_fail(); + * if (x) { + * std::string str = x.value(); + * } else { + * status_t err = x.error(); + * } + */ + +namespace iorap { +namespace detail { + // Use perfect forwarding for expected_data constructors with overloading. + struct expected_tag{}; + struct expected_tag_right : public expected_tag { + static constexpr bool is_right_v = true; + }; + struct expected_tag_error : public expected_tag { + static constexpr bool is_right_v = false; + }; + + template <typename T, typename E, bool DefineDestructor> + struct expected_data; + + // This doesn't always work because this code could be instantiated with a non-trivial T/E, + // and then the union becomes invalid. + template <typename T, typename E> + struct expected_data<T, E, /*DefineDestructor*/true> { + // Mark everything 'constexpr' to keep the code the same as the other partial specialization. + + template <typename U> + constexpr expected_data(U&& either, expected_tag_right) + : right_{std::forward<U>(either)}, is_right_{true} {} + + template <typename U> + constexpr expected_data(U&& either, expected_tag_error) + : error_{std::forward<U>(either)}, is_right_{false} {} + + constexpr bool has_value() const { + return is_right_; + } + + constexpr const T& value() const { + return right_; + } + + constexpr T& value() { + return right_; + } + + constexpr const E& error() const { + return error_; + } + + constexpr E& error() { + return error_; + } + + // Using an "anonymous union" here allows non-trivial types to be stored. + union { + T right_; + E error_; + }; + + bool is_right_; + + // Below code differs slightly by handling non-trivial constructors/destructors. + bool moved_from_{false}; + + // Note: Destructors cannot be templated, so it is illegal to use SFINAE to try to + // conditionalize this destructor somehow. + ~expected_data() { + if (moved_from_) { return; } + if (is_right_) { + right_.~T(); + } else { + error_.~E(); + } + } + + expected_data(expected_data&& other) + noexcept( + noexcept(T(std::move(other.right_))) && + noexcept(E(std::move(other.error_))) + ) { + DCHECK_UB_NOT_NULL(&other) << __PRETTY_FUNCTION__; + DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__; + if (other.is_right_) { + new (&right_) T(std::move(other.right_)); + } else { + new (&error_) E(std::move(other.error_)); + } + other.moved_from_ = true; + is_right_ = other.is_right_; + } + + expected_data(const expected_data& other) { + DCHECK_UB_NOT_NULL(&other) << __PRETTY_FUNCTION__; + DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__; + if (other.is_right_) { + new (&right_) T(other.right_); + } else { + new (&error_) E(other.error_); + } + is_right_ = other.is_right_; + } + + expected_data& operator=(const expected_data& other) { + DCHECK_UB_NOT_NULL(&other) << __PRETTY_FUNCTION__; + DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__; + + if (this == &other) { + return *this; + } + + if (other.is_right_) { + if (!is_right_) { + error_.~E(); + new (&right_) T(other.right_); + } else { + right_ = other.right_; + } + } else { + if (is_right_) { + right_.~T(); + new (&error_) E(other.error_); + } else { + error_ = other.error_; + } + } + is_right_ = other.is_right_; + + return *this; + } + + expected_data& operator=(expected_data&& other) { + DCHECK_UB_NOT_NULL(&other) << __PRETTY_FUNCTION__; + DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__; + + if (this == &other) { + return *this; + } + + if (other.is_right_) { + if (!is_right_) { + error_.~E(); + new (&right_) T(std::move(other.right_)); + } else { + right_ = std::move(other.right_); + } + } else { + if (is_right_) { + right_.~T(); + new (&error_) E(std::move(other.error_)); + } else { + error_ = std::move(other.error_); + } + } + + other.moved_from_ = true; + is_right_ = other.is_right_; + + return *this; + } + }; + + // Trivial-destructor copy of the above struct. + // + // A separate copy is required because otherwise compilation fails with an error about + // the union having an implicitly deleted constructor. + // + // Having this implementation gives us the property that + // + // (is_trivially_destructible<T> && is_trivially_destructible<E> + // ==> is_trivially_destructible<expected<T, E>>) + template <typename T, typename E> + struct expected_data<T, E, /*DefineDestructor*/false> { + template <typename U> + constexpr expected_data(U&& either, expected_tag_right) + : right_{std::forward<U>(either)}, is_right_{true} {} + + template <typename U> + constexpr expected_data(U&& either, expected_tag_error) + : error_{std::forward<U>(either)}, is_right_{false} {} + + constexpr bool has_value() const { + return is_right_; + } + + constexpr const T& value() const { + return right_; + } + + constexpr T& value() { + return right_; + } + + constexpr const E& error() const { + return error_; + } + + constexpr E& error() { + return error_; + } + + // Using an "anonymous union" here allows non-trivial types to be stored. + union { + T right_; + E error_; + }; + + bool is_right_; + + ~expected_data() = default; + }; + + // Select between trivial and non-trivial implementations. Trivial implementations + // are more optimized and constexpr-compatible. + template <typename T, typename E> + using expected_pick_data_t = + expected_data<T, E, + !(std::is_trivially_destructible_v<T> && std::is_trivially_destructible_v<E>) >; +} // namespace detail + +template <typename E> +struct unexpected; + +// Subset of std::experimental::expected proposal (p0323r3). +template <typename T, typename E> +struct expected { + // Never-empty: expected<T,E> values have either 'T' or 'E' in them. + template <typename U = T, typename _ = std::enable_if_t<std::is_default_constructible_v<U>>> + constexpr expected() noexcept(noexcept(T{})) : expected(T{}) {} + + constexpr expected(const T& value) : data_{value, detail::expected_tag_right{}} {} + constexpr expected(T&& value) : data_{std::move(value), detail::expected_tag_right{}} {} + constexpr expected(const E& error) : data_{error, detail::expected_tag_error{}} {} + constexpr expected(E&& error) : data_{std::move(error), detail::expected_tag_error{}} {} + + template <typename G = E> + constexpr expected(unexpected<G> const& u) : expected{u.value()} {} + + template <typename G = E> + constexpr expected(unexpected<G>&& u) : expected{std::move(u.value())} {} + + explicit constexpr operator bool() const { + return has_value(); + } + + constexpr bool has_value() const { + return data_.has_value(); + } + + constexpr const T& operator*() const { + return data_.value(); + } + + constexpr T& operator*() { + return data_.value(); + } + + constexpr const T* _Nonnull operator->() const { + return &data_.value(); + } + + constexpr T* _Nonnull operator->() { + return &data_.value(); + } + + constexpr T& value() & { + CHECK(has_value()); + return data_.value(); + } + + constexpr const T& value() const & { + CHECK(has_value()); + return data_.value(); + } + + constexpr T&& value() && { + CHECK(has_value()); + return std::move(data_.value()); + } + + constexpr const T& value() const && { + CHECK(has_value()); + return std::move(data_.value()); + } + + constexpr E& error() { + DCHECK(!has_value()); + return data_.error(); + } + + constexpr const E& error() const { + DCHECK(!has_value()); + return data_.error(); + } + + // TODO: other functions such as operator=, unexpected, etc. + private: + detail::expected_pick_data_t<T, E> data_; +}; + +// TODO: move to tests file +namespace { + struct TestType { + TestType() {} + ~TestType() {} + }; + struct TestType2 : TestType {}; + + static_assert(std::is_trivially_destructible_v<expected<int, /*error*/double> >); + static_assert(!std::is_trivially_destructible_v<expected<TestType, /*error*/double> >); + static_assert(!std::is_trivially_destructible_v<expected<int, /*error*/TestType> >); + static_assert(!std::is_trivially_destructible_v<expected<TestType, /*error*/TestType2> >); + + // Ensure expected is constexpr-compatible. + struct TestCase { + static constexpr auto t1 = expected<int, double>{}; + }; +} // namespace <anonymous> + +template <typename E> +struct unexpected { + unexpected() = delete; + constexpr explicit unexpected(const E& error) : error_{error} {} + constexpr explicit unexpected(E&& error) : error_{std::move(error)} {} + constexpr const E& value() const& { return error_; } + constexpr E& value() & { return error_; } + constexpr E&& value() && { return std::move(error_); } + constexpr E const&& value() const&& { return std::move(error_); } + private: + E error_; +}; + +template <class E> +constexpr bool operator==(const unexpected<E>& x, const unexpected<E>& y) { + return x.value() == y.value(); +} + +template <class E> +constexpr bool operator!=(const unexpected<E>& x, const unexpected<E>& y) { + return !(x == y); +} + +// TODO: move below codes to separate utils file +// +// future C++20 implementation of std::identity +struct identity { + template <typename U> + constexpr auto operator()(U&& v) const noexcept { + return std::forward<U>(v); + } +}; + +// Given a lambda [...](auto&& var) {...} +// apply std::forward to 'var' to achieve perfect forwarding. +// +// Note that this doesn't work when var is a template type, i.e. +// template <typename T> +// void func(T&& tvar) {...} +// +// It would be invalid to use this macro with 'tvar' in that context. +#define IORAP_FORWARD_LAMBDA(var) std::forward<decltype(var)>(var) + +// Borrowed non-null pointer, i.e. we do not own the lifetime. +// +// Function calls: This pointer is not used past the call. +// Struct fields: This pointer is not used past the lifetime of the struct. +template <class T, class = std::enable_if_t<std::is_pointer<T>::value>> +using borrowed = T _Nonnull; +// TODO: need a DCHECK or high warning levels, since null is technically well-defined. + +} // namespace iorap + +#endif // IORAP_SRC_COMMON_EXPECTED_H_ diff --git a/src/common/introspection.h b/src/common/introspection.h new file mode 100644 index 0000000..af64a20 --- /dev/null +++ b/src/common/introspection.h @@ -0,0 +1,207 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_COMMON_INTROSPECTION_H +#define IORAP_COMMON_INTROSPECTION_H + +/* + * Provide zero-cost compile-time introspection of struct member fields. + * + * Example: + * + * // Declaration + * struct PackageEvent { + * + * int type; + * std::string package_uri; + * std::string package_name; + * }; + * + * IORAP_INTROSPECT_ADAPT_STRUCT(PackageEvent, type, package_uri, package_name); + * + * // Usage + * { + * std::stringstream str; + * for_each_member_field(PackageEvent{123,"hello","world"}, [&](auto&& val) { + * str << val << ","; + * } + * CHECK_EQ("123,hello,world,"s, str.str()); + * } + */ + +#include "common/macros.h" +#include "common/type.h" + +#include <tuple> + +namespace iorap { +namespace introspect { + +template <auto value> +struct member_type; + +// Compile-time introspection data for a member-to-pointer. +// +// Example: +// using package_uri_member_type = member_type<&PackageEvent::&package_uri> +// int type = package_uri_member_type::value(PackageEvent{123,"hello","world"}); +// CHECK_EQ(type, 123); +template <typename T, typename F, F T::*member> +struct member_type<member> { + // The type of the struct this field is located in, e.g. 'struct XYZ {...}' -> XYZ. + static constexpr auto struct_t = type_c<T>; + // The type of the field, e.g. 'struct XYZ { int x; }' -> int. + static constexpr auto type = type_c<F>; + + // Allow a 'const U', 'volatile U', 'U&' etc here. + // Returns the value inside of 'U'. + template <typename U> + static constexpr decltype(auto) value(U&& v) { + static_assert(std::is_same_v<T, std::decay_t<U>>, "U must be cvref of T"); + + using U_noref = std::remove_reference_t<U>; + + // This casts from the regular non-const pointer-to-member to a potentially const/volatile + // pointer-to-member. + F U_noref::*safer_member = member; + + // Now dereference it, + return v.*safer_member; + // TODO: are we properly returning && for rvalue, & for lvalue refs, etc? + } + + static constexpr void set_value(typename decltype(struct_t)::type& s, + typename decltype(type)::type&& value) { + s.*member = std::forward<typename decltype(type)::type>(value); + } +}; + +// Given a self : T, where T has introspection-enabled support, T has some +// members m1, m2, m3, ... , mN. +// +// Invokes fun(self.*m1); fun(self.*m2); fun(self.*m3); ... ; fun(self.*mN). +template <typename T, typename F> +static constexpr void for_each_member_field_value(T&& self, F&& fun) { + constexpr auto members = introspect_members(type_c<std::decay_t<T>>); + // std::tuple<member_type<A>, member_type<B>, ...> + + // Warning: Don't use 'v=std::forward<V>(v)' as that actually captures-by-value. + for_each(members, [&fun, &self](auto&& type) mutable { + // Note that 'type' is a member_type + fun(type.value(std::forward<T>(self))); + }); +} + +// Given a self : T, where T has introspection-enabled support, T has some +// members m1, m2, m3, ... , mN. The basic_type of each member is t1, t2, t3, ..., tN. +// +// Invokes +// self.*m1 = fun(self, t1); +// self.*m2 = fun(self, t2); +// self.*m3 = fun(self, t3); +// ...; +// self.*mN = fun(self, tN). +template <typename T, typename F> +static constexpr void for_each_member_field_set_value(T&& self, F&& fun) { + constexpr auto members = introspect_members(type_c<std::decay_t<T>>); + // std::tuple<member_type<A>, member_type<B>, ...> + + // Warning: Don't use 'v=std::forward<V>(v)' as that actually captures-by-value. + for_each(members, [&fun, &self](auto&& type) mutable { + // Note that 'type' is a member_type + type.set_value(std::forward<T>(self), fun(type.type)); + }); +} + +} +} + +// Add compile-time introspection capabilities to a pre-existing struct or class. +// +// Arguments: Name, [Member1, Member2, ... MemberN] +// +// Example: +// +// struct Rectangle { +// int height; +// int width; +// }; +// +// IORAP_INTROSPECT_ADAPT_STRUCT(Rectangle, height, width); +// +// See also for_each_member_field_value. +#define IORAP_INTROSPECT_ADAPT_STRUCT(/*name, [member1, member2, member3, ...]*/...) \ + IORAP_INTROSPECT_ADAPT_STRUCT_IMPL(IORAP_PP_NARG(__VA_ARGS__), __VA_ARGS__) + +#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL(N, ...) \ + IORAP_PP_CONCAT(IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_, N)(__VA_ARGS__) + +// This simple implementation relies on the 'introspect_members' function being overloaded +// for the type<T> values. ADL is then applied to resolve the exact overload for any T, +// thus allowing this function definition to be in any namespace. + +// The auto signature must conform to: +// introspect_members(type<T>) -> std::tuple<member_type1, member_type_2, ...> + +// TODO: it would be nice to capture the name of the member as a string literal. +#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_1(TYPE) \ + static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ + return std::make_tuple();\ + } +#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_2(TYPE, m1) \ + static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ + return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{}\ + );\ + } + +#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_3(TYPE, m1, m2) \ + static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ + return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{},\ + ::iorap::introspect::member_type<&TYPE::m2>{}\ + ); \ + } + +#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_4(TYPE, m1, m2, m3) \ + static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ + return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{},\ + ::iorap::introspect::member_type<&TYPE::m2>{},\ + ::iorap::introspect::member_type<&TYPE::m3>{}\ + ); \ + } + +#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_5(TYPE, m1, m2, m3, m4) \ + static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ + return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{},\ + ::iorap::introspect::member_type<&TYPE::m2>{},\ + ::iorap::introspect::member_type<&TYPE::m3>{},\ + ::iorap::introspect::member_type<&TYPE::m4>{}\ + ); \ + } + +#define IORAP_INTROSPECT_ADAPT_STRUCT_IMPL_6(TYPE, m1, m2, m3, m4, m5) \ + static constexpr auto introspect_members(::iorap::introspect::type<TYPE>) { \ + return std::make_tuple(::iorap::introspect::member_type<&TYPE::m1>{},\ + ::iorap::introspect::member_type<&TYPE::m2>{},\ + ::iorap::introspect::member_type<&TYPE::m3>{},\ + ::iorap::introspect::member_type<&TYPE::m4>{},\ + ::iorap::introspect::member_type<&TYPE::m5>{}\ + ); \ + } + +// TODO: Consider using IORAP_PP_MAP + + +#endif // IORAP_COMMON_INTROSPECTION_H diff --git a/src/common/loggers.h b/src/common/loggers.h new file mode 100644 index 0000000..1dbb2dc --- /dev/null +++ b/src/common/loggers.h @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_SRC_COMMON_LOGGERS +#define IORAP_SRC_COMMON_LOGGERS + +#include <android-base/logging.h> + +namespace iorap { +namespace common { + +// Log to both Stderr and Logd for convenience when running this from the command line. +class StderrAndLogdLogger { + public: + explicit StderrAndLogdLogger(android::base::LogId default_log_id = android::base::MAIN) +#ifdef __ANDROID__ + : logd_(default_log_id) +#endif + { + } + + void operator()(::android::base::LogId id, + ::android::base::LogSeverity sev, + const char* tag, + const char* file, + unsigned int line, + const char* message) { +#ifdef __ANDROID__ + logd_(id, sev, tag, file, line, message); +#endif + StderrLogger(id, sev, tag, file, line, message); + } + + private: +#ifdef __ANDROID__ + ::android::base::LogdLogger logd_; +#endif +}; + +} // namespace iorap +} // namespace common + +#endif diff --git a/src/common/macros.h b/src/common/macros.h new file mode 100644 index 0000000..1b5b38b --- /dev/null +++ b/src/common/macros.h @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_COMMON_MACROS_H_ +#define IORAP_COMMON_MACROS_H_ + +// Expands to the # of arguments passed to the macro. +// For example `IORAP_PP_NARG(a,b,c)` -> 3 +// For example `IORAP_PP_NARG(x)` -> 1 +// +// The # of arguments must be >0 and <64. +#define IORAP_PP_NARG(...) \ + IORAP_PP_NARG_IMPL(__VA_ARGS__, 64, 63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1,) // NOLINT + +// Implementation explanation: Equivalent of: +// lst = __VA_ARGS__ + 64..1 +// return lst[64] +// The variadic arguments logically shift-off the hardcoded 64..1 list, revealing +// the # of arguments that were shifted. +#define IORAP_PP_NARG_IMPL(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15, e16, e17, e18, e19, e20, e21, e22, e23, e24, e25, e26, e27, e28, e29, e30, e31, e32, e33, e34, e35, e36, e37, e38, e39, e40, e41, e42, e43, e44, e45, e46, e47, e48, e49, e50, e51, e52, e53, e54, e55, e56, e57, e58, e59, e60, e61, e62, e63, N, ...) N + +#define IORAP_PP_CONCAT(lhs, rhs) \ + IORAP_PP_CONCAT_IMPL(lhs, rhs) + +#define IORAP_PP_CONCAT_IMPL(lhs, rhs) \ + lhs ## rhs + +// Expands to call FN around every argument passed it. +// MAP(f, a1, a2, .., an) -> f(a1) f(a2) f(a3) ... f(an) +#define IORAP_PP_MAP(FN, ...) IORAP_PP_MAP_IMPL(IORAP_PP_NARG(__VA_ARGS__), /*sep=blank*/, FN, __VA_ARGS__) +// Expands to call FN around every argument passed it. +// Every non-first/non-last function call also has a separator prefixed before it. +// MAP_SEP(f, sep, a1, a2, .., an) -> f(a1) sep() f(a2) sep() f(a3) sep() ... sep() f(an-1) f(an) +#define IORAP_PP_MAP_SEP(FN, sep, ...) IORAP_PP_MAP_IMPL(IORAP_PP_NARG(__VA_ARGS__), FN, sep, __VA_ARGS__) +#define IORAP_PP_MAP_IMPL(N, FN, sep, ...) IORAP_PP_CONCAT(IORAP_PP_MAP_IMPL_, N)(FN, sep, __VA_ARGS__) +#define IORAP_PP_MAP_IMPL_1(FN, sep, a1) FN(a1) +#define IORAP_PP_MAP_IMPL_2(FN, sep, a1, a2) FN(a1) sep() FN(a2) +#define IORAP_PP_MAP_IMPL_3(FN, sep, a1, a2, a3) FN(a1) sep() FN(a2) sep() FN(a3) +#define IORAP_PP_MAP_IMPL_4(FN, sep, a1, a2, a3, a4) FN(a1) sep() FN(a2) sep() FN(a3) sep() FN(a4) + +// Deferred comma. Useful with the above MAP function when you need a comma separator. +#define IORAP_PP_COMMA() , +// All arguments are ignored. +#define IORAP_PP_NOP(...) + +#endif // IORAP_COMMON_MACROS_H_
\ No newline at end of file diff --git a/src/common/printer.h b/src/common/printer.h new file mode 100644 index 0000000..272e459 --- /dev/null +++ b/src/common/printer.h @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_UTILS_PRINTER_H_ +#define IORAP_UTILS_PRINTER_H_ + +#include <utils/Printer.h> + +#include <string.h> + +namespace iorap::common { + +class StderrLogPrinter : public ::android::Printer { +public: + // Create a printer using the specified logcat and log priority + // - Unless ignoreBlankLines is false, print blank lines to logcat + // (Note that the default ALOG behavior is to ignore blank lines) + StderrLogPrinter(const char* logtag, + android_LogPriority priority = ANDROID_LOG_DEBUG, + const char* prefix = nullptr, + bool ignore_blank_lines = false) + : log_printer_{logtag, priority, prefix, ignore_blank_lines} { + logtag_ = logtag; + prefix_ = prefix; + ignore_blank_lines_ = ignore_blank_lines; + } + + // Print the specified line to logcat. No \n at the end is necessary. + virtual void printLine(const char* string) override { + if (ignore_blank_lines_ && strlen(string) == 0) { + return; + } + std::cerr << logtag_ << ": "; + if (prefix_ != nullptr) { + std::cerr << prefix_; + } + std::cerr << string << std::endl; + log_printer_.printLine(string); + } + private: + ::android::LogPrinter log_printer_; + const char* logtag_; + const char* prefix_; + bool ignore_blank_lines_; +}; + +} // namespace iorap::common + +#endif // IORAP_UTILS_PRINTER_H_ diff --git a/src/common/property.h b/src/common/property.h new file mode 100644 index 0000000..d910193 --- /dev/null +++ b/src/common/property.h @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2020 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_UTILS_PROPERTY_H_ +#define IORAP_UTILS_PROPERTY_H_ + +#include <android-base/properties.h> +#include <server_configurable_flags/get_flags.h> + +namespace iorap::common { + +constexpr const char* ph_namespace = "runtime_native_boot"; + +inline bool IsTracingEnabled(const std::string& default_value) { + return server_configurable_flags::GetServerConfigurableFlag( + ph_namespace, + "iorap_perfetto_enable", + ::android::base::GetProperty("iorapd.perfetto.enable", default_value)) == "true"; +} + +inline bool IsReadAheadEnabled(const std::string& default_value) { + return server_configurable_flags::GetServerConfigurableFlag( + ph_namespace, + "iorap_readahead_enable", + ::android::base::GetProperty("iorapd.readahead.enable", default_value)) == "true"; +} + +inline bool ExcludeDexFiles(bool default_value) { + return ::android::base::GetBoolProperty("iorapd.exclude_dex_files", default_value); +} + +} // namespace iorap::common + +#endif // IORAP_UTILS_PROPERTY_H_ diff --git a/src/common/rx_async.h b/src/common/rx_async.h new file mode 100644 index 0000000..d16f5ae --- /dev/null +++ b/src/common/rx_async.h @@ -0,0 +1,77 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_COMMON_RX_ASYNC_H_ +#define IORAP_SRC_COMMON_RX_ASYNC_H_ + +#include "common/async_pool.h" + +#include <rxcpp/rx.hpp> + +namespace iorap::common { + +// Helper functions to operate with rx chains asynchronously. +class RxAsync { + public: + // Subscribe to the observable on a new thread asynchronously. + // If no observe_on/subscribe_on is used, the chain will execute + // on that new thread. + // + // Returns the composite_subscription which can be used to + // unsubscribe from if we want to abort the chain early. + template <typename T, typename U> + static rxcpp::composite_subscription SubscribeAsync( + AsyncPool& async_pool, + T&& observable, + U&& subscriber) { + rxcpp::composite_subscription subscription; + + async_pool.LaunchAsync([subscription, // safe copy: ref-counted + observable=std::forward<T>(observable), + subscriber=std::forward<U>(subscriber)]() mutable { + observable + .as_blocking() + .subscribe(subscription, + std::forward<decltype(subscriber)>(subscriber)); + }); + + return subscription; + } + + template <typename T, typename U, typename E> + static rxcpp::composite_subscription SubscribeAsync( + AsyncPool& async_pool, + T&& observable, + U&& on_next, + E&& on_error) { + rxcpp::composite_subscription subscription; + + async_pool.LaunchAsync([subscription, // safe copy: ref-counted + observable=std::forward<T>(observable), + on_next=std::forward<U>(on_next), + on_error=std::forward<E>(on_error)]() mutable { + observable + .as_blocking() + .subscribe(subscription, + std::forward<decltype(on_next)>(on_next), + std::forward<decltype(on_error)>(on_error)); + }); + + return subscription; + } +}; + +} // namespace iorap::common + +#endif // IORAP_SRC_COMMON_RX_ASYNC_H_ diff --git a/src/common/trace.h b/src/common/trace.h new file mode 100644 index 0000000..a07d7bc --- /dev/null +++ b/src/common/trace.h @@ -0,0 +1,48 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_COMMON_TRACE_H_ +#define IORAP_COMMON_TRACE_H_ + +#include <cutils/trace.h> + +#include <cstdio> +#include <sstream> + +namespace iorap { + +// TODO: refactor into utils/Trace.h + +class ScopedFormatTrace { + public: + template <typename ... Args> + ScopedFormatTrace(uint64_t tag, const char* fmt, Args&&... args) : tag_{tag} { + char buffer[1024]; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-security" + snprintf(buffer, sizeof(buffer), fmt, args...); +#pragma GCC diagnostic pop + atrace_begin(tag, buffer); + } + + ~ScopedFormatTrace() { + atrace_end(tag_); + } + private: + uint64_t tag_; +}; + +} // namespace iorap + +#endif // IORAP_COMMON_TRACE_H_ diff --git a/src/common/type.h b/src/common/type.h new file mode 100644 index 0000000..e856d73 --- /dev/null +++ b/src/common/type.h @@ -0,0 +1,175 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_COMMON_TYPE_H +#define IORAP_COMMON_TYPE_H + +#include <cstdint> +#include <tuple> + +namespace iorap { +namespace introspect { + +/* + * Simple types-as-value abstractions. + * + * Allow types to be passed as regular function parameters instead of using 'template' type + * parameters. + * + * This enables the following, more concise, pattern: + * ---------------------------- + * + * Traditional metaprogramming with template parameters: + * + * template <typename ... Args> + * struct get_num_params { + * static constexpr size_t value = sizeof...(Args); + * }; + * + * typename get_num_params<decltype("hello"), decltype("world")>::value == 2 + * typename get_num_params<decltype(int), decltype(int), decltype(int), decltype(int)>::value == 4 + * + * Alternative metaprogramming with values: + * + * constexpr auto get_num_params = [](auto&&... val) { return sizeof...(val); }; + * + * get_num_params("hello", "world") == 2 + * get_num_params(0,0,0,0) == 4 + */ + +/* + * A fully instantiated type wrapper. + * + * basic_type<T> is intended for overloading functions between different basic_types. + * type_c<T> is intended for instantiating new type wrappers as a short-hand (and not requiring + * typename). + * + * For basic_type<T> in particular it allows one to overload on basic_type<T> to handle specific + * types, and there's no requirement that T be constexpr, be default constructible, and no + * template specializations is necessary. + * + * void foo(basic_type<int>) { + * printf("int"); + * } + * + * template <typename T> + * void foo(basic_type<T>) { + * printf("everything else"); + * } + * + * as opposed to this verbosity + * + * template <typename T> + * struct foo { + * void operator() { + * printf("everything else"); + * } + * }; + * + * template <> + * struct foo<int> { + * void operator() { + * printf("int"); + * } + * }; + * + * OR this super-hack which works in rare situations + * + * void foo(int) { + * printf("int"); + * } + * + * template <typename T> + * void foo(T&&) { + * printf("everything else") + * } + * + * Note that invoking the last foo(T&&) is particularly challenging. declval<T> fails at compilation + * with a static_assert, so a real value has to be constructed that is immediately discarded. + */ +template <typename T> +struct basic_type { + using type = T; +}; + +template <typename T> +struct type_impl { + struct _ : basic_type<T> { }; +}; + +template <typename T> +using type = basic_type<T>; // typename type_impl<T>::_; // subclass of basic_type<T> +// TODO: why doesn't using type_impl::_ work with ADL? + +template <typename T> +using type_t = type<T>; + +template <typename T> +constexpr auto type_c = type<T>{}; + +template <auto X> +struct value_constant { + static constexpr auto value = X; +}; + +template <int X> +constexpr auto int_c = value_constant<X>{}; + +template <typename T> +constexpr bool dependent_false_v = false; + +// Emit a static_assert(false) if the else branch in an 'if constexpr' is taken. +// Needs a type as the first parameter. +#define STATIC_FAIL(T, msg) static_assert(::iorap::introspect::dependent_false_v<T>, msg) +// Emit a static_assert(false) if an else branch in an 'if constexpr' is taken, used with +// (e.g. auto) values instead of types. +#define STATIC_FAIL_DT(var, msg) STATIC_FAIL(decltype(var), msg) + +template <size_t i, typename Tuple, typename F> +static constexpr void for_each_impl(Tuple&& t, F&& f) { + if constexpr (i == std::tuple_size<std::decay_t<Tuple>>::value) { + return; + } else { + f(std::get<i>(std::forward<Tuple>(t))); + for_each_impl<i+1>(std::forward<Tuple>(t), std::forward<F>(f)); + } +} + +// for each Tuple<a1,a2,...,aN> invoke { f(a1); f(a2); ... ; f(aN); } +template <typename Tuple, typename F> +static constexpr void for_each(Tuple&& t, F&& f) { + return for_each_impl<0u>(std::forward<Tuple>(t), std::forward<F>(f)); +} + +// Perfect forwarding for structured binding. +// +// Example: +// auto&& [a,b] = whatever; +// return aliasing_forward<T>(a); +template <typename T, typename U> +constexpr decltype(auto) aliasing_forward(U&& val) noexcept { + if constexpr (std::is_lvalue_reference_v<T>) { + return val; + } else { + return std::move(val); + } +} + + +} // namespace introspect +} // namespace iorap + +#endif // IORAP_COMMON_TYPE_H diff --git a/src/compiler/compiler.cc b/src/compiler/compiler.cc new file mode 100644 index 0000000..a546c5c --- /dev/null +++ b/src/compiler/compiler.cc @@ -0,0 +1,992 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "compiler/compiler.h" + +#include "common/debug.h" +#include "common/expected.h" + +#include "perfetto/rx_producer.h" // TODO: refactor BinaryWireProtobuf to separate header. +#include "inode2filename/inode.h" +#include "inode2filename/search_directories.h" +#include "serialize/protobuf_io.h" + +#include <android-base/unique_fd.h> +#include <android-base/parseint.h> +#include <android-base/file.h> + +#include <perfetto/trace/trace.pb.h> // ::perfetto::protos::Trace +#include <perfetto/trace/trace_packet.pb.h> // ::perfetto::protos::TracePacket + +#include "rxcpp/rx.hpp" +#include <iostream> +#include <fstream> +#include <optional> +#include <utility> +#include <regex> + +#include <sched.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <syscall.h> +#include <fcntl.h> +#include <unistd.h> + +namespace iorap::compiler { + +using Inode = iorap::inode2filename::Inode; +using InodeResult = iorap::inode2filename::InodeResult; +using SearchDirectories = iorap::inode2filename::SearchDirectories; + +template <typename T> +using ProtobufPtr = iorap::perfetto::ProtobufPtr<T>; + +struct PerfettoTraceProtoInfo { + /* The perfetto trace proto. */ + ::iorap::perfetto::PerfettoTraceProto proto; + /* + * The timestamp limit of the trace. + * It's used to truncate the trace file. + */ + uint64_t timestamp_limit_ns; + /* + * The pid of the app. + * If positive, it's used to filter out other page cache events. + */ + int32_t pid; +}; + +struct PerfettoTracePtrInfo { + /* Deserialized protobuf data containing the perfetto trace. */ + ProtobufPtr<::perfetto::protos::Trace> trace_ptr; + /* + * The timestamp limit of the trace. + * It's used to truncate the trace file. + */ + uint64_t timestamp_limit_ns; + /* + * The pid of the app. + * If positive, it's used to filter out other page cache events. + */ + int32_t pid; +}; + +// Attempt to read protobufs from the filenames. +// Emits one (or none) protobuf for each filename, in the same order as the filenames. +// On any errors, the items are dropped (errors are written to the error LOG). +// +// All work is done on the same Coordinator as the Subscriber. +template <typename ProtoT /*extends MessageLite*/> +auto/*observable<PerfettoTracePtrInfo>*/ ReadProtosFromFileNames( + rxcpp::observable<CompilationInput> file_infos) { + using BinaryWireProtoT = ::iorap::perfetto::PerfettoTraceProto; + + return file_infos + .map([](const CompilationInput& file_info) -> + std::optional<PerfettoTraceProtoInfo> { + LOG(VERBOSE) << "compiler::ReadProtosFromFileNames " << file_info.filename + << " TimeStampLimit "<< file_info.timestamp_limit_ns + << " Pid " << file_info.pid << " [begin]"; + std::optional<BinaryWireProtoT> maybe_proto = + BinaryWireProtoT::ReadFullyFromFile(file_info.filename); + if (!maybe_proto) { + LOG(ERROR) << "Failed to read file: " << file_info.filename; + return std::nullopt; + } + return {{std::move(maybe_proto.value()), file_info.timestamp_limit_ns, file_info.pid}}; + }) + .filter([](const std::optional<PerfettoTraceProtoInfo>& proto_info) { + return proto_info.has_value(); + }) + .map([](std::optional<PerfettoTraceProtoInfo>& proto_info) -> + PerfettoTraceProtoInfo { + return proto_info.value(); + }) // TODO: refactor to something that flattens the optional, and logs in one operator. + .map([](PerfettoTraceProtoInfo& proto_info) -> + std::optional<PerfettoTracePtrInfo> { + std::optional<ProtobufPtr<ProtoT>> t = proto_info.proto.template MaybeUnserialize<ProtoT>(); + if (!t) { + LOG(ERROR) << "Failed to parse protobuf: "; // TODO: filename. + return std::nullopt; + } + return {{std::move(t.value()), proto_info.timestamp_limit_ns, proto_info.pid}}; + }) + .filter([](const std::optional<PerfettoTracePtrInfo>& trace_info) { + return trace_info.has_value(); + }) + .map([](std::optional<PerfettoTracePtrInfo>& trace_info) -> + PerfettoTracePtrInfo { + LOG(VERBOSE) << "compiler::ReadProtosFromFileNames [success]"; + return trace_info.value(); + // TODO: protobufs have no move constructor. this might be inefficient? + }); + +/* + return filenames + .map([](const std::string& filename) { + LOG(VERBOSE) << "compiler::ReadProtosFromFileNames " << filename << " [begin]"; + std::optional<BinaryWireProtoT> maybe_proto = + BinaryWireProtoT::ReadFullyFromFile(filename); + if (!maybe_proto) { + LOG(ERROR) << "Failed to read file: " << filename; + } + + std::unique_ptr<BinaryWireProtoT> ptr; + if (maybe_proto) { + ptr.reset(new BinaryWireProtoT{std::move(*maybe_proto)}); + } + return ptr; + }) + .filter([](const std::unique_ptr<BinaryWireProtoT>& proto) { + return proto != nullptr; + }) + .map([](std::unique_ptr<BinaryWireProtoT>& proto) { + std::optional<ProtoT> t = proto->template MaybeUnserialize<ProtoT>(); + if (!t) { + LOG(ERROR) << "Failed to parse protobuf: "; // TODO: filename. + } + return t; + }) + .filter([](const std::optional<ProtoT>& proto) { + return proto.has_value(); + }) + .map([](std::optional<ProtoT> proto) -> ProtoT { + LOG(VERBOSE) << "compiler::ReadProtosFromFileNames [success]"; + return std::move(proto.value()); + // TODO: protobufs have no move constructor. this might be inefficient? + }); + */ +} + +auto/*observable<PerfettoTracePtrInfo>*/ ReadPerfettoTraceProtos( + std::vector<CompilationInput> file_infos) { + auto filename_obs = rxcpp::observable<>::iterate(std::move(file_infos)); + rxcpp::observable<PerfettoTracePtrInfo> obs = + ReadProtosFromFileNames<::perfetto::protos::Trace>(std::move(filename_obs)); + return obs; +} + +// A flattened data representation of an MmFileMap*FtraceEvent. +// This representation is used for streaming processing. +// +// Note: Perfetto applies a 'union' over all possible fields on all possible devices +// (and uses the max sizeof per-field). +// +// Since all protobuf fields are optional, fields not present on a particular device are always +// null +struct PageCacheFtraceEvent { + /* + * Ftrace buffer-specific + */ + uint32_t cpu; // e.g. 0-7 for the cpu core number. + + /* + * Ftrace-event general data + */ + + // Nanoseconds since an epoch. + // Epoch is configurable by writing into trace_clock. + // By default this timestamp is CPU local. + uint64_t timestamp; + // Kernel pid (do not confuse with userspace pid aka tgid) + uint32_t pid; + + // Tagged by our code while parsing the ftraces: + uint64_t timestamp_relative; // timestamp relative to first ftrace within a Trace protobuf. + bool add_to_page_cache; // AddToPageCache=true, DeleteFromPageCache=false. + + /* + * mm_filemap-specific data + * + * Fields are common: + * - MmFilemapAddToPageCacheFtraceEvent + * - MmFilemapDeleteFromPageCacheFtraceEvent + */ + uint64_t pfn; // page frame number (physical) - null on some devices, e.g. marlin + uint64_t i_ino; // inode number (use in conjunction with s_dev) + uint64_t index; // offset into file: this is a multiple of the page size (usually 4096). + uint64_t s_dev; // (dev_t) device number + uint64_t page; // struct page*. - null on some devices, e.g. blueline. + + Inode inode() const { + return Inode::FromDeviceAndInode(static_cast<dev_t>(s_dev), + static_cast<ino_t>(i_ino)); + } +}; + +std::ostream& operator<<(std::ostream& os, const PageCacheFtraceEvent& e) { + os << "{"; + os << "cpu:" << e.cpu << ","; + os << "timestamp:" << e.timestamp << ","; + os << "pid:" << e.pid << ","; + os << "timestamp_relative:" << e.timestamp_relative << ","; + os << "add_to_page_cache:" << e.add_to_page_cache << ","; + os << "pfn:" << e.pfn << ","; + os << "i_ino:" << e.i_ino << ","; + os << "index:" << e.index << ","; + os << "s_dev:" << e.s_dev << ","; + os << "page:" << e.page; + os << "}"; + + return os; +} + +/* + * Gets the start timestamp. + * + * It is the minimium timestamp. + */ +std::optional<uint64_t> GetStartTimestamp(const ::perfetto::protos::Trace& trace) { + std::optional<uint64_t> timestamp_relative_start; + // Traverse each timestamp to get the minimium one. + for (const ::perfetto::protos::TracePacket& packet : trace.packet()) { + if (packet.has_timestamp()) { + timestamp_relative_start = timestamp_relative_start? + std::min(*timestamp_relative_start, packet.timestamp()) : packet.timestamp(); + } + if (!packet.has_ftrace_events()) { + continue; + } + const ::perfetto::protos::FtraceEventBundle& ftrace_event_bundle = + packet.ftrace_events(); + for (const ::perfetto::protos::FtraceEvent& event : ftrace_event_bundle.event()) { + if (event.has_timestamp()) { + timestamp_relative_start = timestamp_relative_start? + std::min(*timestamp_relative_start, event.timestamp()) : event.timestamp(); + } + } + } + return timestamp_relative_start; +} + +/* + * sample blueline output: + * + * $ adb shell cat /d/tracing/events/filemap/mm_filemap_add_to_page_cache/format + * + * name: mm_filemap_add_to_page_cache + * ID: 178 + * format: + * field:unsigned short common_type; offset:0; size:2; signed:0; + * field:unsigned char common_flags; offset:2; size:1; signed:0; + * field:unsigned char common_preempt_count; offset:3; size:1; signed:0; + * field:int common_pid; offset:4; size:4; signed:1; + * + * field:unsigned long pfn; offset:8; size:8; signed:0; + * field:unsigned long i_ino; offset:16; size:8; signed:0; + * field:unsigned long index; offset:24; size:8; signed:0; + * field:dev_t s_dev; offset:32; size:4; signed:0; + * + * print fmt: "dev %d:%d ino %lx page=%p pfn=%lu ofs=%lu", ((unsigned int) ((REC->s_dev) >> 20)), + * ((unsigned int) ((REC->s_dev) & ((1U << 20) - 1))), REC->i_ino, + * (((struct page *)(((0xffffffffffffffffUL) - ((1UL) << ((39) - 1)) + 1) - + * ((1UL) << ((39) - 12 - 1 + 6))) - (memstart_addr >> 12)) + (REC->pfn)), + * REC->pfn, REC->index << 12 + */ + +auto /*observable<PageCacheFtraceEvent>*/ SelectPageCacheFtraceEvents( + PerfettoTracePtrInfo trace_info) { + const ::perfetto::protos::Trace& trace = *(trace_info.trace_ptr); + + constexpr bool kDebugFunction = true; + + return rxcpp::observable<>::create<PageCacheFtraceEvent>( + [trace=std::move(trace), + timestamp_limit_ns=trace_info.timestamp_limit_ns, + app_pid=trace_info.pid] + (rxcpp::subscriber<PageCacheFtraceEvent> sub) { + uint64_t timestamp = 0; + uint64_t timestamp_relative = 0; + + std::optional<uint64_t> timestamp_relative_start = GetStartTimestamp(trace); + uint32_t cpu = 0; + uint32_t pid = 0; + bool add_to_page_cache = true; + + auto on_next_page_cache_event = [&](const auto& mm_event) { + PageCacheFtraceEvent out; + out.timestamp = timestamp; + out.cpu = cpu; + out.pid = pid; + + out.timestamp_relative = timestamp_relative; + out.add_to_page_cache = add_to_page_cache; + + out.pfn = mm_event.pfn(); + out.i_ino = mm_event.i_ino(); + out.index = mm_event.index(); + out.s_dev = mm_event.s_dev(); + out.page = mm_event.page(); + + sub.on_next(std::move(out)); + }; + + for (const ::perfetto::protos::TracePacket& packet : trace.packet()) { + // Break out of all loops if we are unsubscribed. + if (!sub.is_subscribed()) { + if (kDebugFunction) LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents unsubscribe"; + return; + } + + if (kDebugFunction) LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents TracePacket"; + + if (packet.has_timestamp()) { + timestamp_relative_start = timestamp_relative_start.value_or(packet.timestamp()); + timestamp = packet.timestamp(); // XX: should we call 'has_timestamp()' ? + } else { + timestamp = 0; + } + + if (packet.has_ftrace_events()) { + const ::perfetto::protos::FtraceEventBundle& ftrace_event_bundle = + packet.ftrace_events(); + + cpu = ftrace_event_bundle.cpu(); // XX: has_cpu ? + + for (const ::perfetto::protos::FtraceEvent& event : ftrace_event_bundle.event()) { + // Break out of all loops if we are unsubscribed. + if (!sub.is_subscribed()) { + return; + } + + if (app_pid >= 0 && + (!event.has_pid() || + event.pid() != static_cast<uint32_t>(app_pid))) { + continue; + } + + if (event.has_timestamp()) { + timestamp = event.timestamp(); + if(timestamp > timestamp_limit_ns) { + LOG(VERBOSE) << "The timestamp is " << timestamp << + ", which exceeds the limit "<< timestamp_limit_ns; + continue; + } + } else { + DCHECK(packet.has_timestamp() == false) + << "Timestamp in outer packet but not inner packet"; + // XX: use timestamp from the perfetto TracePacket ??? + // REVIEWERS: not sure if this is ok, does it use the same clock source and + // is the packet data going to be the same clock sample as the Ftrace event? + } + + if (timestamp_relative_start){ + timestamp_relative = timestamp - *timestamp_relative_start; + } else { + timestamp_relative = 0; + } + + pid = event.pid(); // XX: has_pid ? + + if (event.has_mm_filemap_add_to_page_cache()) { + add_to_page_cache = true; + + const ::perfetto::protos::MmFilemapAddToPageCacheFtraceEvent& mm_event = + event.mm_filemap_add_to_page_cache(); + + on_next_page_cache_event(mm_event); + } else if (event.has_mm_filemap_delete_from_page_cache()) { + add_to_page_cache = false; + + const ::perfetto::protos::MmFilemapDeleteFromPageCacheFtraceEvent& mm_event = + event.mm_filemap_delete_from_page_cache(); + + on_next_page_cache_event(mm_event); + } + } + } else { + if (kDebugFunction) { + LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents no ftrace event bundle"; + } + } + } + + if (kDebugFunction) { + LOG(VERBOSE) << "compiler::SelectPageCacheFtraceEvents#on_completed"; + } + + // Let subscriber know there are no more items. + sub.on_completed(); + }); +} + +auto /*observable<Inode*/ SelectDistinctInodesFromTraces( + rxcpp::observable<PerfettoTracePtrInfo> traces) { + // Emit only unique (s_dev, i_ino) pairs from all Trace protos. + auto obs = traces + .flat_map([](PerfettoTracePtrInfo trace) { + rxcpp::observable<PageCacheFtraceEvent> obs = SelectPageCacheFtraceEvents(std::move(trace)); + // FIXME: dont check this in + // return obs; + //return obs.take(100); // for faster development + return obs; + }) // TODO: Upstream bug? using []()::perfetto::protos::Trace&) causes a compilation error. + .map([](const PageCacheFtraceEvent& event) -> Inode { + return Inode::FromDeviceAndInode(static_cast<dev_t>(event.s_dev), + static_cast<ino_t>(event.i_ino)); + }) + .tap([](const Inode& inode) { + LOG(VERBOSE) << "SelectDistinctInodesFromTraces (pre-distinct): " << inode; + }) + .distinct() // observable<Inode>*/ + ; + + return obs; +} +// TODO: static assert checks for convertible return values. + +auto/*observable<InodeResult>*/ ResolveInodesToFileNames( + rxcpp::observable<Inode> inodes, + inode2filename::InodeResolverDependencies dependencies) { + std::shared_ptr<inode2filename::InodeResolver> inode_resolver = + inode2filename::InodeResolver::Create(std::move(dependencies)); + return inode_resolver->FindFilenamesFromInodes(std::move(inodes)); +} + +using InodeMap = std::unordered_map<Inode, std::string /*filename*/>; +auto /*just observable<InodeMap>*/ ReduceResolvedInodesToMap( + rxcpp::observable<InodeResult> inode_results) { + return inode_results.reduce( + InodeMap{}, + [](InodeMap m, InodeResult result) { + if (result) { + LOG(VERBOSE) << "compiler::ReduceResolvedInodesToMap insert " << result; + m.insert({std::move(result.inode), std::move(result.data.value())}); + } else { + // TODO: side stats for how many of these are failed to resolve? + LOG(WARNING) << "compiler: Failed to resolve inode, " << result; + } + return m; + }, + [](InodeMap m) { + return m; // TODO: use an identity function + }); // emits exactly 1 InodeMap value. +} + +struct ResolvedPageCacheFtraceEvent { + std::string filename; + PageCacheFtraceEvent event; +}; + +std::ostream& operator<<(std::ostream& os, const ResolvedPageCacheFtraceEvent& e) { + os << "{"; + os << "filename:\"" << e.filename << "\","; + os << e.event; + os << "}"; + + return os; +} + +struct CombinedState { + CombinedState() = default; + explicit CombinedState(InodeMap inode_map) : inode_map{std::move(inode_map)} {} + explicit CombinedState(PageCacheFtraceEvent event) : ftrace_event{std::move(event)} {} + + CombinedState(InodeMap inode_map, PageCacheFtraceEvent event) + : inode_map(std::move(inode_map)), + ftrace_event{std::move(event)} {} + + std::optional<InodeMap> inode_map; + std::optional<PageCacheFtraceEvent> ftrace_event; + + bool HasAll() const { + return inode_map.has_value() && ftrace_event.has_value(); + } + + const InodeMap& GetInodeMap() const { + DCHECK(HasAll()); + return inode_map.value(); + } + + InodeMap& GetInodeMap() { + DCHECK(HasAll()); + return inode_map.value(); + } + + const PageCacheFtraceEvent& GetEvent() const { + DCHECK(HasAll()); + return ftrace_event.value(); + } + + PageCacheFtraceEvent& GetEvent() { + DCHECK(HasAll()); + return ftrace_event.value(); + } + + void Merge(CombinedState&& other) { + if (other.inode_map) { + inode_map = std::move(other.inode_map); + } + if (other.ftrace_event) { + ftrace_event = std::move(other.ftrace_event); + } + } +}; + +std::ostream& operator<<(std::ostream& os, const CombinedState& s) { + os << "CombinedState{inode_map:"; + if (s.inode_map) { + os << "|sz=" << (s.inode_map.value().size()) << "|"; + } else { + os << "(null)"; + } + os << ",event:"; + if (s.ftrace_event) { + //os << s.ftrace_event.value().timestamp << "ns"; + os << s.ftrace_event.value(); + } else { + os << "(null)"; + } + os << "}"; + return os; +} + +auto/*observable<ResolvedPageCacheFtraceEvent>*/ ResolvePageCacheEntriesFromProtos( + rxcpp::observable<PerfettoTracePtrInfo> traces, + inode2filename::InodeResolverDependencies dependencies) { + + // 1st chain = emits exactly 1 InodeMap. + + // [proto, proto, proto...] -> [inode, inode, inode, ...] + auto/*observable<Inode>*/ distinct_inodes = SelectDistinctInodesFromTraces(traces); + rxcpp::observable<Inode> distinct_inodes_obs = distinct_inodes.as_dynamic(); + // [inode, inode, inode, ...] -> [(inode, {filename|error}), ...] + auto/*observable<InodeResult>*/ inode_names = ResolveInodesToFileNames(distinct_inodes_obs, + std::move(dependencies)); + // rxcpp has no 'join' operators, so do a manual join with concat. + auto/*observable<InodeMap>*/ inode_name_map = ReduceResolvedInodesToMap(inode_names); + + // 2nd chain = emits all PageCacheFtraceEvent + auto/*observable<PageCacheFtraceEvent>*/ page_cache_ftrace_events = traces + .flat_map([](PerfettoTracePtrInfo trace) { + rxcpp::observable<PageCacheFtraceEvent> obs = SelectPageCacheFtraceEvents(std::move(trace)); + return obs; + }); + + auto inode_name_map_precombine = inode_name_map + .map([](InodeMap inode_map) { + LOG(VERBOSE) << "compiler::ResolvePageCacheEntriesFromProtos#inode_name_map_precombine "; + return CombinedState{std::move(inode_map)}; + }); + + auto page_cache_ftrace_events_precombine = page_cache_ftrace_events + .map([](PageCacheFtraceEvent event) { + LOG(VERBOSE) + << "compiler::ResolvePageCacheEntriesFromProtos#page_cache_ftrace_events_precombine " + << event; + return CombinedState{std::move(event)}; + }); + + // Combine 1st+2nd chain. + // + // concat subscribes to each observable, waiting until its completed, before subscribing + // to the next observable and waiting again. + // + // During all this, every #on_next is immediately forwarded to the downstream observables. + // In our case, we want to block until InodeNameMap is ready, and re-iterate all ftrace events. + auto/*observable<ResolvedPageCacheFtraceEvent>*/ resolved_events = inode_name_map_precombine + .concat(page_cache_ftrace_events_precombine) + .scan(CombinedState{}, + [](CombinedState current_state, CombinedState delta_state) { + LOG(VERBOSE) << "compiler::ResolvePageCacheEntriesFromProtos#scan " + << "current=" << current_state << "," + << "delta=" << delta_state; + // IT0 = (,) + (InodeMap,) + // IT1 = (InodeMap,) + (,Event) + // IT2..N = (InodeMap,Event1) + (,Event2) + current_state.Merge(std::move(delta_state)); + return current_state; + }) + .filter([](const CombinedState& state) { + return state.HasAll(); + }) + .map([](CombinedState& state) -> std::optional<ResolvedPageCacheFtraceEvent> { + PageCacheFtraceEvent& event = state.GetEvent(); + const InodeMap& inode_map = state.GetInodeMap(); + + auto it = inode_map.find(event.inode()); + if (it != inode_map.end()) { + std::string filename = it->second; + LOG(VERBOSE) << "compiler::ResolvePageCacheEntriesFromProtos combine_latest " << event; + return ResolvedPageCacheFtraceEvent{std::move(filename), std::move(event)}; + } else { + LOG(ERROR) << "compiler: FtraceEvent's inode did not have resolved filename: " << event; + return std::nullopt; + } + }) + .filter( + [](std::optional<ResolvedPageCacheFtraceEvent> maybe_event) { + return maybe_event.has_value(); + }) + .map([](std::optional<ResolvedPageCacheFtraceEvent> maybe_event) { + return std::move(maybe_event.value()); + }); + // -> observable<ResolvedPageCacheFtraceEvent> + + return resolved_events; +} + +namespace detail { +bool multiless_one(const std::string& a, const std::string& b) { + return std::lexicographical_compare(a.begin(), a.end(), + b.begin(), b.end()); +} + +template <typename T> +constexpr bool multiless_one(T&& a, T&& b) { // a < b + using std::less; // ADL + return less<std::decay_t<T>>{}(std::forward<T>(a), std::forward<T>(b)); +} + +constexpr bool multiless() { + return false; // [] < [] is always false. +} + +template <typename T, typename ... Args> +constexpr bool multiless(T&& a, T&& b, Args&&... args) { + if (a != b) { + return multiless_one(std::forward<T>(a), std::forward<T>(b)); + } else { + return multiless(std::forward<Args>(args)...); + } +} + +} // namespace detail + +// Return [A0...An] < [B0...Bn] ; vector-like scalar comparison of each field. +// Arguments are passed in the order A0,B0,A1,B1,...,An,Bn. +template <typename ... Args> +constexpr bool multiless(Args&&... args) { + return detail::multiless(std::forward<Args>(args)...); +} + +struct CompilerPageCacheEvent { + std::string filename; + uint64_t timestamp_relative; // use relative timestamp because absolute values aren't comparable + // across different trace protos. + // relative timestamps can be said to be 'approximately' comparable. + // assuming we compare the same application startup's trace times. + bool add_to_page_cache; // AddToPageCache=true, DeleteFromPageCache=false. + uint64_t index; // offset into file: this is a multiple of the page size (usually 4096). + + // All other data from the ftrace is dropped because we don't currently use it in the + // compiler algorithms. + + CompilerPageCacheEvent() = default; + CompilerPageCacheEvent(const ResolvedPageCacheFtraceEvent& resolved) + : CompilerPageCacheEvent(resolved.filename, resolved.event) { + } + + CompilerPageCacheEvent(ResolvedPageCacheFtraceEvent&& resolved) + : CompilerPageCacheEvent(std::move(resolved.filename), std::move(resolved.event)) { + } + + // Compare all fields (except the timestamp field). + static bool LessIgnoringTimestamp(const CompilerPageCacheEvent& a, + const CompilerPageCacheEvent& b) { + return multiless(a.filename, b.filename, + a.add_to_page_cache, b.add_to_page_cache, + a.index, b.index); + } + + // Compare all fields. Timestamps get highest precedence. + bool operator<(const CompilerPageCacheEvent& rhs) const { + return multiless(timestamp_relative, rhs.timestamp_relative, + filename, rhs.filename, + add_to_page_cache, rhs.add_to_page_cache, + index, rhs.index); + } + + private: + CompilerPageCacheEvent(std::string filename, const PageCacheFtraceEvent& event) + : filename(std::move(filename)), + timestamp_relative(event.timestamp_relative), + add_to_page_cache(event.add_to_page_cache), + index(event.index) { + } +}; + +std::ostream& operator<<(std::ostream& os, const CompilerPageCacheEvent& e) { + os << "{"; + os << "filename:\"" << e.filename << "\","; + os << "timestamp:" << e.timestamp_relative << ","; + os << "add_to_page_cache:" << e.add_to_page_cache << ","; + os << "index:" << e.index; + os << "}"; + return os; +} + +// Filter an observable chain of 'ResolvedPageCacheFtraceEvent' +// into an observable chain of 'ResolvedPageCacheFtraceEvent'. +// +// Any items emitted by the input chain that match the regular expression +// specified by blacklist_filter are not emitted into the output chain. +auto/*observable<ResolvedPageCacheFtraceEvent>*/ ApplyBlacklistToPageCacheEvents( + rxcpp::observable<ResolvedPageCacheFtraceEvent> resolved_events, + std::optional<std::string> blacklist_filter) { + bool has_re = blacklist_filter.has_value(); + // default regex engine is ecmascript. + std::regex reg_exp{blacklist_filter ? *blacklist_filter : std::string("")}; + + return resolved_events.filter( + [reg_exp, has_re](const ResolvedPageCacheFtraceEvent& event) { + if (!has_re) { + return true; + } + // Remove any entries that match the regex in --blacklist-filter/-bf. + bool res = std::regex_search(event.filename, reg_exp); + if (res) { + LOG(VERBOSE) << "Blacklist filter removed '" << event.filename << "' from chain."; + } + return !res; + }); +} + +// Compile an observable chain of 'ResolvedPageCacheFtraceEvent' into +// an observable chain of distinct, timestamp-ordered, CompilerPageCacheEvent. +// +// This is a reducing operation: No items are emitted until resolved_events is completed. +auto/*observable<CompilerPageCacheEvent>*/ CompilePageCacheEvents( + rxcpp::observable<ResolvedPageCacheFtraceEvent> resolved_events) { + + struct CompilerPageCacheEventIgnoringTimestampLess { + bool operator()(const CompilerPageCacheEvent& lhs, + const CompilerPageCacheEvent& rhs) const { + return CompilerPageCacheEvent::LessIgnoringTimestamp(lhs, rhs); + } + }; + + // Greedy O(N) compilation algorithm. + // + // This produces an inoptimal result (e.g. a small timestamp + // that might occur only 1% of the time nevertheless wins out), but the + // algorithm itself is quite simple, and doesn't require any heuristic tuning. + + // First pass: *Merge* into set that ignores the timestamp value for order, but retains + // the smallest timestamp value if the same key is re-inserted. + using IgnoreTimestampForOrderingSet = + std::set<CompilerPageCacheEvent, CompilerPageCacheEventIgnoringTimestampLess>; + // Second pass: *Sort* data by smallest timestamp first. + using CompilerPageCacheEventSet = + std::set<CompilerPageCacheEvent>; + + return resolved_events + .map( + [](ResolvedPageCacheFtraceEvent event) { + // Drop all the extra metadata like pid, cpu, etc. + // When we merge we could keep a list of the original data, but there is no advantage + // to doing so. + return CompilerPageCacheEvent{std::move(event)}; + } + ) + .reduce( + IgnoreTimestampForOrderingSet{}, + [](IgnoreTimestampForOrderingSet set, CompilerPageCacheEvent event) { + // Add each event to the set, keying by everything but the timestamp. + // If the key is already inserted, re-insert with the smaller timestamp value. + auto it = set.find(event); + + if (it == set.end()) { + // Need to insert new element. + set.insert(std::move(event)); + } else if (it->timestamp_relative > event.timestamp_relative) { + // Replace existing element: the new element has a smaller timestamp. + it = set.erase(it); + // Amortized O(1) time if insertion happens in the position before the hint. + set.insert(it, std::move(event)); + } // else: Skip insertion. Element already present with the minimum timestamp. + + return set; + }, + [](IgnoreTimestampForOrderingSet set) { + // Extract all elements from 'set', re-insert into 'ts_set'. + // The values are now ordered by timestamp (and then the rest of the fields). + CompilerPageCacheEventSet ts_set; + ts_set.merge(std::move(set)); + + + std::shared_ptr<CompilerPageCacheEventSet> final_set{ + new CompilerPageCacheEventSet{std::move(ts_set)}}; + return final_set; + // return ts_set; + }) // observable<CompilerPageCacheEventSet> (just) + .flat_map( + [](std::shared_ptr<CompilerPageCacheEventSet> final_set) { + // TODO: flat_map seems to make a copy of the parameter _every single iteration_ + // without the shared_ptr it would just make a copy of the set every time it went + // through the iterate function. + // Causing absurdly slow compile times x1000 slower than we wanted. + // TODO: file a bug upstream and/or fix upstream. + CompilerPageCacheEventSet& ts_set = *final_set; + // [](CompilerPageCacheEventSet& ts_set) { + LOG(DEBUG) << "compiler: Merge-pass completed (" << ts_set.size() << " entries)."; + //return rxcpp::sources::iterate(std::move(ts_set)); + return rxcpp::sources::iterate(ts_set).map([](CompilerPageCacheEvent e) { return e; }); + } + ); // observable<CompilerPageCacheEvent> +} + +/** Makes a vector of info that includes filename and timestamp limit. */ +std::vector<CompilationInput> MakeCompilationInputs( + std::vector<std::string> input_file_names, + std::vector<uint64_t> timestamp_limit_ns, + std::vector<int32_t> pids){ + // If the timestamp limit is empty, set the limit to max value + // for each trace file. + if (timestamp_limit_ns.empty()) { + for (size_t i = 0; i < input_file_names.size(); i++) { + timestamp_limit_ns.push_back(std::numeric_limits<uint64_t>::max()); + } + } + + // If the pids is empty, set all of them to -1. Because negative pid means any. + if (pids.empty()) { + for (size_t i = 0; i < input_file_names.size(); i++) { + pids.push_back(-1); + } + } + + DCHECK_EQ(input_file_names.size(), timestamp_limit_ns.size()); + std::vector<CompilationInput> file_infos; + for (size_t i = 0; i < input_file_names.size(); i++) { + file_infos.push_back({input_file_names[i], timestamp_limit_ns[i], pids[i]}); + } + return file_infos; +} + +bool PerformCompilation(std::vector<CompilationInput> perfetto_traces, + std::string output_file_name, + bool output_proto, + std::optional<std::string> blacklist_filter, + inode2filename::InodeResolverDependencies dependencies) { + auto trace_protos = ReadPerfettoTraceProtos(std::move(perfetto_traces)); + auto resolved_events = ResolvePageCacheEntriesFromProtos(std::move(trace_protos), + std::move(dependencies)); + auto filtered_events = + ApplyBlacklistToPageCacheEvents(std::move(resolved_events), blacklist_filter); + auto compiled_events = CompilePageCacheEvents(std::move(filtered_events)); + + std::ofstream ofs; + if (!output_file_name.empty()) { + + if (!output_proto) { + ofs.open(output_file_name); + + if (!ofs) { + LOG(ERROR) << "compiler: Failed to open output file for writing: " << output_file_name; + return false; + } + } + } + + auto trace_file_proto = serialize::ArenaPtr<serialize::proto::TraceFile>::Make(); + + // Fast lookup of filename -> FileIndex id. + std::unordered_map<std::string, int64_t /*file handle id*/> file_path_map; + int64_t file_handle_id = 0; + + int counter = 0; + compiled_events + // .as_blocking() + .tap([&](CompilerPageCacheEvent& event) { + if (!output_proto) { + return; + } + + if (!event.add_to_page_cache) { + // Skip DeleteFromPageCache events, they are only used for intermediate. + return; + } + + DCHECK(trace_file_proto->mutable_index() != nullptr); + serialize::proto::TraceFileIndex& index = *trace_file_proto->mutable_index(); + int64_t file_handle; + + // Add TraceFileIndexEntry if it doesn't exist. + + auto it = file_path_map.find(event.filename); + if (it == file_path_map.end()) { + file_handle = file_handle_id++; + file_path_map[event.filename] = file_handle; + + serialize::proto::TraceFileIndexEntry* entry = index.add_entries(); + DCHECK(entry != nullptr); + entry->set_id(file_handle); + entry->set_file_name(event.filename); + + if (kIsDebugBuild) { + int i = static_cast<int>(file_handle); + const serialize::proto::TraceFileIndexEntry& entry_ex = index.entries(i); + DCHECK_EQ(entry->id(), entry_ex.id()); + DCHECK_EQ(entry->file_name(), entry_ex.file_name()); + } + } else { + file_handle = it->second; + } + int kPageSize = 4096; // TODO: don't hardcode the page size. + + int entry_size = trace_file_proto->list().entries_size(); + bool merged = false; + if (entry_size > 0) { + serialize::proto::TraceFileEntry* entry = + trace_file_proto->mutable_list()->mutable_entries(entry_size-1); + if (entry->index_id() == file_handle && + entry->file_offset() + entry->file_length() == + static_cast<int64_t>(event.index) * kPageSize) { + entry->set_file_length(entry->file_length() + kPageSize); + merged = true; + } + } + + if (!merged) { + // Add TraceFileEntry. + DCHECK(trace_file_proto->mutable_list() != nullptr); + serialize::proto::TraceFileEntry* entry = trace_file_proto->mutable_list()->add_entries(); + DCHECK(entry != nullptr); + + entry->set_index_id(file_handle); + // Page index -> file offset in bytes. + entry->set_file_offset(static_cast<int64_t>(event.index) * kPageSize); + entry->set_file_length(kPageSize); + } + }) + .subscribe([&](CompilerPageCacheEvent event) { + if (!output_proto) { + if (output_file_name.empty()) { + LOG(INFO) << "CompilerPageCacheEvent" << event << std::endl; + } else { + ofs << event << "\n"; // TODO: write in proto format instead. + } + } + ++counter; + }); + + if (output_proto) { + LOG(DEBUG) << "compiler: WriteFully to begin into " << output_file_name; + ::google::protobuf::MessageLite& message = *trace_file_proto.get(); + if (auto res = serialize::ProtobufIO::WriteFully(message, output_file_name); !res) { + errno = res.error(); + PLOG(ERROR) << "compiler: Failed to write protobuf to file: " << output_file_name; + return false; + } else { + LOG(INFO) << "compiler: Wrote protobuf " << output_file_name; + } + } + + LOG(DEBUG) << "compiler: Compilation completed (" << counter << " events)."; + + return true; +} + +} // namespace iorap::compiler diff --git a/src/compiler/compiler.h b/src/compiler/compiler.h new file mode 100644 index 0000000..243ab36 --- /dev/null +++ b/src/compiler/compiler.h @@ -0,0 +1,72 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_COMPILER_COMPILER_H_ +#define IORAP_SRC_COMPILER_COMPILER_H_ + +#include "inode2filename/inode_resolver.h" + +#include <optional> +#include <string> +#include <vector> + +namespace iorap::compiler { +struct CompilationInput { + /* The name of the perfetto trace. */ + std::string filename; + /* + * The timestamp limit of the trace. + * It's used to truncate the trace file. + */ + uint64_t timestamp_limit_ns; + + /* + * The pid of the app. + * If positive, it's used to filter out other page cache events. + */ + int32_t pid; +}; + +// Compile one or more perfetto TracePacket protobufs that are stored on the filesystem +// by the filenames given with `input_file_names` and timestamp limit given with +// timestamp_limit_ns. +// +// For each input file name and timestamp limit, ignore any events from the input that +// exceed the associated timestamp limit. +// +// If blacklist_filter is non-null, ignore any file entries whose file path matches the +// regular expression in blacklist_filter. +// +// The result is stored into the file path specified by `output_file_name`. +// +// This is a blocking function which returns only when compilation finishes. The return value +// determines success/failure. +// +// Operation is transactional -- that is if there is a failure, `output_file_name` is untouched. +bool PerformCompilation(std::vector<iorap::compiler::CompilationInput> perfetto_traces, + std::string output_file_name, + bool output_proto, + std::optional<std::string> blacklist_filter, + inode2filename::InodeResolverDependencies dependencies); + +// The size and order of timestamp_limit_ns should match that of +// input_file_names, if not empty. +// If timestamp_limit_ns is empty, will use the max uint64_t. +std::vector<CompilationInput> MakeCompilationInputs( + std::vector<std::string> input_file_names, + std::vector<uint64_t> timestamp_limit_ns, + std::vector<int32_t> pids); +} + +#endif // IORAP_SRC_COMPILER_COMPILER_H_ diff --git a/src/compiler/main.cc b/src/compiler/main.cc new file mode 100644 index 0000000..3036936 --- /dev/null +++ b/src/compiler/main.cc @@ -0,0 +1,258 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/cmd_utils.h" +#include "common/debug.h" +#include "compiler/compiler.h" +#include "inode2filename/inode_resolver.h" + +#include <android-base/parseint.h> +#include <android-base/logging.h> + +#include <iostream> +#include <limits> +#include <optional> +#include <string> + +#if defined(IORAP_COMPILER_MAIN) + +namespace iorap::compiler { + +// Log everything to stderr. +// Log errors and higher to logd. +class StderrAndLogdErrorLogger { + public: + explicit StderrAndLogdErrorLogger(android::base::LogId default_log_id = android::base::MAIN) +#ifdef __ANDROID__ + : logd_(default_log_id) +#endif + { + } + + void operator()(::android::base::LogId id, + ::android::base::LogSeverity sev, + const char* tag, + const char* file, + unsigned int line, + const char* message) { +#ifdef __ANDROID__ + if (static_cast<int>(sev) >= static_cast<int>(::android::base::ERROR)) { + logd_(id, sev, tag, file, line, message); + } +#endif + StderrLogger(id, sev, tag, file, line, message); + } + + private: +#ifdef __ANDROID__ + ::android::base::LogdLogger logd_; +#endif +}; + +void Usage(char** argv) { + std::cerr << "Usage: " << argv[0] << " [--output-proto=output.pb] input1.pb [input2.pb ...]" << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Request a compilation of multiple inputs (format: PerfettoTraceProto)." << std::endl; + std::cerr << " The result is a TraceFile, representing a merged compiled trace with inodes resolved." << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Optional flags:" << std::endl; + std::cerr << " --help,-h Print this Usage." << std::endl; + std::cerr << " --denylist-filter,-df Specify regex acting as a denylist filter." + << std::endl; + std::cerr << " Filepaths matching this regex are removed from the output file." << std::endl; + std::cerr << " --output-text,-ot Output ascii text instead of protobuf (default off)." << std::endl; + std::cerr << " --output-proto $,-op $ TraceFile tracebuffer output file (default stdout)." << std::endl; + std::cerr << " --inode-textcache $,-it $ Resolve inode->filename from textcache (disables diskscan)." << std::endl; + std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; + std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; + std::cerr << " --pid,-p Set the pid for the compiled trace" << std::endl; + std::cerr << " --timestamp_limit_ns,-tl Set the limit timestamp in nanoseconds for the compiled trace. " + "The order and size of the timestamp should match that of " + "the input trace files. If not specified at all, All of" + "the timestamps are set to max."<< std::endl; + exit(1); +} + +int Main(int argc, char** argv) { + android::base::InitLogging(argv); + android::base::SetLogger(StderrAndLogdErrorLogger{}); + + bool wait_for_keystroke = false; + bool enable_verbose = false; + + std::optional<std::string> arg_blacklist_filter; + std::string arg_output_proto; + bool arg_output_text = false; + std::optional<std::string> arg_inode_textcache; + + std::vector<uint64_t> timestamp_limit_ns; + std::vector<int32_t> pids; + + if (argc == 1) { + // Need at least 1 input file to do anything. + Usage(argv); + } + + std::vector<std::string> arg_input_filenames; + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + bool has_arg_next = (arg+1)<argc; + std::string arg_next = has_arg_next ? argv[arg+1] : ""; + + if (argstr == "--help" || argstr == "-h") { + Usage(argv); + } else if (argstr == "--output-proto" || argstr == "-op") { + if (!has_arg_next) { + std::cerr << "Missing --output-proto <value>" << std::endl; + return 1; + } + arg_output_proto = arg_next; + ++arg; + } else if (argstr == "--output-text" || argstr == "-ot") { + arg_output_text = true; + } else if (argstr == "--inode-textcache" || argstr == "-it") { + if (!has_arg_next) { + std::cerr << "Missing --inode-textcache <value>" << std::endl; + return 1; + } + arg_inode_textcache = arg_next; + ++arg; + } else if (argstr == "--denylist-filter" || argstr == "-df") { + if (!has_arg_next) { + std::cerr << "Missing --denylist-filter <value>" << std::endl; + return 1; + } + arg_blacklist_filter = arg_next; + ++arg; + } + else if (argstr == "--verbose" || argstr == "-v") { + enable_verbose = true; + } else if (argstr == "--wait" || argstr == "-w") { + wait_for_keystroke = true; + } else if (argstr == "--pid" || argstr == "-p") { + if (!has_arg_next) { + std::cerr << "Missing --pid <value>" << std::endl; + return 1; + } + int32_t pid; + if (!::android::base::ParseInt<int32_t>(arg_next, &pid)) { + std::cerr << "Invalid --pid "<< arg_next << std::endl; + return 1; + } + pids.push_back(pid); + ++arg; + } else if (argstr == "--timestamp_limit_ns" || argstr == "-tl") { + if (!has_arg_next) { + std::cerr << "Missing --timestamp_limit_ns <value>" << std::endl; + return 1; + } + uint64_t timestamp; + if (!::android::base::ParseUint<uint64_t>(arg_next, ×tamp)) { + std::cerr << "Invalid --timestamp-limit-ns "<< arg_next << std::endl; + return 1; + } + timestamp_limit_ns.push_back(timestamp); + ++arg; + } else { + arg_input_filenames.push_back(argstr); + } + } + + if (!timestamp_limit_ns.empty() && + timestamp_limit_ns.size() != arg_input_filenames.size()) { + std::cerr << "The size of timestamp limits doesn't match the size of input files." + << std::endl; + return 1; + } + + if (!pids.empty() && pids.size() != arg_input_filenames.size()) { + std::cerr << "The size of pids doesn't match the size of input files." + << std::endl; + return 1; + } + if (enable_verbose) { + android::base::SetMinimumLogSeverity(android::base::VERBOSE); + + LOG(VERBOSE) << "Verbose check"; + LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; + } else { + android::base::SetMinimumLogSeverity(android::base::DEBUG); + } + + // Useful to attach a debugger... + // 1) $> iorap.cmd.compiler -w <args> + // 2) $> gdbclient <pid> + if (wait_for_keystroke) { + LOG(INFO) << "Self pid: " << getpid(); + LOG(INFO) << "Press any key to continue..."; + std::cin >> wait_for_keystroke; + } + + auto system_call = std::make_unique<SystemCallImpl>(); + + using namespace inode2filename; + + InodeResolverDependencies ir_dependencies; + // Passed from command-line. + if (arg_inode_textcache) { + ir_dependencies.data_source = DataSourceKind::kTextCache; + ir_dependencies.text_cache_filename = arg_inode_textcache; + ir_dependencies.verify = VerifyKind::kNone; // required for determinism. + } else { + ir_dependencies.data_source = DataSourceKind::kDiskScan; + LOG(WARNING) << "--inode-textcache unspecified. " + << "Inodes will be resolved by scanning the disk, which makes compilation " + << "non-deterministic."; + } + // TODO: add command line. + ir_dependencies.root_directories.push_back("/system"); + ir_dependencies.root_directories.push_back("/apex"); + ir_dependencies.root_directories.push_back("/data"); + ir_dependencies.root_directories.push_back("/vendor"); + ir_dependencies.root_directories.push_back("/product"); + ir_dependencies.root_directories.push_back("/metadata"); + // Hardcoded. + if (iorap::common::GetBoolEnvOrProperty("iorap.inode2filename.out_of_process", true)) { + ir_dependencies.process_mode = ProcessMode::kOutOfProcessIpc; + } else { + ir_dependencies.process_mode = ProcessMode::kInProcessDirect; + } + ir_dependencies.system_call = /*borrowed*/system_call.get(); + + int return_code = 0; + std::vector<CompilationInput> perfetto_traces = + MakeCompilationInputs(arg_input_filenames, timestamp_limit_ns, pids); + return_code = + !PerformCompilation(std::move(perfetto_traces), + std::move(arg_output_proto), + !arg_output_text, + arg_blacklist_filter, + std::move(ir_dependencies)); + + // Uncomment this if we want to leave the process around to inspect it from adb shell. + // sleep(100000); + + // 0 -> successfully wrote the proto out to file. + // 1 -> failed along the way (#on_error and also see the error logs). + return return_code; +} + +} // namespace iorap::compiler + +int main(int argc, char** argv) { + return ::iorap::compiler::Main(argc, argv); +} + +#endif // IORAP_COMPILER_MAIN diff --git a/src/db/app_component_name.h b/src/db/app_component_name.h new file mode 100644 index 0000000..e364163 --- /dev/null +++ b/src/db/app_component_name.h @@ -0,0 +1,130 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_DB_APP_COMPONENT_NAME_H_ +#define IORAP_SRC_DB_APP_COMPONENT_NAME_H_ + +namespace iorap::db { + +struct AppComponentName { + std::string package; + std::string activity_name; + + // Turns the activity name to the fully qualified name. + // For example, if the activity name is ".MainActivity" and the package is + // foo.bar. Then the fully qualified name is foo.bar.MainActivity. + AppComponentName Canonicalize() const { + if (!activity_name.empty() && activity_name[0] == '.') { + return {package, package + activity_name}; + } + return {package, activity_name}; + } + + static bool HasAppComponentName(const std::string& s) { + return s.find('/') != std::string::npos; + } + + // "com.foo.bar/.A" -> {"com.foo.bar", ".A"} + static AppComponentName FromString(const std::string& s) { + constexpr const char delimiter = '/'; + + if (!HasAppComponentName(s)) { + return {std::move(s), ""}; + } + + std::string package = s.substr(0, s.find(delimiter)); + + std::string activity_name = s; + activity_name.erase(0, s.find(delimiter) + sizeof(delimiter)); + + return {std::move(package), std::move(activity_name)}; + } + + // {"com.foo.bar", ".A"} -> "com.foo.bar/.A" + std::string ToString() const { + return package + "/" + activity_name; + } + + /* + * '/' is encoded into %2F + * '%' is encoded into %25 + * + * This allows the component name to be be used as a file name + * ('/' is illegal due to being a path separator) with minimal + * munging. + */ + + // "com.foo.bar%2F.A%25" -> {"com.foo.bar", ".A%"} + static AppComponentName FromUrlEncodedString(const std::string& s) { + std::string cpy = s; + Replace(cpy, "%2F", "/"); + Replace(cpy, "%25", "%"); + + return FromString(cpy); + } + + // {"com.foo.bar", ".A%"} -> "com.foo.bar%2F.A%25" + std::string ToUrlEncodedString() const { + std::string s = ToString(); + Replace(s, "%", "%25"); + Replace(s, "/", "%2F"); + return s; + } + + /* + * '/' is encoded into @@ + * '%' is encoded into ^^ + * + * Two purpose: + * 1. This allows the package name to be used as a file name + * ('/' is illegal due to being a path separator) with minimal + * munging. + * 2. This allows the package name to be used in .mk file because + * '%' is a special char and cannot be easily escaped in Makefile. + * + * This is a workround for test purpose. + * Only package name is used because activity name varies on + * different testing framework. + * Hopefully, the double "@@" and "^^" are not used in other cases. + */ + // {"com.foo.bar", ".A%"} -> "com.foo.bar" + std::string ToMakeFileSafeEncodedPkgString() const { + std::string s = package; + Replace(s, "/", "@@"); + Replace(s, "%", "^^"); + return s; + } + + private: + static bool Replace(std::string& str, const std::string& from, const std::string& to) { + // TODO: call in a loop to replace all occurrences, not just the first one. + const size_t start_pos = str.find(from); + if (start_pos == std::string::npos) { + return false; + } + + str.replace(start_pos, from.length(), to); + + return true; +} +}; + +inline std::ostream& operator<<(std::ostream& os, const AppComponentName& name) { + os << name.ToString(); + return os; +} + +} // namespace iorap::db + +#endif // IORAP_SRC_DB_APP_COMPONENT_NAME_H_ diff --git a/src/db/clean_up.cc b/src/db/clean_up.cc new file mode 100644 index 0000000..de1412d --- /dev/null +++ b/src/db/clean_up.cc @@ -0,0 +1,115 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "db/clean_up.h" + +#include <android-base/file.h> + +#include <cstdio> +#include <filesystem> +#include <fstream> +#include <iostream> +#include <limits> +#include <optional> +#include <string> +#include <vector> + +#include "db/file_models.h" +#include "db/models.h" + +namespace iorap::db { + +void CleanUpFilesForActivity(const db::DbHandle& db, + const db::VersionedComponentName& vcn) { + LOG(DEBUG) << "Clean up files for activity " << vcn.GetActivity(); + // Remove perfetto traces. + std::vector<db::RawTraceModel> raw_traces = + db::RawTraceModel::SelectByVersionedComponentName(db, vcn); + for (db::RawTraceModel raw_trace : raw_traces) { + std::string file_path = raw_trace.file_path; + LOG(DEBUG) << "Remove file: " << file_path; + std::filesystem::remove(file_path.c_str()); + raw_trace.Delete(); + } + + // Remove compiled traces. + std::optional<db::PrefetchFileModel> prefetch_file = + db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn); + + if (prefetch_file) { + std::string file_path = prefetch_file->file_path; + LOG(DEBUG) << "Remove file: " << file_path; + std::filesystem::remove(file_path.c_str()); + prefetch_file->Delete(); + } +} + +void CleanUpFilesForPackage(const db::DbHandle& db, + int package_id, + const std::string& package_name, + int64_t version) { + LOG(DEBUG) << "Clean up files for package " << package_name << " with version " + << version; + std::vector<db::ActivityModel> activities = + db::ActivityModel::SelectByPackageId(db, package_id); + + for (db::ActivityModel activity : activities) { + db::VersionedComponentName vcn{package_name, activity.name, version}; + CleanUpFilesForActivity(db, vcn); + } +} + +void CleanUpFilesForPackage(const db::DbHandle& db, + const std::string& package_name, + int64_t version) { + std::optional<PackageModel> package = + PackageModel::SelectByNameAndVersion(db, package_name.c_str(), version); + + if (!package) { + LOG(DEBUG) << "No package to clean up " << package_name << " with version " << version; + return; + } + + CleanUpFilesForPackage(db, package->id, package_name, version); +} + +void CleanUpFilesForDb(const db::DbHandle& db) { + std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db); + for (db::PackageModel package : packages) { + CleanUpFilesForPackage(db, package.id, package.name, package.version); + } +} + +void CleanUpFilesForPackage(const std::string& db_path, + const std::string& package_name) { + iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path); + db::DbHandle db{db_schema.db()}; + CleanUpFilesForPackage(db, package_name); +} + + +void CleanUpFilesForPackage(const db::DbHandle& db, + const std::string& package_name) { + std::vector<PackageModel> packages = PackageModel::SelectByName(db, package_name.c_str());; + + for (PackageModel& package : packages) { + CleanUpFilesForPackage(db, package.id, package.name, package.version); + } + + if (packages.empty()) { + LOG(DEBUG) << "No package rows to clean up " << package_name; + } +} + +} // namespace iorap::db diff --git a/src/db/clean_up.h b/src/db/clean_up.h new file mode 100644 index 0000000..08b7bde --- /dev/null +++ b/src/db/clean_up.h @@ -0,0 +1,54 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_DB_CLEANER_H_ +#define IORAP_SRC_DB_CLEANER_H_ + +#include <android/content/pm/IPackageManagerNative.h> + +#include <string> +#include <vector> + +#include "binder/package_version_map.h" +#include "db/file_models.h" + +namespace iorap::db { + +// Clean up perfetto traces and compiled traces in disk and rows +// in raw_traces and prefetch_files in the db. +void CleanUpFilesForDb(const db::DbHandle& db); + +// Clean up perfetto traces and compiled traces in disk and rows +// in raw_traces and prefetch_files in the db for a package id. +void CleanUpFilesForPackage(const db::DbHandle& db, + int package_id, + const std::string& package_name, + int64_t version); + +// Clean up all package rows (and files) associated with a package by name. +void CleanUpFilesForPackage(const std::string& db_path, + const std::string& package_name); + +// Clean up all package rows (and files) associated with a package by name. +void CleanUpFilesForPackage(const db::DbHandle& db, + const std::string& package_name); +// Clean up perfetto traces and compiled traces in disk and rows +// in raw_traces and prefetch_files in the db for a package name +// and version. +void CleanUpFilesForPackage(const db::DbHandle& db, + const std::string& package_name, + int64_t version); +} // namespace iorap::db + +#endif // IORAP_SRC_DB_CLEANER_H_ diff --git a/src/db/file_models.cc b/src/db/file_models.cc new file mode 100644 index 0000000..d962d49 --- /dev/null +++ b/src/db/file_models.cc @@ -0,0 +1,194 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/cmd_utils.h" +#include "db/file_models.h" +#include "db/models.h" + +#include <android-base/file.h> +#include <android-base/properties.h> + +#include <algorithm> +#include <sstream> +#include <sys/stat.h> +#include <sys/types.h> +#include <time.h> +#include <unistd.h> + +namespace iorap::db { + +static constexpr const char* kRootPathProp = "iorapd.root.dir"; +static const unsigned int kPerfettoMaxTraces = + ::android::base::GetUintProperty("iorapd.perfetto.max_traces", /*default*/2u); + +static uint64_t GetTimeNanoseconds() { + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + + uint64_t now_ns = (now.tv_sec * 1000000000LL + now.tv_nsec); + return now_ns; +} + +static bool IsDir(const std::string& dirpath) { + struct stat st; + if (stat(dirpath.c_str(), &st) == 0) { + if (S_ISDIR(st.st_mode)) { + return true; + } + } + return false; +} + +// Given some path /a/b/c create all of /a, /a/b, /a/b/c/ recursively. +static bool MkdirWithParents(const std::string& path) { + size_t prev_end = 0; + while (prev_end < path.size()) { + size_t next_end = path.find('/', prev_end + 1); + + std::string dir_path = path.substr(0, next_end); + if (!IsDir(dir_path)) { +#if defined(_WIN32) + int ret = mkdir(dir_path.c_str()); +#else + mode_t old_mask = umask(0); + // The user permission is 5 to allow system server to + // read the files. No other users could do that because + // the upper directory only allows system server and iorapd + // to access. Also selinux rules prevent other users to + // read files here. + int ret = mkdir(dir_path.c_str(), 0755); + umask(old_mask); +#endif + if (ret != 0) { + PLOG(ERROR) << "failed to create dir " << dir_path; + return false; + } + } + prev_end = next_end; + + if (next_end == std::string::npos) { + break; + } + } + return true; +} + +FileModelBase::FileModelBase(VersionedComponentName vcn) + : vcn_{std::move(vcn)} { + root_path_ = common::GetEnvOrProperty(kRootPathProp, + /*default*/"/data/misc/iorapd"); +} + +std::string FileModelBase::BaseDir() const { + std::stringstream ss; + + ss << root_path_ << "/" << vcn_.GetPackage() << "/"; + ss << vcn_.GetVersion(); + ss << "/"; + ss << vcn_.GetActivity() << "/"; + ss << SubDir(); + + return ss.str(); +} + +std::string FileModelBase::FilePath() const { + std::stringstream ss; + ss << BaseDir(); + ss << "/"; + ss << BaseFile(); + + return ss.str(); +} + +bool FileModelBase::MkdirWithParents() { + LOG(VERBOSE) << "MkdirWithParents: " << BaseDir(); + return ::iorap::db::MkdirWithParents(BaseDir()); +} + +PerfettoTraceFileModel::PerfettoTraceFileModel(VersionedComponentName vcn, + uint64_t timestamp) + : FileModelBase{std::move(vcn)}, timestamp_{timestamp} { +} + +PerfettoTraceFileModel PerfettoTraceFileModel::CalculateNewestFilePath(VersionedComponentName vcn) { + uint64_t timestamp = GetTimeNanoseconds(); + return PerfettoTraceFileModel{vcn, timestamp}; +} + +std::string PerfettoTraceFileModel::BaseFile() const { + std::stringstream ss; + ss << timestamp_ << ".perfetto_trace.pb"; + return ss.str(); +} + +bool PerfettoTraceFileModel::NeedMorePerfettoTraces(DbHandle& db, + VersionedComponentName vcn) { + std::vector<RawTraceModel> raw_traces = + RawTraceModel::SelectByVersionedComponentName(db, vcn); + + size_t raw_traces_size = raw_traces.size(); + LOG(VERBOSE) << "The number of perfetto traces is " + << raw_traces_size + << " The cap is " + << kPerfettoMaxTraces ; + return raw_traces_size < kPerfettoMaxTraces; +} + +void PerfettoTraceFileModel::DeleteOlderFiles(DbHandle& db, VersionedComponentName vcn) { + std::vector<RawTraceModel> raw_traces = + RawTraceModel::SelectByVersionedComponentName(db, vcn); + + if (WOULD_LOG(VERBOSE)) { + size_t raw_traces_size = raw_traces.size(); + for (size_t i = 0; i < raw_traces_size; ++i) { + LOG(VERBOSE) << "DeleteOlderFiles - selected " << raw_traces[i]; + } + LOG(VERBOSE) << "DeleteOlderFiles - queried total " << raw_traces_size << " records"; + } + + size_t items_to_delete = 0; + if (raw_traces.size() > kPerfettoMaxTraces) { + items_to_delete = raw_traces.size() - kPerfettoMaxTraces; + } else { + LOG(VERBOSE) << "DeleteOlderFiles - don't delete older raw traces, too few files: " + << " wanted at least " << kPerfettoMaxTraces << ", but got " << raw_traces.size(); + } + + for (size_t i = 0; i < items_to_delete; ++i) { + RawTraceModel& raw_trace = raw_traces[i]; // sorted ascending -> items to delete are first. + std::string err_msg; + + if (!::android::base::RemoveFileIfExists(raw_trace.file_path, /*out*/&err_msg)) { + LOG(ERROR) << "Failed to remove raw trace file: " << raw_trace.file_path + << ", reason: " << err_msg; + } else { + raw_trace.Delete(); + LOG(DEBUG) << "Deleted raw trace for " << vcn << " at " << raw_trace.file_path; + } + } +} + +CompiledTraceFileModel::CompiledTraceFileModel(VersionedComponentName vcn) + : FileModelBase{std::move(vcn)} { +} + +CompiledTraceFileModel CompiledTraceFileModel::CalculateNewestFilePath(VersionedComponentName vcn) { + return CompiledTraceFileModel{vcn}; +} + +std::string CompiledTraceFileModel::BaseFile() const { + return "compiled_trace.pb"; +} + +} // namespace iorap::db diff --git a/src/db/file_models.h b/src/db/file_models.h new file mode 100644 index 0000000..984c206 --- /dev/null +++ b/src/db/file_models.h @@ -0,0 +1,125 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_DB_FILE_MODELS_H_ +#define IORAP_SRC_DB_FILE_MODELS_H_ + +#include <optional> +#include <ostream> +#include <string> + +namespace iorap::db { + +struct VersionedComponentName { + VersionedComponentName(std::string package, + std::string activity, + int64_t version) + : package_{std::move(package)}, + activity_{std::move(activity)}, + version_{version} { + } + + const std::string& GetPackage() const { + return package_; + } + + const std::string& GetActivity() const { + return activity_; + } + + int GetVersion() const { + return version_; + } + + private: + std::string package_; + std::string activity_; + int64_t version_; +}; + +inline std::ostream& operator<<(std::ostream& os, const VersionedComponentName& vcn) { + os << vcn.GetPackage() << "/" << vcn.GetActivity() << "@" << vcn.GetVersion(); + return os; +} + +class FileModelBase { + protected: + FileModelBase(VersionedComponentName vcn); + + virtual std::string SubDir() const = 0; + // Include the last file component only (/a/b/c.txt -> c.txt) + virtual std::string BaseFile() const = 0; + + virtual ~FileModelBase() {} + + public: + virtual std::string ModelName() const = 0; + // Return the absolute file path to this FileModel. + std::string FilePath() const; + + // Include the full path minus the basefile (/a/b/c.txt -> /a/b) + std::string BaseDir() const; + + // Create the base dir recursively (e.g. mkdir -p $basedir). + bool MkdirWithParents(); + private: + VersionedComponentName vcn_; + std::string subdir_; + std::string filename_; + std::string root_path_; +}; + +inline std::ostream& operator<<(std::ostream& os, const FileModelBase& fmb) { + os << fmb.ModelName() << "{'" << fmb.FilePath() << "'}"; + return os; +} + +class DbHandle; + +class PerfettoTraceFileModel : public FileModelBase { + protected: + virtual std::string ModelName() const override { return "PerfettoTrace"; } + virtual std::string SubDir() const override { return "raw_traces"; } + virtual std::string BaseFile() const override; + + PerfettoTraceFileModel(VersionedComponentName vcn, + uint64_t timestamp); + + public: + static PerfettoTraceFileModel CalculateNewestFilePath(VersionedComponentName vcn); + static void DeleteOlderFiles(DbHandle& db, VersionedComponentName vcn); + static bool NeedMorePerfettoTraces(DbHandle& db, VersionedComponentName vcn); + + virtual ~PerfettoTraceFileModel() {} + private: + uint64_t timestamp_; +}; + +class CompiledTraceFileModel : public FileModelBase { + protected: + virtual std::string ModelName() const override { return "CompiledTrace"; } + virtual std::string SubDir() const override { return "compiled_traces"; } + virtual std::string BaseFile() const override; + + CompiledTraceFileModel(VersionedComponentName vcn); + + public: + static CompiledTraceFileModel CalculateNewestFilePath(VersionedComponentName vcn); + + virtual ~CompiledTraceFileModel() {} +}; + +} // namespace iorap::db + +#endif // IORAP_SRC_DB_FILE_MODELS_H_ diff --git a/src/db/main.cc b/src/db/main.cc new file mode 100644 index 0000000..2edd961 --- /dev/null +++ b/src/db/main.cc @@ -0,0 +1,230 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/debug.h" +#include "common/loggers.h" +#include "db/app_component_name.h" +#include "db/models.h" + +#include <android-base/parseint.h> +#include <android-base/logging.h> + +#include <iostream> +#include <optional> +#include <string_view> +#include <string> +#include <vector> + +#include <sqlite3.h> + +#include <signal.h> + +namespace iorap::db { + +void Usage(char** argv) { + std::cerr << "Usage: " << argv[0] << " <path-to-sqlite.db>" << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Interface with the iorap sqlite database and issue commands." << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Optional flags:" << std::endl; + std::cerr << " --help,-h Print this Usage." << std::endl; + std::cerr << " --register-raw-trace,-rrt Register raw trace file path." << std::endl; + std::cerr << " --register-compiled-trace,-rct Register compiled trace file path." << std::endl; + std::cerr << " --insert-component,-ic Add component if it doesn't exist." << std::endl; + std::cerr << " --initialize,-i Initialize new database." << std::endl; + std::cerr << " --rescan,-rs Update all from canonical directories." << std::endl; + std::cerr << " --prune,-pr Remove any stale file paths." << std::endl; + std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; + std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; + exit(1); +} + +void error_log_sqlite_callback(void *pArg, int iErrCode, const char *zMsg) { + LOG(ERROR) << "SQLite error (" << iErrCode << "): " << zMsg; +} + +const constexpr int64_t kNoVersion = -1; + +int Main(int argc, char** argv) { + // Go to system logcat + stderr when running from command line. + android::base::InitLogging(argv, iorap::common::StderrAndLogdLogger{android::base::SYSTEM}); + + bool wait_for_keystroke = false; + bool enable_verbose = false; + + bool command_format_text = false; // false = binary. + + int arg_input_fd = -1; + int arg_output_fd = -1; + + std::vector<std::string> arg_input_filenames; + bool arg_use_sockets = false; + + std::vector<std::pair<std::string,std::string>> arg_register_raw_trace; + std::vector<std::pair<std::string,std::string>> arg_register_compiled_trace; + + std::vector<std::string> arg_insert_component; + + bool arg_initialize = false; + bool arg_rescan = false; + bool arg_prune = false; + + LOG(VERBOSE) << "argparse: argc=" << argc; + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + bool has_arg_next = (arg+1)<argc; + std::string arg_next = has_arg_next ? argv[arg+1] : ""; + + bool has_arg_next_next = (arg+2)<argc; + std::string arg_next_next = has_arg_next_next ? argv[arg+2] : ""; + + LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; + + if (argstr == "--help" || argstr == "-h") { + Usage(argv); + } else if (argstr == "--register-raw-trace" || argstr == "-rrt") { + if (!has_arg_next_next) { + LOG(ERROR) << "--register-raw-trace <component/name> <filepath>"; + Usage(argv); + } + arg_register_raw_trace.push_back({arg_next, arg_next_next}); + } else if (argstr == "--register-compiled-trace" || argstr == "-rct") { + if (!has_arg_next_next) { + LOG(ERROR) << "--register-compiled-trace <component/name> <filepath>"; + Usage(argv); + } + arg_register_compiled_trace.push_back({arg_next, arg_next_next}); + } else if (argstr == "--insert-component" || argstr == "-ic") { + if (!has_arg_next) { + LOG(ERROR) << "--insert-component <component/name>"; + Usage(argv); + } + arg_insert_component.push_back(arg_next); + } else if (argstr == "--initialize" || argstr == "-i") { + arg_initialize = true; + } else if (argstr == "--rescan" || argstr == "-rs") { + arg_rescan = true; + } else if (argstr == "--prune" || argstr == "-pr") { + arg_prune = true; + } else if (argstr == "--verbose" || argstr == "-v") { + enable_verbose = true; + } else if (argstr == "--wait" || argstr == "-w") { + wait_for_keystroke = true; + } else { + arg_input_filenames.push_back(argstr); + } + } + + if (arg_input_filenames.empty()) { + LOG(ERROR) << "Missing positional filename to a sqlite database."; + Usage(argv); + } + + if (enable_verbose) { + android::base::SetMinimumLogSeverity(android::base::VERBOSE); + + LOG(VERBOSE) << "Verbose check"; + LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; + } else { + android::base::SetMinimumLogSeverity(android::base::DEBUG); + } + + LOG(VERBOSE) << "argparse: argc=" << argc; + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + + LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; + } + + // Useful to attach a debugger... + // 1) $> iorap.cmd.readahead -w <args> + // 2) $> gdbclient <pid> + if (wait_for_keystroke) { + LOG(INFO) << "Self pid: " << getpid(); + + raise(SIGSTOP); + // LOG(INFO) << "Press any key to continue..."; + // std::cin >> wait_for_keystroke; + } + + // auto system_call = std::make_unique<SystemCallImpl>(); + // TODO: mock readahead calls? + // + // Uncomment this if we want to leave the process around to inspect it from adb shell. + // sleep(100000); + + int return_code = 0; + + LOG(VERBOSE) << "Hello world"; + + + do { + SchemaModel schema_model = SchemaModel::GetOrCreate(arg_input_filenames[0]); + DbHandle db = schema_model.db(); + if (arg_initialize) { + // Drop tables and restart from scratch. All rows are effectively dropped. + schema_model.Reinitialize(); + } + + for (const auto& component_and_file_name : arg_register_raw_trace) { + AppComponentName component_name = AppComponentName::FromString(component_and_file_name.first); + const std::string& file_path = component_and_file_name.second; + + LOG(VERBOSE) << "--register-raw-trace " << component_name << ", file_path: " << file_path; + + std::optional<ActivityModel> activity = + ActivityModel::SelectOrInsert(db, + component_name.package, + kNoVersion, + component_name.activity_name); + DCHECK(activity.has_value()); + LOG(DEBUG) << "Component selected/inserted: " << *activity; + } + + for (const std::string& component : arg_insert_component) { + AppComponentName component_name = AppComponentName::FromString(component); + + LOG(VERBOSE) << "raw component: " << component; + LOG(VERBOSE) << "package: " << component_name.package; + LOG(VERBOSE) << "activity name: " << component_name.activity_name; + + LOG(VERBOSE) << "--insert-component " << component_name; + + std::optional<ActivityModel> activity = + ActivityModel::SelectOrInsert(db, + component_name.package, + kNoVersion, + component_name.activity_name); + + DCHECK(activity.has_value()); + LOG(DEBUG) << "Component selected/inserted: " << *activity; + } + } while (false); + + LOG(VERBOSE) << "main: Terminating"; + + // 0 -> successfully executed all commands. + // 1 -> failed along the way (#on_error and also see the error logs). + return return_code; +} + +} // namespace iorap::db + +#if defined(IORAP_DB_MAIN) +int main(int argc, char** argv) { + return ::iorap::db::Main(argc, argv); +} +#endif // IORAP_DB_MAIN diff --git a/src/db/models.cc b/src/db/models.cc new file mode 100644 index 0000000..851a061 --- /dev/null +++ b/src/db/models.cc @@ -0,0 +1,21 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "db/models.h" + +namespace iorap::db { + +std::optional<DbHandle> SchemaModel::s_singleton_; + +} // namespace iorap::db diff --git a/src/db/models.h b/src/db/models.h new file mode 100644 index 0000000..79823fd --- /dev/null +++ b/src/db/models.h @@ -0,0 +1,1140 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_DB_MODELS_H_ +#define IORAP_SRC_DB_MODELS_H_ + +#include "clean_up.h" +#include "file_models.h" + +#include <android-base/logging.h> +#include <utils/String8.h> + +#include <filesystem> +#include <iostream> +#include <optional> +#include <ostream> +#include <string> +#include <sstream> +#include <type_traits> +#include <vector> + +#include <sqlite3.h> + +namespace iorap::db { + +const constexpr int kDbVersion = 3; + +struct SqliteDbDeleter { + void operator()(sqlite3* db) { + if (db != nullptr) { + LOG(VERBOSE) << "sqlite3_close for: " << db; + sqlite3_close(db); + } + } +}; + +class DbHandle { + public: + // Take over ownership of sqlite3 db. + explicit DbHandle(sqlite3* db) + : db_{std::shared_ptr<sqlite3>{db, SqliteDbDeleter{}}}, + mutex_{std::make_shared<std::mutex>()} { + } + + sqlite3* get() { + return db_.get(); + } + + operator sqlite3*() { + return db_.get(); + } + + std::mutex& mutex() { + return *mutex_.get(); + } + + private: + std::shared_ptr<sqlite3> db_; + std::shared_ptr<std::mutex> mutex_; +}; + +class ScopedLockDb { + public: + ScopedLockDb(std::mutex& mutex) : mutex(mutex) { + mutex.lock(); + } + + ScopedLockDb(DbHandle& handle) : ScopedLockDb(handle.mutex()) { + } + + ~ScopedLockDb() { + mutex.unlock(); + } + private: + std::mutex& mutex; +}; + +class DbStatement { + public: + template <typename ... Args> + static DbStatement Prepare(DbHandle db, const std::string& sql, Args&&... args) { + return Prepare(db, sql.c_str(), std::forward<Args>(args)...); + } + + template <typename ... Args> + static DbStatement Prepare(DbHandle db, const char* sql, Args&&... args) { + DCHECK(db.get() != nullptr); + DCHECK(sql != nullptr); + + // LOG(VERBOSE) << "Prepare DB=" << db.get(); + + sqlite3_stmt* stmt = nullptr; + int rc = sqlite3_prepare_v2(db.get(), sql, -1, /*out*/&stmt, nullptr); + + DbStatement db_stmt{db, stmt}; + DCHECK(db_stmt.CheckOk(rc)) << sql; + db_stmt.BindAll(std::forward<Args>(args)...); + + return db_stmt; + } + + DbStatement(DbHandle db, sqlite3_stmt* stmt) : db_(db), stmt_(stmt) { + } + + sqlite3_stmt* get() { + return stmt_; + } + + DbHandle db() { + return db_; + } + + // Successive BindAll calls *do not* start back at the 0th bind position. + template <typename T, typename ... Args> + void BindAll(T&& arg, Args&&... args) { + Bind(std::forward<T>(arg)); + BindAll(std::forward<Args>(args)...); + } + + void BindAll() {} + + template <typename T> + void Bind(const std::optional<T>& value) { + if (value) { + Bind(*value); + } else { + BindNull(); + } + } + + void Bind(bool value) { + CheckOk(sqlite3_bind_int(stmt_, bind_counter_++, value)); + } + + void Bind(int value) { + CheckOk(sqlite3_bind_int(stmt_, bind_counter_++, value)); + } + + void Bind(uint64_t value) { + CheckOk(sqlite3_bind_int64(stmt_, bind_counter_++, static_cast<int64_t>(value))); + } + + void Bind(const char* value) { + if (value != nullptr) { + //sqlite3_bind_text(stmt_, /*index*/bind_counter_++, value, -1, SQLITE_STATIC); + CheckOk(sqlite3_bind_text(stmt_, /*index*/bind_counter_++, value, -1, SQLITE_TRANSIENT)); + } else { + BindNull(); + } + } + + void Bind(const std::string& value) { + Bind(value.c_str()); + } + + template <typename E, typename = std::enable_if_t<std::is_enum_v<E>>> + void Bind(E value) { + Bind(static_cast<std::underlying_type_t<E>>(value)); + } + + void BindNull() { + CheckOk(sqlite3_bind_null(stmt_, bind_counter_++)); + } + + int Step() { + ++step_counter_; + return sqlite3_step(stmt_); + } + + bool Step(int expected) { + int rc = Step(); + if (rc != expected) { + LOG(ERROR) << "SQLite error: " << sqlite3_errmsg(db_.get()); + return false; + } + return true; + } + + // Successive ColumnAll calls start at the 0th column again. + template <typename T, typename ... Args> + void ColumnAll(T& first, Args&... rest) { + Column(first); + ColumnAll(rest...); + // Reset column counter back to 0 + column_counter_ = 0; + } + + void ColumnAll() {} + + template <typename T> + void Column(std::optional<T>& value) { + T tmp; + Column(/*out*/tmp); + + if (!tmp) { // disambiguate 0 and NULL + const unsigned char* text = sqlite3_column_text(stmt_, column_counter_ - 1); + if (text == nullptr) { + value = std::nullopt; + return; + } + } + value = std::move(tmp); + } + + template <typename E, typename = std::enable_if_t<std::is_enum_v<E>>> + void Column(E& value) { + std::underlying_type_t<E> tmp; + Column(/*out*/tmp); + value = static_cast<E>(tmp); + } + + void Column(bool& value) { + value = sqlite3_column_int(stmt_, column_counter_++); + } + + void Column(int& value) { + value = sqlite3_column_int(stmt_, column_counter_++); + } + + void Column(uint64_t& value) { + value = static_cast<uint64_t>(sqlite3_column_int64(stmt_, column_counter_++)); + } + + void Column(std::string& value) { + const unsigned char* text = sqlite3_column_text(stmt_, column_counter_++); + + DCHECK(text != nullptr) << "Column should be marked NOT NULL, otherwise use optional<string>"; + if (text == nullptr) { + LOG(ERROR) << "Got NULL back for column " << column_counter_-1 + << "; is this column marked NOT NULL?"; + value = "(((null)))"; // Don't segfault, keep going. + return; + } + + value = std::string{reinterpret_cast<const char*>(text)}; + } + + const char* ExpandedSql() { + char* p = sqlite3_expanded_sql(stmt_); + if (p == nullptr) { + return "(nullptr)"; + } + return p; + } + + const char* Sql() { + const char* p = sqlite3_sql(stmt_); + if (p == nullptr) { + return "(nullptr)"; + } + return p; + } + + + DbStatement(DbStatement&& other) + : db_{other.db_}, stmt_{other.stmt_}, bind_counter_{other.bind_counter_}, + step_counter_{other.step_counter_} { + other.db_ = DbHandle{nullptr}; + other.stmt_ = nullptr; + } + + ~DbStatement() { + if (stmt_ != nullptr) { + DCHECK_GT(step_counter_, 0) << " forgot to call Step()?"; + sqlite3_finalize(stmt_); + } + } + + private: + bool CheckOk(int rc, int expected = SQLITE_OK) { + if (rc != expected) { + LOG(ERROR) << "Got error for SQL query: '" << Sql() << "'" + << ", expanded: '" << ExpandedSql() << "'"; + LOG(ERROR) << "Failed SQLite api call (" << rc << "): " << sqlite3_errstr(rc); + } + return rc == expected; + } + + DbHandle db_; + sqlite3_stmt* stmt_; + int bind_counter_ = 1; + int step_counter_ = 0; + int column_counter_ = 0; +}; + +class DbQueryBuilder { + public: + // Returns the row ID that was inserted last. + template <typename... Args> + static std::optional<int> Insert(DbHandle db, const std::string& sql, Args&&... args) { + ScopedLockDb lock{db}; + + sqlite3_int64 last_rowid = sqlite3_last_insert_rowid(db.get()); + DbStatement stmt = DbStatement::Prepare(db, sql, std::forward<Args>(args)...); + + if (!stmt.Step(SQLITE_DONE)) { + return std::nullopt; + } + + last_rowid = sqlite3_last_insert_rowid(db.get()); + DCHECK_GT(last_rowid, 0); + + return static_cast<int>(last_rowid); + } + + template <typename... Args> + static bool Delete(DbHandle db, const std::string& sql, Args&&... args) { + return ExecuteOnce(db, sql, std::forward<Args>(args)...); + } + + template <typename... Args> + static bool Update(DbHandle db, const std::string& sql, Args&&... args) { + return ExecuteOnce(db, sql, std::forward<Args>(args)...); + } + + template <typename... Args> + static bool ExecuteOnce(DbHandle db, const std::string& sql, Args&&... args) { + ScopedLockDb lock{db}; + + DbStatement stmt = DbStatement::Prepare(db, sql, std::forward<Args>(args)...); + + if (!stmt.Step(SQLITE_DONE)) { + return false; + } + + return true; + } + + template <typename... Args> + static bool SelectOnce(DbStatement& stmt, Args&... args) { + int rc = stmt.Step(); + switch (rc) { + case SQLITE_ROW: + stmt.ColumnAll(/*out*/args...); + return true; + case SQLITE_DONE: + return false; + default: + LOG(ERROR) << "Failed to step (" << rc << "): " << sqlite3_errmsg(stmt.db()); + return false; + } + } +}; + +class Model { + public: + DbHandle db() const { + return db_; + } + + Model(DbHandle db) : db_{db} { + } + + private: + DbHandle db_; +}; + +class SchemaModel : public Model { + public: + static SchemaModel GetOrCreate(std::string location) { + int rc = sqlite3_config(SQLITE_CONFIG_LOG, ErrorLogCallback, /*data*/nullptr); + + if (rc != SQLITE_OK) { + LOG(FATAL) << "Failed to configure logging"; + } + + sqlite3* db = nullptr; + bool is_deprecated = false; + if (location != ":memory:") { + // Try to open DB if it already exists. + rc = sqlite3_open_v2(location.c_str(), /*out*/&db, SQLITE_OPEN_READWRITE, /*vfs*/nullptr); + + if (rc == SQLITE_OK) { + LOG(INFO) << "Opened existing database at '" << location << "'"; + SchemaModel schema{DbHandle{db}, location}; + if (schema.Version() == kDbVersion) { + return schema; + } else { + LOG(DEBUG) << "The version is old, reinit the db." + << " old version is " + << schema.Version() + << " and new version is " + << kDbVersion; + CleanUpFilesForDb(schema.db()); + is_deprecated = true; + } + } + } + + if (is_deprecated) { + // Remove the db and recreate it. + // TODO: migrate to a newer version without deleting the old one. + std::filesystem::remove(location.c_str()); + } + + // Create a new DB if one didn't exist already. + rc = sqlite3_open(location.c_str(), /*out*/&db); + + if (rc != SQLITE_OK) { + LOG(FATAL) << "Failed to open DB: " << sqlite3_errmsg(db); + } + + SchemaModel schema{DbHandle{db}, location}; + schema.Reinitialize(); + // TODO: migrate versions upwards when we rev the schema version + + int old_version = schema.Version(); + LOG(VERBOSE) << "Loaded schema version: " << old_version; + + return schema; + } + + void MarkSingleton() { + s_singleton_ = db(); + } + + static DbHandle GetSingleton() { + DCHECK(s_singleton_.has_value()); + return *s_singleton_; + } + + void Reinitialize() { + const char* sql_to_initialize = R"SQLC0D3( + DROP TABLE IF EXISTS schema_versions; + DROP TABLE IF EXISTS packages; + DROP TABLE IF EXISTS activities; + DROP TABLE IF EXISTS app_launch_histories; + DROP TABLE IF EXISTS raw_traces; + DROP TABLE IF EXISTS prefetch_files; +)SQLC0D3"; + char* err_msg = nullptr; + int rc = sqlite3_exec(db().get(), + sql_to_initialize, + /*callback*/nullptr, + /*arg*/0, + /*out*/&err_msg); + if (rc != SQLITE_OK) { + LOG(FATAL) << "Failed to drop tables: " << err_msg ? err_msg : "nullptr"; + } + + CreateSchema(); + LOG(INFO) << "Reinitialized database at '" << location_ << "'"; + } + + int Version() { + std::string query = "SELECT MAX(version) FROM schema_versions;"; + DbStatement stmt = DbStatement::Prepare(db(), query); + + int return_value = 0; + if (!DbQueryBuilder::SelectOnce(stmt, /*out*/return_value)) { + LOG(ERROR) << "Failed to query schema version"; + return -1; + } + + return return_value; + } + + protected: + SchemaModel(DbHandle db, std::string location) : Model{db}, location_(location) { + } + + private: + static std::optional<DbHandle> s_singleton_; + + void CreateSchema() { + const char* sql_to_initialize = R"SQLC0D3( + CREATE TABLE schema_versions( + version INTEGER NOT NULL + ); + + CREATE TABLE packages( + id INTEGER NOT NULL, + name TEXT NOT NULL, + version INTEGER NOT NULL, + + PRIMARY KEY(id) + ); + + CREATE TABLE activities( + id INTEGER NOT NULL, + name TEXT NOT NULL, + package_id INTEGER NOT NULL, + + PRIMARY KEY(id), + FOREIGN KEY (package_id) REFERENCES packages (id) ON DELETE CASCADE + ); + + CREATE TABLE app_launch_histories( + id INTEGER NOT NULL PRIMARY KEY, + activity_id INTEGER NOT NULL, + -- 1:Cold, 2:Warm, 3:Hot + temperature INTEGER CHECK (temperature IN (1, 2, 3)) NOT NULL, + trace_enabled INTEGER CHECK(trace_enabled in (TRUE, FALSE)) NOT NULL, + readahead_enabled INTEGER CHECK(trace_enabled in (TRUE, FALSE)) NOT NULL, + -- absolute timestamp since epoch + intent_started_ns INTEGER CHECK(intent_started_ns IS NULL or intent_started_ns >= 0), + -- absolute timestamp since epoch + total_time_ns INTEGER CHECK(total_time_ns IS NULL or total_time_ns >= 0), + -- absolute timestamp since epoch + report_fully_drawn_ns INTEGER CHECK(report_fully_drawn_ns IS NULL or report_fully_drawn_ns >= 0), + -- pid of the app + pid INTEGER CHECK(pid IS NULL or pid >= 0), + + FOREIGN KEY (activity_id) REFERENCES activities (id) ON DELETE CASCADE + ); + + CREATE TABLE raw_traces( + id INTEGER NOT NULL PRIMARY KEY, + history_id INTEGER NOT NULL, + file_path TEXT NOT NULL, + + FOREIGN KEY (history_id) REFERENCES app_launch_histories (id) ON DELETE CASCADE + ); + + CREATE TABLE prefetch_files( + id INTEGER NOT NULL PRIMARY KEY, + activity_id INTEGER NOT NULL, + file_path TEXT NOT NULL, + + FOREIGN KEY (activity_id) REFERENCES activities (id) ON DELETE CASCADE + ); +)SQLC0D3"; + + char* err_msg = nullptr; + int rc = sqlite3_exec(db().get(), + sql_to_initialize, + /*callback*/nullptr, + /*arg*/0, + /*out*/&err_msg); + + if (rc != SQLITE_OK) { + LOG(FATAL) << "Failed to create tables: " << err_msg ? err_msg : "nullptr"; + } + + const char* sql_to_insert_schema_version = R"SQLC0D3( + INSERT INTO schema_versions VALUES(%d) + )SQLC0D3"; + rc = sqlite3_exec(db().get(), + android::String8::format(sql_to_insert_schema_version, + kDbVersion), + /*callback*/nullptr, + /*arg*/0, + /*out*/&err_msg); + + if (rc != SQLITE_OK) { + LOG(FATAL) << "Failed to insert the schema version: " + << err_msg ? err_msg : "nullptr"; + } + } + + static void ErrorLogCallback(void *pArg, int iErrCode, const char *zMsg) { + LOG(ERROR) << "SQLite error (" << iErrCode << "): " << zMsg; + } + + std::string location_; +}; + +class PackageModel : public Model { + protected: + PackageModel(DbHandle db) : Model{db} { + } + + public: + static std::optional<PackageModel> SelectById(DbHandle db, int id) { + ScopedLockDb lock{db}; + int original_id = id; + + std::string query = "SELECT * FROM packages WHERE id = ?1 LIMIT 1;"; + DbStatement stmt = DbStatement::Prepare(db, query, id); + + PackageModel p{db}; + if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) { + return std::nullopt; + } + + return p; + } + + static std::vector<PackageModel> SelectByName(DbHandle db, const char* name) { + ScopedLockDb lock{db}; + + std::string query = "SELECT * FROM packages WHERE name = ?1;"; + DbStatement stmt = DbStatement::Prepare(db, query, name); + + std::vector<PackageModel> packages; + + PackageModel p{db}; + while (DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) { + packages.push_back(p); + } + + return packages; + } + + static std::optional<PackageModel> SelectByNameAndVersion(DbHandle db, + const char* name, + int version) { + ScopedLockDb lock{db}; + + std::string query = + "SELECT * FROM packages WHERE name = ?1 AND version = ?2 LIMIT 1;"; + DbStatement stmt = DbStatement::Prepare(db, query, name, version); + + PackageModel p{db}; + if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) { + return std::nullopt; + } + + return p; + } + + static std::vector<PackageModel> SelectAll(DbHandle db) { + ScopedLockDb lock{db}; + + std::string query = "SELECT * FROM packages;"; + DbStatement stmt = DbStatement::Prepare(db, query); + + std::vector<PackageModel> packages; + PackageModel p{db}; + while (DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.version)) { + packages.push_back(p); + } + + return packages; + } + + static std::optional<PackageModel> Insert(DbHandle db, + std::string name, + int version) { + const char* sql = "INSERT INTO packages (name, version) VALUES (?1, ?2);"; + + std::optional<int> inserted_row_id = + DbQueryBuilder::Insert(db, sql, name, version); + if (!inserted_row_id) { + return std::nullopt; + } + + PackageModel p{db}; + p.name = name; + p.version = version; + p.id = *inserted_row_id; + + return p; + } + + bool Delete() { + const char* sql = "DELETE FROM packages WHERE id = ?"; + + return DbQueryBuilder::Delete(db(), sql, id); + } + + int id; + std::string name; + int version; +}; + +inline std::ostream& operator<<(std::ostream& os, const PackageModel& p) { + os << "PackageModel{id=" << p.id << ",name=" << p.name << ","; + os << "version="; + os << p.version; + os << "}"; + return os; +} + +class ActivityModel : public Model { + protected: + ActivityModel(DbHandle db) : Model{db} { + } + + public: + static std::optional<ActivityModel> SelectById(DbHandle db, int id) { + ScopedLockDb lock{db}; + int original_id = id; + + std::string query = "SELECT * FROM activities WHERE id = ? LIMIT 1;"; + DbStatement stmt = DbStatement::Prepare(db, query, id); + + ActivityModel p{db}; + if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.package_id)) { + return std::nullopt; + } + + return p; + } + + static std::optional<ActivityModel> SelectByNameAndPackageId(DbHandle db, + const char* name, + int package_id) { + ScopedLockDb lock{db}; + + std::string query = "SELECT * FROM activities WHERE name = ? AND package_id = ? LIMIT 1;"; + DbStatement stmt = DbStatement::Prepare(db, query, name, package_id); + + ActivityModel p{db}; + if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.package_id)) { + return std::nullopt; + } + + return p; + } + + static std::vector<ActivityModel> SelectByPackageId(DbHandle db, + int package_id) { + ScopedLockDb lock{db}; + + std::string query = "SELECT * FROM activities WHERE package_id = ?;"; + DbStatement stmt = DbStatement::Prepare(db, query, package_id); + + std::vector<ActivityModel> activities; + ActivityModel p{db}; + while (DbQueryBuilder::SelectOnce(stmt, p.id, p.name, p.package_id)) { + activities.push_back(p); + } + + return activities; + } + + static std::optional<ActivityModel> Insert(DbHandle db, + std::string name, + int package_id) { + const char* sql = "INSERT INTO activities (name, package_id) VALUES (?1, ?2);"; + + std::optional<int> inserted_row_id = + DbQueryBuilder::Insert(db, sql, name, package_id); + if (!inserted_row_id) { + return std::nullopt; + } + + ActivityModel p{db}; + p.id = *inserted_row_id; + p.name = name; + p.package_id = package_id; + + return p; + } + + // Try to select by package_name+activity_name, otherwise insert into both tables. + // Package version is ignored for selects. + static std::optional<ActivityModel> SelectOrInsert( + DbHandle db, + std::string package_name, + int package_version, + std::string activity_name) { + std::optional<PackageModel> package = PackageModel::SelectByNameAndVersion(db, + package_name.c_str(), + package_version); + if (!package) { + package = PackageModel::Insert(db, package_name, package_version); + DCHECK(package.has_value()); + } + + std::optional<ActivityModel> activity = + ActivityModel::SelectByNameAndPackageId(db, + activity_name.c_str(), + package->id); + if (!activity) { + activity = Insert(db, activity_name, package->id); + // XX: should we really return an optional here? This feels like it should never fail. + } + + return activity; + } + + int id; + std::string name; + int package_id; // PackageModel::id +}; + +inline std::ostream& operator<<(std::ostream& os, const ActivityModel& p) { + os << "ActivityModel{id=" << p.id << ",name=" << p.name << ","; + os << "package_id=" << p.package_id << "}"; + return os; +} + +class AppLaunchHistoryModel : public Model { + protected: + AppLaunchHistoryModel(DbHandle db) : Model{db} { + } + + public: + enum class Temperature : int32_t { + kUninitialized = -1, // Note: Not a valid SQL value. + kCold = 1, + kWarm = 2, + kHot = 3, + }; + + static std::optional<AppLaunchHistoryModel> SelectById(DbHandle db, int id) { + ScopedLockDb lock{db}; + int original_id = id; + + std::string query = "SELECT * FROM app_launch_histories WHERE id = ? LIMIT 1;"; + DbStatement stmt = DbStatement::Prepare(db, query, id); + + AppLaunchHistoryModel p{db}; + if (!DbQueryBuilder::SelectOnce(stmt, + p.id, + p.activity_id, + p.temperature, + p.trace_enabled, + p.readahead_enabled, + p.intent_started_ns, + p.total_time_ns, + p.report_fully_drawn_ns, + p.pid)) { + return std::nullopt; + } + + return p; + } + + // Selects the activity history for an activity id. + // The requirements are: + // * Should be cold run. + // * Pefetto trace is enabled. + // * intent_start_ns is *NOT* null. + static std::vector<AppLaunchHistoryModel> SelectActivityHistoryForCompile( + DbHandle db, + int activity_id) { + ScopedLockDb lock{db}; + std::string query = "SELECT * FROM app_launch_histories " + "WHERE activity_id = ?1 AND" + " temperature = 1 AND" + " trace_enabled = TRUE AND" + " intent_started_ns IS NOT NULL;"; + DbStatement stmt = DbStatement::Prepare(db, query, activity_id); + std::vector<AppLaunchHistoryModel> result; + + AppLaunchHistoryModel p{db}; + while (DbQueryBuilder::SelectOnce(stmt, + p.id, + p.activity_id, + p.temperature, + p.trace_enabled, + p.readahead_enabled, + p.intent_started_ns, + p.total_time_ns, + p.report_fully_drawn_ns, + p.pid)) { + result.push_back(p); + } + return result; + } + + static std::optional<AppLaunchHistoryModel> Insert(DbHandle db, + int activity_id, + AppLaunchHistoryModel::Temperature temperature, + bool trace_enabled, + bool readahead_enabled, + std::optional<uint64_t> intent_started_ns, + std::optional<uint64_t> total_time_ns, + std::optional<uint64_t> report_fully_drawn_ns, + int32_t pid) + { + const char* sql = "INSERT INTO app_launch_histories (activity_id, temperature, trace_enabled, " + "readahead_enabled, intent_started_ns, " + "total_time_ns, report_fully_drawn_ns, pid) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8);"; + + std::optional<int> inserted_row_id = + DbQueryBuilder::Insert(db, + sql, + activity_id, + temperature, + trace_enabled, + readahead_enabled, + intent_started_ns, + total_time_ns, + report_fully_drawn_ns, + pid); + if (!inserted_row_id) { + return std::nullopt; + } + + AppLaunchHistoryModel p{db}; + p.id = *inserted_row_id; + p.activity_id = activity_id; + p.temperature = temperature; + p.trace_enabled = trace_enabled; + p.readahead_enabled = readahead_enabled; + p.intent_started_ns = intent_started_ns; + p.total_time_ns = total_time_ns; + p.report_fully_drawn_ns = report_fully_drawn_ns; + p.pid = pid; + + return p; + } + + static bool UpdateReportFullyDrawn(DbHandle db, + int history_id, + uint64_t report_fully_drawn_ns) + { + const char* sql = "UPDATE app_launch_histories " + "SET report_fully_drawn_ns = ?1 " + "WHERE id = ?2;"; + + bool result = DbQueryBuilder::Update(db, sql, report_fully_drawn_ns, history_id); + + if (!result) { + LOG(ERROR)<< "Failed to update history_id:"<< history_id + << ", report_fully_drawn_ns: " << report_fully_drawn_ns; + } + return result; + } + + int id; + int activity_id; // ActivityModel::id + Temperature temperature = Temperature::kUninitialized; + bool trace_enabled; + bool readahead_enabled; + std::optional<uint64_t> intent_started_ns; + std::optional<uint64_t> total_time_ns; + std::optional<uint64_t> report_fully_drawn_ns; + int32_t pid; +}; + +inline std::ostream& operator<<(std::ostream& os, const AppLaunchHistoryModel& p) { + os << "AppLaunchHistoryModel{id=" << p.id << "," + << "activity_id=" << p.activity_id << "," + << "temperature=" << static_cast<int>(p.temperature) << "," + << "trace_enabled=" << p.trace_enabled << "," + << "readahead_enabled=" << p.readahead_enabled << "," + << "intent_started_ns="; + if (p.intent_started_ns) { + os << *p.intent_started_ns; + } else { + os << "(nullopt)"; + } + os << ","; + os << "total_time_ns="; + if (p.total_time_ns) { + os << *p.total_time_ns; + } else { + os << "(nullopt)"; + } + os << ","; + os << "report_fully_drawn_ns="; + if (p.report_fully_drawn_ns) { + os << *p.report_fully_drawn_ns; + } else { + os << "(nullopt)"; + } + os << ","; + os << "pid=" << p.pid; + os << "}"; + return os; +} + +class RawTraceModel : public Model { + protected: + RawTraceModel(DbHandle db) : Model{db} { + } + + public: + + // Return raw_traces, sorted ascending by the id. + static std::vector<RawTraceModel> SelectByVersionedComponentName(DbHandle db, + VersionedComponentName vcn) { + ScopedLockDb lock{db}; + + const char* sql = + "SELECT raw_traces.id, raw_traces.history_id, raw_traces.file_path " + "FROM raw_traces " + "INNER JOIN app_launch_histories ON raw_traces.history_id = app_launch_histories.id " + "INNER JOIN activities ON activities.id = app_launch_histories.activity_id " + "INNER JOIN packages ON packages.id = activities.package_id " + "WHERE packages.name = ? AND activities.name = ? AND packages.version = ?" + "ORDER BY raw_traces.id ASC"; + + DbStatement stmt = DbStatement::Prepare(db, + sql, + vcn.GetPackage(), + vcn.GetActivity(), + vcn.GetVersion()); + + std::vector<RawTraceModel> results; + + RawTraceModel p{db}; + while (DbQueryBuilder::SelectOnce(stmt, p.id, p.history_id, p.file_path)) { + results.push_back(p); + } + + return results; + } + + static std::optional<RawTraceModel> SelectByHistoryId(DbHandle db, int history_id) { + ScopedLockDb lock{db}; + + const char* sql = + "SELECT id, history_id, file_path " + "FROM raw_traces " + "WHERE history_id = ?1 " + "LIMIT 1;"; + + DbStatement stmt = DbStatement::Prepare(db, sql, history_id); + + RawTraceModel p{db}; + if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.history_id, p.file_path)) { + return std::nullopt; + } + + return p; + } + + static std::optional<RawTraceModel> Insert(DbHandle db, + int history_id, + std::string file_path) { + const char* sql = "INSERT INTO raw_traces (history_id, file_path) VALUES (?1, ?2);"; + + std::optional<int> inserted_row_id = + DbQueryBuilder::Insert(db, sql, history_id, file_path); + if (!inserted_row_id) { + return std::nullopt; + } + + RawTraceModel p{db}; + p.id = *inserted_row_id; + p.history_id = history_id; + p.file_path = file_path; + + return p; + } + + bool Delete() { + const char* sql = "DELETE FROM raw_traces WHERE id = ?"; + + return DbQueryBuilder::Delete(db(), sql, id); + } + + int id; + int history_id; + std::string file_path; +}; + +inline std::ostream& operator<<(std::ostream& os, const RawTraceModel& p) { + os << "RawTraceModel{id=" << p.id << ",history_id=" << p.history_id << ","; + os << "file_path=" << p.file_path << "}"; + return os; +} + +class PrefetchFileModel : public Model { + protected: + PrefetchFileModel(DbHandle db) : Model{db} { + } + + public: + static std::optional<PrefetchFileModel> SelectByVersionedComponentName( + DbHandle db, + VersionedComponentName vcn) { + ScopedLockDb lock{db}; + + const char* sql = + "SELECT prefetch_files.id, prefetch_files.activity_id, prefetch_files.file_path " + "FROM prefetch_files " + "INNER JOIN activities ON activities.id = prefetch_files.activity_id " + "INNER JOIN packages ON packages.id = activities.package_id " + "WHERE packages.name = ? AND activities.name = ? AND packages.version = ?"; + + DbStatement stmt = DbStatement::Prepare(db, + sql, + vcn.GetPackage(), + vcn.GetActivity(), + vcn.GetVersion()); + + PrefetchFileModel p{db}; + + if (!DbQueryBuilder::SelectOnce(stmt, p.id, p.activity_id, p.file_path)) { + return std::nullopt; + } + + return p; + } + + static std::vector<PrefetchFileModel> SelectAll(DbHandle db) { + ScopedLockDb lock{db}; + + std::string query = + "SELECT prefetch_files.id, prefetch_files.activity_id, prefetch_files.file_path " + "FROM prefetch_files"; + DbStatement stmt = DbStatement::Prepare(db, query); + + std::vector<PrefetchFileModel> prefetch_files; + PrefetchFileModel p{db}; + while (DbQueryBuilder::SelectOnce(stmt, p.id, p.activity_id, p.file_path)) { + prefetch_files.push_back(p); + } + + return prefetch_files; + } + + static std::optional<PrefetchFileModel> Insert(DbHandle db, + int activity_id, + std::string file_path) { + const char* sql = "INSERT INTO prefetch_files (activity_id, file_path) VALUES (?1, ?2);"; + + std::optional<int> inserted_row_id = + DbQueryBuilder::Insert(db, sql, activity_id, file_path); + if (!inserted_row_id) { + return std::nullopt; + } + + PrefetchFileModel p{db}; + p.id = *inserted_row_id; + p.activity_id = activity_id; + p.file_path = file_path; + + return p; + } + + bool Delete() { + const char* sql = "DELETE FROM prefetch_files WHERE id = ?"; + + return DbQueryBuilder::Delete(db(), sql, id); + } + + int id; + int activity_id; + std::string file_path; +}; + +inline std::ostream& operator<<(std::ostream& os, const PrefetchFileModel& p) { + os << "PrefetchFileModel{id=" << p.id << ",activity_id=" << p.activity_id << ","; + os << "file_path=" << p.file_path << "}"; + return os; +} + +} // namespace iorap::db + +#endif // IORAP_SRC_DB_MODELS_H_ diff --git a/src/inode2filename/data_source.cc b/src/inode2filename/data_source.cc new file mode 100644 index 0000000..5fcce73 --- /dev/null +++ b/src/inode2filename/data_source.cc @@ -0,0 +1,187 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "inode2filename/data_source.h" + +#include "common/cmd_utils.h" +#include "inode2filename/inode_resolver.h" +#include "inode2filename/search_directories.h" + +#include <android-base/logging.h> + +#include <fstream> +#include <stdio.h> + +namespace rx = rxcpp; + +namespace iorap::inode2filename { + +std::vector<std::string> ToArgs(DataSourceKind data_source_kind) { + const char* value = nullptr; + + switch (data_source_kind) { + case DataSourceKind::kDiskScan: + value = "diskscan"; + break; + case DataSourceKind::kTextCache: + value = "textcache"; + break; + case DataSourceKind::kBpf: + value = "bpf"; + break; + } + + std::vector<std::string> args; + iorap::common::AppendNamedArg(args, "--data-source", value); + return args; +} + +std::vector<std::string> ToArgs(const DataSourceDependencies& deps) { + std::vector<std::string> args; + + iorap::common::AppendArgsRepeatedly(args, ToArgs(deps.data_source)); + // intentionally skip system_call; it does not have a command line equivalent + iorap::common::AppendNamedArgRepeatedly(args, "--root", deps.root_directories); + + if (deps.text_cache_filename) { + iorap::common::AppendNamedArg(args, "--textcache", *(deps.text_cache_filename)); + } + + return args; +} + +class DiskScanDataSource : public DataSource { + public: + DiskScanDataSource(DataSourceDependencies dependencies) + : DataSource(std::move(dependencies)) { + DCHECK_NE(dependencies_.root_directories.size(), 0u) << "Root directories can't be empty"; + } + + virtual rxcpp::observable<InodeResult> EmitInodes() const override { + SearchDirectories searcher{/*borrow*/dependencies_.system_call}; + return searcher.ListAllFilenames(dependencies_.root_directories); + } + + // Since not all Inodes emitted are the ones searched for, doing additional stat(2) calls here + // would be redundant. + // + // We set the device number to a dummy value here (-1) so that InodeResolver + // can fill it in later with stat(2). This is effectively the same thing as always doing + // verification. + virtual bool ResultIncludesDeviceNumber() const { return false; }; + + virtual ~DiskScanDataSource() {} +}; + +static inline void LeftTrim(/*inout*/std::string& s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { + return !std::isspace(ch); + })); +} + +class TextCacheDataSource : public DataSource { + public: + TextCacheDataSource(DataSourceDependencies dependencies) + : DataSource(std::move(dependencies)) { + DCHECK(dependencies_.text_cache_filename.has_value()) << "Must have text cache filename"; + } + + virtual rxcpp::observable<InodeResult> EmitInodes() const override { + const std::string& file_name = *dependencies_.text_cache_filename; + + return rx::observable<>::create<InodeResult>( + [file_name](rx::subscriber<InodeResult> dest) { + LOG(VERBOSE) << "TextCacheDataSource (lambda)"; + + std::ifstream ifs{file_name}; + + if (!ifs.is_open()) { + dest.on_error(rxcpp::util::make_error_ptr( + std::ios_base::failure("Could not open specified text cache filename"))); + return; + } + + while (dest.is_subscribed() && ifs) { + LOG(VERBOSE) << "TextCacheDataSource (read line)"; + // TODO. + + uint64_t device_number = 0; + uint64_t inode_number = 0; + uint64_t file_size = 0; + + // Parse lines of form: + // "$device_number $inode $filesize $filename..." + // This format conforms to system/extras/pagecache/pagecache.py + + ifs >> device_number; + ifs >> inode_number; + ifs >> file_size; + (void)file_size; // Not used in iorapd. + + std::string value_filename; + std::getline(/*inout*/ifs, /*out*/value_filename); + + if (value_filename.empty()) { + // Ignore empty lines. + continue; + } + + // getline, unlike ifstream, does not ignore spaces. + // There's always at least 1 space in a textcache output file. + // However, drop *all* spaces on the left since filenames starting with a space are + // ambiguous to us. + LeftTrim(/*inout*/value_filename); + + Inode inode = Inode::FromDeviceAndInode(static_cast<dev_t>(device_number), + static_cast<ino_t>(inode_number)); + + LOG(VERBOSE) << "TextCacheDataSource (on_next) " << inode << "->" << value_filename; + dest.on_next(InodeResult::makeSuccess(inode, value_filename)); + } + + dest.on_completed(); + } + ); + + // TODO: plug into the filtering and verification graph. + } + + virtual ~TextCacheDataSource() {} +}; + +DataSource::DataSource(DataSourceDependencies dependencies) + : dependencies_{std::move(dependencies)} { + DCHECK(dependencies_.system_call != nullptr); +} + +std::shared_ptr<DataSource> DataSource::Create(DataSourceDependencies dependencies) { + switch (dependencies.data_source) { + case DataSourceKind::kDiskScan: + return std::shared_ptr<DataSource>{new DiskScanDataSource{std::move(dependencies)}}; + case DataSourceKind::kTextCache: + return std::shared_ptr<DataSource>{new TextCacheDataSource{std::move(dependencies)}}; + case DataSourceKind::kBpf: + // TODO: BPF-based data source. + LOG(FATAL) << "Not implemented yet"; + return nullptr; + default: + LOG(FATAL) << "Invalid data source value"; + return nullptr; + } +} + +void DataSource::StartRecording() {} +void DataSource::StopRecording() {} + +} // namespace iorap::inode2filename diff --git a/src/inode2filename/data_source.h b/src/inode2filename/data_source.h new file mode 100644 index 0000000..36c8383 --- /dev/null +++ b/src/inode2filename/data_source.h @@ -0,0 +1,74 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_INODE2FILENAME_DATA_SOURCE_H_ +#define IORAP_SRC_INODE2FILENAME_DATA_SOURCE_H_ + +#include "inode2filename/inode_result.h" +#include "inode2filename/system_call.h" + +#include <rxcpp/rx.hpp> + +#include <memory> +#include <optional> +#include <string> +#include <vector> +namespace iorap::inode2filename { + +enum class DataSourceKind { + kDiskScan, + kTextCache, + kBpf, +}; + +std::vector<std::string> ToArgs(DataSourceKind data_source_kind); + +struct DataSourceDependencies { + DataSourceKind data_source = DataSourceKind::kDiskScan; + borrowed<SystemCall*> system_call = nullptr; + + // kDiskScan-specific options. Other data sources ignore these fields. + std::vector<std::string> root_directories; + // kTextCache-specific options. Other data sources ignore these fields. + std::optional<std::string> text_cache_filename; +}; + +std::vector<std::string> ToArgs(const DataSourceDependencies& deps); + +class DataSource : std::enable_shared_from_this<DataSource> { + public: + static std::shared_ptr<DataSource> Create(DataSourceDependencies dependencies); + + virtual void StartRecording(); // XX: feels like this should be BPF-specific. + virtual void StopRecording(); + + // Emit all inode->filename mappings (i.e. an infinite lazy list). + // The specific order is determined by the extra dependency options. + // + // The work must terminate if all subscriptions are removed. + virtual rxcpp::observable<InodeResult> EmitInodes() const = 0; + + // Does the InodeResult include a valid device number? + // If returning false, the InodeResolver fills in the missing device number with stat(2). + virtual bool ResultIncludesDeviceNumber() const { return true; }; + + protected: + virtual ~DataSource() {} + DataSource(DataSourceDependencies dependencies); + DataSourceDependencies dependencies_; +}; + +} // namespace iorap::inode2filename + +#endif // IORAP_SRC_INODE2FILENAME_DATA_SOURCE_H_ diff --git a/src/inode2filename/inode.cc b/src/inode2filename/inode.cc new file mode 100644 index 0000000..cc665ea --- /dev/null +++ b/src/inode2filename/inode.cc @@ -0,0 +1,87 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "inode2filename/inode.h" + +#include <android-base/logging.h> +#include <android-base/parseint.h> +#include <android-base/strings.h> + +#include <string> +#include <vector> + +#include <sys/sysmacros.h> + +using android::base::ParseUint; + +namespace iorap::inode2filename { + +// TODO: refactor to return expected<Inode, string> +bool Inode::Parse(const std::string& str, Inode* out, std::string* error_msg) { + DCHECK(out != nullptr); + DCHECK(error_msg != nullptr); + + // Major:minor:inode OR dev_t@inode + std::vector<std::string> lst_pair = android::base::Split(str, "@"); + if (lst_pair.size() == 2) { + size_t dev_whole = 0; + if (!ParseUint(lst_pair[0], &dev_whole)) { + *error_msg = "Failed to parse the whole device id as uint."; + return false; + } + + dev_t dev_w = static_cast<dev_t>(dev_whole); + out->device_major = major(dev_w); + out->device_minor = minor(dev_w); + + if (!ParseUint(lst_pair[1], &out->inode)) { + *error_msg = "Failed to parse inode as uint."; + return false; + } + + return true; + } + + std::vector<std::string> lst = android::base::Split(str, ":"); + + if (lst.size() != 3) { + *error_msg = "Too few : separated items"; + return false; + } + + if (!ParseUint(lst[0], &out->device_major)) { + *error_msg = "Failed to parse 0th element as a uint"; + return false; + } + + if (!ParseUint(lst[1], &out->device_minor)) { + *error_msg = "Failed to parse 1st element as a uint"; + return false; + } + + if (!ParseUint(lst[2], &out->inode)) { + *error_msg = "Failed to parse 2nd element as a uint"; + return false; + } + + return true; +} + +} // namespace iorap::inode2filename + +// Ensure our pollution-free aliases match the typedefs in the system headers. +static_assert(std::is_same_v<iorap::inode2filename::dev_t, dev_t>); +static_assert(std::is_same_v<iorap::inode2filename::ino_t, ino_t>); + +// TODO: consider some tests for major, minor, etc.
\ No newline at end of file diff --git a/src/inode2filename/inode.h b/src/inode2filename/inode.h new file mode 100644 index 0000000..f336a14 --- /dev/null +++ b/src/inode2filename/inode.h @@ -0,0 +1,150 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_INODE2FILENAME_INODE_H_ +#define IORAP_SRC_INODE2FILENAME_INODE_H_ + +#include <functional> +#include <ostream> +#include <string> + +#include <stddef.h> + +namespace iorap::inode2filename { + +// Avoid polluting headers. +#if defined(__ANDROID__) +# if !defined(__LP64__) +/* This historical accident means that we had a 32-bit dev_t on 32-bit architectures. */ +using dev_t = uint32_t; +# else +using dev_t = uint64_t; +# endif +using ino_t = unsigned long; +#else +# if !defined(__x86_64__) +using dev_t = unsigned long long; +using ino_t = unsigned long long; +# else +using dev_t = unsigned long; +using ino_t = unsigned long; +# endif +#endif + +#ifdef makedev +#undef makedev +#endif + +/** Combines `major` and `minor` into a device number. */ +constexpr inline dev_t makedev(unsigned int major, unsigned int minor) { + return + (((major) & 0xfffff000ULL) << 32) | (((major) & 0xfffULL) << 8) | + (((minor) & 0xffffff00ULL) << 12) | (((minor) & 0xffULL)); +} + +#ifdef major +#undef major +#endif + +/** Extracts the major part of a device number. */ +constexpr inline unsigned int major(dev_t dev) { + return + ((unsigned) ((((unsigned long long) (dev) >> 32) & 0xfffff000) | (((dev) >> 8) & 0xfff))); +} + +#ifdef minor +#undef minor +#endif + +/** Extracts the minor part of a device number. */ +constexpr inline unsigned int minor(dev_t dev) { + return + ((unsigned) ((((dev) >> 12) & 0xffffff00) | ((dev) & 0xff))); +}; +// Note: above definitions copied from sysmacros.h, to avoid polluting global namespace in a header. + +/* + * A convenient datum representing a (dev_t, ino_t) tuple. + * + * ino_t values may be reused across different devices (e.g. different partitions), + * so we need the full tuple to uniquely identify an inode on a system. + */ +struct Inode { + size_t device_major; // dev_t = makedev(major, minor) + size_t device_minor; + size_t inode; // ino_t = inode + + // "Major:minor:inode" OR "dev_t@inode" + static bool Parse(const std::string& str, /*out*/Inode* out, /*out*/std::string* error_msg); + + constexpr bool operator==(const Inode& rhs) const { + return device_major == rhs.device_major && + device_minor == rhs.device_minor && + inode == rhs.inode; + } + + constexpr bool operator!=(const Inode& rhs) const { + return !(*this == rhs); + } + + Inode() = default; + constexpr Inode(size_t device_major, size_t device_minor, size_t inode) + : device_major{device_major}, device_minor{device_minor}, inode{inode} { + } + + static constexpr Inode FromDeviceAndInode(dev_t dev, ino_t inode) { + return Inode{major(dev), minor(dev), static_cast<size_t>(inode)}; + } + + constexpr dev_t GetDevice() const { + return makedev(device_major, device_minor); + } + + constexpr ino_t GetInode() const { + return static_cast<ino_t>(inode); + } +}; + +inline std::ostream& operator<<(std::ostream& os, const Inode& inode) { + os << inode.device_major << ":" << inode.device_minor << ":" << inode.inode; + return os; +} + +} // namespace iorap::inode2filename + +namespace std { + template <> + struct hash<iorap::inode2filename::Inode> { + using argument_type = iorap::inode2filename::Inode; + using result_type = size_t; + result_type operator()(argument_type const& s) const noexcept { + // Hash the inode by using only the inode#. Ignore devices, we are extremely unlikely + // to ever collide on the devices. + result_type const h1 = std::hash<size_t>{}(s.inode); + return h1; + } + }; +} // namespace std + +namespace rxcpp { +template <class T, typename> +struct filtered_hash; + +// support for the 'distinct' rx operator. +template <> +struct filtered_hash<iorap::inode2filename::Inode, void> : std::hash<iorap::inode2filename::Inode> { +}; +} // namespace rxcpp + +#endif // IORAP_SRC_INODE2FILENAME_INODE_H_ diff --git a/src/inode2filename/inode_resolver.cc b/src/inode2filename/inode_resolver.cc new file mode 100644 index 0000000..5e2945d --- /dev/null +++ b/src/inode2filename/inode_resolver.cc @@ -0,0 +1,207 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "inode2filename/inode_resolver.h" + +#include "common/cmd_utils.h" +#include "inode2filename/out_of_process_inode_resolver.h" +#include "inode2filename/search_directories.h" + +#include <android-base/logging.h> + +#include <fstream> +#include <stdio.h> + +namespace rx = rxcpp; + +namespace iorap::inode2filename { + +std::vector<std::string> ToArgs(ProcessMode process_mode) { + const char* value = nullptr; + + switch (process_mode) { + case ProcessMode::kInProcessDirect: + value = "in"; + break; + case ProcessMode::kInProcessIpc: + value = "in-ipc"; + break; + case ProcessMode::kOutOfProcessIpc: + value = "out"; + break; + } + + std::vector<std::string> args; + iorap::common::AppendNamedArg(args, "--process-mode", value); + return args; +} + +std::vector<std::string> ToArgs(VerifyKind verify_kind) { + const char* value = nullptr; + + switch (verify_kind) { + case VerifyKind::kNone: + value = "none"; + break; + case VerifyKind::kStat: + value = "stat"; + break; + } + + std::vector<std::string> args; + iorap::common::AppendNamedArg(args, "--verify", value); + return args; +} + +std::vector<std::string> ToArgs(const InodeResolverDependencies& deps) { + std::vector<std::string> args = ToArgs(*static_cast<const DataSourceDependencies*>(&deps)); + iorap::common::AppendArgsRepeatedly(args, ToArgs(deps.process_mode)); + iorap::common::AppendArgsRepeatedly(args, ToArgs(deps.verify)); + + return args; +} + +struct InodeResolver::Impl { + Impl(InodeResolverDependencies dependencies) + : dependencies_{std::move(dependencies)} { + DCHECK(dependencies_.system_call != nullptr); + data_source_ = DataSource::Create(/*downcast*/dependencies_); + } + Impl(InodeResolverDependencies dependencies, std::shared_ptr<DataSource> data_source) + : dependencies_{std::move(dependencies)} { + DCHECK(dependencies_.system_call != nullptr); + data_source_ = std::move(data_source); + DCHECK(data_source_ != nullptr); + } + InodeResolverDependencies dependencies_; + std::shared_ptr<DataSource> data_source_; +}; + +InodeResolver::InodeResolver(InodeResolverDependencies dependencies) + : impl_(new InodeResolver::Impl{std::move(dependencies)}) { +} + +InodeResolver::InodeResolver(InodeResolverDependencies dependencies, + std::shared_ptr<DataSource> data_source) + : impl_(new InodeResolver::Impl{std::move(dependencies), std::move(data_source)}) { +} + +std::shared_ptr<InodeResolver> InodeResolver::Create(InodeResolverDependencies dependencies) { + if (dependencies.process_mode == ProcessMode::kInProcessDirect) { + return std::shared_ptr<InodeResolver>{ + new InodeResolver{std::move(dependencies)}}; + } else if (dependencies.process_mode == ProcessMode::kOutOfProcessIpc) { + return std::shared_ptr<InodeResolver>{ + new OutOfProcessInodeResolver{std::move(dependencies)}}; + } else { + CHECK(false); + } + return nullptr; +} + +std::shared_ptr<InodeResolver> InodeResolver::Create(InodeResolverDependencies dependencies, + std::shared_ptr<DataSource> data_source) { + if (dependencies.process_mode == ProcessMode::kInProcessDirect) { + return std::shared_ptr<InodeResolver>{ + new InodeResolver{std::move(dependencies), std::move(data_source)}}; + } else if (dependencies.process_mode == ProcessMode::kOutOfProcessIpc) { + CHECK(false); // directly providing a DataSource only makes sense in-process + } else { + CHECK(false); + } + return nullptr; +} + +rxcpp::observable<InodeResult> + InodeResolver::FindFilenamesFromInodes(rxcpp::observable<Inode> inodes) const { + + // It's inefficient to search for inodes until the full search list is available, + // so first reduce to a vector so we can access all the inodes simultaneously. + return inodes.reduce(std::vector<Inode>{}, + [](std::vector<Inode> vec, Inode inode) { + vec.push_back(inode); + return vec; + }, + [](std::vector<Inode> v) { + return v; // TODO: use an identity function + }) + .flat_map([self=shared_from_this()](std::vector<Inode> vec) { + // All borrowed values (e.g. SystemCall) must outlive the observable. + return self->FindFilenamesFromInodes(vec); + } + ); +} + +rxcpp::observable<InodeResult> +InodeResolver::FindFilenamesFromInodes(std::vector<Inode> inodes) const { + const DataSource& data_source = *impl_->data_source_; + const InodeResolverDependencies& dependencies = impl_->dependencies_; + + // Get lazy list of inodes from the data source. + rxcpp::observable<InodeResult> all_inodes = impl_->data_source_->EmitInodes(); + + // Filter it according to the source+dependency requirements. + // Unsubscribe from 'all_inodes' early if all inodes are matched early. + const bool needs_device_number = !data_source.ResultIncludesDeviceNumber(); + const bool needs_verification = dependencies.verify == VerifyKind::kStat; + SearchDirectories search{impl_->dependencies_.system_call}; + return search.FilterFilenamesForSpecificInodes(all_inodes, + inodes, + needs_device_number, + needs_verification); +} + +rxcpp::observable<InodeResult> +InodeResolver::EmitAll() const { + const DataSource& data_source = *impl_->data_source_; + const InodeResolverDependencies& dependencies = impl_->dependencies_; + + // Get lazy list of inodes from the data source. + rxcpp::observable<InodeResult> all_inodes = impl_->data_source_->EmitInodes(); + + // Apply verification and fill-in missing device numbers. + const bool needs_device_number = !data_source.ResultIncludesDeviceNumber(); + const bool needs_verification = dependencies.verify == VerifyKind::kStat; + SearchDirectories search{impl_->dependencies_.system_call}; + return search.EmitAllFilenames(all_inodes, + needs_device_number, + needs_verification); +} + +InodeResolver::~InodeResolver() { + // std::unique_ptr requires complete types, but we hide the definition in the header. + delete impl_; + // XX: Does this work if we just force the dtor definition into the .cc file with a unique_ptr? +} + +InodeResolverDependencies& InodeResolver::GetDependencies() { + return impl_->dependencies_; +} + +const InodeResolverDependencies& InodeResolver::GetDependencies() const { + return impl_->dependencies_; +} + +void InodeResolver::StartRecording() { + impl_->data_source_->StartRecording(); +} + +void InodeResolver::StopRecording() { + impl_->data_source_->StopRecording(); +} + +// TODO: refactor more code from search_directories into this file. +// XX: do we also need a DataSink class? lets see if recording gets more complicated. + +} // namespace iorap::inode2filename diff --git a/src/inode2filename/inode_resolver.h b/src/inode2filename/inode_resolver.h new file mode 100644 index 0000000..218640f --- /dev/null +++ b/src/inode2filename/inode_resolver.h @@ -0,0 +1,121 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_INODE2FILENAME_INODE_RESOLVER_H_ +#define IORAP_SRC_INODE2FILENAME_INODE_RESOLVER_H_ + +#include "common/expected.h" +#include "inode2filename/data_source.h" +#include "inode2filename/inode.h" +#include "inode2filename/system_call.h" + +#include <rxcpp/rx.hpp> + +#include <memory> +#include <optional> +#include <string> +#include <vector> +namespace iorap::inode2filename { + +enum class ProcessMode { + // Test modes: + kInProcessDirect, // Execute the code directly. + kInProcessIpc, // Execute code via IPC layer using multiple threads. + // Shipping mode: + kOutOfProcessIpc, // Execute code via fork+exec with IPC. + + // Note: in-process system-wide stat(2)/readdir/etc is blocked by selinux. + // Attempting to call the test modes will fail with -EPERM. + // + // Use fork+exec mode in shipping configurations, which spawns inode2filename + // as a separate command. +}; + +enum class VerifyKind { + kNone, + kStat, +}; + +std::vector<std::string> ToArgs(VerifyKind verify_kind); + +struct InodeResolverDependencies : public DataSourceDependencies { + ProcessMode process_mode = ProcessMode::kInProcessDirect; + VerifyKind verify{VerifyKind::kStat}; // Filter out results that aren't up-to-date with stat(2) ? +}; + +std::vector<std::string> ToArgs(const InodeResolverDependencies& deps); + +// Create an rx-observable chain that allows searching for inode->filename mappings given +// a set of inode keys. +class InodeResolver : public std::enable_shared_from_this<InodeResolver> { + public: + static std::shared_ptr<InodeResolver> Create(InodeResolverDependencies dependencies, + std::shared_ptr<DataSource> data_source); // nonnull + + // Convenience function for above: Uses DataSource::Create for the data-source. + static std::shared_ptr<InodeResolver> Create(InodeResolverDependencies dependencies); + + // Search the associated data source to map each inode in 'inodes' to a file path. + // + // Observes DataSource::EmitInodes(), which is unsubscribed from early once all inodes are found. + // + // Notes: + // * Searching does not begin until all 'inodes' are observed to avoid rescanning. + // * If the observable is unsubscribed to prior to completion, searching will halt. + // + // Post-condition: All emitted results are in inodes, and all inodes are in emitted results. + rxcpp::observable<InodeResult> + FindFilenamesFromInodes(rxcpp::observable<Inode> inodes) const; + // TODO: feels like we could turn this into a general helper? + // Convenience function for above. + virtual rxcpp::observable<InodeResult> + FindFilenamesFromInodes(std::vector<Inode> inodes) const; + + // Enumerate *all* inodes available from the data source, associating it with a filepath. + // + // Depending on the data source (e.g. diskscan), it can take a very long time for this observable + // to complete. The intended use-case is for development/debugging, not for production. + // + // Observes DataSource::EmitInodes() until it reaches #on_completed. + // + // Notes: + // * If the observable is unsubscribed to prior to completion, searching will halt. + virtual rxcpp::observable<InodeResult> + EmitAll() const; + + // Notifies the DataSource to begin recording. + // Some DataSources may be continuously refreshing, but only if recording is enabled. + // To get the most up-to-date data, toggle recording before reading the inodes out. + void StartRecording(); // XX: feels like this should be BPF-specific. + + // Notifies the DataSource to stop recording. + // Some DataSources may be continuously refreshing, but only if recording is enabled. + // The snapshot of data returned by e.g. #EmitAll would then not change outside of recording. + void StopRecording(); + + virtual ~InodeResolver(); + private: + struct Impl; + Impl* impl_; + + protected: + InodeResolver(InodeResolverDependencies dependencies); + InodeResolver(InodeResolverDependencies dependencies, std::shared_ptr<DataSource> data_source); + InodeResolverDependencies& GetDependencies(); + const InodeResolverDependencies& GetDependencies() const; +}; + +} + +#endif // IORAP_SRC_INODE2FILENAME_INODE_RESOLVER_H_ diff --git a/src/inode2filename/inode_result.cc b/src/inode2filename/inode_result.cc new file mode 100644 index 0000000..139b038 --- /dev/null +++ b/src/inode2filename/inode_result.cc @@ -0,0 +1,62 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "inode2filename/inode_result.h" + +#include <string.h> + +namespace iorap::inode2filename { + +std::optional<std::string_view> InodeResult::ErrorMessage() const { + if (data) { + return std::nullopt; + } + + const int err_no = data.error(); + return std::string_view{ + [=]() -> const char * { + switch (err_no) { + case InodeResult::kCouldNotFindFilename: + return "Could not find filename"; + case InodeResult::kVerificationFailed: + return "Verification failed"; + default: + return strerror(err_no); + } + }() + }; +} + +std::ostream& operator<<(std::ostream& os, const InodeResult& result) { + os << "InodeResult{"; + if (result) { + os << "OK,"; + } else { + os << "ERR,"; + } + + os << result.inode << ","; + + if (result) { + os << "\"" << result.data.value() << "\""; + } else { + os << result.data.error(); + os << " (" << *result.ErrorMessage() << ")"; + } + + os << "}"; + return os; +} + +} // namespace iorap::inode2filename
\ No newline at end of file diff --git a/src/inode2filename/inode_result.h b/src/inode2filename/inode_result.h new file mode 100644 index 0000000..df352fb --- /dev/null +++ b/src/inode2filename/inode_result.h @@ -0,0 +1,82 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_INODE2FILENAME_INODE_RESULT_H_ +#define IORAP_SRC_INODE2FILENAME_INODE_RESULT_H_ + +#include "common/expected.h" +#include "inode2filename/inode.h" +#include "inode2filename/inode_result.h" +#include "inode2filename/system_call.h" + +#include <rxcpp/rx.hpp> + +#include <memory> +#include <optional> +#include <string> +#include <vector> +namespace iorap::inode2filename { + +// Tuple of (Inode -> (Filename|Errno)) +struct InodeResult { + // We set this error when all root directories have been searched and + // yet we still could not find a corresponding filename for the inode under search. + static constexpr int kCouldNotFindFilename = ENOKEY; + + // An initial inode->filename mapping was found, but subsequent verification for it failed. + static constexpr int kVerificationFailed = EKEYEXPIRED; + + // There is always an inode, but sometimes we may fail to resolve the filename. + Inode inode; + // Value: Contains the filename (with a root directory as a prefix). + // Error: Contains the errno, usually one of the above, otherwise some system error. + iorap::expected<std::string /*filename*/, int /*errno*/> data; + + static InodeResult makeSuccess(Inode inode, std::string filename) { + return InodeResult{inode, std::move(filename)}; + } + + static InodeResult makeFailure(Inode inode, int err_no) { + return InodeResult{inode, iorap::unexpected{err_no}}; + } + + constexpr explicit operator bool() const { + return data.has_value(); + } + + constexpr bool operator==(const InodeResult& other) const { + if (inode == other.inode) { + if (data && other.data) { + return *data == *other.data; + } else if (!data && !other.data) { + return data.error() == other.data.error(); + } + // TODO: operator== for expected + } + return false; + } + + constexpr bool operator!=(const InodeResult& other) const { + return !(*this == other); + } + + // Returns a human-readable error message, or 'nullopt' if there was no error. + std::optional<std::string_view> ErrorMessage() const; +}; + +std::ostream& operator<<(std::ostream& os, const InodeResult& result); + +} // namespace iorap::inode2filename + +#endif // IORAP_SRC_INODE2FILENAME_INODE_RESULT_H_
\ No newline at end of file diff --git a/src/inode2filename/main.cc b/src/inode2filename/main.cc new file mode 100644 index 0000000..986f6ac --- /dev/null +++ b/src/inode2filename/main.cc @@ -0,0 +1,455 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/debug.h" +#include "common/expected.h" +#include "inode2filename/inode_resolver.h" + +#include <android-base/strings.h> + +#include <iostream> +#include <fstream> +#include <sstream> +#include <string_view> + +#if defined(IORAP_INODE2FILENAME_MAIN) + +namespace iorap::inode2filename { + +void Usage(char** argv) { + std::cerr << "Usage: " << argv[0] << " <options> <<inode_syntax>> [inode_syntax1 inode_syntax2 ...]" << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Block until all inodes have been read in, then begin searching for filenames for those inodes." << std::endl; + std::cerr << " Results are written immediately as they are available, and once all inodes are found, " << std::endl; + std::cerr << " the program will terminate." << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Inode syntax: ('dev_t@inode' | 'major:minor:inode')" << std::endl; + std::cerr << "" << std::endl; // CLI-only flags. + std::cerr << " --help,-h Print this Usage." << std::endl; + std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; + std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; + std::cerr << "" << std::endl; // General flags. + std::cerr << " --all,-a Enumerate all inode->filename mappings in the dataset (default off)." << std::endl; + std::cerr << " All <<inode_syntaxN>> arguments are ignored." << std::endl; + std::cerr << " --data-source=, Choose a data source (default 'diskscan')." << std::endl; + std::cerr << " -ds " << std::endl; + std::cerr << " diskscan Scan disk recursively using readdir." << std::endl; + std::cerr << " textcache Use the file from the '--output-format=text'." << std::endl; + std::cerr << " bpf Query kernel BPF maps (experimental)." << std::endl; + std::cerr << " --output=,-o Choose an output file (default 'stdout')." << std::endl; + std::cerr << " --output-format=, Choose an output format (default 'log')." << std::endl; + std::cerr << " -of " << std::endl; + std::cerr << " log Log human-readable, non-parsable format to stdout+logcat." << std::endl; + std::cerr << " textcache Results are in the same format as system/extras/pagecache." << std::endl; + std::cerr << " ipc Results are in a binary inter-process communications format" << std::endl; + std::cerr << " --process-mode=, Choose a process mode (default 'in'). Test-oriented." << std::endl; + std::cerr << " -pm " << std::endl; + std::cerr << " in Use a single process to do the work in." << std::endl; + std::cerr << " out Out-of-process work (forks into a -pm=in)." << std::endl; + std::cerr << " --verify=,-vy Verification modes for the data source (default 'stat')." << std::endl; + std::cerr << " stat Use stat(2) call to validate data inodes are up-to-date. " << std::endl; + std::cerr << " none Trust that the data-source is up-to-date without checking." << std::endl; + std::cerr << "" << std::endl; // --data-source=<?> specific flags. + std::cerr << " Data-source-specific commands:" << std::endl; + std::cerr << " --data-source=diskscan" << std::endl; + std::cerr << " --root=,-r Add root directory (default '.'). Repeatable." << std::endl; + std::cerr << " --data-source=textcache" << std::endl; + std::cerr << " --textcache=,-tc Name of file that contains the textcache." << std::endl; + std::cerr << "" << std::endl; // Programmatic flags. + std::cerr << " --in-fd=# Input file descriptor. Default input is from argv." << std::endl; + std::cerr << " --out-fd=# Output file descriptor. Default stdout." << std::endl; + exit(1); +} + +static fruit::Component<SystemCall> GetSystemCallComponent() { + return fruit::createComponent().bind<SystemCall, SystemCallImpl>(); +} + +std::optional<DataSourceKind> ParseDataSourceKind(std::string_view str) { + if (str == "diskscan") { + return DataSourceKind::kDiskScan; + } else if (str == "textcache") { + return DataSourceKind::kTextCache; + } else if (str == "bpf") { + return DataSourceKind::kBpf; + } + return std::nullopt; +} + +enum class OutputFormatKind { + kLog, + kTextCache, + kIpc, +}; + +std::optional<OutputFormatKind> ParseOutputFormatKind(std::string_view str) { + if (str == "log") { + return OutputFormatKind::kLog; + } else if (str == "textcache") { + return OutputFormatKind::kTextCache; + } else if (str == "ipc") { + return OutputFormatKind::kIpc; + } + return std::nullopt; +} + +std::optional<VerifyKind> ParseVerifyKind(std::string_view str) { + if (str == "none") { + return VerifyKind::kNone; + } else if (str == "stat") { + return VerifyKind::kStat; + } + return std::nullopt; +} + +std::optional<ProcessMode> ParseProcessMode(std::string_view str) { + if (str == "in") { + return ProcessMode::kInProcessDirect; + } else if (str == "out") { + return ProcessMode::kOutOfProcessIpc; + } + return std::nullopt; +} + +bool StartsWith(std::string_view haystack, std::string_view needle) { + return haystack.size() >= needle.size() + && haystack.compare(0, needle.size(), needle) == 0; +} + +bool EndsWith(std::string_view haystack, std::string_view needle) { + return haystack.size() >= needle.size() + && haystack.compare(haystack.size() - needle.size(), haystack.npos, needle) == 0; +} + +bool StartsWithOneOf(std::string_view haystack, + std::string_view needle, + std::string_view needle2) { + return StartsWith(haystack, needle) || StartsWith(haystack, needle2); +} + +enum ParseError { + kParseSkip, + kParseFailed, +}; + +std::optional<std::string> ParseNamedArgument(std::initializer_list<std::string> names, + std::string argstr, + std::optional<std::string> arg_next, + /*inout*/ + int* arg_pos) { + for (const std::string& name : names) { + { + // Try parsing only 'argstr' for '--foo=bar' type parameters. + std::vector<std::string> split_str = ::android::base::Split(argstr, "="); + if (split_str.size() >= 2) { + /* + std::cerr << "ParseNamedArgument(name=" << name << ", argstr='" + << argstr << "')" << std::endl; + */ + + if (split_str[0] + "=" == name) { + return split_str[1]; + } + } + } + //if (EndsWith(name, "=")) { + // continue; + /*} else */ { + // Try parsing 'argstr arg_next' for '-foo bar' type parameters. + if (argstr == name) { + ++(*arg_pos); + + if (arg_next) { + return arg_next; + } else { + // Missing argument, e.g. '-foo' was the last token in the argv. + std::cerr << "Missing " << name << " flag value." << std::endl; + exit(1); + } + } + } + } + + return std::nullopt; +} + +int main(int argc, char** argv) { + android::base::InitLogging(argv); + android::base::SetLogger(android::base::StderrLogger); + + bool all = false; + bool wait_for_keystroke = false; + bool enable_verbose = false; + std::vector<std::string> root_directories; + std::vector<Inode> inode_list; + int recording_time_sec = 0; + + DataSourceKind data_source = DataSourceKind::kDiskScan; + OutputFormatKind output_format = OutputFormatKind::kLog; + VerifyKind verify = VerifyKind::kStat; + ProcessMode process_mode = ProcessMode::kInProcessDirect; + + std::optional<std::string> output_filename; + std::optional<int /*fd*/> in_fd, out_fd; // input-output file descriptors [for fork+exec]. + std::optional<std::string> text_cache_filename; + + if (argc == 1) { + Usage(argv); + } + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + std::optional<std::string> arg_next; + if ((arg + 1) < argc) { + arg_next = argv[arg+1]; + } + + if (argstr == "--help" || argstr == "-h") { + Usage(argv); + } else if (auto val = ParseNamedArgument({"--root=", "-r"}, argstr, arg_next, /*inout*/&arg); + val) { + root_directories.push_back(*val); + } else if (argstr == "--verbose" || argstr == "-v") { + enable_verbose = true; + } else if (argstr == "--wait" || argstr == "-w") { + wait_for_keystroke = true; + } + else if (argstr == "--all" || argstr == "-a") { + all = true; + } else if (auto val = ParseNamedArgument({"--data-source=", "-ds"}, + argstr, + arg_next, + /*inout*/&arg); + val) { + auto ds = ParseDataSourceKind(*val); + if (!ds) { + std::cerr << "Invalid --data-source=<value>" << std::endl; + return 1; + } + data_source = *ds; + } else if (auto val = ParseNamedArgument({"--output=", "-o"}, + argstr, + arg_next, + /*inout*/&arg); + val) { + output_filename = *val; + } else if (auto val = ParseNamedArgument({"--process-mode=", "-pm"}, + argstr, + arg_next, + /*inout*/&arg); + val) { + auto pm = ParseProcessMode(*val); + if (!pm) { + std::cerr << "Invalid --process-mode=<value>" << std::endl; + return 1; + } + process_mode = *pm; + } + else if (auto val = ParseNamedArgument({"--output-format=", "-of"}, + argstr, + arg_next, + /*inout*/&arg); + val) { + auto of = ParseOutputFormatKind(*val); + if (!of) { + std::cerr << "Missing --output-format=<value>" << std::endl; + return 1; + } + output_format = *of; + } else if (auto val = ParseNamedArgument({"--verify=", "-vy="}, + argstr, + arg_next, + /*inout*/&arg); + val) { + auto vy = ParseVerifyKind(*val); + if (!vy) { + std::cerr << "Invalid --verify=<value>" << std::endl; + return 1; + } + verify = *vy; + } else if (auto val = ParseNamedArgument({"--textcache=", "-tc"}, + argstr, + arg_next, + /*inout*/&arg); + val) { + text_cache_filename = *val; + } else { + Inode maybe_inode{}; + + std::string error_msg; + if (Inode::Parse(argstr, /*out*/&maybe_inode, /*out*/&error_msg)) { + inode_list.push_back(maybe_inode); + } else { + if (argstr.size() >= 1) { + if (argstr[0] == '-') { + std::cerr << "Unrecognized flag: " << argstr << std::endl; + return 1; + } + } + + std::cerr << "Failed to parse inode (" << argstr << ") because: " << error_msg << std::endl; + return 1; + } + } + } + + if (root_directories.size() == 0) { + root_directories.push_back("."); + } + + if (inode_list.size() == 0 && !all) { + std::cerr << "Provide at least one inode. Or use --all to dump everything." << std::endl; + return 1; + } else if (all && inode_list.size() > 0) { + std::cerr << "[WARNING]: --all flag ignores all inodes passed on command line." << std::endl; + } + + std::ofstream fout; + if (output_filename) { + fout.open(*output_filename); + if (!fout) { + std::cerr << "Failed to open output file for writing: \"" << *output_filename << "\""; + return 1; + } + } else { + fout.open("/dev/null"); // have to open *something* otherwise rdbuf fails. + fout.basic_ios<char>::rdbuf(std::cout.rdbuf()); + } + + if (enable_verbose) { + android::base::SetMinimumLogSeverity(android::base::VERBOSE); + + LOG(VERBOSE) << "Verbose check"; + LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; + + for (auto& inode_num : inode_list) { + LOG(VERBOSE) << "Searching for inode " << inode_num; + } + + LOG(VERBOSE) << "Dumping all inodes? " << all; + } + // else use + // $> ANDROID_LOG_TAGS='*:d' iorap.inode2filename <args> + // which will enable arbitrary log levels. + + // Useful to attach a debugger... + // 1) $> inode2filename -w <args> + // 2) $> gdbclient <pid> + if (wait_for_keystroke) { + LOG(INFO) << "Self pid: " << getpid(); + LOG(INFO) << "Press any key to continue..."; + std::cin >> wait_for_keystroke; + } + + fruit::Injector<SystemCall> injector(GetSystemCallComponent); + + InodeResolverDependencies ir_dependencies; + // Passed from command-line. + ir_dependencies.data_source = data_source; + ir_dependencies.process_mode = process_mode; + ir_dependencies.root_directories = root_directories; + ir_dependencies.text_cache_filename = text_cache_filename; + ir_dependencies.verify = verify; + // Hardcoded. + ir_dependencies.system_call = injector.get<SystemCall*>(); + + std::shared_ptr<InodeResolver> inode_resolver = + InodeResolver::Create(ir_dependencies); + + inode_resolver->StartRecording(); + sleep(recording_time_sec); // TODO: add cli flag for this when we add something that needs it. + inode_resolver->StopRecording(); + + auto/*observable<InodeResult>*/ inode_results = all + ? inode_resolver->EmitAll() + : inode_resolver->FindFilenamesFromInodes(std::move(inode_list)); + + int return_code = 2; + inode_results.subscribe( + /*on_next*/[&return_code, output_format, &fout](const InodeResult& result) { + if (result) { + LOG(DEBUG) << "Inode match: " << result; + if (output_format == OutputFormatKind::kLog) { + fout << "\033[1;32m[OK]\033[0m " + << result.inode + << " \"" << result.data.value() << "\"" << std::endl; + } else if (output_format == OutputFormatKind::kIpc) { + std::stringstream stream; + stream << "K " << result.inode << " " << result.data.value(); + std::string line = stream.str(); + + // Convert the size to 4 bytes. + int32_t size = line.size(); + char buf[sizeof(int32_t)]; + memcpy(buf, &size, sizeof(int32_t)); + fout.write(buf, sizeof(int32_t)); + fout.write(line.c_str(), size); + } else if (output_format == OutputFormatKind::kTextCache) { + // Same format as TextCacheDataSource (system/extras/pagecache/pagecache.py -d) + // "$device_number $inode $filesize $filename..." + const Inode& inode = result.inode; + fout << inode.GetDevice() << " " + << inode.GetInode() + << " -1 " // always -1 for filesize, since we don't track what it is. + << result.data.value() << "\n"; // don't use endl which flushes, for faster writes. + } else { + LOG(FATAL) << "Not implemented this kind of --output-format"; + } + + return_code = 0; + } else { + LOG(DEBUG) << "Failed to match inode: " << result; + if (output_format == OutputFormatKind::kLog) { + fout << "\033[1;31m[ERR]\033[0m " + << result.inode + << " '" << *result.ErrorMessage() << "'" << std::endl; + } else if (output_format == OutputFormatKind::kIpc) { + std::stringstream stream; + stream << "E " << result.inode << " " << result.data.error() << std::endl; + std::string line = stream.str(); + + // Convert the size to 4 bytes. + int32_t size = line.size(); + char buf[sizeof(int32_t)]; + memcpy(buf, &size, sizeof(int32_t)); + fout.write(buf, sizeof(int32_t)); + fout.write(line.c_str(), size); + } + else if (output_format == OutputFormatKind::kTextCache) { + // Don't add bad results to the textcache. They are dropped. + } else { + LOG(FATAL) << "Not implemented this kind of --output-format"; + } + } + }, /*on_error*/[&return_code](rxcpp::util::error_ptr error) { + // Usually occurs very early on before we see the first result. + // In this case the error is terminal so we just end up exiting out soon. + return_code = 3; + LOG(ERROR) << "Critical error: " << rxcpp::util::what(error); + }); + + // 0 -> found at least a single match, + // 1 -> bad parameters, + // 2 -> could not find any matches, + // 3 -> rxcpp on_error. + return return_code; +} + +} // namespace iorap::inode2filename + +int main(int argc, char** argv) { + return ::iorap::inode2filename::main(argc, argv); +} + +#endif diff --git a/src/inode2filename/out_of_process_inode_resolver.cc b/src/inode2filename/out_of_process_inode_resolver.cc new file mode 100644 index 0000000..f409fd2 --- /dev/null +++ b/src/inode2filename/out_of_process_inode_resolver.cc @@ -0,0 +1,402 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/cmd_utils.h" +#include "inode2filename/inode_resolver.h" +#include "inode2filename/out_of_process_inode_resolver.h" + +#include <android-base/file.h> +#include <android-base/logging.h> +#include <android-base/unique_fd.h> + +#include <sstream> +#include <stdio.h> +#include <unistd.h> + +namespace rx = rxcpp; + +namespace iorap::inode2filename { + +using ::android::base::unique_fd; +using ::android::base::borrowed_fd; + +static const char* GetCommandFileName() { + // Avoid ENOENT by execve by specifying the absolute path of inode2filename. +#ifdef __ANDROID__ + return "/system/bin/iorap.inode2filename"; +#else + static const char* file_name = nullptr; + + if (file_name == nullptr) { + char* out_dir = getenv("ANDROID_HOST_OUT"); + static std::string file_name_str = out_dir; + if (out_dir != nullptr) { + file_name_str += "/bin/"; + } else { + // Assume it's in the same directory as the binary we are in. + std::string self_path; + CHECK(::android::base::Readlink("/proc/self/exe", /*out*/&self_path)); + + std::string self_dir = ::android::base::Dirname(self_path); + file_name_str = self_dir + "/"; + } + file_name_str += "iorap.inode2filename"; + + file_name = file_name_str.c_str(); + } + + return file_name; +#endif +} + +std::error_code ErrorCodeFromErrno() { + int err = errno; + return std::error_code(err, std::system_category()); +} + +std::ios_base::failure IosBaseFailureWithErrno(const char* message) { + std::error_code ec = ErrorCodeFromErrno(); + return std::ios_base::failure(message, ec); +} + +static constexpr bool kDebugFgets = false; + +int32_t ReadLineLength(FILE* stream, bool* eof) { + char buf[sizeof(int32_t)]; + size_t count = fread(buf, 1, sizeof(int32_t), stream); + if (feof(stream)) { + // If reaching the end of the stream when trying to read the first int, just + // return. This is legitimate, because after reading the last line, the next + // iteration will reach this. + *eof = true; + return 0; + } + int32_t length; + memcpy(&length, buf, sizeof(int32_t)); + return length; +} + +// The steam is like [size1][file1][size2][file2]...[sizeN][fileN]. +std::string ReadOneLine(FILE* stream, bool* eof) { + DCHECK(stream != nullptr); + DCHECK(eof != nullptr); + + int32_t length = ReadLineLength(stream, eof); + if (length <= 0) { + PLOG(ERROR) << "unexpected 0 length line."; + *eof = true; + return ""; + } + + std::string str(length, '\0'); + size_t count = fread(&str[0], sizeof(char), length, stream); + if (feof(stream) || ferror(stream) || count != (uint32_t)length) { + // error! :( + PLOG(ERROR) << "unexpected end of the line during fread"; + *eof = true; + return ""; + } + return str; +} + +static inline void LeftTrim(/*inout*/std::string& s) { + s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { + return !std::isspace(ch); + })); +} + +// Parses an --output-format=ipc kind of line into an InodeResult. +// Returns nullopt if parsing failed. +std::optional<InodeResult> ParseFromLine(const std::string& line) { + // inode <- INT:INT:INT + // line_ok <- 'K ' inode ' ' STRING + // line_err <- 'E ' inode ' ' INT + // + // result <- line_ok | line_err + + std::stringstream ss{line}; + + bool result_ok = false; + + std::string ok_or_error; + ss >> ok_or_error; + + if (ss.fail()) { + return std::nullopt; + } + + if (ok_or_error == "K") { + result_ok = true; + } else if (ok_or_error == "E") { + result_ok = false; + } else { + return std::nullopt; + } + + std::string inode_s; + ss >> inode_s; + + if (ss.fail()) { + return std::nullopt; + } + + Inode inode; + + std::string inode_parse_error_msg; + if (!Inode::Parse(inode_s, /*out*/&inode, /*out*/&inode_parse_error_msg)) { + return std::nullopt; + } + + if (result_ok == false) { + int error_code; + ss >> error_code; + + if (ss.fail()) { + return std::nullopt; + } + + return InodeResult::makeFailure(inode, error_code); + } else if (result_ok == true) { + std::string rest_of_line; + ss >> rest_of_line; + + // parse " string with potential spaces" + // into "string with potential spaces" + LeftTrim(/*inout*/rest_of_line); + + if (ss.fail()) { + return std::nullopt; + } + + const std::string& file_name = rest_of_line; + + return InodeResult::makeSuccess(inode, file_name); + } + + return std::nullopt; +} + +struct OutOfProcessInodeResolver::Impl { + Impl() { + } + + private: + // Create the argv we will pass to the forked inode2filename, corresponds to #EmitAll. + std::vector<std::string> CreateArgvAll(const InodeResolverDependencies& deps) const { + std::vector<std::string> argv; + argv.push_back("--all"); + + return CreateArgv(deps, std::move(argv)); + } + + // Create the argv we will pass to the forked inode2filename, corresponds to + // #FindFilenamesFromInodes. + std::vector<std::string> CreateArgvFind(const InodeResolverDependencies& deps, + const std::vector<Inode>& inodes) const { + std::vector<std::string> argv; + iorap::common::AppendArgsRepeatedly(argv, inodes); + + return CreateArgv(deps, std::move(argv)); + } + + std::vector<std::string> CreateArgv(const InodeResolverDependencies& deps, + std::vector<std::string> append_argv) const { + InodeResolverDependencies deps_oop = deps; + deps_oop.process_mode = ProcessMode::kInProcessDirect; + + std::vector<std::string> argv = ToArgs(deps_oop); + + argv.push_back("--output-format=ipc"); + + if (iorap::common::GetBoolEnvOrProperty("iorap.inode2filename.log.verbose", false)) { + argv.push_back("--verbose"); + } + + iorap::common::AppendArgsRepeatedly(argv, std::move(append_argv)); + + return argv; + } + + public: + // fork+exec into inode2filename with 'inodes' as the search list. + // Each result is parsed into a dest#on_next(result). + // If a fatal error occurs, dest#on_error is called once and no other callbacks are called. + void EmitFromCommandFind(rxcpp::subscriber<InodeResult>& dest, + const InodeResolverDependencies& deps, + const std::vector<Inode>& inodes) { + // Trivial case: complete immediately. + // Executing inode2filename with empty search list will just print the --help menu. + if (inodes.empty()) { + dest.on_completed(); + } + + std::vector<std::string> argv = CreateArgvFind(deps, inodes); + EmitFromCommandWithArgv(/*inout*/dest, std::move(argv), inodes.size()); + } + + // fork+exec into inode2filename with --all (listing *all* inodes). + // Each result is parsed into a dest#on_next(result). + // If a fatal error occurs, dest#on_error is called once and no other callbacks are called. + void EmitFromCommandAll(rxcpp::subscriber<InodeResult>& dest, + const InodeResolverDependencies& deps) { + std::vector<std::string> argv = CreateArgvAll(deps); + EmitFromCommandWithArgv(/*inout*/dest, std::move(argv), /*result_count*/std::nullopt); + } + + private: + void EmitFromCommandWithArgv(rxcpp::subscriber<InodeResult>& dest, + std::vector<std::string> argv_vec, + std::optional<size_t> result_count) { + unique_fd pipe_reader, pipe_writer; + if (!::android::base::Pipe(/*out*/&pipe_reader, /*out*/&pipe_writer)) { + dest.on_error( + rxcpp::util::make_error_ptr( + IosBaseFailureWithErrno("Failed to create out-going pipe for inode2filename"))); + return; + } + + pid_t child = fork(); + if (child == -1) { + dest.on_error( + rxcpp::util::make_error_ptr( + IosBaseFailureWithErrno("Failed to fork process for inode2filename"))); + return; + } else if (child > 0) { // we are the caller of this function + LOG(DEBUG) << "forked into a process for inode2filename , pid = " << child; + } else { + // we are the child that was forked + + const char* kCommandFileName = GetCommandFileName(); + + std::stringstream argv; // for debugging. + for (std::string arg : argv_vec) { + argv << arg << ' '; + } + LOG(DEBUG) << "fork+exec: " << kCommandFileName << " " << argv.str(); + + // Redirect only stdout. stdin is unused, stderr is same as parent. + if (dup2(pipe_writer.get(), STDOUT_FILENO) == -1) { + // Trying to call #on_error does not make sense here because we are in a forked process, + // the only thing we can do is crash definitively. + PLOG(FATAL) << "Failed to dup2 for inode2filename"; + } + + std::unique_ptr<const char *[]> argv_ptr = + common::VecToArgv(kCommandFileName, argv_vec); + + if (execve(kCommandFileName, + (char **)argv_ptr.get(), + /*envp*/nullptr) == -1) { + // Trying to call #on_error does not make sense here because we are in a forked process, + // the only thing we can do is crash definitively. + PLOG(FATAL) << "Failed to execve process for inode2filename"; + } + // This should never return. + } + + // Immediately close the writer end of the pipe because we never use it. + pipe_writer.reset(); + + // Convert pipe(reader) file descriptor into FILE*. + std::unique_ptr<FILE, int(*)(FILE*)> file_reader{ + ::android::base::Fdopen(std::move(pipe_reader), /*mode*/"r"), fclose}; + if (!file_reader) { + dest.on_error( + rxcpp::util::make_error_ptr( + IosBaseFailureWithErrno("Failed to fdopen for inode2filename"))); + return; + } + + size_t actual_result_count = 0; + + bool file_eof = false; + while (!file_eof) { + std::string inode2filename_line = ReadOneLine(file_reader.get(), /*out*/&file_eof); + + if (inode2filename_line.empty()) { + if (!file_eof) { + // Ignore empty lines. + LOG(WARNING) << "inode2filename: got empty line"; + } + continue; + } + + LOG(DEBUG) << "inode2filename output-line: " << inode2filename_line; + + std::optional<InodeResult> res = ParseFromLine(inode2filename_line); + if (!res) { + std::string error_msg = "Invalid output: "; + error_msg += inode2filename_line; + dest.on_error( + rxcpp::util::make_error_ptr(std::ios_base::failure(error_msg))); + return; + } + dest.on_next(*res); + + ++actual_result_count; + } + + LOG(DEBUG) << "inode2filename output-eof"; + + // Ensure that the # of inputs into the rx stream match the # of outputs. + // This is validating the post-condition of FindFilenamesFromInodes. + if (result_count && actual_result_count != *result_count) { + std::stringstream ss; + ss << "Invalid number of results, expected: " << *result_count; + ss << ", actual: " << actual_result_count; + + dest.on_error( + rxcpp::util::make_error_ptr(std::ios_base::failure(ss.str()))); + return; + } + + CHECK(child > 0); // we are in the parent process, parse the IPC output of inode2filename + dest.on_completed(); + } +}; + +rxcpp::observable<InodeResult> + OutOfProcessInodeResolver::FindFilenamesFromInodes(std::vector<Inode> inodes) const { + return rxcpp::observable<>::create<InodeResult>( + [self=std::static_pointer_cast<const OutOfProcessInodeResolver>(shared_from_this()), + inodes=std::move(inodes)]( + rxcpp::subscriber<InodeResult> s) { + self->impl_->EmitFromCommandFind(s, self->GetDependencies(), inodes); + }); +} + +rxcpp::observable<InodeResult> + OutOfProcessInodeResolver::EmitAll() const { + auto self = std::static_pointer_cast<const OutOfProcessInodeResolver>(shared_from_this()); + CHECK(self != nullptr); + CHECK(self->impl_ != nullptr); + + return rxcpp::observable<>::create<InodeResult>( + [self](rxcpp::subscriber<InodeResult> s) { + CHECK(self != nullptr); + CHECK(self->impl_ != nullptr); + self->impl_->EmitFromCommandAll(s, self->GetDependencies()); + }); +} + +OutOfProcessInodeResolver::OutOfProcessInodeResolver(InodeResolverDependencies dependencies) + : InodeResolver{std::move(dependencies)}, impl_{new Impl{}} { +} + +OutOfProcessInodeResolver::~OutOfProcessInodeResolver() { + // std::unique_ptr requires complete types, but we hide the definition in the header. + delete impl_; +} + +} // namespace iorap::inode2filename diff --git a/src/inode2filename/out_of_process_inode_resolver.h b/src/inode2filename/out_of_process_inode_resolver.h new file mode 100644 index 0000000..d41e93f --- /dev/null +++ b/src/inode2filename/out_of_process_inode_resolver.h @@ -0,0 +1,53 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_INODE2FILENAME_OUT_OF_PROCESS_INDOE_RESOLVER_H_ +#define IORAP_SRC_INODE2FILENAME_OUT_OF_PROCESS_INDOE_RESOLVER_H_ + +#include "common/expected.h" +#include "inode2filename/inode_resolver.h" + +namespace iorap::inode2filename { + +// Create an InodeResolver that fork+exec+pipes into the binary 'iorap.inode2filename' +// and transmits the results back via an IPC mechanism. +// +// This is instantiated by InodeResolver::Create + ProcessMode::kOutOfProcessIpc +class OutOfProcessInodeResolver : public InodeResolver { + public: + virtual rxcpp::observable<InodeResult> + FindFilenamesFromInodes(std::vector<Inode> inodes) const override; + + virtual rxcpp::observable<InodeResult> + EmitAll() const override; + + OutOfProcessInodeResolver(InodeResolverDependencies dependencies); + ~OutOfProcessInodeResolver(); + private: + struct Impl; + Impl* impl_; +}; + +// Reads one line data from the stream. +// Each line is in the format of "<4 bytes line length><state> <inode info> <file path>" +// The <4 bytes line length> is the length rest data of "<state> <inode info> <file path>". +// The return string is "<state> <inode info> <file path>". +// For example: for "<size>K 253:9:6 ./test", the return value is +// "K 253:9:6 ./test". The <size> is encoded in the first 4 bytes. +// Note: there is no newline in the end of each line and the line shouldn't be +// empty unless there is some error. +std::string ReadOneLine(FILE* stream, bool* eof); +} + +#endif // IORAP_SRC_INODE2FILENAME_OUT_OF_PROCESS_INDOE_RESOLVER_H_ diff --git a/src/inode2filename/search_directories.cc b/src/inode2filename/search_directories.cc new file mode 100644 index 0000000..1d31671 --- /dev/null +++ b/src/inode2filename/search_directories.cc @@ -0,0 +1,1366 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/debug.h" +#include "inode2filename/search_directories.h" +#include "inode2filename/system_call.h" + +#include <android-base/file.h> +#include <android-base/logging.h> +#include <android-base/scopeguard.h> +#include <android-base/stringprintf.h> +#include <android-base/unique_fd.h> + +#include "rxcpp/rx.hpp" + +#include <iostream> +#include <stdio.h> +#include <fstream> +#include <vector> +#include <optional> + +#include <signal.h> +#include <stdlib.h> +#include <unistd.h> + +#include <sys/types.h> + +#ifdef __ANDROID__ +#include <sys/sysmacros.h> +#endif + +#include <sys/stat.h> +#include <fcntl.h> +#include <poll.h> +#include <dirent.h> + +#include <unordered_map> + +namespace rx = rxcpp; +using android::base::unique_fd; // NOLINT +using android::base::StringPrintf; // NOLINT + +namespace iorap::inode2filename { + +#define DEBUG_INODE_SET 0 + +// A multimap of 'ino_t -> List[Inode]' (where the value Inodes have the same ino_t as the key). +// +// A flat list of Inodes is turned into the above map, then keys can be removed one at a time +// until the InodeSet eventually becomes empty. +struct InodeSet { + + InodeSet() = default; +#if DEBUG_INODE_SET + InodeSet(const InodeSet& other) { + LOG(INFO) << "InodeSet-copyctor"; + set_ = other.set_; + } + + InodeSet(InodeSet&& other) { + LOG(INFO) << "InodeSet-movector"; + set_ = std::move(other.set_); + } + + InodeSet& operator=(const InodeSet& other) { + LOG(INFO) << "InodeSet-opassign-copy"; + set_ = other.set_; + return *this; + } + + InodeSet& operator=(InodeSet&& other) { + LOG(INFO) << "InodeSet-opassign-move"; + set_ = std::move(other.set_); + return *this; + } +#else + InodeSet(InodeSet&& other) = default; + InodeSet& operator=(InodeSet&& other) = default; + // Copying InodeSet can be very expensive, refuse to even allow compiling such code. + InodeSet(const InodeSet& other) = delete; + InodeSet& operator=(const InodeSet& other) = delete; +#endif + + struct ValueRange { + auto/*Iterable<Inode>*/ begin() { + return begin_; + } + + auto/*Iterable<Inode>*/ end() { + return end_; + } + + bool empty() const { + return begin_ == end_; + } + + explicit operator bool() const { + return !empty(); + } + + std::unordered_multimap<ino_t, Inode>::iterator begin_, end_; + + friend std::ostream& operator<<(std::ostream& os, const ValueRange& s); + }; + + // Create an observable that emits the remaining inodes in the map. + // + // Mutation functions must not be called until this observable + // has been finished emitting all values (e.g. with on_completed) since that + // would cause the underlying iterators to go into an undefined state. + auto/*observable<Inode>*/ IterateValues() const { + return rxcpp::observable<>::iterate(set_).map( // XX: should we use identity_immediate here? + [](const std::pair<const ino_t, Inode>& pair) { + return pair.second; + } + ); + // TODO: this would be more efficient as a range-v3 view. + } + + constexpr bool Empty() const { + return set_.empty(); + } + + static InodeSet OfList(const std::vector<Inode>& list) { + InodeSet new_inode_set; + std::unordered_multimap<ino_t, Inode>* map = &new_inode_set.set_; + + for (const Inode& inode : list) { + map->insert({inode.inode, inode}); + } + + return new_inode_set; + } + + // Return an optional list of 'Inode' structs whose 'inode' field matches the 'inode' parameter. + // Returns an empty range if there was nothing found. + ValueRange FindInodeList(ino_t inode) { + auto range = set_.equal_range(inode); + return ValueRange{range.first, range.second}; + } + + // Match all fields of an Inode against a 'struct stat' stat_buf. + // + // The returned Inode (if any) is removed from the InodeSet; it will not be returned by + // FindInodeList in future calls. + std::optional<Inode> FindAndRemoveInodeInList(ValueRange inode_list, + const struct stat& stat_buf) { + LOG(VERBOSE) << "FindAndRemoveInodeInList " << inode_list << ", " + << "stat_buf{st_dev=" << stat_buf.st_dev << ",st_ino=" << stat_buf.st_ino << "}"; + + auto /*iterator*/ found = std::find_if(inode_list.begin(), + inode_list.end(), + [&](const std::pair<ino_t, Inode>& pair) { + const Inode& inode = pair.second; + if (inode.inode != stat_buf.st_ino) { + return false; + } + + dev_t inode_dev = + makedev(static_cast<int>(inode.device_major), static_cast<int>(inode.device_minor)); + + // Inodes could be the same across different devices. + // Also match the device id. + if (inode_dev != stat_buf.st_dev) { + LOG(VERBOSE) << "InodeSet:FindAndRemoveInodeInList matched ino: " << inode.inode + << " but not device" + << ", expected dev: " << stat_buf.st_dev + << ", actual dev: " << inode_dev; + return false; + } + return true; + }); + + if (found != inode_list.end()) { + Inode inode = found->second; + LOG(VERBOSE) << "InodeSet:FindAndRemoveInodeInList *success* inode+device " << inode; + DCHECK(found->second.inode == stat_buf.st_ino); + // Erase the inode from the list. This is important. + set_.erase(found); + return inode; + } + + return std::nullopt; + } + + // Match all fields of an Inode against another Inode. + // + // The returned Inode (if any) is removed from the InodeSet; it will not be returned by + // FindInodeList in future calls. + std::optional<Inode> FindAndRemoveInodeInList(ValueRange inode_list, + const Inode& inode) { + LOG(VERBOSE) << "FindAndRemoveInodeInList " << inode_list << ", " + << inode << "}"; + + auto /*iterator*/ found = std::find_if(inode_list.begin(), + inode_list.end(), + [&](const std::pair<ino_t, Inode>& pair) { + return inode == pair.second; + }); + + if (found != inode_list.end()) { + Inode inode = found->second; + LOG(VERBOSE) << "InodeSet:FindAndRemoveInodeInList *success* inode+device " << inode; + DCHECK_EQ(found->second, inode); + // Erase the inode from the list. This is important. + set_.erase(found); + return inode; + } + + return std::nullopt; + } + + // TODO: equality and string operators for testing/logging. + private: + // Explanation: readdir returns a 'file' -> 'ino_t inode' mapping. + // + // However inodes can be reused on different partitions (but they have a different device number). + // To handle this edge case, and to avoid calling stat whenever the inode definitely doesn't match + // store the inodes into a single-key,multi-value container. + // + // This enables fast scanning of readdir results by matching just the 'inode' portion, + // then calling stat only when the inode portion definitely matches to confirm the device. + + // There are no single-key multi-value containers in standard C++, so pretend + // we have one by writing this simple facade around an unordered set. + // + // We expect that the vector size is usually size=1 (or 2 or 3) since the # of devices + // is fixed by however many partitions there are on the system, AND the same inode # + // would have to be reused across a different file. + std::unordered_multimap<ino_t, Inode> set_; // TODO: Rename to map_. + + friend std::ostream& operator<<(std::ostream& os, const InodeSet& s); +}; + +std::ostream& operator<<(std::ostream& os, const InodeSet& s) { + os << "InodeSet{"; + for (const auto& kv : s.set_) { + // e.g. "123=>(1:2:123)" ... its expected for the 'ino_t' portion to be repeated. + os << "" << kv.first << "=>(" << kv.second << "),"; + } + os << "}"; + return os; +} + +std::ostream& operator<<(std::ostream& os, const InodeSet::ValueRange& v) { + // Don't want to make a const and non const version of ValueRange. + InodeSet::ValueRange& s = const_cast<InodeSet::ValueRange&>(v); + + os << "InodeSet::ValueRange{"; + for (const auto& kv : s) { + // e.g. "123=>(1:2:123)" ... its expected for the 'ino_t' portion to be repeated. + os << "" << kv.first << "=>(" << kv.second << "),"; + } + os << "}"; + return os; +} + +void search_for_inodes_in(std::vector<Inode>& inode_list, const std::string& dirpath); + +enum DirectoryEntryErrorCode { + kInvalid, // not a real error code. to detect bad initialization. + kOpenDir, // opendir failed. + kReadDir, // readdir failed. + kDtUnknown, // d_type was DT_UNKNOWN error. +}; + +struct DirectoryEntryError { + DirectoryEntryErrorCode code; + int err_no; + std::string filename; +}; + +std::ostream& operator<<(std::ostream& os, const DirectoryEntryError& e) { + os << "DirectoryEntryError{" + << static_cast<int>(e.code) << "," << e.err_no << "," << e.filename << "}"; + return os; + // TODO: pretty-print code and err-no +} + +static common::DebugCounter gDebugDirectoryEntryCounter{}; +static constexpr bool kDebugDirectoryEntry = false; + +#define DIRECTORY_ENTRY_MOVE_DCHECK() \ + DCHECK_EQ(other.moved_from_, false) << __PRETTY_FUNCTION__ << "CNT:" << other.debug_counter_; +#define DIRECTORY_ENTRY_TRACE_CTOR() \ + if (kDebugDirectoryEntry) LOG(VERBOSE) << __PRETTY_FUNCTION__ << "@CNT:" << debug_counter_ + +struct DirectoryEntry { + using ResultT = iorap::expected<DirectoryEntry, DirectoryEntryError>; + using ObservableT = rx::observable<ResultT>; + + static constexpr ino_t kInvalidIno = std::numeric_limits<ino_t>::max(); + static constexpr auto kInvalidFileName = ""; + + // Path to file, the prefix is one of the root directories. + std::string filename{kInvalidFileName}; + // Inode number of the file. Not unique across different devices. + ino_t d_ino{kInvalidIno}; + // File type (DT_LNK, DT_REG, DT_DIR, or DT_UNKNOWN) + unsigned char d_type{DT_UNKNOWN}; // Note: not seen outside of sentinel roots. + // TODO: Consider invariant checks for valid combinations of above fields? + + // Debug-only flags. + bool moved_from_{false}; + size_t debug_counter_{0}; + + private: + // TODO: remove default constructor? + // + // SEEMS TO BE USED by std::vector etc. FIX DAT. + DirectoryEntry() noexcept { + debug_counter_ = gDebugDirectoryEntryCounter++; + DIRECTORY_ENTRY_TRACE_CTOR(); + } + public: + DirectoryEntry(std::string filename, ino_t d_ino, unsigned char d_type) noexcept + : filename{std::move(filename)}, + d_ino{d_ino}, + d_type{d_type} { + debug_counter_ = gDebugDirectoryEntryCounter++; + DIRECTORY_ENTRY_TRACE_CTOR(); + } + + DirectoryEntry(const DirectoryEntry& other) noexcept { + // Do not use member-initialization syntax so that this DCHECK can execute first. + DIRECTORY_ENTRY_MOVE_DCHECK(); + + filename = other.filename; + d_ino = other.d_ino; + d_type = other.d_type; + children_paths_ = other.children_paths_; + children_initialized_ = other.children_initialized_; + debug_counter_ = other.debug_counter_; + DIRECTORY_ENTRY_TRACE_CTOR(); + } + + DirectoryEntry& operator=(const DirectoryEntry& other) noexcept { + if (this == &other) { + return *this; + } + + DIRECTORY_ENTRY_MOVE_DCHECK(); + + filename = other.filename; + d_ino = other.d_ino; + d_type = other.d_type; + children_paths_ = other.children_paths_; + children_initialized_ = other.children_initialized_; + debug_counter_ = other.debug_counter_; + DIRECTORY_ENTRY_TRACE_CTOR(); + + return *this; + } + + DirectoryEntry& operator=(DirectoryEntry&& other) noexcept { + if (this == &other) { + return *this; + } + + DIRECTORY_ENTRY_MOVE_DCHECK(); + + filename = std::move(other.filename); + d_ino = other.d_ino; + d_type = other.d_type; + children_paths_ = std::move(other.children_paths_); + children_initialized_ = other.children_initialized_; + debug_counter_ = other.debug_counter_; + DIRECTORY_ENTRY_TRACE_CTOR(); + + return *this; + } + + DirectoryEntry(DirectoryEntry&& other) noexcept { + DIRECTORY_ENTRY_MOVE_DCHECK(); + other.moved_from_ = true; + + filename = std::move(other.filename); + d_ino = other.d_ino; + d_type = other.d_type; + children_paths_ = std::move(other.children_paths_); + children_initialized_ = other.children_initialized_; + debug_counter_ = other.debug_counter_; + DIRECTORY_ENTRY_TRACE_CTOR(); + } + + // Create a sentinel (root of roots) whose children entries are those specified by + // children_paths. + static DirectoryEntry CreateSentinel(std::vector<std::string> children_paths) { + DirectoryEntry e; + e.d_type = DT_DIR; + ++gDebugDirectoryEntryCounter; + + for (std::string& child_path : children_paths) { + // TODO: Should we call Stat on the child path here to reconstitute the ino_t for a root dir? + // Otherwise it can look a little strange (i.e. the root dir itself will never match + // the searched inode). + // + // Probably not too big of a problem in practice. + DirectoryEntry child_entry{std::move(child_path), kInvalidIno, DT_DIR}; + ResultT child_entry_as_result{std::move(child_entry)}; + e.children_paths_.push_back(std::move(child_entry_as_result)); + } + + e.children_initialized_ = true; + + return e; + } + + // Return an observable which emits the direct children only. + // The children entries are now read from disk (with readdir) if they weren't read previously. + std::vector<ResultT> GetChildrenEntries(borrowed<SystemCall*> system_call) const& { + BuildChildrenPaths(system_call); + return children_paths_; + } + + // Return an observable which emits the direct children only. + // The children entries are now read from disk (with readdir) if they weren't read previously. + // Movable overload. + std::vector<ResultT> GetChildrenEntries(borrowed<SystemCall*> system_call) && { + BuildChildrenPaths(system_call); + return std::move(children_paths_); + } + + // Returns a (lazy) observable that emits every single node, in pre-order, + // rooted at this tree. + // + // New entries are only read from disk (with e.g. readdir) when more values are pulled + // from the observable. Only the direct children of any entry are read at any time. + // + // The emission can be stopped prematurely by unsubscribing from the observable. + // This means the maximum amount of 'redundant' IO reads is bounded by the children count + // of all entries emitted thus far minus entries actually emitted. + ObservableT GetSubTreePreOrderEntries(borrowed<SystemCall*> system_call) const; + + private: + // Out-of-line definition to avoid circular type dependency. + void BuildChildrenPaths(borrowed<SystemCall*> system_call) const; + + // We need to lazily initialize children_paths_ only when we try to read them. + // + // Assuming the underlying file system doesn't change (which isn't strictly true), + // the directory children are referentially transparent. + // + // In practice we do not need to distinguish between the file contents changing out + // from under us in this code, so we don't need the more strict requirements. + mutable std::vector<ResultT> children_paths_; + mutable bool children_initialized_{false}; + + friend std::ostream& operator<<(std::ostream& os, const DirectoryEntry& d); +}; + +std::ostream& operator<<(std::ostream& os, const DirectoryEntry& d) { + os << "DirectoryEntry{" << d.filename << ",ino:" << d.d_ino << ",type:" << d.d_type << "}"; + return os; +} + +using DirectoryEntryResult = DirectoryEntry::ResultT; + +// Read all directory entries and return it as a vector. This must be an eager operation, +// as readdir is not re-entrant. +// +// This could be considered as a limitation from the 'observable' perspective since +// one can end up reading unnecessary extra directory entries that are then never consumed. +// +// The following entries are skipped: +// - '.' self +// - ".." parent +// +// All DT types except the following are removed: +// * DT_LNK - symbolic link (empty children) +// * DT_REG - regular file (empty children) +// * DT_DIR - directory (has children) +static std::vector<DirectoryEntryResult> + ReadDirectoryEntriesFromDirectoryPath(std::string dirpath, borrowed<SystemCall*> system_call) { + DIR *dirp; + struct dirent *dp; + + LOG(VERBOSE) << "ReadDirectoryEntriesFromDirectoryPath(" << dirpath << ")"; + + if ((dirp = system_call->opendir(dirpath.c_str())) == nullptr) { + PLOG(ERROR) << "Couldn't open directory: " << dirpath; + return {DirectoryEntryError{kOpenDir, errno, dirpath}}; + } + + // Read all the results up front because readdir is not re-entrant. + std::vector<DirectoryEntryResult> results; + + // Get full path + the directory entry path. + auto child_path = [&] { return dirpath + "/" + dp->d_name; }; + + do { + errno = 0; + if ((dp = system_call->readdir(dirp)) != nullptr) { + if (dp->d_type == DT_DIR) { + if (strcmp(".", dp->d_name) == 0 || strcmp("..", dp->d_name) == 0) { + LOG(VERBOSE) << "Skip self/parent: " << dp->d_name; + continue; + } + + LOG(VERBOSE) << "Find entry " << child_path() + << ", ino: " << dp->d_ino << ", type: " << dp->d_type; + results.push_back(DirectoryEntry{child_path(), + static_cast<ino_t>(dp->d_ino), + dp->d_type}); + } else if (dp->d_type == DT_UNKNOWN) { + // This seems bad if it happens. We should probably do something about this. + LOG(WARNING) << "Found unknown DT entry: " << child_path(); + + results.push_back(DirectoryEntryError{kDtUnknown, /*errno*/0, child_path()}); + } else if (dp->d_type == DT_LNK || dp->d_type == DT_REG) { + // Regular non-directory file entry. + results.push_back(DirectoryEntry{child_path(), + static_cast<ino_t>(dp->d_ino), + dp->d_type}); + } else { + // Block device, character device, socket, etc... + LOG(VERBOSE) << "Skip DT entry of type: " << dp->d_type << " " << child_path(); + } + } else if (errno != 0) { + PLOG(ERROR) << "Error reading directory entry in " << dirpath; + + results.push_back(DirectoryEntryError{kReadDir, errno, dirpath}); + } + } while (dp != nullptr); + + if (system_call->closedir(dirp) < 0) { + PLOG(ERROR) << "Failed to close directory " << dirpath; + } + + return results; +} + +void DirectoryEntry::BuildChildrenPaths(borrowed<SystemCall*> system_call) const { + if (children_initialized_) { + return; + } + + if (d_type == DT_DIR) { + children_paths_ = ReadDirectoryEntriesFromDirectoryPath(filename, system_call); + // TODO: consider using dependency injection here to substitute this function during testing? + } +} + +struct InodeSearchParameters { + std::vector<Inode> inode_list; + std::vector<std::string> root_dirs; +}; + +// [IN] +// observable: expected<Value, Error>, ... +// [OUT] +// observable: Value, ... +// +// Any encountered 'Error' items are dropped after logging. +template <typename T> +auto MapExpectedOrLog(T&& observable, + ::android::base::LogSeverity log_level) { + return observable.filter([log_level](const auto& result) { + if (result) { + return true; + } else { + LOG(log_level) << result.error(); + return false; + } + }).map([](auto&& result) { + return IORAP_FORWARD_LAMBDA(result).value(); + }); +} + +template <typename T> +auto MapExpectedOrLogError(T&& observable) { + return MapExpectedOrLog(std::forward<T>(observable), ::android::base::ERROR); +} + +template <typename T> +auto MapOptionalOrDrop(T&& observable) { + return observable.filter([](const auto& result) { + return result.has_value(); + }).map([](auto&& result) { + return IORAP_FORWARD_LAMBDA(result).value(); + }); + // TODO: static_assert this isn't used with an unexpected. +} + +template <typename T, typename F> +auto VisitValueOrLogError(T&& expected, F&& visit_func, const char* error_prefix = "") { + if (!expected) { + LOG(ERROR) << error_prefix << " " << expected.error(); + } else { + visit_func(std::forward<T>(expected).value()); + } + // TODO: Could be good to make this more monadic by returning an optional. +} + +template <typename TSimple, typename T, typename F> +void TreeTraversalPreOrderObservableImpl(rx::subscriber<TSimple> dest, T&& node, F&& fn) { + LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (begin) " << __PRETTY_FUNCTION__; + + if (!dest.is_subscribed()) { + LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (unsubscribed)"; + return; + } else { + LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (on_next node)"; + + // Copy the node here. This is less bad than it seems since we haven't yet + // calculated its children (except in the root), so its just doing a shallow memcpy (sizeof(T)). + // + // This assumes the children are calculated lazily, otherwise we'd need to have a separate + // NodeBody class which only holds the non-children elements. + + TSimple copy = std::forward<T>(node); + dest.on_next(std::move(copy)); + + if (!node.has_value()) { + return; + } + + // Whenever we call 'on_next' also check if we end up unsubscribing. + // This avoids the expensive call into the children. + if (!dest.is_subscribed()) { + LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (post-self unsubscribe)"; + return; + } + + // Eagerly get the childrem, moving them instead of copying them. + auto&& children = fn(std::forward<T>(node)); + for (auto&& child : children) { + TreeTraversalPreOrderObservableImpl(dest, IORAP_FORWARD_LAMBDA(child), fn); + // TODO: double check this is doing the std::move properly for rvalues. + + if (!dest.is_subscribed()) { + LOG(VERBOSE) << "TreeTraversalPreOrderObservableImpl (unsubscribed in children)"; + break; + } + }; + } +} + +// Creates an observable over all the nodes in the tree rooted at node. +// fn is a function that returns the children of that node. +// +// The items are emitted left-to-right pre-order, and stop early if the +// observable is unsubscribed from. +// +// Implementation requirement: +// typeof(node) -> expected<V, E> or optional<V> or similar. +// fn(node) -> iterable<typeof(node)> +// +// preorder(self): +// visit(self) +// for child in fn(self): +// preorder(child) +template <typename T, typename F> +auto/*observable<T>*/ TreeTraversalPreOrderObservable(T&& node, F&& fn) { + LOG(VERBOSE) << "TreeTraversalPreOrderObservable: " << __PRETTY_FUNCTION__; + + using T_simple = std::decay_t<T>; + return rx::observable<>::create<T_simple>( + // Copy node to avoid lifetime issues. + [node=node,fn=std::forward<F>(fn)](rx::subscriber<T_simple> dest) { + LOG(VERBOSE) << "TreeTraversalPreOrderObservable (lambda)"; + TreeTraversalPreOrderObservableImpl<T_simple>(dest, + std::move(node), + std::move(fn)); + dest.on_completed(); + } + ); +} + +DirectoryEntry::ObservableT + DirectoryEntry::GetSubTreePreOrderEntries(borrowed<SystemCall*> system_call) const { + return TreeTraversalPreOrderObservable( + DirectoryEntryResult{*this}, + [system_call=system_call](auto/*DirectoryEntryResult*/&& result) + -> std::vector<DirectoryEntryResult> { + if (!result) { + LOG(VERBOSE) << "GetSubTreePreOrderEntries (no value return)"; + // Cannot have children when it was an error. + return {}; + } + return + IORAP_FORWARD_LAMBDA(result) + .value() + .GetChildrenEntries(system_call); + }); +} + +struct StatError { + int err_no; + std::string path_name; +}; + +std::ostream& operator<<(std::ostream& os, const StatError& e) { + os << "StatError{" << e.err_no << "," << e.path_name + << ": " << strerror(e.err_no) << "}"; + return os; +} + +template <typename U = void> // suppress unused warning. +static iorap::expected<struct stat, StatError> Stat(const std::string& path_name, + borrowed<SystemCall*> system_call) { + struct stat statbuf{}; + + // Call stat(2) in live code. Overridden in test code. + if (system_call->stat(path_name.c_str(), /*out*/&statbuf) == 0) { + return statbuf; + } else { + return iorap::unexpected(StatError{errno, path_name}); + } +} + +using StatResult = iorap::expected<struct stat, StatError>; + +// An inode's corresponding filename on the system. +struct SearchMatch { + Inode inode; + // Relative path joined with a root directory. + // + // Use absolute path root dirs to get back absolute path filenames. + // If relative, this is relative to the current working directory. + std::string filename; +}; + +std::ostream& operator<<(std::ostream& os, const SearchMatch& s) { + os << "SearchMatch{" << s.inode << ", " << s.filename << "}"; + return os; +} + +struct SearchState { + // Emit 'match' Inodes corresponding to the ones here. + InodeSet inode_set; + + // An inode matching one of the ones in inode_set was discovered in the most-recently + // emitted SearchState. + // + // The InodeSet removes any matching 'Inode'. + std::optional<SearchMatch> match; + + SearchState() = default; + SearchState(SearchState&& other) = default; + + // Do not copy this because copying InodeSet is excruciatingly slow. + SearchState(const SearchState& other) = delete; + + // TODO: make sure this doesn't copy [inodes], as that would be unnecessarily expensive. +}; + +std::ostream& operator<<(std::ostream& os, const SearchState& s) { + os << "SearchState{match:"; + // Print the 'match' first. The InodeSet could be very large so it could be truncated in logs. + if (s.match) { + os << s.match.value(); + } else { + os << "(none)"; + } + os << ", inode_set:" << s.inode_set << "}"; + return os; +} + +// TODO: write operator<< etc. + +// Return a lazy observable that will search for all filenames whose inodes +// match the inodes in inode_search_list. +// +// Every unmatched inode will be emitted as an unexpected at the end of the stream. +auto/*[observable<InodeResult>, connectable]*/ SearchDirectoriesForMatchingInodes( + std::vector<std::string> root_dirs, + std::vector<Inode> inode_search_list, + borrowed<SystemCall*> system_call) { + + // Create a (lazy) observable that will emit each DirectoryEntry that is a recursive subchild + // of root_dirs. Emission will be stopped when its unsubscribed from. + // + // This is done by calling readdir(3) lazily. + auto/*obs<DirectoryEntry>*/ find_all_subdir_entries = ([&]() { + DirectoryEntry sentinel = DirectoryEntry::CreateSentinel(std::move(root_dirs)); + auto/*obs<DirectoryEntryResult*/ results = sentinel.GetSubTreePreOrderEntries(system_call); + + // Drop any errors by logging them to logcat. "Unwrap" the expected into the underlying data. + auto/*obs<DirectoryEntry*>*/ expected_drop_errors = MapExpectedOrLogError(std::move(results)); + return expected_drop_errors; + })(); + + // DirectoryEntry is missing the dev_t portion, so we may need to call scan(2) again + // to confirm the dev_t. We skip calling scan(2) when the ino_t does not match. + // InodeSet lets us optimally avoid calling scan(2). + std::shared_ptr<SearchState> initial = std::make_shared<SearchState>(); + initial->inode_set = InodeSet::OfList(inode_search_list); + + auto/*[observable<SearchState>,Connectable]*/ search_state_results = find_all_subdir_entries.scan( + std::move(initial), + [system_call=system_call](std::shared_ptr<SearchState> search_state, + const DirectoryEntry& dir_entry) { + LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#Scan " + << dir_entry << ", state: " << *search_state; + + search_state->match = std::nullopt; + + InodeSet* inodes = &search_state->inode_set; + + // Find all the possible inodes across different devices. + InodeSet::ValueRange inode_list = inodes->FindInodeList(dir_entry.d_ino); + + // This directory doesn't correspond to any inodes we are searching for. + if (!inode_list) { + return search_state; + } + + StatResult maybe_stat = Stat(dir_entry.filename, system_call); + VisitValueOrLogError(maybe_stat, [&](const struct stat& stat_buf) { + // Try to match the specific inode. Usually this will not result in a match (nullopt). + std::optional<Inode> inode = inodes->FindAndRemoveInodeInList(inode_list, stat_buf); + + if (inode) { + search_state->match = SearchMatch{inode.value(), dir_entry.filename}; + } + }); + + return search_state; + } + // Avoid exhausting a potentially 'infinite' stream of files by terminating as soon + // as we find every single inode we care about. + ).take_while([](std::shared_ptr<SearchState> state) { + // Also emit the last item that caused the search set to go empty. + bool cond = !state->inode_set.Empty() || state->match; + + if (WOULD_LOG(VERBOSE)) { + static int kCounter = 0; + LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#take_while (" << kCounter++ << + ",is_empty:" + << state->inode_set.Empty() << ", match:" << state->match.has_value(); + } + // Minor O(1) implementation inefficiency: + // (Too minor to fix but it can be strange if looking at the logs or readdir traces). + // + // Note, because we return 'true' after the search set went empty, + // the overall stream graph still pulls from search_state_results exactly once more: + // + // This means that for cond to go to false, we would've read one extra item and then discarded + // it. If that item was the first child of a directory, that means we essentially did + // one redundant pass of doing a readdir. + // In other words if the search set goes to empty while the current item is a directory, + // + // it will definitely readdir on it at least once as we try to get the first child in + // OnTreeTraversal. + // + // This could be fixed with a 'take_until(Predicate)' operator which doesn't discard + // the last item when the condition becomes false. However rxcpp seems to lack this operator, + // whereas RxJava has it. + + if (!cond) { + LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#take_while " + << "should now terminate for " << *state; + } + + return cond; + }).publish(); + // The publish here is mandatory. The stream is consumed twice (once by matched and once by + // unmatched streams). Without the publish, once all items from 'matched' were consumed it would + // start another instance of 'search_state_results' (i.e. it appears as if the search + // is restarted). + // + // By using 'publish', the search_state_results is effectively shared by both downstream nodes. + // Note that this also requires the subscriber to additionally call #connect on the above stream, + // otherwise no work will happen. + + // Lifetime notes: + // + // The the 'SearchState' is emitted into both below streams simultaneously. + // The 'unmatched_inode_values' only touches the inode_set. + // The 'matched_inode_values' only touches the match. + // Either stream can 'std::move' from those fields because they don't move each other's fields. + auto/*observable<InodeResult>*/ matched_inode_values = search_state_results + .filter([](std::shared_ptr<SearchState> search_state) { + return search_state->match.has_value(); }) + .map([](std::shared_ptr<SearchState> search_state) { + return std::move(search_state->match.value()); }) + // observable<SearchMatch> + .map([](SearchMatch search_match) { + return InodeResult::makeSuccess(search_match.inode, std::move(search_match.filename)); + }); // observable<InodeResult> + + auto/*observable<?>*/ unmatched_inode_values = search_state_results + // The 'last' SearchState is the one that contains all the remaining inodes. + .take_last(1) // observable<SearchState> + .flat_map([](std::shared_ptr<SearchState> search_state) { + LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#unmatched -- flat_map"; + // Aside: Could've used a move here if the inodes weren't so lightweight already. + return search_state->inode_set.IterateValues(); }) + // observable<Inode> + .map([](const Inode& inode) { + LOG(VERBOSE) << "SearchDirectoriesForMatchingInodes#unmatched -- map"; + return InodeResult::makeFailure(inode, InodeResult::kCouldNotFindFilename); + }); + // observable<InodeResult> + + // The matched and unmatched InodeResults are emitted together. + // Use merge, not concat, because we need both observables to be subscribed to simultaneously. + + auto/*observable<InodeResult*/ all_inode_results = + matched_inode_values.merge(unmatched_inode_values); + + // Now that all mid-stream observables have been connected, turn the Connectable observable + // into a regular observable. + + // The caller has to call 'connect' on the search_state_results after subscribing + // and before any work can actually start. + return std::make_pair(all_inode_results, search_state_results); +} + +rxcpp::observable<InodeResult> SearchDirectories::FindFilenamesFromInodes( + std::vector<std::string> root_directories, + std::vector<Inode> inode_list, + SearchMode mode) const { + DCHECK(mode == SearchMode::kInProcessDirect) << " other modes not implemented yet"; + + auto/*observable[2]*/ [inode_results, connectable] = SearchDirectoriesForMatchingInodes( + std::move(root_directories), + std::move(inode_list), + system_call_); + + return inode_results.ref_count(connectable); +} + +// I think we could avoid this with auto_connect, which rxcpp doesn't seem to have. +// +// I can't figure out any other way to avoid this, or at least to allow connecting +// on the primary observable (instead of a secondary side-observable). +// +// If using the obvious publish+ref_count then the unmerged stream gets no items emitted into it. +// If tried to ref_count later, everything turns into no-op. +// If trying to call connect too early, the subscribe is missed. +template <typename T> +struct RxAnyConnectableFromObservable : public SearchDirectories::RxAnyConnectable { + virtual void connect() override { + observable.connect(); + } + + virtual ~RxAnyConnectableFromObservable() {} + + RxAnyConnectableFromObservable(rxcpp::connectable_observable<T> observable) + : observable(observable) { + } + + rxcpp::connectable_observable<T> observable; +}; + +// Type deduction helper. +template <typename T> +std::unique_ptr<SearchDirectories::RxAnyConnectable> + MakeRxAnyConnectableFromObservable(rxcpp::connectable_observable<T> observable) { + SearchDirectories::RxAnyConnectable* ptr = new RxAnyConnectableFromObservable<T>{observable}; + return std::unique_ptr<SearchDirectories::RxAnyConnectable>{ptr}; +} + +std::pair<rxcpp::observable<InodeResult>, std::unique_ptr<SearchDirectories::RxAnyConnectable>> + SearchDirectories::FindFilenamesFromInodesPair( + std::vector<std::string> root_directories, + std::vector<Inode> inode_list, + SearchMode mode) const { + DCHECK(mode == SearchMode::kInProcessDirect) << " other modes not implemented yet"; + + auto/*observable[2]*/ [inode_results, connectable] = SearchDirectoriesForMatchingInodes( + std::move(root_directories), + std::move(inode_list), + system_call_); + + std::unique_ptr<SearchDirectories::RxAnyConnectable> connectable_ptr = + MakeRxAnyConnectableFromObservable(connectable.as_dynamic()); + + return {inode_results, std::move(connectable_ptr)}; +} + +rxcpp::observable<InodeResult> + SearchDirectories::FindFilenamesFromInodes(std::vector<std::string> root_directories, + rxcpp::observable<Inode> inodes, + SearchMode mode) const { + + // It's inefficient to search for inodes until the full search list is available, + // so first reduce to a vector so we can access all the inodes simultaneously. + return inodes.reduce(std::vector<Inode>{}, + [](std::vector<Inode> vec, Inode inode) { + vec.push_back(inode); + return vec; + }, + [](std::vector<Inode> v){ + return v; // TODO: use an identity function + }) + .flat_map([root_directories=std::move(root_directories), mode, self=*this] + (std::vector<Inode> vec) { + // All borrowed values (e.g. SystemCall) must outlive the observable. + return self.FindFilenamesFromInodes(root_directories, vec, mode); + } + ); +} + +auto/*[observable<InodeResult>]*/ EmitAllInodesFromDirectories( + std::vector<std::string> root_dirs, + borrowed<SystemCall*> system_call) { + + // Create a (lazy) observable that will emit each DirectoryEntry that is a recursive subchild + // of root_dirs. Emission will be stopped when its unsubscribed from. + // + // This is done by calling readdir(3) lazily. + auto/*obs<DirectoryEntry>*/ find_all_subdir_entries = ([&]() { + DirectoryEntry sentinel = DirectoryEntry::CreateSentinel(std::move(root_dirs)); + auto/*obs<DirectoryEntryResult*/ results = sentinel.GetSubTreePreOrderEntries(system_call); + + // Drop any errors by logging them to logcat. "Unwrap" the expected into the underlying data. + auto/*obs<DirectoryEntry*>*/ expected_drop_errors = MapExpectedOrLogError(std::move(results)); + return expected_drop_errors; + })(); + + // Fill in -1 for the dev_t since readdir only returns the ino_t. + // The caller of this function is expected to call stat(2) later on to fill in + // the full data. + return find_all_subdir_entries.map([](DirectoryEntry e) { + return InodeResult::makeSuccess(Inode::FromDeviceAndInode(-1, e.d_ino), std::move(e.filename)); + }); +} + +rxcpp::observable<InodeResult> + SearchDirectories::ListAllFilenames(std::vector<std::string> root_directories) const { + // TODO: refactor implementation into DiskScanDataSource. + return EmitAllInodesFromDirectories(std::move(root_directories), + /*borrowed*/system_call_); +} + +struct FilterState { + // Emit 'match' Inodes corresponding to the ones here. + InodeSet inode_set; + + // An inode matching one of the ones in inode_set was discovered in the most-recently + // emitted SearchState. + // + // The InodeSet removes any matching 'Inode'. + std::optional<InodeResult> match; + + FilterState() = default; + FilterState(FilterState&& other) = default; + + // Copying the InodeSet is expensive, so forbid any copies. + FilterState(const FilterState& other) = delete; +}; + +std::ostream& operator<<(std::ostream& os, const FilterState& s) { + os << "FilterState{match:"; + // Print the 'match' first. The InodeSet could be very large so it could be truncated in logs. + if (s.match) { + os << s.match.value(); + } else { + os << "(none)"; + } + os << ", inode_set:" << s.inode_set << "}"; + return os; +} + +rxcpp::observable<InodeResult> SearchDirectories::FilterFilenamesForSpecificInodes( + rxcpp::observable<InodeResult> all_inodes, + std::vector<Inode> inode_list, + bool missing_device_number, // missing dev_t portion? + bool needs_verification) const { + // TODO: refactor into InodeResolver + + borrowed<SystemCall*> system_call = system_call_; + + // InodeResult may be missing the dev_t portion, so we may need to call scan(2) again + // to confirm the dev_t. We skip calling scan(2) when the ino_t does not match. + // InodeSet lets us optimally avoid calling scan(2). + std::shared_ptr<FilterState> initial = std::make_shared<FilterState>(); + initial->inode_set = InodeSet::OfList(inode_list); + + auto/*[observable<FilterState>,Connectable]*/ filter_state_results = all_inodes.scan( + std::move(initial), + [system_call, missing_device_number] + (std::shared_ptr<FilterState> filter_state, InodeResult inode_result) { + LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#Scan " + << inode_result << ", state: " << *filter_state; + + filter_state->match = std::nullopt; + + InodeSet* inodes = &filter_state->inode_set; + + // Find all the possible (dev_t, ino_t) potential needles given an ino_t in the haystack. + InodeSet::ValueRange inode_list = inodes->FindInodeList(inode_result.inode.inode); + + // This inode result doesn't correspond to any inodes we are searching for. + if (!inode_list) { + // Drop the result and keep going. + return filter_state; + } + + if (missing_device_number) { + // Need to fill in dev_t by calling stat(2). + VisitValueOrLogError(std::move(inode_result.data), [&](std::string filename) { + StatResult maybe_stat = Stat(filename, system_call); + VisitValueOrLogError(maybe_stat, [&](const struct stat& stat_buf) { + // Try to match the specific inode. Usually this will not result in a match (nullopt). + std::optional<Inode> inode = inodes->FindAndRemoveInodeInList(inode_list, stat_buf); + + if (inode) { + filter_state->match = InodeResult::makeSuccess(inode.value(), std::move(filename)); + } + }); + + // Note: stat errors are logged here to make the error closer to the occurrence. + // In theory, we could just return it as an InodeResult but then the error would + // just get logged elsewhere. + }); + } else { + // Trust the dev_t in InodeResult is valid. Later passes can verify it. + + // Try to match the specific inode. Usually this will not result in a match (nullopt). + std::optional<Inode> inode = + inodes->FindAndRemoveInodeInList(inode_list, inode_result.inode); + + if (inode) { + filter_state->match = inode_result; + } + + // Note that the InodeResult doesn't necessarily need to have a valid filename here. + // If the earlier pass returned an error-ed result, this will forward the error code. + } + + return filter_state; + } + // Avoid exhausting a potentially 'infinite' stream of files by terminating as soon + // as we find every single inode we care about. + ).take_while([](std::shared_ptr<FilterState> state) { + // Also emit the last item that caused the search set to go empty. + bool cond = !state->inode_set.Empty() || state->match; + + if (WOULD_LOG(VERBOSE)) { + static int kCounter = 0; + LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#take_while (" << kCounter++ << + ",is_empty:" + << state->inode_set.Empty() << ", match:" << state->match.has_value(); + } + // Minor O(1) implementation inefficiency: + // (Too minor to fix but it can be strange if looking at the logs or readdir traces). + // + // Note, because we return 'true' after the search set went empty, + // the overall stream graph still pulls from filter_state_results exactly once more: + // + // This means that for cond to go to false, we would've read one extra item and then discarded + // it. If that item was the first child of a directory, that means we essentially did + // one redundant pass of doing a readdir. + // In other words if the search set goes to empty while the current item is a directory, + // + // it will definitely readdir on it at least once as we try to get the first child in + // OnTreeTraversal. + // + // This could be fixed with a 'take_until(Predicate)' operator which doesn't discard + // the last item when the condition becomes false. However rxcpp seems to lack this operator, + // whereas RxJava has it. + + if (!cond) { + LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#take_while " + << "should now terminate for " << *state; + } + + return cond; + }).publish(); + // The publish here is mandatory. The stream is consumed twice (once by matched and once by + // unmatched streams). Without the publish, once all items from 'matched' were consumed it would + // start another instance of 'filter_state_results' (i.e. it appears as if the search + // is restarted). + // + // By using 'publish', the filter_state_results is effectively shared by both downstream nodes. + // Note that this also requires the subscriber to additionally call #connect on the above stream, + // otherwise no work will happen. + + // Lifetime notes: + // + // The the 'FilterState' is emitted into both below streams simultaneously. + // The 'unmatched_inode_values' only touches the inode_set. + // The 'matched_inode_values' only touches the match. + // Either stream can 'std::move' from those fields because they don't move each other's fields. + auto/*observable<InodeResult>*/ matched_inode_values = filter_state_results + .filter([](std::shared_ptr<FilterState> filter_state) { + return filter_state->match.has_value(); }) + .map([](std::shared_ptr<FilterState> filter_state) { + return std::move(filter_state->match.value()); }); + // observable<InodeResult> + + auto/*observable<?>*/ unmatched_inode_values = filter_state_results + // The 'last' FilterState is the one that contains all the remaining inodes. + .take_last(1) // observable<FilterState> + .flat_map([](std::shared_ptr<FilterState> filter_state) { + LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#unmatched -- flat_map"; + // Aside: Could've used a move here if the inodes weren't so lightweight already. + return filter_state->inode_set.IterateValues(); }) + // observable<Inode> + .map([](const Inode& inode) { + LOG(VERBOSE) << "FilterFilenamesForSpecificInodes#unmatched -- map"; + return InodeResult::makeFailure(inode, InodeResult::kCouldNotFindFilename); + }); + // observable<InodeResult> + + // The matched and unmatched InodeResults are emitted together. + // Use merge, not concat, because we need both observables to be subscribed to simultaneously. + + auto/*observable<InodeResult*/ all_inode_results = + matched_inode_values.merge(unmatched_inode_values); + + // Verify the inode results by calling stat(2). + // Unverified results are turned into an error. + + auto/*observable<InodeResult>*/ verified_inode_results = + all_inode_results.map([needs_verification, system_call](InodeResult result) { + if (!needs_verification || !result) { + // Skip verification if requested, or if the result didn't have a filename. + return result; + } + + const std::string& filename = result.data.value(); + StatResult maybe_stat = Stat(filename, system_call); + + if (maybe_stat) + { + if (result.inode == Inode::FromDeviceAndInode(maybe_stat->st_dev, maybe_stat->st_ino)) { + return result; + } else { + LOG(WARNING) + << "FilterFilenamesForSpecificInodes#verified fail out-of-date inode: " << result; + return InodeResult::makeFailure(result.inode, InodeResult::kVerificationFailed); + } + } else { + // Forward stat errors directly, as it could be a missing security rule, + // but turn -ENOENT into casual verification errors. + const StatError& err = maybe_stat.error(); + int error_code = err.err_no; + if (err.err_no == ENOENT) { + error_code = InodeResult::kVerificationFailed; + + // TODO: Don't LOG(WARNING) here because this could be very common if we + // access the data much much later after the initial results were read in. + LOG(WARNING) + << "FilterFilenamesForSpecificInodes#verified fail out-of-date filename: " << result; + } else { + LOG(ERROR) + << "FilterFilenamesForSpecificInodes#verified stat(2) failure: " << err; + } + + return InodeResult::makeFailure(result.inode, error_code); + } + }); + + // Now that all mid-stream observables have been connected, turn the Connectable observable + // into a regular observable. + return verified_inode_results.ref_count(filter_state_results); +} + +rxcpp::observable<InodeResult> SearchDirectories::EmitAllFilenames( + rxcpp::observable<InodeResult> all_inodes, + bool missing_device_number, // missing dev_t portion? + bool needs_verification) const { + // TODO: refactor into InodeResolver + + borrowed<SystemCall*> system_call = system_call_; + + // InodeResult may be missing the dev_t portion, so we may need to call scan(2) again + // to confirm the dev_t. + + using EmitAllState = std::optional<InodeResult>; + + auto/*[observable<FilterState>,Connectable]*/ all_inode_results = all_inodes.map( + [system_call, missing_device_number](InodeResult inode_result) { + LOG(VERBOSE) << "EmitAllFilenames#map " + << inode_result; + + // Could fail if the device number is missing _and_ stat(2) fails. + EmitAllState match = std::nullopt; + + if (missing_device_number) { + // Need to fill in dev_t by calling stat(2). + VisitValueOrLogError(std::move(inode_result.data), [&](std::string filename) { + StatResult maybe_stat = Stat(filename, system_call); + VisitValueOrLogError(maybe_stat, [&](const struct stat& stat_buf) { + Inode inode = Inode::FromDeviceAndInode(stat_buf.st_dev, stat_buf.st_ino); + match = InodeResult::makeSuccess(inode, std::move(filename)); + }); + + // Note: stat errors are logged here to make the error closer to the occurrence. + // In theory, we could just return it as an InodeResult but then the error would + // just get logged elsewhere. + }); + } else { + // Trust the dev_t in InodeResult is valid. Later passes can verify it. + match = std::move(inode_result); + + // Note that the InodeResult doesn't necessarily need to have a valid filename here. + // If the earlier pass returned an error-ed result, this will forward the error code. + } + + return match; // implicit move. + } + ); + + auto/*observable<InodeResult>*/ matched_inode_values = all_inode_results + .filter([](const EmitAllState& filter_state) { return filter_state.has_value(); }) + .map([](EmitAllState& filter_state) { return std::move(filter_state.value()); }); + // observable<InodeResult> + + // Verify the inode results by calling stat(2). + // Unverified results are turned into an error. + + auto/*observable<InodeResult>*/ verified_inode_results = + matched_inode_values.map([needs_verification, system_call](InodeResult result) { + if (!needs_verification || !result) { + // Skip verification if requested, or if the result didn't have a filename. + return result; + } + + const std::string& filename = result.data.value(); + StatResult maybe_stat = Stat(filename, system_call); + + if (maybe_stat) + { + if (result.inode == Inode::FromDeviceAndInode(maybe_stat->st_dev, maybe_stat->st_ino)) { + return result; + } else { + LOG(WARNING) + << "EmitAllFilenames#verified fail out-of-date inode: " << result; + return InodeResult::makeFailure(result.inode, InodeResult::kVerificationFailed); + } + } else { + // Forward stat errors directly, as it could be a missing security rule, + // but turn -ENOENT into casual verification errors. + const StatError& err = maybe_stat.error(); + int error_code = err.err_no; + if (err.err_no == ENOENT) { + error_code = InodeResult::kVerificationFailed; + + // TODO: Don't LOG(WARNING) here because this could be very common if we + // access the data much much later after the initial results were read in. + LOG(WARNING) + << "EmitAllFilenames#verified fail out-of-date filename: " << result; + } else { + LOG(ERROR) + << "EmitAllFilenames#verified stat(2) failure: " << err; + } + + return InodeResult::makeFailure(result.inode, error_code); + } + }); + + // TODO: refactor this function some more with the Find(inode_set) equivalent. + + // Now that all mid-stream observables have been connected, turn the Connectable observable + // into a regular observable. + return verified_inode_results; +} + +} // namespace iorap::inode2filename diff --git a/src/inode2filename/search_directories.h b/src/inode2filename/search_directories.h new file mode 100644 index 0000000..0b4af78 --- /dev/null +++ b/src/inode2filename/search_directories.h @@ -0,0 +1,142 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_INODE2FILENAME_SEARCH_DIRECTORIES_H_ +#define IORAP_SRC_INODE2FILENAME_SEARCH_DIRECTORIES_H_ + +#include "inode2filename/inode.h" +#include "inode2filename/system_call.h" +#include "inode2filename/inode_resolver.h" + +#include <fruit/fruit.h> + +#include <rxcpp/rx.hpp> +namespace iorap::inode2filename { + +// TODO: rename. +using SearchMode = ProcessMode; + +struct SearchDirectories { + // Type-erased subset of rxcpp::connectable_observable<?> + struct RxAnyConnectable { + // Connects to the underlying observable. + // + // This kicks off the graph, streams begin emitting items. + // This method will block until all items have been fully emitted + // and processed by any subscribers. + virtual void connect() = 0; + + virtual ~RxAnyConnectable(){} + }; + + + // Create a cold observable of inode results (a lazy stream) corresponding + // to the inode search list. + // + // A depth-first search is done on each of the root directories (in order), + // until all inodes have been found (or until all directories have been exhausted). + // + // Some internal errors may occur during emission that aren't part of an InodeResult; + // these will be sent to the error logcat and dropped. + // + // Calling this function does not begin the search. + // The returned observable will begin the search after subscribing to it. + // + // The emitted InodeResult stream has these guarantees: + // - All inodes in inode_list will eventually be emitted exactly once in an InodeResult + // - When all inodes are found, directory traversal is halted. + // - The order of emission can be considered arbitrary. + // + // Lifetime rules: + // - The observable must be fully consumed before deleting any of the SearchDirectory's + // borrowed constructor parameters (e.g. the SystemCall). + // - SearchDirectory itself can be deleted at any time after creating an observable. + rxcpp::observable<InodeResult> + FindFilenamesFromInodes(std::vector<std::string> root_directories, + std::vector<Inode> inode_list, + SearchMode mode) const; + + // Create a cold observable of inode results (a lazy stream) corresponding + // to the inode search list. + // + // A depth-first search is done on each of the root directories (in order), + // until all inodes have been found (or until all directories have been exhausted). + // + // Some internal errors may occur during emission that aren't part of an InodeResult; + // these will be sent to the error logcat and dropped. + // + // Calling this function does not begin the search. + // The returned observable will begin the search after subscribing to it. + // + // The emitted InodeResult stream has these guarantees: + // - All inodes in inode_list will eventually be emitted exactly once in an InodeResult + // - When all inodes are found, directory traversal is halted. + // - The order of emission can be considered arbitrary. + // + // Lifetime rules: + // - The observable must be fully consumed before deleting any of the SearchDirectory's + // borrowed constructor parameters (e.g. the SystemCall). + // - SearchDirectory itself can be deleted at any time after creating an observable. + std::pair<rxcpp::observable<InodeResult>, std::unique_ptr<RxAnyConnectable>> + FindFilenamesFromInodesPair(std::vector<std::string> root_directories, + std::vector<Inode> inode_list, + SearchMode mode) const; + + // No items on the output stream will be emitted until 'inodes' completes. + // + // The current algorithm is a naive DFS, so if it began too early it would either + // miss the search items or require traversal restarts. + // + // See above for more details. + rxcpp::observable<InodeResult> + FindFilenamesFromInodes(std::vector<std::string> root_directories, + rxcpp::observable<Inode> inodes, + SearchMode mode) const; + + rxcpp::observable<InodeResult> + ListAllFilenames(std::vector<std::string> root_directories) const; + + rxcpp::observable<InodeResult> FilterFilenamesForSpecificInodes( + // haystack that will be subscribed to until all in inode_list are found. + rxcpp::observable<InodeResult> all_inodes, + // key list: traverse all_inodes until we emit all results from inode_list. + std::vector<Inode> inode_list, + // all_inodes have a missing device number: use stat(2) to fill it in. + bool missing_device_number, + bool needs_verification) const; + + rxcpp::observable<InodeResult> EmitAllFilenames( + // haystack that will be subscribed to until all in inode_list are found. + rxcpp::observable<InodeResult> all_inodes, + // all_inodes have a missing device number: use stat(2) to fill it in. + bool missing_device_number, + bool needs_verification) const; + + // Any borrowed parameters here can also be borrowed by the observables returned by the above + // member functions. + // + // The observables must be fully consumed within the lifetime of the borrowed parameters. + INJECT(SearchDirectories(borrowed<SystemCall*> system_call)) + : system_call_(system_call) {} + + // TODO: is there a way to get rid of this second RxAnyConnectable parameter? + private: + // This gets passed around to lazy lambdas, so we must finish consuming any observables + // before the injected system call is deleted. + borrowed<SystemCall*> system_call_; +}; + +} // namespace iorap::inode2filename + +#endif // IORAP_SRC_INODE2FILENAME_SEARCH_DIRECTORIES_H_ diff --git a/src/inode2filename/system_call.h b/src/inode2filename/system_call.h new file mode 100644 index 0000000..43c371f --- /dev/null +++ b/src/inode2filename/system_call.h @@ -0,0 +1,73 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_INODE2FILENAME_SYSTEM_CALL_H_ +#define IORAP_SRC_INODE2FILENAME_SYSTEM_CALL_H_ + +#include <fruit/fruit.h> + +#include <dirent.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +// Abstract out the system calls behind a virtual interface: +// This enables us to use dependency injection to provide mock implementations +// during tests. +struct SystemCall { + // stat(2) + virtual int stat(const char *pathname, struct stat *statbuf) = 0; + + // opendir(3) + virtual DIR *opendir(const char *name) = 0; + + // readdir(3) + virtual struct dirent *readdir(DIR *dirp) = 0; + + // closedir(3) + virtual int closedir(DIR *dirp) = 0; + + virtual ~SystemCall() {} +}; + +// "Live" implementation that calls down to libc. +struct SystemCallImpl : public SystemCall { + // Marks this constructor as the one to use for injection. + INJECT(SystemCallImpl()) = default; + + // stat(2) + virtual int stat(const char *pathname, struct stat *statbuf) override { + return ::stat(pathname, statbuf); + } + + // opendir(3) + virtual DIR *opendir(const char *name) override { + return ::opendir(name); + } + + // readdir(3) + virtual struct dirent *readdir(DIR *dirp) override { + return ::readdir(dirp); + } + + // closedir(3) + virtual int closedir(DIR *dirp) override { + return ::closedir(dirp); + } + + virtual ~SystemCallImpl() {} +}; + +#endif // IORAP_SRC_INODE2FILENAME_SYSTEM_CALL_H_ + diff --git a/src/iorapd/main.cc b/src/iorapd/main.cc new file mode 100644 index 0000000..a3a63f2 --- /dev/null +++ b/src/iorapd/main.cc @@ -0,0 +1,82 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "binder/iiorap_impl.h" +#include "common/debug.h" +#include "common/loggers.h" +#include "common/property.h" +#include "db/models.h" +#include "manager/event_manager.h" + +#include <android-base/logging.h> +#include <android-base/properties.h> +#include <binder/IPCThreadState.h> +#include <server_configurable_flags/get_flags.h> +#include <utils/Trace.h> + +#include <stdio.h> + +static constexpr const char* kServiceName = iorap::binder::IIorapImpl::getServiceName(); + +int main(int /*argc*/, char** argv) { + bool tracing_allowed = iorap::common::IsTracingEnabled(/*default_value=*/"false"); + bool readahead_allowed = iorap::common::IsReadAheadEnabled(/*default_value*/"false"); + if (!tracing_allowed && !readahead_allowed) { + LOG(INFO) << "Turn off IORap because both tracing and prefetching are off."; + return 0; + } + + if (android::base::GetBoolProperty("iorapd.log.verbose", iorap::kIsDebugBuild)) { + // Show verbose logs if the property is enabled or if we are a debug build. + setenv("ANDROID_LOG_TAGS", "*:v", /*overwrite*/ 1); + } + + // Logs go to system logcat. + android::base::InitLogging(argv, iorap::common::StderrAndLogdLogger{android::base::SYSTEM}); + + LOG(INFO) << kServiceName << " (the prefetchening) firing up"; + { + android::ScopedTrace trace_db_init{ATRACE_TAG_ACTIVITY_MANAGER, "IorapNativeService::db_init"}; + iorap::db::SchemaModel db_schema = + iorap::db::SchemaModel::GetOrCreate( + android::base::GetProperty("iorapd.db.location", + "/data/misc/iorapd/sqlite.db")); + db_schema.MarkSingleton(); + } + + std::shared_ptr<iorap::manager::EventManager> event_manager; + { + android::ScopedTrace trace_start{ATRACE_TAG_ACTIVITY_MANAGER, "IorapNativeService::start"}; + + // TODO: use fruit for this DI. + event_manager = + iorap::manager::EventManager::Create(); + if (!iorap::binder::IIorapImpl::Start(event_manager)) { + LOG(ERROR) << "Unable to start IorapNativeService"; + exit(1); + } + } + + // This must be logged after all other initialization has finished. + LOG(INFO) << kServiceName << " (the prefetchening) readied up"; + + event_manager->Join(); // TODO: shutdown somewhere? + // Block until something else shuts down the binder service. + android::IPCThreadState::self()->joinThreadPool(); + LOG(INFO) << kServiceName << " shutting down"; + + return 0; +} diff --git a/src/maintenance/controller.cc b/src/maintenance/controller.cc new file mode 100644 index 0000000..8929057 --- /dev/null +++ b/src/maintenance/controller.cc @@ -0,0 +1,633 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "compiler/compiler.h" +#include "maintenance/controller.h" + +#include "common/cmd_utils.h" +#include "common/debug.h" +#include "common/expected.h" +#include "common/trace.h" + +#include "db/models.h" +#include "inode2filename/inode.h" +#include "inode2filename/search_directories.h" +#include "prefetcher/read_ahead.h" + +#include <android-base/file.h> +#include <utils/Printer.h> + +#include <chrono> +#include <ctime> +#include <iostream> +#include <filesystem> +#include <fstream> +#include <limits> +#include <mutex> +#include <optional> +#include <vector> +#include <string> +#include <sys/wait.h> + +namespace iorap::maintenance { + +const constexpr int64_t kCompilerCheckIntervalMs = 10; +static constexpr size_t kMinTracesForCompilation = 1; +const constexpr char* kDenyListFilterDexFiles = "[.](art|oat|odex|vdex|dex)$"; + +struct LastJobInfo { + time_t last_run_ns_{0}; + size_t activities_last_compiled_{0}; +}; + +LastJobInfo last_job_info_; +std::mutex last_job_info_mutex_; + +// Gets the path of output compiled trace. +db::CompiledTraceFileModel CalculateNewestFilePath( + const std::string& package_name, + const std::string& activity_name, + int version) { + db::VersionedComponentName versioned_component_name{ + package_name, activity_name, version}; + + db::CompiledTraceFileModel output_file = + db::CompiledTraceFileModel::CalculateNewestFilePath(versioned_component_name); + + return output_file; +} + +using ArgString = const char*; + +static constexpr const char kCommandFileName[] = "/system/bin/iorap.cmd.compiler"; + +int Exec::Execve(const std::string& pathname, + std::vector<std::string>& argv_vec, + char *const envp[]) { + std::unique_ptr<ArgString[]> argv_ptr = + common::VecToArgv(kCommandFileName, argv_vec); + + return execve(pathname.c_str(), (char**)argv_ptr.get(), envp); +} + +pid_t Exec::Fork() { + return fork(); +} + +// Represents the parameters used when fork+exec compiler. +struct CompilerForkParameters { + std::vector<std::string> input_pbs; + std::vector<uint64_t> timestamp_limit_ns; + std::string output_proto; + std::vector<int32_t> pids; + ControllerParameters controller_params; + + CompilerForkParameters(const std::vector<compiler::CompilationInput>& perfetto_traces, + const std::string& output_proto, + ControllerParameters controller_params) : + output_proto(output_proto), controller_params(controller_params) { + for (compiler::CompilationInput perfetto_trace : perfetto_traces) { + input_pbs.push_back(perfetto_trace.filename); + timestamp_limit_ns.push_back(perfetto_trace.timestamp_limit_ns); + pids.push_back(perfetto_trace.pid); + } + } +}; + +std::vector<std::string> MakeCompilerParams(const CompilerForkParameters& params) { + std::vector<std::string> argv; + ControllerParameters controller_params = params.controller_params; + + common::AppendArgsRepeatedly(argv, params.input_pbs); + common::AppendArgsRepeatedly(argv, "--timestamp_limit_ns", params.timestamp_limit_ns); + common::AppendArgsRepeatedly(argv, "--pid", params.pids); + + if (controller_params.output_text) { + argv.push_back("--output-text"); + } + + common::AppendArgs(argv, "--output-proto", params.output_proto); + + if (controller_params.inode_textcache) { + common::AppendArgs(argv, "--inode-textcache", *controller_params.inode_textcache); + } + + if (controller_params.verbose) { + argv.push_back("--verbose"); + } + + if (controller_params.exclude_dex_files) { + common::AppendArgs(argv, "--denylist-filter", kDenyListFilterDexFiles); + } + + return argv; +} + +// Sets a watch dog for the given pid and kill it if timeout. +std::thread SetTimeoutWatchDog(pid_t pid, int64_t timeout_ms, std::atomic<bool>& cancel_watchdog) { + std::thread watchdog_thread{[pid, timeout_ms, &cancel_watchdog]() { + std::chrono::time_point start = std::chrono::system_clock::now(); + std::chrono::milliseconds timeout(timeout_ms); + while (!cancel_watchdog) { + int status = kill(pid, 0); + if (status != 0) { + LOG(DEBUG) << "Process (" << pid << ") doesn't exist now."; + break; + } + std::chrono::time_point cur = std::chrono::system_clock::now(); + if (cur - start > timeout) { + LOG(INFO) << "Process (" << pid << ") is timeout!"; + LOG(INFO) << "start time: " + << std::chrono::system_clock::to_time_t(start) + << " end time: " + << std::chrono::system_clock::to_time_t(cur) + << " timeout: " + << timeout_ms; + kill(pid, SIGKILL); + break; + } + usleep(kCompilerCheckIntervalMs * 1000); + } + }}; + + return watchdog_thread; +} + +bool StartViaFork(const CompilerForkParameters& params) { + const ControllerParameters& controller_params = params.controller_params; + pid_t child = controller_params.exec->Fork(); + + if (child == -1) { + LOG(FATAL) << "Failed to fork a process for compilation"; + } else if (child > 0) { // we are the caller of this function + LOG(DEBUG) << "forked into a process for compilation , pid = " << child; + + int64_t compiler_timeout_ms = + android::base::GetIntProperty("iorapd.maintenance.compiler_timeout_ms", + /*default*/ 10 * 60 * 1000); // 10 min + std::atomic<bool> cancel_watchdog(false); + std::thread watchdog_thread = SetTimeoutWatchDog(child, compiler_timeout_ms, cancel_watchdog); + int wstatus; + waitpid(child, /*out*/&wstatus, /*options*/0); + + // Terminate the thread after the compiler process is killed or done. + LOG(DEBUG) << "Terminate the watch dog thread."; + cancel_watchdog = true; + watchdog_thread.join(); + + if (!WIFEXITED(wstatus)) { + LOG(ERROR) << "Child terminated abnormally, status: " << WEXITSTATUS(wstatus); + return false; + } + + int status = WEXITSTATUS(wstatus); + LOG(DEBUG) << "Child terminated, status: " << status; + if (status == 0) { + LOG(DEBUG) << "Iorap compilation succeeded"; + return true; + } else { + LOG(ERROR) << "Iorap compilation failed"; + return false; + } + } else { + // we are the child that was forked. + std::vector<std::string> argv_vec = MakeCompilerParams(params); + std::unique_ptr<ArgString[]> argv_ptr = + common::VecToArgv(kCommandFileName, argv_vec); + + std::stringstream argv; // for debugging. + for (std::string arg : argv_vec) { + argv << arg << ' '; + } + LOG(DEBUG) << "fork+exec: " << kCommandFileName << " " << argv.str(); + + controller_params.exec->Execve(kCommandFileName, + argv_vec, + /*envp*/nullptr); + // This should never return. + } + return false; +} + +// Gets the perfetto trace infos in the histories. +std::vector<compiler::CompilationInput> GetPerfettoTraceInfo( + const db::DbHandle& db, + const std::vector<db::AppLaunchHistoryModel>& histories) { + std::vector<compiler::CompilationInput> perfetto_traces; + + for(db::AppLaunchHistoryModel history : histories) { + // Get perfetto trace. + std::optional<db::RawTraceModel> raw_trace = + db::RawTraceModel::SelectByHistoryId(db, history.id); + if (!raw_trace) { + // This is normal: non-cold launches do not have traces. + continue; + } + + if (!history.pid) { + LOG(DEBUG) << "Missing pid for history " << history.id; + continue; + } + + uint64_t timestamp_limit = std::numeric_limits<uint64_t>::max(); + // Get corresponding timestamp limit. + if (history.report_fully_drawn_ns) { + timestamp_limit = *history.report_fully_drawn_ns; + } else if (history.total_time_ns) { + timestamp_limit = *history.total_time_ns; + } else { + LOG(DEBUG) << " No timestamp exists. Using the max value."; + } + perfetto_traces.push_back({raw_trace->file_path, timestamp_limit, history.pid}); + } + return perfetto_traces; +} + +// Helper struct for printing vector. +template <class T> +struct VectorPrinter { + std::vector<T>& values; +}; + +std::ostream& operator<<(std::ostream& os, + const struct compiler::CompilationInput& perfetto_trace) { + os << "file_path: " << perfetto_trace.filename << " " + << "timestamp_limit: " << perfetto_trace.timestamp_limit_ns; + return os; +} + +template <class T> +std::ostream& operator<<(std::ostream& os, const struct VectorPrinter<T>& printer) { + os << "[\n"; + for (T i : printer.values) { + os << i << ",\n"; + } + os << "]\n"; + return os; +} + +// Compiled the perfetto traces for an activity. +bool CompileActivity(const db::DbHandle& db, + int package_id, + const std::string& package_name, + const std::string& activity_name, + int version, + const ControllerParameters& params) { + ScopedFormatTrace atrace_compile_package(ATRACE_TAG_PACKAGE_MANAGER, + "Compile activity %s", + activity_name.c_str()); + + LOG(DEBUG) << "CompileActivity: " << package_name << "/" << activity_name << "@" << version; + + db::CompiledTraceFileModel output_file = + CalculateNewestFilePath(package_name, activity_name, version); + + std::string file_path = output_file.FilePath(); + + if (!params.recompile) { + if (std::filesystem::exists(file_path)) { + LOG(DEBUG) << "compiled trace exists in " << file_path; + + db::VersionedComponentName vcn{package_name, activity_name, version}; + std::optional<db::PrefetchFileModel> prefetch_file = + db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn); + if (prefetch_file) { + return true; + } else { + LOG(WARNING) << "Missing corresponding prefetch_file db row for " << vcn; + // let it go and compile again. we'll insert the prefetch_file at the bottom. + } + } + } + + std::optional<db::ActivityModel> activity = + db::ActivityModel::SelectByNameAndPackageId(db, activity_name.c_str(), package_id); + if (!activity) { + LOG(ERROR) << "Cannot find activity for package_id: " << package_id + <<" activity_name: " <<activity_name; + return false; + } + + int activity_id = activity->id; + + std::vector<db::AppLaunchHistoryModel> histories = + db::AppLaunchHistoryModel::SelectActivityHistoryForCompile(db, activity_id); + + { + std::vector<compiler::CompilationInput> perfetto_traces = + GetPerfettoTraceInfo(db, histories); + + if (perfetto_traces.size() < params.min_traces) { + LOG(DEBUG) << "The number of perfetto traces is " << perfetto_traces.size() + <<", which is less than " << params.min_traces; + return false; + } + + { + std::lock_guard<std::mutex> last_job_info_guard{last_job_info_mutex_}; + last_job_info_.activities_last_compiled_++; + } + + // Show the compilation config. + LOG(DEBUG) << "Try to compiled package_id: " << package_id + << " package_name: " << package_name + << " activity_name: " << activity_name + << " version: " << version + << " file_path: " << file_path + << " verbose: " << params.verbose + << " perfetto_traces: " + << VectorPrinter<compiler::CompilationInput>{perfetto_traces}; + if (params.inode_textcache) { + LOG(DEBUG) << "inode_textcache: " << *params.inode_textcache; + } + + CompilerForkParameters compiler_params{perfetto_traces, file_path, params}; + + if (!output_file.MkdirWithParents()) { + LOG(ERROR) << "Compile activity failed. Failed to mkdirs " << file_path; + return false; + } + + ScopedFormatTrace atrace_compile_fork(ATRACE_TAG_PACKAGE_MANAGER, + "Fork+exec iorap.cmd.compiler", + activity_name.c_str()); + if (!StartViaFork(compiler_params)) { + LOG(ERROR) << "Compilation failed for package_id:" << package_id + << " activity_name: " << activity_name; + return false; + } + } + + std::optional<db::PrefetchFileModel> compiled_trace = + db::PrefetchFileModel::Insert(db, activity_id, file_path); + if (!compiled_trace) { + LOG(ERROR) << "Cannot insert compiled trace activity_id: " << activity_id + << " file_path: " << file_path; + return false; + } + return true; +} + +// Compiled the perfetto traces for activities in an package. +bool CompilePackage(const db::DbHandle& db, + const std::string& package_name, + int version, + const ControllerParameters& params) { + ScopedFormatTrace atrace_compile_package(ATRACE_TAG_PACKAGE_MANAGER, + "Compile package %s", + package_name.c_str()); + + std::optional<db::PackageModel> package = + db::PackageModel::SelectByNameAndVersion(db, package_name.c_str(), version); + + if (!package) { + LOG(ERROR) << "Cannot find package for package_name: " + << package_name + << " and version " + << version; + return false; + } + + std::vector<db::ActivityModel> activities = + db::ActivityModel::SelectByPackageId(db, package->id); + + bool ret = true; + for (db::ActivityModel activity : activities) { + if (!CompileActivity(db, package->id, package->name, activity.name, version, params)) { + ret = false; + } + } + return ret; +} + +// Compiled the perfetto traces for packages in a device. +bool CompileAppsOnDevice(const db::DbHandle& db, const ControllerParameters& params) { + { + std::lock_guard<std::mutex> last_job_info_guard{last_job_info_mutex_}; + last_job_info_.activities_last_compiled_ = 0; + } + + std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db); + bool ret = true; + for (db::PackageModel package : packages) { + if (!CompilePackage(db, package.name, package.version, params)) { + ret = false; + } + } + + { + std::lock_guard<std::mutex> last_job_info_guard{last_job_info_mutex_}; + last_job_info_.last_run_ns_ = time(nullptr); + } + + return ret; +} + +// Compiled the perfetto traces for a single package in a device. +bool CompileSingleAppOnDevice(const db::DbHandle& db, + const ControllerParameters& params, + const std::string& package_name) { + std::vector<db::PackageModel> packages = db::PackageModel::SelectByName(db, package_name.c_str()); + bool ret = true; + for (db::PackageModel package : packages) { + if (!CompilePackage(db, package.name, package.version, params)) { + ret = false; + } + } + + return ret; +} + +bool Compile(const std::string& db_path, const ControllerParameters& params) { + iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path); + db::DbHandle db{db_schema.db()}; + return CompileAppsOnDevice(db, params); +} + +bool Compile(const std::string& db_path, + const std::string& package_name, + int version, + const ControllerParameters& params) { + iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path); + db::DbHandle db{db_schema.db()}; + return CompilePackage(db, package_name, version, params); +} + +bool Compile(const std::string& db_path, + const std::string& package_name, + const std::string& activity_name, + int version, + const ControllerParameters& params) { + iorap::db::SchemaModel db_schema = db::SchemaModel::GetOrCreate(db_path); + db::DbHandle db{db_schema.db()}; + + std::optional<db::PackageModel> package = + db::PackageModel::SelectByNameAndVersion(db, package_name.c_str(), version); + + if (!package) { + LOG(ERROR) << "Cannot find package with name " + << package_name + << " and version " + << version; + return false; + } + return CompileActivity(db, package->id, package_name, activity_name, version, params); +} + +static std::string TimeToString(time_t the_time) { + tm tm_buf{}; + tm* tm_ptr = localtime_r(&the_time, &tm_buf); + + if (tm_ptr != nullptr) { + char time_buffer[256]; + strftime(time_buffer, sizeof(time_buffer), "%a %b %d %H:%M:%S %Y", tm_ptr); + return std::string{time_buffer}; + } else { + return std::string{"(nullptr)"}; + } +} + +static std::string GetTimestampForPrefetchFile(const db::PrefetchFileModel& prefetch_file) { + std::filesystem::path path{prefetch_file.file_path}; + + std::error_code ec{}; + auto last_write_time = std::filesystem::last_write_time(path, /*out*/ec); + if (ec) { + return std::string("Failed to get last write time: ") + ec.message(); + } + + time_t time = decltype(last_write_time)::clock::to_time_t(last_write_time); + + std::string time_str = TimeToString(time); + return time_str; +} + +void DumpPackageActivity(const db::DbHandle& db, + ::android::Printer& printer, + const db::PackageModel& package, + const db::ActivityModel& activity) { + int package_id = package.id; + const std::string& package_name = package.name; + int package_version = package.version; + const std::string& activity_name = activity.name; + db::VersionedComponentName vcn{package_name, activity_name, package_version}; + + // com.google.Settings/com.google.Settings.ActivityMain@1234567890 + printer.printFormatLine(" %s/%s@%d", + package_name.c_str(), + activity_name.c_str(), + package_version); + + std::optional<db::PrefetchFileModel> prefetch_file = + db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn); + + std::vector<db::AppLaunchHistoryModel> histories = + db::AppLaunchHistoryModel::SelectActivityHistoryForCompile(db, activity.id); + std::vector<compiler::CompilationInput> perfetto_traces = + GetPerfettoTraceInfo(db, histories); + + if (prefetch_file) { + bool exists_on_disk = std::filesystem::exists(prefetch_file->file_path); + + std::optional<size_t> prefetch_byte_sum = + prefetcher::ReadAhead::PrefetchSizeInBytes(prefetch_file->file_path); + + if (exists_on_disk) { + printer.printFormatLine(" Compiled Status: Usable compiled trace"); + } else { + printer.printFormatLine(" Compiled Status: Prefetch file deleted from disk."); + } + + if (prefetch_byte_sum) { + printer.printFormatLine(" Bytes to be prefetched: %zu", *prefetch_byte_sum); + } else { + printer.printFormatLine(" Bytes to be prefetched: (bad file path)" ); + } + + printer.printFormatLine(" Time compiled: %s", + GetTimestampForPrefetchFile(*prefetch_file).c_str()); + printer.printFormatLine(" %s", prefetch_file->file_path.c_str()); + } else { + size_t size = perfetto_traces.size(); + + if (size >= kMinTracesForCompilation) { + printer.printFormatLine(" Compiled Status: Raw traces pending compilation (%zu)", + perfetto_traces.size()); + } else { + size_t remaining = kMinTracesForCompilation - size; + printer.printFormatLine(" Compiled Status: Need %zu more traces for compilation", + remaining); + } + } + + printer.printFormatLine(" Raw traces:"); + printer.printFormatLine(" Trace count: %zu", perfetto_traces.size()); + + for (compiler::CompilationInput& compilation_input : perfetto_traces) { + std::string& raw_trace_file_name = compilation_input.filename; + + printer.printFormatLine(" %s", raw_trace_file_name.c_str()); + } +} + +void DumpPackage(const db::DbHandle& db, + ::android::Printer& printer, + db::PackageModel package) { + std::vector<db::ActivityModel> activities = + db::ActivityModel::SelectByPackageId(db, package.id); + + for (db::ActivityModel& activity : activities) { + DumpPackageActivity(db, printer, package, activity); + } +} + +void DumpAllPackages(const db::DbHandle& db, ::android::Printer& printer) { + printer.printLine("Package history in database:"); + + std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db); + for (db::PackageModel package : packages) { + DumpPackage(db, printer, package); + } + + printer.printLine(""); +} + +void Dump(const db::DbHandle& db, ::android::Printer& printer) { + bool locked = last_job_info_mutex_.try_lock(); + + LastJobInfo info = last_job_info_; + + printer.printFormatLine("Background job:"); + if (!locked) { + printer.printLine(""""" (possible deadlock)"); + } + if (info.last_run_ns_ != time_t{0}) { + std::string time_str = TimeToString(info.last_run_ns_); + + printer.printFormatLine(" Last run at: %s", time_str.c_str()); + } else { + printer.printFormatLine(" Last run at: (None)"); + } + printer.printFormatLine(" Activities last compiled: %zu", info.activities_last_compiled_); + + printer.printLine(""); + + if (locked) { + last_job_info_mutex_.unlock(); + } + + DumpAllPackages(db, printer); +} + +} // namespace iorap::maintenance diff --git a/src/maintenance/controller.h b/src/maintenance/controller.h new file mode 100644 index 0000000..750d0b4 --- /dev/null +++ b/src/maintenance/controller.h @@ -0,0 +1,116 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_MAINTENANCE_COMPILER_CONTROLLER_H_ +#define IORAP_SRC_MAINTENANCE_COMPILER_CONTROLLER_H_ + +#include "db/file_models.h" +#include "inode2filename/inode_resolver.h" + +#include <string> +#include <vector> + +namespace android { +class Printer; +} // namespace android + +namespace iorap::maintenance { + +// Enabling mock for testing purpose. +class IExec { + public: + virtual int Execve(const std::string& pathname, + std::vector<std::string>& argv_vec, + char *const envp[]) = 0; + virtual int Fork() = 0; + virtual ~IExec() = default; +}; + +class Exec : public IExec { + public: + virtual int Execve(const std::string& pathname, + std::vector<std::string>& argv_vec, + char *const envp[]); + virtual int Fork(); +}; + +// Represents the parameters used for compilation controller. +struct ControllerParameters { + bool output_text; + // The path of inode2filepath file. + std::optional<std::string> inode_textcache; + bool verbose; + bool recompile; + uint64_t min_traces; + std::shared_ptr<IExec> exec; + bool exclude_dex_files; + + ControllerParameters(bool output_text, + std::optional<std::string> inode_textcache, + bool verbose, + bool recompile, + uint64_t min_traces, + std::shared_ptr<IExec> exec, + bool exclude_dex_files) : + output_text(output_text), + inode_textcache(inode_textcache), + verbose(verbose), + recompile(recompile), + min_traces(min_traces), + exec(exec), + exclude_dex_files(exclude_dex_files) { + } +}; + +// Control the compilation of perfetto traces in the sqlite db. +// +// The strategy now is to compile all the existing perfetto traces for an activity +// and skip ones if the number of perfetto traces is less than the min_traces. +// +// By default, the program doesn't replace the existing compiled trace, it just +// return true. To force replace the existing compiled trace, set `force` to true. +// +// The timestamp limit of the each perfetto trace is determined by `report_fully_drawn_ns` +// timestamp. If it doesn't exists, use `total_time_ns`. If neither of them exists, +// use the max. + +// Compile all activities of all packages in the database. +bool Compile(const std::string& db_path, const ControllerParameters& params); + +// Compile all activities in the package. +// If the version is not given, an arbitrary package that has the same name is used. +bool Compile(const std::string& db_path, + const std::string& package_name, + int version, + const ControllerParameters& params); + +// Compile trace for the activity. +// If the version is not given, an arbitrary package has the same name is used. +bool Compile(const std::string& db_path, + const std::string& package_name, + const std::string& activity_name, + int version, + const ControllerParameters& params); +// Visible for testing. +bool CompileAppsOnDevice(const db::DbHandle& db, const ControllerParameters& params); + +bool CompileSingleAppOnDevice(const db::DbHandle& db, + const ControllerParameters& params, + const std::string& package_name); + +void Dump(const db::DbHandle& db, ::android::Printer& printer); + +} // iorap::maintenance + +#endif // IORAP_SRC_MAINTENANCE_COMPILER_CONTROLLER_H_ diff --git a/src/maintenance/db_cleaner.cc b/src/maintenance/db_cleaner.cc new file mode 100644 index 0000000..56eff70 --- /dev/null +++ b/src/maintenance/db_cleaner.cc @@ -0,0 +1,66 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "maintenance/db_cleaner.h" + +#include <android-base/file.h> + +#include <cstdio> +#include <filesystem> +#include <fstream> +#include <iostream> +#include <limits> +#include <optional> +#include <string> +#include <vector> + +#include "db/clean_up.h" +#include "db/file_models.h" +#include "db/models.h" + +namespace iorap::maintenance { + +// Enable foreign key restriction. +const constexpr char* kForeignKeyOnSql = "PRAGMA foreign_keys = ON;"; + +void CleanUpDatabase(const db::DbHandle& db, + std::shared_ptr<binder::PackageVersionMap> version_map) { + std::vector<db::PackageModel> packages = db::PackageModel::SelectAll(db); + // Enable cascade deletion. + if (!db::DbQueryBuilder::ExecuteOnce(db, kForeignKeyOnSql)) { + LOG(ERROR) << "Fail to turn on foreign key restraint!"; + } + + for (db::PackageModel package : packages) { + std::optional<int64_t> version = version_map->Find(package.name); + if (!version) { + LOG(DEBUG) << "Fail to find version for package " << package.name + << " with version " << package.version + << ". The package manager may be down."; + continue; + } + // Package is cleanup if it + // * is not in the version map, it may be uninstalled + // * has an different version with the latest one + if (*version != package.version) { + db::CleanUpFilesForPackage(db, package.id, package.name, package.version); + if (!package.Delete()) { + LOG(ERROR) << "Fail to delete package " << package.name + << " with version " << package.version; + } + } + } +} + +} // namespace iorap::maintenance diff --git a/src/maintenance/db_cleaner.h b/src/maintenance/db_cleaner.h new file mode 100644 index 0000000..1168bd5 --- /dev/null +++ b/src/maintenance/db_cleaner.h @@ -0,0 +1,34 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_MAINTENANCE_VERSION_UPDATE_H_ +#define IORAP_SRC_MAINTENANCE_VERSION_UPDATE_H_ + +#include <android/content/pm/IPackageManagerNative.h> + +#include <string> +#include <vector> + +#include "binder/package_version_map.h" +#include "db/file_models.h" + +namespace iorap::maintenance { + +// Clean up the database. +// Remove all relevant data for old-version packages. +void CleanUpDatabase(const db::DbHandle& db, + std::shared_ptr<binder::PackageVersionMap> version_map); +} // namespace iorap::maintenance + +#endif // IORAP_SRC_MAINTENANCE_VERSION_UPDATE_H_ diff --git a/src/maintenance/main.cc b/src/maintenance/main.cc new file mode 100644 index 0000000..8b66069 --- /dev/null +++ b/src/maintenance/main.cc @@ -0,0 +1,198 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/debug.h" +#include "compiler/compiler.h" +#include "maintenance/controller.h" +#include "db/clean_up.h" + +#include <android-base/parseint.h> +#include <android-base/properties.h> +#include <android-base/logging.h> + +#include <iostream> +#include <optional> + +#if defined(IORAP_MAINTENANCE_MAIN) + +namespace iorap::maintenance { + +void Usage(char** argv) { + std::cerr << "Usage: " << argv[0] << " <path of sqlite db>" << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Compile the perfetto trace for an package and activity." << std::endl; + std::cerr << " The info of perfetto trace is stored in the sqlite db." << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Optional flags:" << std::endl; + std::cerr << " --package $,-p $ Package name." << std::endl; + std::cerr << " --version $,-ve $ Package version." << std::endl; + std::cerr << " --activity $,-a $ Activity name." << std::endl; + std::cerr << " --inode-textcache $,-it $ Resolve inode->filename from textcache." << std::endl; + std::cerr << " --help,-h Print this Usage." << std::endl; + std::cerr << " --recompile,-r Force re-compilation, which replace the existing compiled trace ." << std::endl; + std::cerr << " --purge-package,-pp Purge all files associated with a package." << std::endl; + std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; + std::cerr << " --output-text,-ot Output ascii text instead of protobuf (default off)." << std::endl; + std::cerr << " --min_traces,-mt The min number of perfetto traces needed " + << "for compilation (default 1)." << std::endl; + std::cerr << " --exclude-dex-files,-edf Set of exclude dex files" << std::endl; + exit(1); +} + + +int Main(int argc, char** argv){ + android::base::InitLogging(argv); + android::base::SetLogger(android::base::StderrLogger); + + if (argc == 1) { + // Need at least 1 input file to do anything. + Usage(argv); + } + + std::vector<std::string> arg_input_filenames; + std::optional<std::string> arg_package; + std::optional<std::string> arg_purge_package; + int arg_version = -1; + std::optional<std::string> arg_activity; + std::optional<std::string> arg_inode_textcache; + bool recompile = false; + bool enable_verbose = false; + bool arg_output_text = false; + uint64_t arg_min_traces = 1; + bool exclude_dex_files = false; + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + bool has_arg_next = (arg+1)<argc; + std::string arg_next = has_arg_next ? argv[arg+1] : ""; + + if (argstr == "--help" || argstr == "-h") { + Usage(argv); + } else if (argstr == "--package" || argstr == "-p") { + if (!has_arg_next) { + std::cerr << "Missing --package <value>" << std::endl; + return 1; + } + arg_package = arg_next; + ++arg; + } else if (argstr == "--version" || argstr == "-ve") { + if (!has_arg_next) { + std::cerr << "Missing --version <value>" << std::endl; + return 1; + } + int version; + if (!android::base::ParseInt<int>(arg_next, &version)) { + std::cerr << "Invalid --version " << arg_next << std::endl; + return 1; + } + arg_version = version; + ++arg; + } else if (argstr == "--activity" || argstr == "-a") { + if (!has_arg_next) { + std::cerr << "Missing --activity <value>" << std::endl; + return 1; + } + arg_activity = arg_next; + ++arg; + } else if (argstr == "--inode-textcache" || argstr == "-it") { + if (!has_arg_next) { + std::cerr << "Missing --inode-textcache <value>" << std::endl; + return 1; + } + arg_inode_textcache = arg_next; + ++arg; + } else if (argstr == "--purge-package" || argstr == "-pp") { + if (!has_arg_next) { + std::cerr << "Missing --purge-package <value>" << std::endl; + return 1; + } + arg_purge_package = arg_next; + ++arg; + } else if (argstr == "--verbose" || argstr == "-v") { + enable_verbose = true; + } else if (argstr == "--recompile" || argstr == "-r") { + recompile = true; + } else if (argstr == "--output-text" || argstr == "-ot") { + arg_output_text = true; + } else if (argstr == "--min_traces" || argstr == "-mt") { + if (!has_arg_next) { + std::cerr << "Missing --min_traces <value>" << std::endl; + return 1; + } + arg_min_traces = std::stoul(arg_next); + ++arg; + } else if (argstr == "--exclude-dex-files" || argstr == "-edf") { + exclude_dex_files = true; + } else { + arg_input_filenames.push_back(argstr); + } + } + + if (arg_input_filenames.empty()) { + LOG(ERROR) << "Missing filename to a sqlite database."; + Usage(argv); + } else if (arg_input_filenames.size() > 1) { + LOG(ERROR) << "More than one filename to a sqlite database."; + Usage(argv); + } + + std::string db_path = arg_input_filenames[0]; + + if (enable_verbose) { + android::base::SetMinimumLogSeverity(android::base::VERBOSE); + + LOG(VERBOSE) << "Verbose check"; + LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; + } else { + android::base::SetMinimumLogSeverity(android::base::DEBUG); + } + + if (arg_purge_package) { + db::CleanUpFilesForPackage(db_path, *arg_purge_package); + return 0; + // Don't do any more work because SchemaModel can only be created once. + } + + maintenance::ControllerParameters params{ + arg_output_text, + arg_inode_textcache, + enable_verbose, + recompile, + arg_min_traces, + std::make_shared<Exec>(), + exclude_dex_files}; + + int ret_code = 0; + if (arg_package && arg_activity) { + ret_code = !Compile(std::move(db_path), + std::move(*arg_package), + std::move(*arg_activity), + arg_version, + params); + } else if (arg_package) { + ret_code = !Compile(std::move(db_path), std::move(*arg_package), arg_version, params); + } else { + ret_code = !Compile(std::move(db_path), params); + } + return ret_code; +} + +} // iorap::maintenance + +int main(int argc, char** argv) { + return ::iorap::maintenance::Main(argc, argv); +} + + +#endif // IORAP_MAINTENANCE_MAIN diff --git a/src/manager/event_manager.cc b/src/manager/event_manager.cc new file mode 100644 index 0000000..0242c10 --- /dev/null +++ b/src/manager/event_manager.cc @@ -0,0 +1,1401 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "binder/package_version_map.h" +#include "common/debug.h" +#include "common/expected.h" +#include "common/printer.h" +#include "common/rx_async.h" +#include "common/property.h" +#include "common/trace.h" +#include "db/app_component_name.h" +#include "db/file_models.h" +#include "db/models.h" +#include "maintenance/controller.h" +#include "maintenance/db_cleaner.h" +#include "manager/event_manager.h" +#include "perfetto/rx_producer.h" +#include "prefetcher/read_ahead.h" +#include "prefetcher/task_id.h" + +#include <android-base/chrono_utils.h> +#include <android-base/strings.h> +#include <android-base/properties.h> +#include <rxcpp/rx.hpp> +#include <server_configurable_flags/get_flags.h> +#include <utils/misc.h> +#include <utils/Trace.h> + +#include <atomic> +#include <filesystem> +#include <functional> +#include <type_traits> +#include <unordered_map> + +using rxcpp::observe_on_one_worker; + +namespace iorap::manager { + +using binder::AppLaunchEvent; +using binder::DexOptEvent; +using binder::JobScheduledEvent; +using binder::RequestId; +using binder::TaskResult; + +using common::AsyncPool; +using common::RxAsync; + +using perfetto::PerfettoStreamCommand; +using perfetto::PerfettoTraceProto; + +using db::AppComponentName; + +const constexpr bool kExcludeDexFilesDefault = true; + +static std::atomic<bool> s_tracing_allowed{false}; +static std::atomic<bool> s_readahead_allowed{false}; +static std::atomic<uint64_t> s_min_traces{3}; + +struct PackageBlacklister { + // "x.y.z;foo.bar.baz" colon-separated list of substrings + PackageBlacklister(std::string blacklist_string) { + LOG(VERBOSE) << "Configuring package blacklister with string: " << blacklist_string; + + std::vector<std::string> split = ::android::base::Split(blacklist_string, ";"); + + // Ignore any l/r whitespace or empty strings. + for (const std::string& s : split) { + std::string t = ::android::base::Trim(s); + if (!t.empty()) { + LOG(INFO) << "Blacklisted package: " << t << "; will not optimize."; + packages_.push_back(t); + } + } + } + + PackageBlacklister() = default; + + bool IsBlacklisted(const std::string& package_name) const { + return std::find(packages_.begin(), packages_.end(), package_name) != packages_.end(); + } + + bool IsBlacklisted(const AppComponentName& component_name) const { + return IsBlacklisted(component_name.package); + } + + bool IsBlacklisted(const std::optional<AppComponentName>& component_name) const { + return component_name.has_value() && IsBlacklisted(component_name->package); + } + + private: + std::vector<std::string> packages_; +}; + +using PackageVersionMap = std::unordered_map<std::string, int64_t>; + +// Main logic of the #OnAppLaunchEvent scan method. +// +// All functions are called from the same thread as the event manager +// functions. +// +// This is a data type, it's moved (std::move) around from one iteration +// of #scan to another. +struct AppLaunchEventState { + std::optional<AppComponentName> component_name_; + // Sequence ID is shared amongst the same app launch sequence, + // but changes whenever a new app launch sequence begins. + size_t sequence_id_ = static_cast<size_t>(-1); + std::optional<AppLaunchEvent::Temperature> temperature_; + + // Push data to perfetto rx chain for associating + // the raw_trace with the history_id. + std::optional<rxcpp::subscriber<int>> history_id_subscriber_; + rxcpp::observable<int> history_id_observable_; + + std::optional<uint64_t> intent_started_ns_; + std::optional<uint64_t> total_time_ns_; + + // Used by kReportFullyDrawn to find the right history_id. + // We assume no interleaving between different sequences. + // This assumption is checked in the Java service code. + std::optional<uint64_t> recent_history_id_; + + // labeled as 'shared' due to rx not being able to handle move-only objects. + // lifetime: in practice equivalent to unique_ptr. + std::shared_ptr<prefetcher::ReadAhead> read_ahead_; + bool allowed_readahead_{true}; + bool is_read_ahead_{false}; + std::optional<prefetcher::TaskId> read_ahead_task_; + + bool allowed_tracing_{true}; + bool is_tracing_{false}; + std::optional<rxcpp::composite_subscription> rx_lifetime_; + std::vector<rxcpp::composite_subscription> rx_in_flight_; + + PackageBlacklister package_blacklister_{}; + + borrowed<perfetto::RxProducerFactory*> perfetto_factory_; // not null + borrowed<observe_on_one_worker*> thread_; // not null + borrowed<observe_on_one_worker*> io_thread_; // not null + borrowed<AsyncPool*> async_pool_; // not null + + std::shared_ptr<binder::PackageVersionMap> version_map_; + + explicit AppLaunchEventState(borrowed<perfetto::RxProducerFactory*> perfetto_factory, + bool allowed_readahead, + bool allowed_tracing, + PackageBlacklister package_blacklister, + borrowed<observe_on_one_worker*> thread, + borrowed<observe_on_one_worker*> io_thread, + borrowed<AsyncPool*> async_pool, + std::shared_ptr<binder::PackageVersionMap> version_map) + : read_ahead_{std::make_shared<prefetcher::ReadAhead>()} + { + perfetto_factory_ = perfetto_factory; + DCHECK(perfetto_factory_ != nullptr); + + allowed_readahead_ = allowed_readahead; + allowed_tracing_ = allowed_tracing; + + package_blacklister_ = package_blacklister; + + thread_ = thread; + DCHECK(thread_ != nullptr); + + io_thread_ = io_thread; + DCHECK(io_thread_ != nullptr); + + async_pool_ = async_pool; + DCHECK(async_pool_ != nullptr); + + version_map_ = version_map; + DCHECK(version_map_ != nullptr); + } + + // Updates the values in this struct only as a side effect. + // + // May create and fire a new rx chain on the same threads as passed + // in by the constructors. + void OnNewEvent(const AppLaunchEvent& event) { + LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent: " << event; + + android::ScopedTrace trace_db_init{ATRACE_TAG_ACTIVITY_MANAGER, + "IorapNativeService::OnAppLaunchEvent"}; + + using Type = AppLaunchEvent::Type; + + DCHECK_GE(event.sequence_id, 0); + sequence_id_ = static_cast<size_t>(event.sequence_id); + allowed_readahead_ = s_readahead_allowed; + allowed_tracing_ = s_tracing_allowed; + + switch (event.type) { + case Type::kIntentStarted: { + const std::string& package_name = event.intent_proto->component().package_name(); + const std::string& class_name = event.intent_proto->component().class_name(); + AppComponentName component_name{package_name, class_name}; + component_name = component_name.Canonicalize(); + component_name_ = component_name; + + if (package_blacklister_.IsBlacklisted(component_name)) { + LOG(DEBUG) << "kIntentStarted: package " << component_name.package + << " ignored due to blacklisting."; + break; + } + + // Create a new history ID chain for each new app start-up sequence. + auto history_id_observable = rxcpp::observable<>::create<int>( + [&](rxcpp::subscriber<int> subscriber) { + history_id_subscriber_ = std::move(subscriber); + LOG(VERBOSE) << " set up the history id subscriber "; + }) + .tap([](int history_id) { LOG(VERBOSE) << " tap rx history id = " << history_id; }) + .replay(1); // Remember the history id in case we subscribe too late. + + history_id_observable_ = history_id_observable; + + // Immediately turn observable hot, creating the subscriber. + history_id_observable.connect(); + + DCHECK(!IsTracing()); + + // The time should be set before perfetto tracing. + // Record the timestamp even no perfetto tracing is triggered, + // because the tracing may start in the following ActivityLaunched + // event. Otherwise, there will be no starting timestamp and + // trace without starting timestamp is not considered for compilation. + if (event.timestamp_nanos >= 0) { + intent_started_ns_ = event.timestamp_nanos; + } else { + LOG(WARNING) << "Negative event timestamp: " << event.timestamp_nanos; + } + break; + } + case Type::kIntentFailed: + if (package_blacklister_.IsBlacklisted(component_name_)) { + LOG(VERBOSE) << "kIntentFailed: package " << component_name_->package + << " ignored due to blacklisting."; + break; + } + + if (history_id_subscriber_) { + history_id_subscriber_->on_error(rxcpp::util::make_error_ptr( + std::ios_base::failure("Aborting due to intent failed"))); + history_id_subscriber_ = std::nullopt; + } + + break; + case Type::kActivityLaunched: { + // TODO add test in Android framework to verify this. + const std::string& title = + event.activity_record_proto->window_token().window_container().identifier().title(); + if (!AppComponentName::HasAppComponentName(title)) { + // Proto comment claim this is sometimes a window title. + // We need the actual 'package/component' here, so just ignore it if it's a title. + LOG(WARNING) << "App launched without a component name: " << event; + break; + } + + AppComponentName component_name = AppComponentName::FromString(title); + component_name = component_name.Canonicalize(); + component_name_ = component_name; + + if (package_blacklister_.IsBlacklisted(component_name_)) { + LOG(VERBOSE) << "kActivityLaunched: package " << component_name_->package + << " ignored due to blacklisting."; + break; + } + + // Cancel tracing for warm/hot. + // Restart tracing if the activity was unexpected. + + AppLaunchEvent::Temperature temperature = event.temperature; + temperature_ = temperature; + if (temperature != AppLaunchEvent::Temperature::kCold) { + LOG(DEBUG) << "AppLaunchEventState#OnNewEvent don't trace due to non-cold temperature"; + } else if (!IsTracing() && !IsReadAhead()) { // and the temperature is Cold. + // Start late trace when intent didn't have a component name + LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent need to start new trace"; + + if (allowed_readahead_ && !IsReadAhead()) { + StartReadAhead(sequence_id_, component_name); + } + if (allowed_tracing_ && !IsTracing() && !IsReadAhead()) { + rx_lifetime_ = StartTracing(std::move(component_name)); + } + } else { + // FIXME: match actual component name against intent component name. + // abort traces if they don't match. + + if (allowed_tracing_) { + LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent already tracing"; + } + LOG(VERBOSE) << "AppLaunchEventState#OnNewEvent already doing readahead"; + } + break; + } + case Type::kActivityLaunchFinished: + if (package_blacklister_.IsBlacklisted(component_name_)) { + LOG(VERBOSE) << "kActivityLaunchFinished: package " << component_name_->package + << " ignored due to blacklisting."; + break; + } + + if (event.timestamp_nanos >= 0) { + total_time_ns_ = event.timestamp_nanos; + } + RecordDbLaunchHistory(event.activity_record_proto->proc_id()); + // Finish tracing and collect trace buffer. + // + // TODO: this happens automatically when perfetto finishes its + // trace duration. + if (IsTracing()) { + MarkPendingTrace(); + } + FinishReadAhead(); + break; + case Type::kActivityLaunchCancelled: + if (package_blacklister_.IsBlacklisted(component_name_)) { + LOG(VERBOSE) << "kActivityLaunchCancelled: package " << component_name_->package + << " ignored due to blacklisting."; + break; + } + + // Abort tracing. + AbortTrace(); + AbortReadAhead(); + break; + case Type::kReportFullyDrawn: { + if (package_blacklister_.IsBlacklisted(component_name_)) { + LOG(VERBOSE) << "kReportFullyDrawn: package " << component_name_->package + << " ignored due to blacklisting."; + break; + } + + if (!recent_history_id_) { + LOG(WARNING) << "Dangling kReportFullyDrawn event"; + return; + } + UpdateReportFullyDrawn(*recent_history_id_, event.timestamp_nanos); + recent_history_id_ = std::nullopt; + break; + } + default: + DCHECK(false) << "invalid type: " << event; // binder layer should've rejected this. + LOG(ERROR) << "invalid type: " << event; // binder layer should've rejected this. + } + } + + // Is there an in-flight readahead task currently? + bool IsReadAhead() const { + return read_ahead_task_.has_value(); + } + + // Gets the compiled trace. + // If a compiled trace exists in sqlite, use that one. Otherwise, try + // to find a prebuilt one. + std::optional<std::string> GetCompiledTrace(const AppComponentName& component_name) { + ScopedFormatTrace atrace_get_compiled_trace(ATRACE_TAG_ACTIVITY_MANAGER, "GetCompiledTrace"); + // Firstly, try to find the compiled trace from sqlite. + android::base::Timer timer{}; + db::DbHandle db{db::SchemaModel::GetSingleton()}; + std::optional<int> version = + version_map_->GetOrQueryPackageVersion(component_name.package); + if (!version) { + LOG(DEBUG) << "The version is NULL, maybe package manager is down."; + return std::nullopt; + } + db::VersionedComponentName vcn{component_name.package, + component_name.activity_name, + *version}; + + std::optional<db::PrefetchFileModel> compiled_trace = + db::PrefetchFileModel::SelectByVersionedComponentName(db, vcn); + + std::chrono::milliseconds duration_ms = timer.duration(); + LOG(DEBUG) << "EventManager: Looking up compiled trace done in " + << duration_ms.count() // the count of ticks. + << "ms."; + + if (compiled_trace) { + if (std::filesystem::exists(compiled_trace->file_path)) { + return compiled_trace->file_path; + } else { + LOG(DEBUG) << "Compiled trace in sqlite doesn't exists. file_path: " + << compiled_trace->file_path; + } + } + + LOG(DEBUG) << "Cannot find compiled trace in sqlite for package_name: " + << component_name.package + << " activity_name: " + << component_name.activity_name; + + // If sqlite doesn't have the compiled trace, try the prebuilt path. + std::string file_path = "/product/iorap-trace/"; + file_path += component_name.ToMakeFileSafeEncodedPkgString(); + file_path += ".compiled_trace.pb"; + + if (std::filesystem::exists(file_path)) { + return file_path; + } + + LOG(DEBUG) << "Prebuilt compiled trace doesn't exists. file_path: " + << file_path; + + return std::nullopt; + } + + void StartReadAhead(size_t id, const AppComponentName& component_name) { + DCHECK(allowed_readahead_); + DCHECK(!IsReadAhead()); + + std::optional<std::string> file_path = GetCompiledTrace(component_name); + if (!file_path) { + LOG(VERBOSE) << "Cannot find a compiled trace."; + return; + } + + prefetcher::TaskId task{id, *file_path}; + read_ahead_->BeginTask(task); + // TODO: non-void return signature? + + read_ahead_task_ = std::move(task); + } + + void FinishReadAhead() { + // if no readahead task exist, do nothing. + if (!IsReadAhead()){ + return; + } + + read_ahead_->FinishTask(*read_ahead_task_); + read_ahead_task_ = std::nullopt; + } + + void AbortReadAhead() { + FinishReadAhead(); + } + + bool IsTracing() const { + return is_tracing_; + } + + std::optional<rxcpp::composite_subscription> StartTracing( + AppComponentName component_name) { + DCHECK(allowed_tracing_); + DCHECK(!IsTracing()); + + std::optional<int> version = + version_map_->GetOrQueryPackageVersion(component_name_->package); + if (!version) { + LOG(DEBUG) << "The version is NULL, maybe package manager is down."; + return std::nullopt; + } + db::VersionedComponentName versioned_component_name{component_name.package, + component_name.activity_name, + *version}; + db::DbHandle db{db::SchemaModel::GetSingleton()}; + { + ScopedFormatTrace atrace_traces_number_check( + ATRACE_TAG_ACTIVITY_MANAGER, "IorapNativeService::CheckPerfettoTracesNnumber"); + // Just return if we have enough perfetto traces. + if (!db::PerfettoTraceFileModel::NeedMorePerfettoTraces( + db, versioned_component_name)) { + return std::nullopt; + } + } + + auto /*observable<PerfettoStreamCommand>*/ perfetto_commands = + rxcpp::observable<>::just(PerfettoStreamCommand::kStartTracing) + // wait 1x + .concat( + // Pick a value longer than the perfetto config delay_ms, so that we send + // 'kShutdown' after tracing has already finished. + rxcpp::observable<>::interval(std::chrono::milliseconds(10000)) + .take(2) // kStopTracing, kShutdown. + .map([](int value) { + // value is 1,2,3,... + return static_cast<PerfettoStreamCommand>(value); // 1,2, ... + }) + ); + + auto /*observable<PerfettoTraceProto>*/ trace_proto_stream = + perfetto_factory_->CreateTraceStream(perfetto_commands); + // This immediately connects to perfetto asynchronously. + // + // TODO: create a perfetto handle earlier, to minimize perfetto startup latency. + + rxcpp::composite_subscription lifetime; + + auto stream_via_threads = trace_proto_stream + .tap([](const PerfettoTraceProto& trace_proto) { + LOG(VERBOSE) << "StartTracing -- PerfettoTraceProto received (1)"; + }) + .combine_latest(history_id_observable_) + .observe_on(*thread_) // All work prior to 'observe_on' is handled on thread_. + .subscribe_on(*thread_) // All work prior to 'observe_on' is handled on thread_. + .observe_on(*io_thread_) // Write data on an idle-class-priority thread. + .tap([](std::tuple<PerfettoTraceProto, int> trace_tuple) { + LOG(VERBOSE) << "StartTracing -- PerfettoTraceProto received (2)"; + }); + + lifetime = RxAsync::SubscribeAsync(*async_pool_, + std::move(stream_via_threads), + /*on_next*/[versioned_component_name] + (std::tuple<PerfettoTraceProto, int> trace_tuple) { + PerfettoTraceProto& trace_proto = std::get<0>(trace_tuple); + int history_id = std::get<1>(trace_tuple); + + db::PerfettoTraceFileModel file_model = + db::PerfettoTraceFileModel::CalculateNewestFilePath(versioned_component_name); + + std::string file_path = file_model.FilePath(); + + ScopedFormatTrace atrace_write_to_file(ATRACE_TAG_ACTIVITY_MANAGER, + "Perfetto Write Trace To File %s", + file_path.c_str()); + + if (!file_model.MkdirWithParents()) { + LOG(ERROR) << "Cannot save TraceBuffer; failed to mkdirs " << file_path; + return; + } + + if (!trace_proto.WriteFullyToFile(file_path)) { + LOG(ERROR) << "Failed to save TraceBuffer to " << file_path; + } else { + LOG(INFO) << "Perfetto TraceBuffer saved to file: " << file_path; + + ScopedFormatTrace atrace_update_raw_traces_table( + ATRACE_TAG_ACTIVITY_MANAGER, + "update raw_traces table history_id = %d", + history_id); + db::DbHandle db{db::SchemaModel::GetSingleton()}; + std::optional<db::RawTraceModel> raw_trace = + db::RawTraceModel::Insert(db, history_id, file_path); + + if (!raw_trace) { + LOG(ERROR) << "Failed to insert raw_traces for " << file_path; + } else { + LOG(VERBOSE) << "Inserted into db: " << *raw_trace; + } + } + }, + /*on_error*/[](rxcpp::util::error_ptr err) { + LOG(ERROR) << "Perfetto trace proto collection error: " << rxcpp::util::what(err); + }); + + is_tracing_ = true; + + return lifetime; + } + + void AbortTrace() { + LOG(VERBOSE) << "AppLaunchEventState - AbortTrace"; + + // if the tracing is not running, do nothing. + if (!IsTracing()){ + return; + } + + is_tracing_ = false; + if (rx_lifetime_) { + // TODO: it would be good to call perfetto Destroy. + + rx_in_flight_.erase(std::remove(rx_in_flight_.begin(), + rx_in_flight_.end(), *rx_lifetime_), + rx_in_flight_.end()); + + LOG(VERBOSE) << "AppLaunchEventState - AbortTrace - Unsubscribe"; + rx_lifetime_->unsubscribe(); + + rx_lifetime_.reset(); + } + } + + void MarkPendingTrace() { + LOG(VERBOSE) << "AppLaunchEventState - MarkPendingTrace"; + DCHECK(is_tracing_); + DCHECK(rx_lifetime_.has_value()); + + if (rx_lifetime_) { + LOG(VERBOSE) << "AppLaunchEventState - MarkPendingTrace - lifetime moved"; + // Don't unsubscribe because that would cause the perfetto TraceBuffer + // to get dropped on the floor. + // + // Instead, we want to let it finish and write it out to a file. + rx_in_flight_.push_back(*std::move(rx_lifetime_)); + rx_lifetime_.reset(); + } else { + LOG(VERBOSE) << "AppLaunchEventState - MarkPendingTrace - lifetime was empty"; + } + + is_tracing_ = false; + // FIXME: how do we clear this vector? + } + + void RecordDbLaunchHistory(int32_t pid) { + std::optional<db::AppLaunchHistoryModel> history = InsertDbLaunchHistory(pid); + + // RecordDbLaunchHistory happens-after kIntentStarted + if (!history_id_subscriber_.has_value()) { + LOG(WARNING) << "Logic error? Should always have a subscriber here."; + return; + } + + // Ensure that the history id rx chain is terminated either with an error or with + // the newly inserted app_launch_histories.id + if (!history) { + history_id_subscriber_->on_error(rxcpp::util::make_error_ptr( + std::ios_base::failure("Failed to insert history id"))); + recent_history_id_ = std::nullopt; + } else { + // Note: we must have already subscribed, or this value will disappear. + LOG(VERBOSE) << "history_id_subscriber on_next history_id=" << history->id; + history_id_subscriber_->on_next(history->id); + history_id_subscriber_->on_completed(); + + recent_history_id_ = history->id; + } + history_id_subscriber_ = std::nullopt; + } + + std::optional<db::AppLaunchHistoryModel> InsertDbLaunchHistory(int32_t pid) { + // TODO: deferred queue into a different lower priority thread. + if (!component_name_ || !temperature_) { + LOG(VERBOSE) << "Skip RecordDbLaunchHistory, no component name available."; + + return std::nullopt; + } + + android::ScopedTrace trace{ATRACE_TAG_ACTIVITY_MANAGER, + "IorapNativeService::RecordDbLaunchHistory"}; + db::DbHandle db{db::SchemaModel::GetSingleton()}; + + using namespace iorap::db; + + std::optional<int> version = + version_map_->GetOrQueryPackageVersion(component_name_->package); + if (!version) { + LOG(DEBUG) << "The version is NULL, maybe package manager is down."; + return std::nullopt; + } + std::optional<ActivityModel> activity = + ActivityModel::SelectOrInsert(db, + component_name_->package, + *version, + component_name_->activity_name); + + if (!activity) { + LOG(WARNING) << "Failed to query activity row for : " << *component_name_; + return std::nullopt; + } + + auto temp = static_cast<db::AppLaunchHistoryModel::Temperature>(*temperature_); + + std::optional<AppLaunchHistoryModel> alh = + AppLaunchHistoryModel::Insert(db, + activity->id, + temp, + IsTracing(), + IsReadAhead(), + intent_started_ns_, + total_time_ns_, + // ReportFullyDrawn event normally occurs after this. Need update later. + /* report_fully_drawn_ns= */ std::nullopt, + pid); + //Repo + if (!alh) { + LOG(WARNING) << "Failed to insert app_launch_histories row"; + return std::nullopt; + } + + LOG(VERBOSE) << "RecordDbLaunchHistory: " << *alh; + return alh; + } + + void UpdateReportFullyDrawn(int history_id, uint64_t timestamp_ns) { + LOG(DEBUG) << "Update kReportFullyDrawn for history_id:" + << history_id + << " timestamp_ns: " + << timestamp_ns; + + android::ScopedTrace trace{ATRACE_TAG_ACTIVITY_MANAGER, + "IorapNativeService::UpdateReportFullyDrawn"}; + db::DbHandle db{db::SchemaModel::GetSingleton()}; + + bool result = + db::AppLaunchHistoryModel::UpdateReportFullyDrawn(db, + history_id, + timestamp_ns); + + if (!result) { + LOG(WARNING) << "Failed to update app_launch_histories row"; + } + } +}; + +struct AppLaunchEventDefender { + binder::AppLaunchEvent::Type last_event_type_{binder::AppLaunchEvent::Type::kUninitialized}; + + enum class Result { + kAccept, // Pass-through the new event. + kOverwrite, // Overwrite the new event with a different event. + kReject // Completely reject the new event, it will not be delivered. + }; + + Result OnAppLaunchEvent(binder::RequestId request_id, + const binder::AppLaunchEvent& event, + binder::AppLaunchEvent* overwrite) { + using Type = binder::AppLaunchEvent::Type; + CHECK(overwrite != nullptr); + + // Ensure only legal transitions are allowed. + switch (last_event_type_) { + case Type::kUninitialized: + case Type::kIntentFailed: + case Type::kActivityLaunchCancelled: + case Type::kReportFullyDrawn: { // From a terminal state, only go to kIntentStarted + if (event.type != Type::kIntentStarted) { + LOG(DEBUG) << "Rejecting transition from " << last_event_type_ << " to " << event.type; + last_event_type_ = Type::kUninitialized; + return Result::kReject; + } else { + LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type; + last_event_type_ = event.type; + return Result::kAccept; + } + } + case Type::kIntentStarted: { + if (event.type == Type::kIntentFailed || + event.type == Type::kActivityLaunched) { + LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type; + last_event_type_ = event.type; + return Result::kAccept; + } else { + LOG(DEBUG) << "Overwriting transition from kIntentStarted to " + << event.type << " into kIntentFailed"; + last_event_type_ = Type::kIntentFailed; + + *overwrite = event; + overwrite->type = Type::kIntentFailed; + return Result::kOverwrite; + } + } + case Type::kActivityLaunched: { + if (event.type == Type::kActivityLaunchFinished || + event.type == Type::kActivityLaunchCancelled) { + LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type; + last_event_type_ = event.type; + return Result::kAccept; + } else { + LOG(DEBUG) << "Overwriting transition from kActivityLaunched to " + << event.type << " into kActivityLaunchCancelled"; + last_event_type_ = Type::kActivityLaunchCancelled; + + *overwrite = event; + overwrite->type = Type::kActivityLaunchCancelled; + return Result::kOverwrite; + } + } + case Type::kActivityLaunchFinished: { + if (event.type == Type::kIntentStarted || + event.type == Type::kReportFullyDrawn) { + LOG(VERBOSE) << "Accept transition from " << last_event_type_ << " to " << event.type; + last_event_type_ = event.type; + return Result::kAccept; + } else { + LOG(DEBUG) << "Rejecting transition from " << last_event_type_ << " to " << event.type; + last_event_type_ = Type::kUninitialized; + return Result::kReject; + } + } + } + } +}; + +// Convert callback pattern into reactive pattern. +struct AppLaunchEventSubject { + using RefWrapper = + std::reference_wrapper<const AppLaunchEvent>; + + AppLaunchEventSubject() {} + + void Subscribe(rxcpp::subscriber<RefWrapper> subscriber) { + DCHECK(ready_ != true) << "Cannot Subscribe twice"; + + subscriber_ = std::move(subscriber); + + // Release edge of synchronizes-with AcquireIsReady. + ready_.store(true); + } + + void OnNext(const AppLaunchEvent& e) { + if (!AcquireIsReady()) { + return; + } + + if (!subscriber_->is_subscribed()) { + return; + } + + /* + * TODO: fix upstream. + * + * Rx workaround: this fails to compile when + * the observable is a reference type: + * + * external/Reactive-Extensions/RxCpp/Rx/v2/src/rxcpp/rx-observer.hpp:354:18: error: multiple overloads of 'on_next' instantiate to the same signature 'void (const iorap::binder::AppLaunchEvent &) const' + * virtual void on_next(T&&) const {}; + * + * external/Reactive-Extensions/RxCpp/Rx/v2/src/rxcpp/rx-observer.hpp:353:18: note: previous declaration is here + * virtual void on_next(T&) const {}; + * + * (The workaround is to use reference_wrapper instead + * of const AppLaunchEvent&) + */ + subscriber_->on_next(std::cref(e)); + + } + + void OnCompleted() { + if (!AcquireIsReady()) { + return; + } + + subscriber_->on_completed(); + } + + private: + bool AcquireIsReady() { + // Synchronizes-with the release-edge in Subscribe. + // This can happen much later, only once the subscription actually happens. + + // However, as far as I know, 'rxcpp::subscriber' is not thread safe, + // (but the observable chain itself can be made thread-safe via #observe_on, etc). + // so we must avoid reading it until it has been fully synchronized. + // + // TODO: investigate rxcpp subscribers and see if we can get rid of this atomics, + // to make it simpler. + return ready_.load(); + } + + // TODO: also track the RequestId ? + + std::atomic<bool> ready_{false}; + + + std::optional<rxcpp::subscriber<RefWrapper>> subscriber_; +}; + +// Convert callback pattern into reactive pattern. +struct JobScheduledEventSubject { + JobScheduledEventSubject() {} + + void Subscribe(rxcpp::subscriber<std::pair<RequestId, JobScheduledEvent>> subscriber) { + DCHECK(ready_ != true) << "Cannot Subscribe twice"; + + subscriber_ = std::move(subscriber); + + // Release edge of synchronizes-with AcquireIsReady. + ready_.store(true); + } + + void OnNext(RequestId request_id, JobScheduledEvent e) { + if (!AcquireIsReady()) { + return; + } + + if (!subscriber_->is_subscribed()) { + return; + } + + subscriber_->on_next(std::pair<RequestId, JobScheduledEvent>{std::move(request_id), std::move(e)}); + + } + + void OnCompleted() { + if (!AcquireIsReady()) { + return; + } + + subscriber_->on_completed(); + } + + private: + bool AcquireIsReady() { + // Synchronizes-with the release-edge in Subscribe. + // This can happen much later, only once the subscription actually happens. + + // However, as far as I know, 'rxcpp::subscriber' is not thread safe, + // (but the observable chain itself can be made thread-safe via #observe_on, etc). + // so we must avoid reading it until it has been fully synchronized. + // + // TODO: investigate rxcpp subscribers and see if we can get rid of this atomics, + // to make it simpler. + return ready_.load(); + } + + // TODO: also track the RequestId ? + + std::atomic<bool> ready_{false}; + + std::optional<rxcpp::subscriber<std::pair<RequestId, JobScheduledEvent>>> subscriber_; +}; + +std::ostream& operator<<(std::ostream& os, const android::content::pm::PackageChangeEvent& event) { + os << "PackageChangeEvent{"; + os << "packageName=" << event.packageName << ","; + os << "version=" << event.version << ","; + os << "lastUpdateTimeMillis=" << event.lastUpdateTimeMillis; + os << "}"; + return os; +} + +class EventManager::Impl { + public: + Impl(/*borrow*/perfetto::RxProducerFactory& perfetto_factory) + : perfetto_factory_(perfetto_factory), + worker_thread_(rxcpp::observe_on_new_thread()), + worker_thread2_(rxcpp::observe_on_new_thread()), + io_thread_(perfetto::ObserveOnNewIoThread()) { + // Try to create version map + RetryCreateVersionMap(); + + iorap::common::StderrLogPrinter printer{"iorapd"}; + RefreshSystemProperties(printer); + + rx_lifetime_ = InitializeRxGraph(); + rx_lifetime_jobs_ = InitializeRxGraphForJobScheduledEvents(); + + android::add_sysprop_change_callback(&Impl::OnSyspropChanged, /*priority*/-10000); + } + + void RetryCreateVersionMap() { + android::base::Timer timer{}; + version_map_ = binder::PackageVersionMap::Create(); + std::chrono::milliseconds duration_ms = timer.duration(); + LOG(DEBUG) << "Got versions for " + << version_map_->Size() + << " packages in " + << duration_ms.count() + << "ms"; + } + + void SetTaskResultCallbacks(std::shared_ptr<TaskResultCallbacks> callbacks) { + DCHECK(callbacks_.expired()); + callbacks_ = callbacks; + } + + void Join() { + async_pool_.Join(); + } + + bool OnAppLaunchEvent(RequestId request_id, + const AppLaunchEvent& event) { + LOG(VERBOSE) << "EventManager::OnAppLaunchEvent(" + << "request_id=" << request_id.request_id << "," + << event; + + // Filter any incoming events through a defender that enforces + // that all state transitions are as contractually documented in + // ActivityMetricsLaunchObserver's javadoc. + AppLaunchEvent overwrite_event{}; + AppLaunchEventDefender::Result result = + app_launch_event_defender_.OnAppLaunchEvent(request_id, event, /*out*/&overwrite_event); + + switch (result) { + case AppLaunchEventDefender::Result::kAccept: + app_launch_event_subject_.OnNext(event); + return true; + case AppLaunchEventDefender::Result::kOverwrite: + app_launch_event_subject_.OnNext(overwrite_event); + return false; + case AppLaunchEventDefender::Result::kReject: + // Intentionally left-empty: we drop the event completely. + return false; + } + + // In theory returns BAD_VALUE to the other side of this binder connection. + // In practice we use 'oneway' flags so this doesn't matter on a regular build. + return false; + } + + bool OnDexOptEvent(RequestId request_id, + const DexOptEvent& event) { + LOG(VERBOSE) << "EventManager::OnDexOptEvent(" + << "request_id=" << request_id.request_id << "," + << event.package_name + << ")"; + + if (common::ExcludeDexFiles(kExcludeDexFilesDefault)) { + LOG(VERBOSE) << "Dex files are excluded. Skip the purging."; + return true; + } + return PurgePackage(event.package_name); + } + + bool OnJobScheduledEvent(RequestId request_id, + const JobScheduledEvent& event) { + LOG(VERBOSE) << "EventManager::OnJobScheduledEvent(" + << "request_id=" << request_id.request_id << ",event=TODO)."; + + job_scheduled_event_subject_.OnNext(std::move(request_id), event); + + return true; // No errors. + } + + bool OnPackageChanged(const android::content::pm::PackageChangeEvent& event) { + LOG(DEBUG) << "Received " << event; + if (event.isDeleted) { + // Do nothing if the package is deleted rignt now. + // The package will be removed from db during maintenance. + return true; + } + // Update the version map. + if (version_map_->Update(event.packageName, event.version)) { + return true; + } + + // Sometimes a package is updated without any version change. + // Clean it up in this case. + db::DbHandle db{db::SchemaModel::GetSingleton()}; + db::CleanUpFilesForPackage(db, event.packageName, event.version); + return true; + } + + void Dump(/*borrow*/::android::Printer& printer) { + ::iorap::prefetcher::ReadAhead::Dump(printer); + ::iorap::perfetto::PerfettoConsumerImpl::Dump(/*borrow*/printer); + ::iorap::maintenance::Dump(db::SchemaModel::GetSingleton(), printer); + } + + rxcpp::composite_subscription InitializeRxGraph() { + LOG(VERBOSE) << "EventManager::InitializeRxGraph"; + + app_launch_events_ = rxcpp::observable<>::create<AppLaunchEventRefWrapper>( + [&](rxcpp::subscriber<AppLaunchEventRefWrapper> subscriber) { + app_launch_event_subject_.Subscribe(std::move(subscriber)); + }); + + rxcpp::composite_subscription lifetime; + + if (!tracing_allowed_) { + LOG(WARNING) << "Tracing disabled by system property"; + } + if (!readahead_allowed_) { + LOG(WARNING) << "Readahead disabled by system property"; + } + + AppLaunchEventState initial_state{&perfetto_factory_, + readahead_allowed_, + tracing_allowed_, + package_blacklister_, + &worker_thread2_, + &io_thread_, + &async_pool_, + version_map_}; + app_launch_events_ + .subscribe_on(worker_thread_) + .scan(std::move(initial_state), + [](AppLaunchEventState state, AppLaunchEventRefWrapper event) { + state.OnNewEvent(event.get()); + return state; + }) + .subscribe(/*out*/lifetime, [](const AppLaunchEventState& state) { + // Intentionally left blank. + (void)state; + }); + + return lifetime; + } + + // Runs the maintenance code to compile perfetto traces to compiled + // trace for a package. + void StartMaintenance(bool output_text, + std::optional<std::string> inode_textcache, + bool verbose, + bool recompile, + uint64_t min_traces, + std::string package_name, + bool should_update_versions) { + ScopedFormatTrace atrace_bg_scope(ATRACE_TAG_PACKAGE_MANAGER, + "Background Job Scope"); + + db::DbHandle db{db::SchemaModel::GetSingleton()}; + if (should_update_versions) { + { + ScopedFormatTrace atrace_update_versions(ATRACE_TAG_PACKAGE_MANAGER, + "Update package versions map cache"); + // Update the version map. + version_map_->UpdateAll(); + } + + { + ScopedFormatTrace atrace_cleanup_db(ATRACE_TAG_PACKAGE_MANAGER, + "Clean up obsolete data in database"); + // Cleanup the obsolete data in the database. + maintenance::CleanUpDatabase(db, version_map_); + } + } + + { + ScopedFormatTrace atrace_compile_apps(ATRACE_TAG_PACKAGE_MANAGER, + "Compile apps on device"); + // Compilation + maintenance::ControllerParameters params{ + output_text, + inode_textcache, + verbose, + recompile, + min_traces, + std::make_shared<maintenance::Exec>(), + common::ExcludeDexFiles(kExcludeDexFilesDefault)}; + + LOG(DEBUG) << "StartMaintenance: min_traces=" << min_traces; + maintenance::CompileSingleAppOnDevice(db, params, package_name); + } + } + + rxcpp::composite_subscription InitializeRxGraphForJobScheduledEvents() { + LOG(VERBOSE) << "EventManager::InitializeRxGraphForJobScheduledEvents"; + + using RequestAndJobEvent = std::pair<RequestId, JobScheduledEvent>; + + job_scheduled_events_ = rxcpp::observable<>::create<RequestAndJobEvent>( + [&](rxcpp::subscriber<RequestAndJobEvent> subscriber) { + job_scheduled_event_subject_.Subscribe(std::move(subscriber)); + }); + + rxcpp::composite_subscription lifetime; + + job_scheduled_events_ + .observe_on(worker_thread_) // async handling. + .tap([this](const RequestAndJobEvent& e) { + LOG(VERBOSE) << "EventManager#JobScheduledEvent#tap(1) - job begins"; + this->NotifyProgress(e.first, TaskResult{TaskResult::State::kBegan}); + + LOG(VERBOSE) << "Compile " << std::get<1>(e).package_name; + StartMaintenance(/*output_text=*/false, + /*inode_textcache=*/std::nullopt, + /*verbose=*/false, + /*recompile=*/false, + s_min_traces, + std::get<1>(e).package_name, + std::get<1>(e).should_update_versions); + + // TODO: probably this shouldn't be emitted until most of the usual DCHECKs + // (for example, validate a job isn't already started, the request is not reused, etc). + // In this way we could block from the client until it sees 'kBegan' and Log.wtf otherwise. + }) + .tap([](const RequestAndJobEvent& e) { + // TODO. Actual work. + LOG(VERBOSE) << "EventManager#JobScheduledEvent#tap(2) - job is being processed"; + + // TODO: abort functionality for in-flight jobs. + // + // maybe something like scan that returns an observable<Job> + flat map to that job. + // then we could unsubscribe from the scan to do a partial abort? need to try it and see if it works. + // + // other option is to create a new outer subscription for each job id which seems less ideal. + }) + .subscribe(/*out*/lifetime, + /*on_next*/ + [this](const RequestAndJobEvent& e) { + LOG(VERBOSE) << "EventManager#JobScheduledEvent#subscribe - job completed"; + this->NotifyComplete(e.first, TaskResult{TaskResult::State::kCompleted}); + } +#if 0 + , + /*on_error*/ + [](rxcpp::util::error_ptr err) { + LOG(ERROR) << "Scheduled job event failed: " << rxcpp::util::what(err); + + //std::shared_ptr<TaskResultCallbacks> callbacks = callbacks_.lock(); + //if (callbacks != nullptr) { + // FIXME: How do we get the request ID back out of the error? Seems like a problem. + // callbacks->OnComplete(, TaskResult{TaskResult::kError}); + // We may have to wrap with an iorap::expected instead of using on_error. + //} + + // FIXME: need to add a 'OnErrorResumeNext' operator? + DCHECK(false) << "forgot to implement OnErrorResumeNext"; + } +#endif + ); + + // TODO: error output should happen via an observable. + + return lifetime; + } + + void NotifyComplete(RequestId request_id, TaskResult result) { + std::shared_ptr<TaskResultCallbacks> callbacks = callbacks_.lock(); + if (callbacks != nullptr) { + callbacks->OnComplete(std::move(request_id), std::move(result)); + } else { + LOG(WARNING) << "EventManager: TaskResultCallbacks may have been released early"; + } + } + + void NotifyProgress(RequestId request_id, TaskResult result) { + std::shared_ptr<TaskResultCallbacks> callbacks = callbacks_.lock(); + if (callbacks != nullptr) { + callbacks->OnProgress(std::move(request_id), std::move(result)); + } else { + LOG(WARNING) << "EventManager: TaskResultCallbacks may have been released early"; + } + } + + static void OnSyspropChanged() { + LOG(DEBUG) << "OnSyspropChanged"; + } + + void RefreshSystemProperties(::android::Printer& printer) { + // TODO: read all properties from one config class. + // PH properties do not work if they contain ".". "_" was instead used here. + tracing_allowed_ = common::IsTracingEnabled(/*default_value=*/"false"); + s_tracing_allowed = tracing_allowed_; + printer.printFormatLine("iorapd.perfetto.enable = %s", tracing_allowed_ ? "true" : "false"); + + readahead_allowed_ = common::IsReadAheadEnabled(/*default_value=*/"false"); + s_readahead_allowed = readahead_allowed_; + printer.printFormatLine("iorapd.readahead.enable = %s", s_readahead_allowed ? "true" : "false"); + + s_min_traces = + ::android::base::GetUintProperty<uint64_t>("iorapd.maintenance.min_traces", /*default*/1); + uint64_t min_traces = s_min_traces; + printer.printFormatLine("iorapd.maintenance.min_traces = %" PRIu64, min_traces); + + printer.printFormatLine("iorapd.exclude_dex_files = %s", + common::ExcludeDexFiles(kExcludeDexFilesDefault) ? "true" : "false"); + + package_blacklister_ = PackageBlacklister{ + /* Colon-separated string list of blacklisted packages, e.g. + * "foo.bar.baz;com.fake.name" would blacklist {"foo.bar.baz", "com.fake.name"} packages. + * + * Blacklisted packages are ignored by iorapd. + */ + server_configurable_flags::GetServerConfigurableFlag( + common::ph_namespace, + "iorap_blacklisted_packages", + ::android::base::GetProperty("iorapd.blacklist_packages", + /*default*/"")) + }; + + LOG(DEBUG) << "RefreshSystemProperties"; + } + + bool PurgePackage(::android::Printer& printer, const std::string& package_name) { + (void)printer; + return PurgePackage(package_name); + } + + bool PurgePackage(const std::string& package_name) { + db::DbHandle db{db::SchemaModel::GetSingleton()}; + db::CleanUpFilesForPackage(db, package_name); + LOG(DEBUG) << "PurgePackage: " << package_name; + return true; + } + + bool CompilePackage(::android::Printer& printer, const std::string& package_name) { + (void)printer; + + ScopedFormatTrace atrace_compile_app(ATRACE_TAG_PACKAGE_MANAGER, + "Compile one app on device"); + + maintenance::ControllerParameters params{ + /*output_text*/false, + /*inode_textcache*/std::nullopt, + WOULD_LOG(VERBOSE), + /*recompile*/false, + s_min_traces, + std::make_shared<maintenance::Exec>(), + common::ExcludeDexFiles(kExcludeDexFilesDefault)}; + + db::DbHandle db{db::SchemaModel::GetSingleton()}; + bool res = maintenance::CompileSingleAppOnDevice(db, std::move(params), package_name); + LOG(DEBUG) << "CompilePackage: " << package_name; + + return res; + } + + bool readahead_allowed_{true}; + + perfetto::RxProducerFactory& perfetto_factory_; + bool tracing_allowed_{true}; + + PackageBlacklister package_blacklister_{}; + + std::weak_ptr<TaskResultCallbacks> callbacks_; // avoid cycles with weakptr. + + using AppLaunchEventRefWrapper = AppLaunchEventSubject::RefWrapper; + rxcpp::observable<AppLaunchEventRefWrapper> app_launch_events_; + AppLaunchEventSubject app_launch_event_subject_; + AppLaunchEventDefender app_launch_event_defender_; + + rxcpp::observable<std::pair<RequestId, JobScheduledEvent>> job_scheduled_events_; + JobScheduledEventSubject job_scheduled_event_subject_; + + rxcpp::observable<RequestId> completed_requests_; + + // regular-priority thread to handle binder callbacks. + observe_on_one_worker worker_thread_; + observe_on_one_worker worker_thread2_; + // low priority idle-class thread for IO operations. + observe_on_one_worker io_thread_; + // async futures pool for async rx operations. + AsyncPool async_pool_; + + rxcpp::composite_subscription rx_lifetime_; // app launch events + rxcpp::composite_subscription rx_lifetime_jobs_; // job scheduled events + + // package version map + std::shared_ptr<binder::PackageVersionMap> version_map_; + +//INTENTIONAL_COMPILER_ERROR_HERE: + // FIXME: + // ok so we want to expose a 'BlockingSubscribe' or a 'Subscribe' or some kind of function + // that the main thread can call. This would subscribe on all the observables we internally + // have here (probably on an event-manager-dedicated thread for simplicity). + // + // ideally we'd just reuse the binder thread to handle the events but I'm not super sure, + // maybe this already works with the identity_current_thread coordination? +}; +using Impl = EventManager::Impl; + +EventManager::EventManager(perfetto::RxProducerFactory& perfetto_factory) + : impl_(new Impl(perfetto_factory)) {} + +std::shared_ptr<EventManager> EventManager::Create() { + static perfetto::PerfettoDependencies::Injector injector{ + perfetto::PerfettoDependencies::CreateComponent + }; + static perfetto::RxProducerFactory producer_factory{ + /*borrow*/injector + }; + return EventManager::Create(/*borrow*/producer_factory); +} + +std::shared_ptr<EventManager> EventManager::Create(perfetto::RxProducerFactory& perfetto_factory) { + std::shared_ptr<EventManager> p{new EventManager{/*borrow*/perfetto_factory}}; + return p; +} + +void EventManager::SetTaskResultCallbacks(std::shared_ptr<TaskResultCallbacks> callbacks) { + return impl_->SetTaskResultCallbacks(std::move(callbacks)); +} + +void EventManager::Join() { + return impl_->Join(); +} + +bool EventManager::OnAppLaunchEvent(RequestId request_id, + const AppLaunchEvent& event) { + return impl_->OnAppLaunchEvent(request_id, event); +} + +bool EventManager::OnDexOptEvent(RequestId request_id, + const DexOptEvent& event) { + return impl_->OnDexOptEvent(request_id, event); +} + +bool EventManager::OnJobScheduledEvent(RequestId request_id, + const JobScheduledEvent& event) { + return impl_->OnJobScheduledEvent(request_id, event); +} + +bool EventManager::OnPackageChanged(const android::content::pm::PackageChangeEvent& event) { + return impl_->OnPackageChanged(event); +} + +void EventManager::Dump(/*borrow*/::android::Printer& printer) { + return impl_->Dump(printer); +} + +void EventManager::RefreshSystemProperties(::android::Printer& printer) { + return impl_->RefreshSystemProperties(printer); +} + +bool EventManager::PurgePackage(::android::Printer& printer, const std::string& package_name) { + return impl_->PurgePackage(printer, package_name); +} + +bool EventManager::CompilePackage(::android::Printer& printer, const std::string& package_name) { + return impl_->CompilePackage(printer, package_name); +} + +} // namespace iorap::manager diff --git a/src/manager/event_manager.h b/src/manager/event_manager.h new file mode 100644 index 0000000..fa09eec --- /dev/null +++ b/src/manager/event_manager.h @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IORAP_MANAGER_EVENT_MANAGER_H_ +#define IORAP_MANAGER_EVENT_MANAGER_H_ + +#include "binder/app_launch_event.h" +#include "binder/dexopt_event.h" +#include "binder/job_scheduled_event.h" +#include "binder/request_id.h" +#include "binder/task_result.h" + +#include <android/content/pm/PackageChangeEvent.h> + +#include <memory> + +namespace android { +class Printer; +} // namespace android + +namespace iorap::perfetto { +struct RxProducerFactory; +} // namespace iorap::perfetto + +namespace iorap::manager { + +// These callbacks are invoked by the EventManager to provide asynchronous notification for the status +// of an event handler. +// +// Calling 'On_Event' in EventManager should be considered merely to start the task. +// Calling 'OnComplete' here is considered to terminate the request (either with a success or error). +// OnProgress is optional, but if it is called it must be called prior to 'OnComplete'. +// +// All callbacks for the same request-id are sequentially consistent. +class TaskResultCallbacks { + public: + virtual void OnProgress(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) {} + virtual void OnComplete(iorap::binder::RequestId request_id, iorap::binder::TaskResult task_result) {} + + virtual ~TaskResultCallbacks() {} +}; + +class EventManager { + public: + static std::shared_ptr<EventManager> Create(); + static std::shared_ptr<EventManager> Create( + /*borrow*/perfetto::RxProducerFactory& perfetto_factory); + void SetTaskResultCallbacks(std::shared_ptr<TaskResultCallbacks> callbacks); + + // Joins any background threads created by EventManager. + void Join(); + + // Handles an AppLaunchEvent: + // + // * Intent starts and app launch starts are treated critically + // and will be handled immediately. This means the caller + // (e.g. the binder pool thread) could be starved in the name + // of low latency. + // + // * Other types are handled in a separate thread. + bool OnAppLaunchEvent(binder::RequestId request_id, + const binder::AppLaunchEvent& event); + + // Handles a DexOptEvent: + // + // Clean up the invalidate traces after package is updated by dexopt. + bool OnDexOptEvent(binder::RequestId request_id, + const binder::DexOptEvent& event); + + + // Handles a JobScheduledEvent: + // + // * Start/stop background jobs (typically for idle maintenance). + // * For example, this could kick off a background compiler. + bool OnJobScheduledEvent(binder::RequestId request_id, + const binder::JobScheduledEvent& event); + + // Handles a PackageChangeEvent: + // + // * The package manager service send this event for package install + // update or delete. + bool OnPackageChanged(const android::content::pm::PackageChangeEvent& event); + + // Print to adb shell dumpsys (for bugreport info). + void Dump(/*borrow*/::android::Printer& printer); + + // A dumpsys --refresh-properties command signaling that we should + // refresh our system properties. + void RefreshSystemProperties(::android::Printer& printer); + + // A dumpsys --purge-package <name> command signaling + // that all db rows and files associated with a package should be deleted. + bool PurgePackage(::android::Printer& printer, const std::string& package_name); + + // A dumpsys --compile-package <name> command signaling + // that a package should be recompiled. + bool CompilePackage(::android::Printer& printer, const std::string& package_name); + + class Impl; + private: + std::unique_ptr<Impl> impl_; + + EventManager(perfetto::RxProducerFactory& perfetto_factory); +}; + +} // namespace iorap::manager + +#endif // IORAP_MANAGER_EVENT_MANAGER_H_ diff --git a/src/perfetto/main.cc b/src/perfetto/main.cc new file mode 100644 index 0000000..0b1056e --- /dev/null +++ b/src/perfetto/main.cc @@ -0,0 +1,244 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//#undef NDEBUG // get DCHECK etc. + + +#include "common/debug.h" +#include "common/expected.h" +#include "perfetto/rx_producer.h" + +#include <android-base/unique_fd.h> +#include <android-base/parseint.h> +#include <android-base/file.h> + +#include "rxcpp/rx.hpp" +#include <iostream> +#include <optional> + +#include <sched.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <syscall.h> +#include <fcntl.h> +#include <unistd.h> + +using namespace iorap::perfetto; // NOLINT + +#if defined(IORAP_PERFETTO_MAIN) + +void Usage(char** argv) { + std::cerr << "Usage: " << argv[0] << " [--config-proto=config.pb] [--duration-ms=5000] [--output-proto=output.pb]" << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Request a perfetto trace, blocking until it's complete. The resulting trace proto" << std::endl; + std::cerr << " is output to stdout as text, or to --output-proto as a binary." << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Optional flags:" << std::endl; + std::cerr << " --help,-h Print this Usage." << std::endl; + std::cerr << " --output-proto $,-op $ Perfetto tracebuffer output file (default stdout)." << std::endl; + std::cerr << " --config-proto $,-cp $ Path to binary protobuf config." << std::endl; + std::cerr << " --duration-ms $,-dm $ How long to run trace for in milliseconds." << std::endl; + std::cerr << " --simple Simplest possible perfetto state transitions (default off)." << std::endl; + std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; + std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; + exit(1); +} + +PerfettoDependencies::Component CreateCommandLinePerfettoDependenciesComponent( + uint32_t duration_ms) { + // TODO: read from command line. + static const uint32_t kBufferSize = 4096; + + // TODO: remove this hack. + static const uint32_t kTraceDurationMs = duration_ms; + + // fruit: using 'bindInstance' causes a segfault every time. +#if 0 + + // fruit: Can't use a stateful lambda, so use bindInstance instead of registerProvider. + auto config = PerfettoDependencies::CreateConfig(duration_ms, + /*deferred_start*/true, + kBufferSize); + + .... bindInstance(config); +#endif + + return fruit::createComponent() + .bind<PerfettoConsumer, PerfettoConsumerImpl>() + .registerProvider([]() /* -> TraceConfig */ { + return PerfettoDependencies::CreateConfig(kTraceDurationMs, + /*deferred_start*/true, + kBufferSize); + }); +} + +static void CollectPerfettoTraceBufferViaAbstractions( + RxProducerFactory& producer_factory, + const std::string& arg_output_proto, + const int arg_duration_ms) { + LOG(VERBOSE) << "CollectPerfettoTraceBufferViaAbstractions"; + + // Don't create a subscriber to emit the PerfettoStreamCommand. + // RxCpp is "greedy" and consumes every possible item emitted (it doesn't support 'pull'). We want + // to operate on a (command,state) iteration every time, just like in a real scenario. + // Adding the 'interval' turns into a non-greedy version (i.e. push). + + // Immediately emit 'kStartTracing', wait and emit kStopTracing, wait and emit kShutdown. + // In reality, there would be a delay between all these events. + auto /*observable<PerfettoStreamCommand>*/ commands = + rxcpp::observable<>::just(PerfettoStreamCommand::kStartTracing) + // wait 1x + .concat( + // Pick a value longer than the perfetto config delay_ms, so that we send + // 'kShutdown' after tracing has already finished. + rxcpp::observable<>::interval(std::chrono::milliseconds(arg_duration_ms * 2)) + .take(2) // kStopTracing, kShutdown. + .map([](int value) { + // value is 1,2,3,... + return static_cast<PerfettoStreamCommand>(value); // 1,2, ... + }) + ); + + auto /*observable<PerfettoTraceProto>*/ trace_proto_stream = + producer_factory.CreateTraceStream(commands); + + trace_proto_stream + .observe_on(ObserveOnNewIoThread()) // Write data on an idle-class-priority thread. + .as_blocking() // Wait for observable to terminate with on_completed or on_error. + .subscribe(/*on_next*/[arg_output_proto] + (PerfettoTraceProto trace_proto) { + if (!trace_proto.WriteFullyToFile(arg_output_proto)) { + LOG(ERROR) << "Failed to save TraceBuffer to " << arg_output_proto; + } else { + LOG(INFO) << "TraceBuffer saved to file: " << arg_output_proto; + LOG(INFO); + LOG(INFO) << "To print this in a human readable form, execute these commands:"; + LOG(INFO) << "$> adb pull '" << arg_output_proto << "'"; + LOG(INFO) << "$> trace_to_text systrace <filename.pb>"; + } + }, + /*on_error*/[](rxcpp::util::error_ptr err) { + LOG(ERROR) << "Perfetto trace proto collection error: " << rxcpp::util::what(err); + }); +} + +namespace iorap::perfetto { +// Reach inside rx_producer.cc +// Not part of any headers because it's internal. +void CollectPerfettoTraceBufferImmediately( + RxProducerFactory& producer_factory, + const std::string& arg_output_proto); +} + +int main(int argc, char** argv) { + android::base::InitLogging(argv); + android::base::SetLogger(android::base::StderrLogger); + + bool wait_for_keystroke = false; + bool enable_verbose = false; + + std::string arg_output_proto; + std::string arg_config_proto; + uint32_t arg_duration_ms = 1000; + bool arg_simple = false; + + if (argc == 1) { + Usage(argv); + } + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + bool has_arg_next = (arg+1)<argc; + std::string arg_next = has_arg_next ? argv[arg+1] : ""; + + if (argstr == "--help" || argstr == "-h") { + Usage(argv); + } else if (argstr == "--output-proto" || argstr == "-op") { + if (!has_arg_next) { + std::cerr << "Missing --output-proto <value>" << std::endl; + return 1; + } + arg_output_proto = arg_next; + ++arg; + } else if (argstr == "--config-proto" || argstr == "-cp") { + if (!has_arg_next) { + std::cerr << "Missing --config-proto <value>" << std::endl; + return 1; + } + arg_config_proto = arg_next; + LOG(WARNING) << "TODO: parse configs from a file, not implemented yet."; + ++arg; + } else if (argstr == "--duration-ms" || argstr == "-dm") { + if (!has_arg_next) { + std::cerr << "Missing --duration-ms <value>" << std::endl; + return 1; + } + if (!android::base::ParseUint(arg_next.c_str(), /*out*/&arg_duration_ms)) { + std::cerr << "Invalid --duration-ms " << arg_next << ", reason: " << strerror(errno); + return 1; + } + ++arg; + } else if (argstr == "--simple") { + arg_simple = true; + } else if (argstr == "--verbose" || argstr == "-v") { + enable_verbose = true; + } else if (argstr == "--wait" || argstr == "-w") { + wait_for_keystroke = true; + } + } + + if (enable_verbose) { + android::base::SetMinimumLogSeverity(android::base::VERBOSE); + + LOG(VERBOSE) << "Verbose check"; + LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; + } + + // Useful to attach a debugger... + // 1) $> iorap-cmd-perfetto -w <args> + // 2) $> gdbclient <pid> + if (wait_for_keystroke) { + LOG(INFO) << "Self pid: " << getpid(); + LOG(INFO) << "Press any key to continue..."; + std::cin >> wait_for_keystroke; + } + + int return_code = 0; + // TODO: convert #on-error into a non-0 return code. + + PerfettoDependencies::Injector injector{ + CreateCommandLinePerfettoDependenciesComponent, + arg_duration_ms + }; + RxProducerFactory rx_producer_factory{/*borrow*/injector}; + + if (arg_simple) { + // To debug any kind of low-level perfetto issues. + CollectPerfettoTraceBufferImmediately(/*inout*/rx_producer_factory, arg_output_proto); + } else { + // To debug our own iorap internal abstractions. + CollectPerfettoTraceBufferViaAbstractions(/*inout*/rx_producer_factory, + arg_output_proto, + arg_duration_ms); + } + + // Uncomment this if we want to leave the process around to inspect it from adb shell. + // sleep(100000); + + // 0 -> successfully wrote the TraceProto out to file. + // 1 -> failed along the way (#on_error and also see the error logs). + return return_code; +} + +#endif diff --git a/src/perfetto/perfetto_consumer.cc b/src/perfetto/perfetto_consumer.cc new file mode 100644 index 0000000..4fe677b --- /dev/null +++ b/src/perfetto/perfetto_consumer.cc @@ -0,0 +1,608 @@ +// Copyright (C) 2020 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "perfetto/perfetto_consumer.h" + +#include "common/trace.h" + +#include <android-base/logging.h> +#include <android-base/properties.h> +#include <utils/Looper.h> +#include <utils/Printer.h> + +#include <limits> +#include <map> +#include <memory> +#include <mutex> +#include <sstream> +#include <vector> + +#include <inttypes.h> +#include <time.h> + +namespace iorap::perfetto { + +using State = PerfettoConsumer::State; +using Handle = PerfettoConsumer::Handle; +static constexpr Handle kInvalidHandle = PerfettoConsumer::kInvalidHandle; +using OnStateChangedCb = PerfettoConsumer::OnStateChangedCb; +using TraceBuffer = PerfettoConsumer::TraceBuffer; + +enum class StateKind { + kUncreated, + kCreated, + kStartedTracing, + kReadTracing, + kTimedOutDestroyed, // same as kDestroyed but timed out. + kDestroyed, // calling kDestroyed before timing out. +}; + +std::ostream& operator<<(std::ostream& os, StateKind kind) { + switch (kind) { + case StateKind::kUncreated: + os << "kUncreated"; + break; + case StateKind::kCreated: + os << "kCreated"; + break; + case StateKind::kStartedTracing: + os << "kStartedTracing"; + break; + case StateKind::kReadTracing: + os << "kReadTracing"; + break; + case StateKind::kTimedOutDestroyed: + os << "kTimedOutDestroyed"; + break; + case StateKind::kDestroyed: + os << "kDestroyed"; + break; + default: + os << "(invalid)"; + break; + } + return os; +} + +std::string ToString(StateKind kind) { + std::stringstream ss; + ss << kind; + return ss.str(); +} + +static constexpr uint64_t kSecToNano = 1000000000LL; + +static uint64_t GetTimeNanoseconds() { + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + + uint64_t now_ns = (now.tv_sec * kSecToNano + now.tv_nsec); + return now_ns; +} + +// Describe the state of our handle in detail for debugging/logging. +struct HandleDescription { + Handle handle_; + StateKind kind_{StateKind::kUncreated}; // Our state. required for correctness. + OnStateChangedCb callback_{nullptr}; // Required for Destroy callbacks. + void* callback_arg_{nullptr}; + + // For dumping to logs: + State state_{State::kSessionNotFound}; // perfetto state + std::optional<uint64_t> started_tracing_ns_; // when StartedTracing last called. + std::optional<uint64_t> read_trace_ns_; // When ReadTrace last called. + std::uint64_t last_transition_ns_{0}; + std::optional<uint64_t> trace_cookie_; // atrace beginning at StartTracing. + bool trace_ended_{false}; // atrace ending at ReadTrace or Destroy. + + HandleDescription(Handle handle): handle_(handle) {} +}; + +// pimpl idiom to hide the implementation details from header +// +// Track and verify that our perfetto usage is sane. +struct PerfettoConsumerImpl::Impl { + Impl() : raw_{new PerfettoConsumerRawImpl{}}, + message_handler_{new TraceMessageHandler{this}} { + std::thread watchdog_thread{ [this]() { + ::android::sp<::android::Looper> looper; + { + std::lock_guard<std::mutex> guard{looper_mutex_}; + looper = ::android::Looper::prepare(/*opts*/0); + looper_ = looper; + } + + static constexpr int kTimeoutMillis = std::numeric_limits<int>::max(); + + while (true) { + // Execute any pending callbacks, otherwise just block forever. + int result = looper->pollAll(kTimeoutMillis); + + if (result == ::android::Looper::POLL_ERROR) { + LOG(ERROR) << "PerfettoConsumerImpl::Looper got a POLL_ERROR"; + } else { + LOG(DEBUG) << "PerfettoConsumerImpl::Looper result was " << result; + } + } + }}; + + // Let thread run freely on its own. + watchdog_thread.detach(); + + // Block until looper_ is prepared. + while (true) { + std::lock_guard<std::mutex> guard{looper_mutex_}; + if (looper_ != nullptr) { + break; + } + } + } + + private: + std::unique_ptr<PerfettoConsumerRawImpl> raw_; + std::map<Handle, HandleDescription> states_; + + // We need this to be a counter to avoid memory leaks. + Handle last_created_{0}; + Handle last_destroyed_{0}; + uint64_t trace_cookie_{0}; + + std::mutex mutex_; // Guard above values. + + ::android::sp<::android::Looper> looper_; + std::mutex looper_mutex_; // Guard looper_. + + struct TraceMessageHandler : public ::android::MessageHandler { + TraceMessageHandler(Impl* impl) : impl_{impl} { + CHECK(impl != nullptr); + } + + Impl* impl_; + + virtual void handleMessage(const ::android::Message& message) override { + impl_->OnTraceMessage(static_cast<Handle>(message.what)); + } + }; + + ::android::sp<TraceMessageHandler> message_handler_; + + public: + Handle Create(const void* config_proto, + size_t config_len, + OnStateChangedCb callback, + void* callback_arg) { + LOG(VERBOSE) << "PerfettoConsumer::Create(" + << "config_len=" << config_len << ")"; + Handle handle = raw_->Create(config_proto, config_len, callback, callback_arg); + + std::lock_guard<std::mutex> guard{mutex_}; + + // Assume every Handle starts at 0 and then increments by 1 every Create. + ++last_created_; + CHECK_EQ(last_created_, handle) << "perfetto handle had unexpected behavior."; + // Without this^ increment-by-1 behavior our detection of untracked state values is broken. + // If we have to, we can go with Untracked=Uncreated|Destroyed but it's better to distinguish + // the two if possible. + + HandleDescription handle_desc{handle}; + handle_desc.handle_ = handle; + handle_desc.callback_ = callback; + handle_desc.callback_arg_ = callback_arg; + UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kCreated); + + // assume we never wrap around due to using int64 + bool inserted = states_.insert({handle, handle_desc}).second; + CHECK(inserted) << "perfetto handle was re-used: " << handle; + + return handle; + } + + void StartTracing(Handle handle) { + LOG(DEBUG) << "PerfettoConsumer::StartTracing(handle=" << handle << ")"; + + uint64_t trace_cookie; + { + std::lock_guard<std::mutex> guard{mutex_}; + + auto it = states_.find(handle); + if (it == states_.end()) { + LOG(ERROR) << "Cannot StartTracing(" << handle << "), untracked handle"; + return; + } + HandleDescription& handle_desc = it->second; + + raw_->StartTracing(handle); + UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kStartedTracing); + } + + // Use a looper here to add a timeout and immediately destroy the trace buffer. + CHECK_LE(static_cast<int64_t>(handle), static_cast<int64_t>(std::numeric_limits<int>::max())); + int message_code = static_cast<int>(handle); + ::android::Message message{message_code}; + + std::lock_guard<std::mutex> looper_guard{looper_mutex_}; + looper_->sendMessageDelayed(static_cast<nsecs_t>(GetPropertyTraceTimeoutNs()), + message_handler_, + message); + } + + TraceBuffer ReadTrace(Handle handle) { + LOG(DEBUG) << "PerfettoConsumer::ReadTrace(handle=" << handle << ")"; + + std::lock_guard<std::mutex> guard{mutex_}; + + auto it = states_.find(handle); + if (it == states_.end()) { + LOG(ERROR) << "Cannot ReadTrace(" << handle << "), untracked handle"; + return TraceBuffer{}; + } + + HandleDescription& handle_desc = it->second; + + TraceBuffer trace_buffer = raw_->ReadTrace(handle); + UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kReadTracing); + + return trace_buffer; + } + + void Destroy(Handle handle) { + HandleDescription handle_desc{handle}; + TryDestroy(handle, /*do_destroy*/true, /*out*/&handle_desc);; + } + + bool TryDestroy(Handle handle, bool do_destroy, /*out*/HandleDescription* handle_desc_out) { + CHECK(handle_desc_out != nullptr); + + LOG(VERBOSE) << "PerfettoConsumer::Destroy(handle=" << handle << ")"; + + std::lock_guard<std::mutex> guard{mutex_}; + + auto it = states_.find(handle); + if (it == states_.end()) { + // Leniency for calling Destroy multiple times. It's not a mistake. + LOG(ERROR) << "Cannot Destroy(" << handle << "), untracked handle"; + return false; + } + + HandleDescription& handle_desc = it->second; + + if (do_destroy) { + raw_->Destroy(handle); + } + UpdateHandleDescription(/*inout*/&handle_desc, StateKind::kDestroyed); + + *handle_desc_out = handle_desc; + + // No longer track this handle to avoid memory leaks. + last_destroyed_ = handle; + states_.erase(it); + + return true; + } + + State PollState(Handle handle) { + // Just pass-through the call, we never use it directly anyway. + return raw_->PollState(handle); + } + + // Either fetch or infer the current handle state from a handle. + // Meant for debugging/logging only. + HandleDescription GetOrInferHandleDescription(Handle handle) { + std::lock_guard<std::mutex> guard{mutex_}; + + auto it = states_.find(handle); + if (it == states_.end()) { + HandleDescription state{handle}; + // If it's untracked it hasn't been created yet, or it was already destroyed. + if (IsDestroyed(handle)) { + UpdateHandleDescription(/*inout*/&state, StateKind::kDestroyed); + } else { + if (!IsUncreated(handle)) { + LOG(WARNING) << "bad state detection"; + } + UpdateHandleDescription(/*inout*/&state, StateKind::kUncreated); + } + return state; + } + return it->second; + } + + void OnTraceMessage(Handle handle) { + LOG(VERBOSE) << "OnTraceMessage(" << static_cast<int64_t>(handle) << ")"; + HandleDescription handle_desc{handle}; + { + std::lock_guard<std::mutex> guard{mutex_}; + + auto it = states_.find(handle); + if (it == states_.end()) { + // Handle values are never re-used, so we can simply ignore the message here + // instead of having to remove it from the message queue. + LOG(VERBOSE) << "OnTraceMessage(" << static_cast<int64_t>(handle) + << ") no longer tracked handle"; + return; + } + handle_desc = it->second; + } + + // First check. Has this trace been active for too long? + uint64_t now_ns = GetTimeNanoseconds(); + if (handle_desc.kind_ == StateKind::kStartedTracing) { + // Ignore other kinds of traces because they don't exhaust perfetto resources. + CHECK(handle_desc.started_tracing_ns_.has_value()) << static_cast<int64_t>(handle); + + uint64_t started_tracing_ns = *handle_desc.started_tracing_ns_; + + if ((now_ns - started_tracing_ns) > GetPropertyTraceTimeoutNs()) { + LOG(WARNING) << "Perfetto Handle timed out after " << (now_ns - started_tracing_ns) << "ns" + << ", forcibly destroying"; + + // Let the callback handler call Destroy. + handle_desc.callback_(handle, State::kTraceFailed, handle_desc.callback_arg_); + } + } + + // Second check. Are there too many traces now? Cull the old traces. + std::vector<HandleDescription> handle_list; + do { + std::lock_guard<std::mutex> guard{mutex_}; + + size_t max_trace_count = GetPropertyMaxTraceCount(); + if (states_.size() > max_trace_count) { + size_t overflow_count = states_.size() - max_trace_count; + LOG(WARNING) << "Too many perfetto handles, overflowed by " << overflow_count + << ", pruning down to " << max_trace_count; + } else { + break; + } + + size_t prune_count = states_.size() - max_trace_count; + auto it = states_.begin(); + for (size_t i = 0; i < prune_count; ++i) { + // Simply prune by handle 1,2,3,4... + // We could do better with a timestamp if we wanted to. + ++it; + handle_list.push_back(it->second); + } + } while (false); + + for (HandleDescription& handle_desc : handle_list) { + LOG(DEBUG) << "Perfetto handle pruned: " << static_cast<int64_t>(handle); + + // Let the callback handler call Destroy. + handle_desc.callback_(handle, State::kTraceFailed, handle_desc.callback_arg_); + } + } + + private: + static uint64_t GetPropertyTraceTimeoutNs() { + static uint64_t value = // property is timeout in seconds + ::android::base::GetUintProperty<uint64_t>("iorapd.perfetto.timeout", /*default*/10); + return value * kSecToNano; + } + + static size_t GetPropertyMaxTraceCount() { + static size_t value = + ::android::base::GetUintProperty<size_t>("iorapd.perfetto.max_traces", /*default*/5); + return value; + } + + void UpdateHandleDescription(/*inout*/HandleDescription* handle_desc, StateKind kind) { + CHECK(handle_desc != nullptr); + handle_desc->kind_ = kind; + handle_desc->state_ = raw_->PollState(handle_desc->handle_); + + handle_desc->last_transition_ns_ = GetTimeNanoseconds(); + if (kind == StateKind::kStartedTracing) { + if (!handle_desc->started_tracing_ns_) { + handle_desc->started_tracing_ns_ = handle_desc->last_transition_ns_; + + handle_desc->trace_cookie_ = ++trace_cookie_; + + atrace_async_begin(ATRACE_TAG_ACTIVITY_MANAGER, + "Perfetto Scoped Trace", + *handle_desc->trace_cookie_); + atrace_int(ATRACE_TAG_ACTIVITY_MANAGER, + "Perfetto::Trace Handle", + static_cast<int32_t>(handle_desc->handle_)); + } + } + + if (kind == StateKind::kReadTracing) { + if (!handle_desc->read_trace_ns_) { + handle_desc->read_trace_ns_ = handle_desc->last_transition_ns_; + + if (handle_desc->trace_cookie_.has_value() && !handle_desc->trace_ended_) { + atrace_async_end(ATRACE_TAG_ACTIVITY_MANAGER, + "Perfetto Scoped Trace", + handle_desc->trace_cookie_.value()); + + handle_desc->trace_ended_ = true; + } + } + } + + // If Destroy is called prior to ReadTrace, mark the atrace as finished. + if (kind == StateKind::kDestroyed && handle_desc->trace_cookie_ && !handle_desc->trace_ended_) { + atrace_async_end(ATRACE_TAG_ACTIVITY_MANAGER, + "Perfetto Scoped Trace", + *handle_desc->trace_cookie_); + handle_desc->trace_ended_ = true; + } + } + + // The following state detection is for debugging only. + // We figure out if something is destroyed, uncreated, or live. + + // Does not distinguish between kTimedOutDestroyed and kDestroyed. + bool IsDestroyed(Handle handle) const { + auto it = states_.find(handle); + if (it != states_.end()) { + // Tracked values are not destroyed yet. + return false; + } + + if (handle == kInvalidHandle) { + return false; + } + + // The following assumes handles are incrementally generated: + if (it == states_.end()) { + // value is in range of [0, last_destroyed] => destroyed. + return handle <= last_destroyed_; + } + + auto min_it = states_.begin(); + if (handle < min_it->first) { + // value smaller than anything tracked: it was destroyed and we stopped tracking it. + return true; + } + + auto max_it = states_.rbegin(); + if (handle > max_it->first) { + // value too big: it's uncreated; + return false; + } + + // else it was a value that was previously tracked within [min,max] but no longer + return true; + } + + bool IsUncreated(Handle handle) const { + auto it = states_.find(handle); + if (it != states_.end()) { + // Tracked values are not uncreated. + return false; + } + + if (handle == kInvalidHandle) { + // Strangely enough, an invalid handle can never be created. + return true; + } + + // The following assumes handles are incrementally generated: + if (it == states_.end()) { + // value is in range of (last_destroyed, inf) => uncreated. + return handle > last_destroyed_; + } + + auto min_it = states_.begin(); + if (handle < min_it->first) { + // value smaller than anything tracked: it was destroyed and we stopped tracking it. + return false; + } + + auto max_it = states_.rbegin(); + if (handle > max_it->first) { + // value too big: it's uncreated; + return true; + } + + // else it was a value that was previously tracked within [min,max] but no longer + return false; + } + + public: + void Dump(::android::Printer& printer) { + // Locking can fail if we dump during a deadlock, so just do a best-effort lock here. + bool is_it_locked = mutex_.try_lock(); + + printer.printFormatLine("Perfetto consumer state:"); + if (!is_it_locked) { + printer.printLine(""""" (possible deadlock)"); + } + printer.printFormatLine(" Last destroyed handle: %" PRId64, last_destroyed_); + printer.printFormatLine(" Last created handle: %" PRId64, last_created_); + printer.printFormatLine(""); + printer.printFormatLine(" In-flight handles:"); + + for (auto it = states_.begin(); it != states_.end(); ++it) { + HandleDescription& handle_desc = it->second; + uint64_t started_tracing = + handle_desc.started_tracing_ns_ ? *handle_desc.started_tracing_ns_ : 0; + printer.printFormatLine(" Handle %" PRId64, handle_desc.handle_); + printer.printFormatLine(" Kind: %s", ToString(handle_desc.kind_).c_str()); + printer.printFormatLine(" Perfetto State: %d", static_cast<int>(handle_desc.state_)); + printer.printFormatLine(" Started tracing at: %" PRIu64, started_tracing); + printer.printFormatLine(" Last transition at: %" PRIu64, + handle_desc.last_transition_ns_); + } + if (states_.empty()) { + printer.printFormatLine(" (None)"); + } + + printer.printFormatLine(""); + + if (is_it_locked) { // u.b. if calling unlock on an unlocked mutex. + mutex_.unlock(); + } + } + + static PerfettoConsumerImpl::Impl* GetImplSingleton() { + static PerfettoConsumerImpl::Impl impl; + return &impl; + } +}; + +// Use a singleton because fruit instantiates a new PerfettoConsumer object for every +// new rx chain in RxProducerFactory. However, we want to track all perfetto transitions globally +// through 1 impl object. +// +// TODO: Avoiding a singleton would mean a more significant refactoring to remove the fruit/perfetto +// usage. + + +// +// Forward all calls to PerfettoConsumerImpl::Impl +// + +PerfettoConsumerImpl::~PerfettoConsumerImpl() { + // delete impl_; // TODO: no singleton +} + +void PerfettoConsumerImpl::Initialize() { + // impl_ = new PerfettoConsumerImpl::Impl(); // TODO: no singleton + impl_ = PerfettoConsumerImpl::Impl::GetImplSingleton(); +} + +void PerfettoConsumerImpl::Dump(::android::Printer& printer) { + PerfettoConsumerImpl::Impl::GetImplSingleton()->Dump(/*borrow*/printer); +} + +PerfettoConsumer::Handle PerfettoConsumerImpl::Create(const void* config_proto, + size_t config_len, + PerfettoConsumer::OnStateChangedCb callback, + void* callback_arg) { + return impl_->Create(config_proto, + config_len, + callback, + callback_arg); +} + +void PerfettoConsumerImpl::StartTracing(PerfettoConsumer::Handle handle) { + impl_->StartTracing(handle); +} + +PerfettoConsumer::TraceBuffer PerfettoConsumerImpl::ReadTrace(PerfettoConsumer::Handle handle) { + return impl_->ReadTrace(handle); +} + +void PerfettoConsumerImpl::Destroy(PerfettoConsumer::Handle handle) { + impl_->Destroy(handle); +} + +PerfettoConsumer::State PerfettoConsumerImpl::PollState(PerfettoConsumer::Handle handle) { + return impl_->PollState(handle); +} + +} // namespace iorap::perfetto diff --git a/src/perfetto/perfetto_consumer.h b/src/perfetto/perfetto_consumer.h new file mode 100644 index 0000000..2d828c0 --- /dev/null +++ b/src/perfetto/perfetto_consumer.h @@ -0,0 +1,115 @@ +// Copyright (C) 2018 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_PERFETTO_PERFETTO_CONSUMER_H_ +#define IORAP_SRC_PERFETTO_PERFETTO_CONSUMER_H_ + +#include <fruit/fruit.h> +#include <perfetto/public/consumer_api.h> // libperfetto + +namespace android { +class Printer; +} // namespace android + +namespace iorap::perfetto { + +// Abstract out the Perfetto C API behind a virtual interface: +// This enables us to use dependency injection to provide mock implementations +// during tests. +struct PerfettoConsumer { + // 1:1 aliasing of type definitions and constants in perfetto/public/consumer_api.h + // Refer to the documentation there. + using State = ::perfetto::consumer::State; + using Handle = ::perfetto::consumer::Handle; + static constexpr Handle kInvalidHandle = ::perfetto::consumer::kInvalidHandle; + using OnStateChangedCb = ::perfetto::consumer::OnStateChangedCb; + using TraceBuffer = ::perfetto::consumer::TraceBuffer; + + // 1:1 forwarding of C-style functions in perfetto/public/consumer_api.h + // Refer to the documentation there. + + virtual Handle Create(const void* config_proto, + size_t config_len, + OnStateChangedCb callback, + void* callback_arg) = 0; + virtual void StartTracing(Handle) = 0; + virtual TraceBuffer ReadTrace(Handle) = 0; + virtual void Destroy(Handle) = 0; + virtual State PollState(Handle) = 0; + + virtual ~PerfettoConsumer() {} +}; + +// "Live" implementation that calls down to libperfetto. +struct PerfettoConsumerRawImpl : public PerfettoConsumer { + // Marks this constructor as the one to use for injection. + INJECT(PerfettoConsumerRawImpl()) = default; + + virtual Handle Create(const void* config_proto, + size_t config_len, + OnStateChangedCb callback, + void* callback_arg) override { + return ::perfetto::consumer::Create(config_proto, + config_len, + callback, + callback_arg); + } + + virtual void StartTracing(Handle handle) override { + ::perfetto::consumer::StartTracing(handle); + } + + virtual TraceBuffer ReadTrace(Handle handle) override { + return ::perfetto::consumer::ReadTrace(handle); + } + + virtual void Destroy(Handle handle) override { + ::perfetto::consumer::Destroy(handle); + } + virtual State PollState(Handle handle) override { + return ::perfetto::consumer::PollState(handle); + } + + virtual ~PerfettoConsumerRawImpl() {} +}; + +// "Safe" implementation that has extra checking around it. +class PerfettoConsumerImpl : public PerfettoConsumer { + public: + // Marks this constructor as the one to use for injection. + INJECT(PerfettoConsumerImpl()) { Initialize(); } + + virtual Handle Create(const void* config_proto, + size_t config_len, + OnStateChangedCb callback, + void* callback_arg) override; + virtual void StartTracing(Handle handle) override; + virtual TraceBuffer ReadTrace(Handle handle) override; + virtual void Destroy(Handle handle) override; + virtual State PollState(Handle handle) override; + + virtual ~PerfettoConsumerImpl(); + + static void Dump(/*borrow*/::android::Printer& printer); + + private: + void Initialize(); + struct Impl; + PerfettoConsumerImpl::Impl* impl_; +}; + +} // namespace iorap::perfetto + +#endif // IORAP_SRC_PERFETTO_PERFETTO_CONSUMER_H_ + diff --git a/src/perfetto/rx_producer.cc b/src/perfetto/rx_producer.cc new file mode 100644 index 0000000..824600c --- /dev/null +++ b/src/perfetto/rx_producer.cc @@ -0,0 +1,939 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/debug.h" +#include "common/expected.h" +#include "perfetto/rx_producer.h" + +#include <android-base/file.h> +#include <android-base/properties.h> +#include <android-base/unique_fd.h> + +#include <iostream> + +#include <sched.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <syscall.h> +#include <fcntl.h> +#include <unistd.h> + +// TODO: move to perfetto code +namespace perfetto { +namespace consumer { + +std::ostream& operator<<(std::ostream& os, State state) { + switch (state) { + case State::kTraceFailed: + os << "kTraceFailed"; + break; + case State::kConnectionError: + os << "kConnectionError"; + break; + case State::kSessionNotFound: + os << "kSessionNotFound"; + break; + case State::kIdle: + os << "kIdle"; + break; + case State::kConnecting: + os << "kConnecting"; + break; + case State::kConfigured: + os << "kConfigured"; + break; + case State::kTracing: + os << "kTracing"; + break; + case State::kTraceEnded: + os << "kTraceEnded"; + break; + default: + os << "(unknown)"; // did someone forget to update this code? + break; + } + return os; +} + +} // namespace consumer +} // namespace perfetto + +namespace iorap::perfetto { + +PerfettoDependencies::Component PerfettoDependencies::CreateComponent() { + // TODO: read from config. + static const uint32_t kTraceDurationMs = + ::android::base::GetUintProperty("iorapd.perfetto.trace_duration_ms", /*default*/5000U); + + static const uint32_t kBufferSize = + ::android::base::GetUintProperty("iorapd.perfetto.buffer_size", /*default*/4096U); + + return fruit::createComponent() + .bind<PerfettoConsumer, PerfettoConsumerImpl>() + .registerProvider([]() /* -> TraceConfig */ { + return CreateConfig(kTraceDurationMs, + /*deferred_start*/false, + kBufferSize); + }); +} + +::perfetto::protos::TraceConfig PerfettoDependencies::CreateConfig(uint32_t duration_ms, + bool deferred_start, + uint32_t buffer_size) { + ::perfetto::protos::TraceConfig trace_config; + + trace_config.set_duration_ms(duration_ms); + trace_config.add_buffers()->set_size_kb(buffer_size); + trace_config.set_deferred_start(deferred_start); + + auto* ds_config = trace_config.add_data_sources()->mutable_config(); + ds_config->set_name("linux.ftrace"); + ds_config->mutable_ftrace_config()->add_ftrace_events( + "mm_filemap_add_to_page_cache"); + ds_config->mutable_ftrace_config()->add_ftrace_events( + "mm_filemap_delete_from_page_cache"); + ds_config->set_target_buffer(0); + + return trace_config; +} + +// RAII-style wrapper around a perfetto handle that calls Destroy +// in a thread-safe manner. +struct PerfettoConsumerHandle { + private: + std::shared_ptr<PerfettoConsumer> consumer_; + PerfettoConsumer::Handle handle_; + + public: + // Takes over ownership of the 'handle'. + // + // Consumer must not be null. + PerfettoConsumerHandle(std::shared_ptr<PerfettoConsumer> consumer, + PerfettoConsumer::Handle handle) + : consumer_{std::move(consumer)}, + handle_{std::move(handle)} { + DCHECK(consumer_ != nullptr); + } + + std::shared_ptr<PerfettoConsumer> GetConsumer() const { + return consumer_; + } + + PerfettoConsumer::Handle GetHandle() const { + return handle_; + } + + ~PerfettoConsumerHandle() { + LOG(VERBOSE) << "PerfettoConsumerHandle::Destroy(" << handle_ << ")"; + consumer_->Destroy(handle_); + } + + bool operator==(const PerfettoConsumerHandle& other) const { + return handle_ == other.handle_ && consumer_ == other.consumer_; + } + + bool operator!=(const PerfettoConsumerHandle& other) const { + return !(*this == other); + } +}; + + +// Snapshot of a single perfetto OnStateChanged callback. +// +// Operate on the PerfettoConsumer to further change the state. +// +// The Handle is kept 'valid' until all references to the PerfettoConsumerHandle +// are dropped to 0. This ensures the Handle is not destroyed too early. All +// direct usages of 'Handle' must be scoped by the PerfettoConsumerHandle. +struct PerfettoStateChange { + public: + using State = ::perfetto::consumer::State; + using Handle = ::perfetto::consumer::Handle; + + State state; // Never invalid. + std::shared_ptr<PerfettoConsumerHandle> perfetto_consumer_and_handle; // Never null. + + // Safety: Use only within scope of the PerfettoStateChange. + Handle GetHandle() const { + // TODO: it would be even safer to wrap all the calls to the handle inside a class, + // instead of exposing this raw Handle. + return perfetto_consumer_and_handle->GetHandle(); + } + + std::shared_ptr<PerfettoConsumer> GetConsumer() const { + return perfetto_consumer_and_handle->GetConsumer(); + } +}; + +std::ostream& operator<<(std::ostream& os, const PerfettoStateChange& state_change) { + os << "PerfettoStateChange{" << state_change.state << "," + << state_change.GetHandle() << "," + << state_change.GetConsumer().get() << "}"; + return os; +} + +// Once created, this acts as a hot observable, emitting 'PerfettoStateChange' transition items. +// Only the 'state' will vary, the handle and perfetto_consumer are always the same value. +// +// Clients only need to handle the success states in #on_next, all failure states will go to +// #on_error. +// +// Upon reaching the appropriate terminal states, either #on_completed or #on_error is called. +// No future callbacks will then occur, so this object should be subsequently deleted. +// +// The Handle is destroyed automatically after the last item is emitted, so it must only be +// manipulated from the #on_next callbacks. Do not save the Handle and use it at other times. +class StateChangedSubject { + public: + using State = ::perfetto::consumer::State; + using Handle = ::perfetto::consumer::Handle; + + // Static members to solve use-after-free bug. + // The object is accessed from not only perfetto thread, but also iorap + // thread. Use this global map to manage it. + static std::mutex state_subject_mutex_; + static std::unordered_map<Handle, StateChangedSubject*> state_subject_map_; + + StateChangedSubject(const ::perfetto::protos::TraceConfig& trace_config, + rxcpp::subscriber<PerfettoStateChange> destination, + std::shared_ptr<PerfettoConsumer> perfetto_consumer) + : deferred_start(trace_config.deferred_start()), + dest(std::move(destination)), + perfetto_consumer_(std::move(perfetto_consumer)) { + DCHECK(perfetto_consumer_ != nullptr); + } + + private: + struct StateChangedError : public std::runtime_error { + explicit StateChangedError(const std::string& what_arg) : std::runtime_error(what_arg) {} + }; + + std::shared_ptr<PerfettoConsumerHandle> handle_; // non-null after bound_ == true. + std::atomic<bool> bound_{false}; // synchronize-with for BindHandle -> OnStateChanged. + + State last_state{State::kIdle}; + bool deferred_start{false}; + + rxcpp::subscriber<PerfettoStateChange> dest; + std::shared_ptr<PerfettoConsumer> perfetto_consumer_; // This is never null. + + void DcheckBadStateTransition(State state, bool fail_unless = false) const { + DCHECK(fail_unless) << "Invalid state transition to " << state << " from " << last_state; + } + + void DcheckValidStateTransition(State state) { + // State must not be out of range. + DCHECK_GE(state, State::kTraceFailed); + DCHECK_LE(state, State::kTraceEnded); + + // Internal state that should never leak out into public perfetto API: + DCHECK_NE(state, State::kIdle); + // These can only be returned by PollState: + DCHECK_NE(state, State::kSessionNotFound); + + // Validate state transitions as per the perfetto API contract. + // See the 'state diagram' in consumer_api.h + switch (last_state) { + case State::kTraceFailed: // Final and unrecoverable. + // b/122548195: this can transition to 'kConnectionError' if selinux is disabled. + if (state == State::kConnectionError) { + LOG(WARNING) << "b/122548195: kTraceFailed is non-terminal, ignoring."; + // This is a bit awkward: rxcpp will drop the #on_error calls if its more than once. + break; + } + DcheckBadStateTransition(state); + break; + case State::kConnectionError: // Final and unrecoverable. + DcheckBadStateTransition(state); + break; + case State::kSessionNotFound: + DcheckBadStateTransition(state); + break; + case State::kIdle: + // OK: we initialized our own state to idle prior to the first callback. + break; + case State::kConnecting: + switch (state) { + case State::kConfigured: + // kConfigured, if |deferred_start| == true in the trace config. + DcheckBadStateTransition(state, deferred_start); + break; + case State::kTracing: + // kTracing, if |deferred_start| == false. + DcheckBadStateTransition(state, !deferred_start); + break; + case State::kConnectionError: + // An error state, e.g. if cannot reach the traced daemon. + break; + default: + // Unconditionally invalid state transitions from kConnecting to anything else. + DcheckBadStateTransition(state); + } + break; + case State::kConfigured: + DCHECK(deferred_start); + if (state != State::kTracing // OK: this is documented. + && state != State::kTraceFailed) { // Undocumented selinux failure. + // Undocumented, but it appears to go directly from Configured->TraceEnded + // it can also go to kTraceFailed if e.g. there's an selinux violation + // however this appears to be underdocumented. + // b/122607276 #2 + + if (state != State::kTraceEnded) { // b/122607276 #1 + DcheckBadStateTransition(state); + } + } + break; + case State::kTracing: + switch (state) { + case State::kTraceEnded: + break; + case State::kTraceFailed: + break; + default: + DcheckBadStateTransition(state); + } + break; + case State::kTraceEnded: + // Cannot transition from terminal state to another state. + DcheckBadStateTransition(state); + break; + + // default: This list is exhaustive + } + } + + constexpr bool IsTerminalState() const { + switch (last_state) { + case State::kTraceFailed: + case State::kConnectionError: + case State::kTraceEnded: + return true; + default: + return false; + } + } + + // Returns true for non-terminal states (i.e. this callback will be invoked again). + // Returns false otherwise. + bool OnStateChanged(Handle handle, State state) { + using namespace ::perfetto::consumer; + + // Block until 'BoundHandle' is called by the other thread. + while (!bound_.load()) {} // seq_cst acquire. + + std::shared_ptr<PerfettoConsumerHandle> handle_ptr = handle_; + DCHECK(handle_ptr != nullptr); + + DCHECK_EQ(handle_ptr->GetHandle(), handle); + DcheckValidStateTransition(state); + + switch (state) { + // Error states (terminal). + case State::kTraceFailed: + EmitError("kTraceFailed"); + break; + case State::kConnectionError: + EmitError("kConnectionError"); + break; + + // Regular transitions (non-terminal). + case State::kConnecting: + case State::kConfigured: + case State::kTracing: + EmitNext(state); + break; + // Regular transitions (terminal). + case State::kTraceEnded: // XX: do we even need to emit the 'TraceEnded' state? + EmitNext(state); + dest.on_completed(); + break; + default: + DcheckBadStateTransition(state); + } + + bool force_non_terminal = false; + + if (last_state == State::kConfigured && state == State::kConnectionError) { + // b/122548195: this can transition to 'kConnectionError' if selinux is disabled. + force_non_terminal = true; + // This function must 'return true' in this buggy case, otherwise we will + // call the destructor too early and subsequent callbacks will crash. + } + + // Remember the state to validate prior state transitions. + last_state = state; + + // The owner of this class should avoid leaking memory once we reach a terminal state. + return !IsTerminalState() || force_non_terminal; + } + + public: + // Thread safety: Called by main thread, terminates the rx stream. + // When this function is invoked, no calls to this class from other threads can occur. + void OnCreateFailed() { + // returned when an invalid handle is passed to PollState(). + last_state = State::kSessionNotFound; + EmitError("Create returned kInvalidHandle"); + } + + // Thread safety: Called by main thread, this could be concurrent to + // 'CallbackOnStateChanged'. + void BindHandle(const std::shared_ptr<PerfettoConsumerHandle>& handle) { + handle_ = handle; + + // Unblock OnStateChanged. + bound_.store(true); // seq_cst release. + } + + + // Called by libperfetto background thread (same one every time) and iorap + // thread. + static void CallbackOnStateChanged(Handle handle, State state, void* callback_arg) { + LOG(VERBOSE) << "CallbackOnStateChanged(handle=" << handle << ",state=" << state + << ",callback_arg=" << callback_arg << ")"; + + // Validate OnStateChanged callback invariants, guaranteed by libperfetto. + DCHECK_NE(handle, ::perfetto::consumer::kInvalidHandle); + + // TODO: the memory ordering guarantees should be explicitly specified in consumer_api.h: + // This isn't specific enough: + // "The callback will be invoked on an internal thread and must not block." + // However looking at the implementation it posts onto a single-thread task runner, + // so this must be the case. + + // This current thread owns 'StateChangedSubject', no other threads must access it. + // Explicit synchronization is not necessary. + + { + std::lock_guard<std::mutex> guard(StateChangedSubject::state_subject_mutex_); + auto it = StateChangedSubject::state_subject_map_.find(handle); + // If the object is already deleted, do nothing. + if (it == StateChangedSubject::state_subject_map_.end()) { + return; + } + + StateChangedSubject* state_subject = it->second; + if (!state_subject->OnStateChanged(handle, state)) { + // Clean up the state tracker when we reach a terminal state. + // This means that no future callbacks will occur anymore. + StateChangedSubject::state_subject_map_.erase(it); + delete state_subject; + } + } + } + + private: + void EmitError(const std::string& msg) { + // Sidenote: Exact error class does not matter, rxcpp only lets us access the error + // as a string (rxcpp::util::what). + // + // Either way, the recovery strategy is identical (log then try and restart). + dest.on_error(rxcpp::util::make_error_ptr(StateChangedError{msg})); + } + + void EmitNext(State state) { + if (WOULD_LOG(VERBOSE) && !dest.is_subscribed()) { + // This is purely for logging: #on_next already filters out items after unsubscription. + LOG(VERBOSE) << "StateChangedSubject#EmitNext(" << state << ") - drop due to unsubscribe"; + } + + auto handle_ptr = handle_; + DCHECK(handle_ptr != nullptr); + + // Non-null guarantee for the items emitted into this stream. + PerfettoStateChange state_change{state, handle_ptr}; + dest.on_next(std::move(state_change)); + } + + // TODO: inherit from rx subject and handle #unsubscribe explicitly, instead + // of just being subject-like? +}; + +std::mutex StateChangedSubject::state_subject_mutex_; +std::unordered_map<::perfetto::consumer::Handle, + StateChangedSubject*> StateChangedSubject::state_subject_map_; + +// Note: The states will be emitted on a separate thread, so e.g. #as_blocking() +// needs to be used to avoid dropping everything on the floor. +// +// Important: The #on_error case must be handled explicitly by the observable, +// because the default behavior is to 'throw' which will cause an std::terminate with -fno-except. +static auto /*[observable<State>, shared_ptr<PerfettoConsumerHandle>]*/ + CreatePerfettoStateStream(::perfetto::protos::TraceConfig perfetto_config, + std::shared_ptr<PerfettoConsumer> perfetto_consumer) { + auto obs = rxcpp::observable<>::create<PerfettoStateChange>( + [perfetto_config = std::move(perfetto_config), perfetto_consumer = std::move(perfetto_consumer)] + (rxcpp::subscriber<PerfettoStateChange> subscriber) { + std::unique_ptr<StateChangedSubject> state_subject{ + new StateChangedSubject{perfetto_config, subscriber, perfetto_consumer}}; + + // Perfetto API requires a pointer to a serialized protobuf, it doesn't accept + // the code-generated object. + std::string perfetto_config_str = perfetto_config.SerializeAsString(); + + ::perfetto::consumer::Handle handle = + perfetto_consumer->Create(perfetto_config_str.data(), + perfetto_config_str.size(), + // executes on the same background thread repeatedly. + &StateChangedSubject::CallbackOnStateChanged, + // inter-thread-move + reinterpret_cast<void*>(state_subject.get())); + // perfetto::consumer::Create synchronizes-with OnStateChanged callback, this means + // we don't need to explicitly synchronize state_subject here so long as we don't access + // it on this thread again. + LOG(DEBUG) << "Create Perfetto handle " << handle; + + if (handle == ::perfetto::consumer::kInvalidHandle) { + LOG(ERROR) << "Failed to create Perfetto handle"; + // No callbacks will occur, so our thread still owns the state subject. + state_subject->OnCreateFailed(); + return; + } + + { + std::lock_guard<std::mutex> guard(StateChangedSubject::state_subject_mutex_); + StateChangedSubject::state_subject_map_[handle] = state_subject.get(); + } + + std::shared_ptr<PerfettoConsumerHandle> safe_handle{ + new PerfettoConsumerHandle{perfetto_consumer, handle}}; + + // Share ownership of the Handle with the StateSubject. + // This way we defer calling 'Destroy' until the callback reaches a terminal state + // *and* all users of the stream are done with the handle. + state_subject->BindHandle(safe_handle); + + // state_subject ownership is taken over by OnStateChanged. + // It will also be touched in a separate thread, so we must never access it here again. + state_subject.release(); + + // 'subscriber#add' is actually a call to register an on_unsubscribe listener. + subscriber.add([safe_handle]() { + LOG(VERBOSE) << "PerfettoStateChange#unsubscribe"; + + // Release our ref-count to the handle. + // safe_handle.reset(); // This happens implicitly. + + // TODO: I think this won't handle the case where we need to shut down early. + // Need to use the explicit kShutdown for that? + }); + + // TODO: this would be an excellent place to shuffle the perfetto config protobuf + // into a global debug state for dumpsys. + }); + + return obs; +} + +template <typename T> +bool BinaryWireProtobuf<T>::WriteFullyToFile(const std::string& path, + bool follow_symlinks) const { + // TODO: it would be great if android::base had a string_view overload to avoid copying + // data into an std::string. + + // u g o + // rw-rw---- + // + // Protobufs can be read/written but not executed. + static constexpr const mode_t kMode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; + + int flags = + O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC | O_BINARY | (follow_symlinks ? 0 : O_NOFOLLOW); + android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), flags, kMode))); + + if (fd == -1) { + PLOG(ERROR) << "BinaryWireProtobuf::WriteFullyToFile open failed"; + return false; + } + + if (!::android::base::WriteFully(fd, data_.data(), size())) { + PLOG(ERROR) << "BinaryWireProtobuf::WriteFullyToFile write failed"; + return CleanUpAfterFailedWrite(path); + } + + return true; +} + +template <typename T> +bool BinaryWireProtobuf<T>::CleanUpAfterFailedWrite(const std::string& path) { + // Something went wrong. Let's not leave a corrupt file lying around. + int saved_errno = errno; + unlink(path.c_str()); + errno = saved_errno; + return false; +} + +template <typename T> +bool BinaryWireProtobuf<T>::WriteStringToFd(int fd) const { + const char* p = reinterpret_cast<const char*>(data_.data()); + size_t left = size(); + while (left > 0) { + ssize_t n = TEMP_FAILURE_RETRY(write(fd, p, left)); + if (n == -1) { + return false; + } + p += n; + left -= n; + } + return true; +} + +template <typename T> +std::optional<BinaryWireProtobuf<T>> BinaryWireProtobuf<T>::ReadFullyFromFile( + const std::string& path, + bool follow_symlinks) { + std::vector<std::byte> data; + + int flags = O_RDONLY | O_CLOEXEC | O_BINARY | (follow_symlinks ? 0 : O_NOFOLLOW); + android::base::unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), flags))); + if (fd == -1) { + return std::nullopt; + } + + if (ReadFdToString(fd.get(), /*out*/&data)) { + return BinaryWireProtobuf<T>{std::move(data)}; + } else { + return std::nullopt; + } +} + +template <typename T> +bool BinaryWireProtobuf<T>::operator==(const BinaryWireProtobuf<T>& other) const { + if (data_.size() != other.data_.size()) { + return false; + } + return std::equal(data_.begin(), data_.end(), other.data_.begin()); +} + +template <typename T> +bool BinaryWireProtobuf<T>::ReadFdToString(int fd, /*out*/std::vector<std::byte>* content) { + DCHECK(content != nullptr); + + content->clear(); + + struct stat sb; + if (fstat(fd, /*out*/&sb) != -1 && sb.st_size > 0) { + content->reserve(sb.st_size); + } + + char buf[BUFSIZ]; + auto it = content->begin(); + ssize_t n; + while ((n = TEMP_FAILURE_RETRY(read(fd, &buf[0], sizeof(buf)))) > 0) { + content->insert(it, + reinterpret_cast<std::byte*>(&buf[0]), + reinterpret_cast<std::byte*>(&buf[n])); + + std::advance(/*inout*/it, static_cast<size_t>(n)); + + static_assert(sizeof(char) == sizeof(std::byte), "sanity check for reinterpret cast"); + } + return (n == 0) ? true : false; +} + +// explicit template instantiation. +template struct BinaryWireProtobuf<::google::protobuf::MessageLite>; +// TODO: refactor this not to need the template instantiation. + +// Copy of the 2.6.18 kernel header (linux/ioprio.h) + +#define IOPRIO_WHO_PROCESS (1) +#define IOPRIO_CLASS_IDLE (3) + +#define IOPRIO_BITS (16) +#define IOPRIO_CLASS_SHIFT (13) +#define IOPRIO_PRIO_MASK ((1UL << IOPRIO_CLASS_SHIFT) - 1) + +#define IOPRIO_PRIO_CLASS(mask) ((mask) >> IOPRIO_CLASS_SHIFT) +#define IOPRIO_PRIO_DATA(mask) ((mask) & IOPRIO_PRIO_MASK) +#define IOPRIO_PRIO_VALUE(class, data) (((class) << IOPRIO_CLASS_SHIFT) | data) + +static int ioprio_get(int which, int who) { + return syscall(SYS_ioprio_get, which, who); +} + +static int ioprio_set(int which, int who, int ioprio) { + return syscall(SYS_ioprio_set, which, who, ioprio); +} + +// An rx Coordination, which will cause a new thread to spawn for each new Worker. +// +// Idle-class priority is set for the CPU and IO priorities on the new thread. +rxcpp::observe_on_one_worker ObserveOnNewIoThread() { + // IO thread factory for idle-priority threads. + // Both the CPU scheduler and the IO scheduler are set to idle. + // + // Use this when needing to schedule disk access from a normal-priority thread onto a + // very low priority thread, but not so low that we need to use a BackgroundJobScheduler. + struct io_thread_factory { + std::thread operator()(std::function<void()> start) const { + return std::thread{ + [start=std::move(start)]() { + // Set IO priority to idle. + do { + int value = ioprio_get(IOPRIO_WHO_PROCESS, /*pid*/0); + if (value == -1) { + PLOG(ERROR) << "io_thread_factory failed ioprio_get"; + break; // Can't set the ioprio, we don't know what data to use. + } + + int data = IOPRIO_PRIO_DATA(value); // priority level + // This appears to be '4' in practice. We may want to raise to + // be the highest-priority within the idle class. + + // idle scheduling class. only access disk when nobody else needs disk. + int res = ioprio_set(IOPRIO_WHO_PROCESS, + /*pid*/0, + IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE, data)); + if (res < 0) { + PLOG(ERROR) << "io_thread_factory failed ioprio_set"; + break; + } + + // Changing the IO priority only has any effect with cfq scheduler: + // $> cat /sys/block/sda/queue/scheduler + LOG(VERBOSE) << "ioprio_set(WHO_PROCESS, class=IDLE, data=" << data << ")"; + } while (false); + + // Set CPU priority to idle. + do { + struct sched_param param{}; + param.sched_priority = 0; // Required to be statically 0 when used with SCHED_IDLE. + + if (sched_setscheduler(/*pid*/0, // current thread, + SCHED_IDLE, + /*in*/¶m) != 0) { + PLOG(ERROR) << "io_thread_factory failed sched_setscheduler"; + break; + } + + LOG(VERBOSE) << "sched_setscheduler(self, IDLE)"; + } while (false); + + // XX: if changing the scheduling is too aggressive (i.e. it causes starvation), + // we may want to stick with the default class and change the nice (priority) levels + // to the minimum. + + // TODO: future work, maybe use cgroups configuration file instead? + + // Call the rxcpp-supplied code. + start(); + } + }; + } + }; + + static rxcpp::schedulers::scheduler thread_scheduler = + rxcpp::schedulers::make_new_thread(io_thread_factory{}); + + static rxcpp::observe_on_one_worker observe_on_io_thread{thread_scheduler}; + + return observe_on_io_thread; +} + +static auto/*observable<PerfettoTraceProto>*/ + CreatePerfettoStream(rxcpp::observable<PerfettoStreamCommand> input, + std::shared_ptr<PerfettoConsumer> perfetto_consumer, + const ::perfetto::protos::TraceConfig& trace_config) { + // XX: should I also take a scheduler for input here??? + + auto /*observable<PerfettoStateChange>*/ perfetto_states = + CreatePerfettoStateStream(trace_config, perfetto_consumer); + + using State = ::perfetto::consumer::State; + + auto/*coordinator*/ serialize_coordinator = rxcpp::observe_on_new_thread(); + // Rx note: + // The optimal thing to do would be to have a lock/unlock for an entire subset of a chain. + // This would avoid creating new threads, and could also be used to intentionally block + // the regular C-callback perfetto thread. + // + // It seems possible to create a coordinator to lock a single operator in a chain, but this + // appears to be unsound. In particular, it doesn't even make life any simpler below because + // it would only apply the synchronization to 'zip' but not 'flat_map' which is unsound. + // + // There is also the built-in 'serialize_new_thread' which seems to create a new thread but + // then never actually uses it, that seems unfortunate and wasteful. + // + // Instead, do the simple thing which is create a new thread and always queue on there. + // Execution an action on that worker is itself unsynchronized, but this doesn't matter since + // the worker is only backed by 1 thread (no 2 schedulables can be executed concurrently + // on the 'observe_new_thread' worker). + return input + .tap([](PerfettoStreamCommand command) { + LOG(VERBOSE) << "CreatePerfettoStreamCommand#tap(command=" << command << ")"; + }) + // Input A, thread tA. Input B, thread tB. Continue execution with (A,B) on thread tC. + .zip(serialize_coordinator, // rest of chain is also executed on the same thread. + perfetto_states) + // Note: zip terminates when either of the streams complete. + .flat_map( + [](std::tuple<PerfettoStreamCommand, PerfettoStateChange> p) { + auto& [command, state_change] = p; + LOG(VERBOSE) << "CreatePerfettoStream#combine(" + << command << "," << state_change << ")"; + if (command == PerfettoStreamCommand::kShutdown) { + // Perfetto: Always safe to call ::perfetto::consumer::Destroy + // at any time. + // + // XX: How do we clean up the StateChangedSubject without racing + // against the callback? It strikes me that we may need a 'kDestroyed' + // state that perfetto can transition to from kConfigured. + LOG(VERBOSE) << "Call Perfetto_Consumer->Destroy"; + state_change.GetConsumer()->Destroy(state_change.GetHandle()); + + // XX: Do we even have any guarantees about not getting more callbacks? + // We could just say 'there can still be spurious output after Shutdown' + // and just ignore it (e.g. Shutdown and immediately unsubscribe). + } else if (command == PerfettoStreamCommand::kStartTracing + && state_change.state == State::kConfigured) { + LOG(VERBOSE) << "Call Perfetto_Consumer->StartTracing"; + state_change.GetConsumer()->StartTracing(state_change.GetHandle()); + } else if (command == PerfettoStreamCommand::kStopTracing && + state_change.state == State::kTraceEnded) { + // TODO: if perfetto actually had a 'StopTracing' we could call that here. + // right now we just pretend it exists, but rely on the config timer instead. + ::perfetto::consumer::TraceBuffer trace_buffer = + state_change.GetConsumer()->ReadTrace(state_change.GetHandle()); + + LOG(VERBOSE) << "Perfetto Trace ended" + << ", addr=" << reinterpret_cast<void*>(trace_buffer.begin) + << ",size= " << trace_buffer.size; + + PerfettoTraceProto wire_proto{trace_buffer.begin, trace_buffer.size}; + return rxcpp::observable<>::just(std::move(wire_proto)).as_dynamic(); + } + return rxcpp::observable<>::empty<PerfettoTraceProto>().as_dynamic(); + } + ); +} + +std::ostream& operator<<(std::ostream& os, PerfettoStreamCommand c) { + switch (c) { + case PerfettoStreamCommand::kStartTracing: + os << "kStartTracing"; + break; + case PerfettoStreamCommand::kStopTracing: + os << "kStopTracing"; + break; + case PerfettoStreamCommand::kShutdown: + os << "kShutdown"; + break; + default: + os << "(unknown)"; + break; + } + return os; +} + +RxProducerFactory::RxProducerFactory(PerfettoDependencies::Injector& injector) + : injector_(injector) { +} + +// TODO: (fruit) maybe this could be streamlined further by avoiding this boilerplate? +rxcpp::observable<PerfettoTraceProto> RxProducerFactory::CreateTraceStream( + rxcpp::observable<PerfettoStreamCommand> commands) { + std::shared_ptr<PerfettoConsumer> perfetto_consumer = + injector_.get<std::shared_ptr<PerfettoConsumer>>(); + const ::perfetto::protos::TraceConfig& trace_config = + injector_.get<::perfetto::protos::TraceConfig>(); + + DCHECK(perfetto_consumer != nullptr); + DCHECK(reinterpret_cast<volatile const void*>(&trace_config) != nullptr); + + return CreatePerfettoStream(commands, + perfetto_consumer, + trace_config); +} + +// For testing/debugging only. +// +// Saves protobuf results in file name specified by 'arg_output_proto'. +void CollectPerfettoTraceBufferImmediately( + RxProducerFactory& producer_factory, + const std::string& arg_output_proto) { + LOG(VERBOSE) << "CollectPerfettoTraceBufferImmediately"; + + std::shared_ptr<PerfettoConsumer> perfetto_consumer = + producer_factory.injector_.get<std::shared_ptr<PerfettoConsumer>>(); + const ::perfetto::protos::TraceConfig& trace_config = + producer_factory.injector_.get<const ::perfetto::protos::TraceConfig&>(); + + auto /*observable<PerfettoStateChange>*/ perfetto_states = + CreatePerfettoStateStream(trace_config, perfetto_consumer); + + perfetto_states + .as_blocking() // Wait for observable to terminate with on_completed or on_error. + .subscribe(/*on_next*/[&](auto state_change) { + LOG(VERBOSE) << "Perfetto post-processed State change: " << state_change; + + using State = ::perfetto::consumer::State; + switch (state_change.state) { + case State::kConnecting: + LOG(VERBOSE) << "Perfetto Tracing is Connecting"; + // Transitional state. No-op. + break; + case State::kConfigured: + state_change.GetConsumer()->StartTracing(state_change.GetHandle()); + break; + case State::kTracing: + LOG(VERBOSE) << "Perfetto Tracing started"; + // Transitional state. No-op. + break; + case State::kTraceEnded: { + ::perfetto::consumer::TraceBuffer trace_buffer = + state_change.GetConsumer()->ReadTrace(state_change.GetHandle()); + + LOG(VERBOSE) << "Perfetto Trace ended" + << ", addr=" << reinterpret_cast<void*>(trace_buffer.begin) + << ",size= " << trace_buffer.size; + + if (!arg_output_proto.empty()) { + std::string trace_buffer_str; + trace_buffer_str.resize(trace_buffer.size); + std::copy(trace_buffer.begin, + trace_buffer.begin + trace_buffer.size, + trace_buffer_str.data()); + if (!android::base::WriteStringToFile(trace_buffer_str, arg_output_proto)) { + LOG(ERROR) << "Failed to save TraceBuffer to " << arg_output_proto; + } else { + LOG(INFO) << "TraceBuffer saved to file: " << arg_output_proto; + LOG(INFO); + LOG(INFO) << "To print this in a human readable form, execute these commands:"; + LOG(INFO) << "$> adb pull '" << arg_output_proto << "'"; + LOG(INFO) << "$> trace_to_text systrace <filename.pb>"; + } + } + + // TODO: something more useful with this TraceBuffer, such as saving it to a file + // and printing the output. + break; + } + default: + // No other states are possible, because they go to #on_error or cause a dcheck. + DCHECK(false) << "Invalid state: " << state_change; + } + + //INTENTIONAL_COMPILER_ERROR_HERE // lets make sure this code actually does a trace. + + }, /*on_error*/[](rxcpp::util::error_ptr err) { + LOG(ERROR) << "Perfetto post-processed state change failed: " << rxcpp::util::what(err); + }, /*on_completed*/[]() { + LOG(VERBOSE) << "Perfetto post-processed State #on_completed"; + }); +} + + +} // namespace iorap::perfetto diff --git a/src/perfetto/rx_producer.h b/src/perfetto/rx_producer.h new file mode 100644 index 0000000..4e93f0d --- /dev/null +++ b/src/perfetto/rx_producer.h @@ -0,0 +1,212 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef IORAP_SRC_PERFETTO_RX_PRODUCER_H_ +#define IORAP_SRC_PERFETTO_RX_PRODUCER_H_ + +#include "perfetto/perfetto_consumer.h" // libiorap + +#include <perfetto/config/trace_config.pb.h> // libperfetto +#include <rxcpp/rx.hpp> + +#include <iosfwd> +#include <functional> +#include <optional> +#include <vector> + +namespace iorap::perfetto { + +struct PerfettoDependencies { + using Component = + fruit::Component<PerfettoConsumer, ::perfetto::protos::TraceConfig>; + using Injector = + fruit::Injector<PerfettoConsumer, ::perfetto::protos::TraceConfig>; + using NormalizedComponent = + fruit::NormalizedComponent<PerfettoConsumer, ::perfetto::protos::TraceConfig>; + + // Create a 'live' component that will talk to perfetto via traced. + static Component CreateComponent(/*TODO: config params*/); + + // Create perfetto.protos.TraceConfig , serialized as a (machine-readable) string. + // + // The following ftrace events are enabled: + // * mm_filemap_add_to_page_cache + // * mm_filemap_delete_from_page_cache + // + // If deferred starting is also enabled, no tracing will begin until + // ::perfetto::consumer::StartTracing is invoked. + static ::perfetto::protos::TraceConfig CreateConfig(uint32_t duration_ms, + bool deferred_start = true, + uint32_t buffer_size = 4096); +}; + +namespace detail { + template <typename T> + struct concept_message_lite_base { + static_assert(std::is_base_of_v<::google::protobuf::MessageLite, T>, + "T must inherit from MessageLite"); + using type = T; + }; + + template <typename T> + using concept_message_lite_base_t = typename concept_message_lite_base<T>::type; +} // namespace detail + +/* + * In Android's version of libprotobuf, move-constructors are not generated. + * This results in a legitimate (~10sec per TracePacket being compiled) slowdown, + * so we need to avoid it everywhere. + * + * 1) Don't copy the protos, move them instead. + * 2) Use 'shared_ptr' because rxcpp won't compile with unique_ptr. + */ +template <typename T> +using ProtobufPtr = std::shared_ptr<detail::concept_message_lite_base_t<const T>>; + +template <typename T> +using ProtobufMutablePtr = std::shared_ptr<detail::concept_message_lite_base_t<T>>; + +// This acts as a lightweight type marker so that we know what data has actually +// encoded under the hood. +template <typename T> +struct BinaryWireProtobuf { + static_assert(std::is_base_of_v<::google::protobuf::MessageLite, T>, + "T should be a base class of MessageLite"); + + std::vector<std::byte>& data() { + return data_; + } + + const std::vector<std::byte>& data() const { + return data_; + } + + size_t size() const { + return data_.size(); + } + + explicit BinaryWireProtobuf(char* data, size_t size) + : BinaryWireProtobuf(reinterpret_cast<std::byte*>(data), size) { + } + + explicit BinaryWireProtobuf(std::byte* data, size_t size) { + data_.resize(size); + std::copy(data, + data + size, + data_.data()); + } + + explicit BinaryWireProtobuf(std::vector<std::byte> data) : data_{std::move(data)} { + } + + // You wouldn't want to accidentally copy a giant multi-megabyte chunk would you? + // BinaryWireProtobuf(const BinaryWireProtobuf& other) = delete; // FIXME: rx likes to copy. + BinaryWireProtobuf(const BinaryWireProtobuf& other) = default; + BinaryWireProtobuf(BinaryWireProtobuf&& other) = default; + + // Important: Deserialization could fail, for example data is truncated or + // some minor disc corruption occurred. + template <typename U> + std::optional<ProtobufPtr<U>> MaybeUnserialize() { + ProtobufMutablePtr<U> unencoded{new U{}}; + + if (!unencoded->ParseFromArray(data_.data(), data_.size())) { + return std::nullopt; + } + + return {std::move(unencoded)}; + } + + bool WriteFullyToFile(const std::string& path, + bool follow_symlinks = false) const; + + static std::optional<BinaryWireProtobuf<T>> ReadFullyFromFile(const std::string& path, + bool follow_symlinks = false); + + bool operator==(const BinaryWireProtobuf<T>& other) const; + bool operator!=(const BinaryWireProtobuf<T>& other) const { + return !(*this == other); + } + + private: + static bool CleanUpAfterFailedWrite(const std::string& path); + bool WriteStringToFd(int fd) const; + + static bool ReadFdToString(int fd, /*out*/std::vector<std::byte>* data); + + std::vector<std::byte> data_; +}; + +//using PerfettoTraceProto = BinaryWireProtobuf<::perfetto::protos::Trace>; +using PerfettoTraceProto = BinaryWireProtobuf<::google::protobuf::MessageLite>; + +enum class PerfettoStreamCommand { + kStartTracing, // -> () | on_error + kStopTracing, // -> on_next(PerfettoTraceProto) | on_error + kShutdown, // -> on_completed | on_error + // XX: should shutdown be converted to use the rx suscriber#unsubscribe instead? +}; + +std::ostream& operator<<(std::ostream& os, PerfettoStreamCommand c); + +struct RxProducerFactory { + // Passing anything by value leads to a lot of pain and headache. + // Pass in the injector by reference because nothing else seems to work. + explicit RxProducerFactory(PerfettoDependencies::Injector& injector); + + // Create a one-shot perfetto observable that will begin + // asynchronously producing a PerfettoTraceProto after the 'kStartTracing' + // command is observed. + // + // libperfetto is immediately primed (i.e. connected in a deferred state) + // upon calling this function, to reduce the latency of 'kStartTracing'. + // + // To finish the trace, push 'kStopTracing'. To cancel or tear down at any + // time, push 'kShutdown'. + // + // The TraceProto may come out at any time after 'kStartTracing', + // this is controlled by duration_ms in the TraceConfig. + // + // TODO: libperfetto should actually stop tracing when we ask it to, + // instead of using a hardcoded time. + // + // The observable may go into #on_error at any time, if the underlying + // libperfetto states transition to a failing state. + // This usually means the OS is not configured correctly. + rxcpp::observable<PerfettoTraceProto> CreateTraceStream( + rxcpp::observable<PerfettoStreamCommand> commands); + + // TODO: is this refactor-able into a subscriber factory that takes + // the commands-observable as a parameter? + + // TODO: infinite perfetto stream. + + private: + // XX: why doesn't this just let me pass in a regular Component? + PerfettoDependencies::Injector& injector_; + + friend void CollectPerfettoTraceBufferImmediately( + RxProducerFactory& producer_factory, + const std::string& arg_output_proto); +}; + +// An rx Coordination, which will cause a new thread to spawn for each new Worker. +// +// Idle-class priority is set for the CPU and IO priorities on the new thread. +// +// TODO: move to separate file +rxcpp::observe_on_one_worker ObserveOnNewIoThread(); + +} // namespace iorap::perfetto +#endif // IORAP_SRC_PERFETTO_RX_PRODUCER_H_ diff --git a/src/prefetcher/main.cc b/src/prefetcher/main.cc new file mode 100644 index 0000000..94ba141 --- /dev/null +++ b/src/prefetcher/main.cc @@ -0,0 +1,190 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/debug.h" +#include "common/loggers.h" +#include "prefetcher/prefetcher_daemon.h" + +#include <android-base/parseint.h> +#include <android-base/logging.h> + +#include <iostream> +#include <optional> +#include <string_view> +#include <string> +#include <vector> + +#include <signal.h> + +#if defined(IORAP_PREFETCHER_MAIN) + +namespace iorap::prefetcher { + +void Usage(char** argv) { + std::cerr << "Usage: " << argv[0] << " [--input-fd=#] [--output-fd=#]" << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Run the readahead daemon which can prefetch files given a command." << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Optional flags:" << std::endl; + std::cerr << " --help,-h Print this Usage." << std::endl; + std::cerr << " --input-fd,-if Input FD (default stdin)." << std::endl; + std::cerr << " --output-fd,-of Output FD (default stdout)." << std::endl; + std::cerr << " --use-sockets,-us Use AF_UNIX sockets (default off)." << std::endl; + std::cerr << " --command-format=[text|binary],-cf (default text)." << std::endl; + std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; + std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; + exit(1); +} + +int Main(int argc, char** argv) { + // Go to system logcat + stderr when running from command line. + android::base::InitLogging(argv, iorap::common::StderrAndLogdLogger{android::base::SYSTEM}); + + bool wait_for_keystroke = false; + bool enable_verbose = false; + + bool command_format_text = false; // false = binary. + + int arg_input_fd = -1; + int arg_output_fd = -1; + + std::vector<std::string> arg_input_filenames; + bool arg_use_sockets = false; + + LOG(VERBOSE) << "argparse: argc=" << argc; + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + bool has_arg_next = (arg+1)<argc; + std::string arg_next = has_arg_next ? argv[arg+1] : ""; + + LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; + + if (argstr == "--help" || argstr == "-h") { + Usage(argv); + } else if (argstr == "--input-fd" || argstr == "-if") { + if (!has_arg_next) { + LOG(ERROR) << "--input-fd=<numeric-value>"; + Usage(argv); + } + if (!::android::base::ParseInt(arg_next, /*out*/&arg_input_fd)) { + LOG(ERROR) << "--input-fd value must be numeric"; + Usage(argv); + } + } else if (argstr == "--output-fd" || argstr == "-of") { + if (!has_arg_next) { + LOG(ERROR) << "--output-fd=<numeric-value>"; + Usage(argv); + } + if (!::android::base::ParseInt(arg_next, /*out*/&arg_output_fd)) { + LOG(ERROR) << "--output-fd value must be numeric"; + Usage(argv); + } + } else if (argstr == "--command-format=" || argstr == "-cf") { + if (!has_arg_next) { + LOG(ERROR) << "--command-format=text|binary"; + Usage(argv); + } + if (arg_next == "text") { + command_format_text = true; + } else if (arg_next == "binary") { + command_format_text = false; + } else { + LOG(ERROR) << "--command-format must be one of {text,binary}"; + Usage(argv); + } + } else if (argstr == "--use-sockets" || argstr == "-us") { + arg_use_sockets = true; + } else if (argstr == "--verbose" || argstr == "-v") { + enable_verbose = true; + } else if (argstr == "--wait" || argstr == "-w") { + wait_for_keystroke = true; + } else { + arg_input_filenames.push_back(argstr); + } + } + + if (enable_verbose) { + android::base::SetMinimumLogSeverity(android::base::VERBOSE); + + LOG(VERBOSE) << "Verbose check"; + LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; + } else { + android::base::SetMinimumLogSeverity(android::base::DEBUG); + } + + LOG(VERBOSE) << "argparse: argc=" << argc; + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + + LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; + } + + // Useful to attach a debugger... + // 1) $> iorap.cmd.readahead -w <args> + // 2) $> gdbclient <pid> + if (wait_for_keystroke) { + LOG(INFO) << "Self pid: " << getpid(); + + raise(SIGSTOP); + // LOG(INFO) << "Press any key to continue..."; + // std::cin >> wait_for_keystroke; + } + + // auto system_call = std::make_unique<SystemCallImpl>(); + // TODO: mock readahead calls? + // + // Uncomment this if we want to leave the process around to inspect it from adb shell. + // sleep(100000); + + int return_code = 0; + + LOG(VERBOSE) << "Hello world"; + + if (arg_input_fd == -1) { + arg_input_fd = STDIN_FILENO; + } + if (arg_output_fd == -1) { + arg_output_fd = STDOUT_FILENO; + } + + PrefetcherForkParameters params{}; + params.input_fd = arg_input_fd; + params.output_fd = arg_output_fd; + params.format_text = command_format_text; + params.use_sockets = arg_use_sockets; + + LOG(VERBOSE) << "main: Starting PrefetcherDaemon: " + << "input_fd=" << params.input_fd + << ",output_fd=" << params.output_fd; + { + PrefetcherDaemon daemon; + // Blocks until receiving an exit command. + daemon.Main(std::move(params)); + } + LOG(VERBOSE) << "main: Terminating"; + + // 0 -> successfully executed all commands. + // 1 -> failed along the way (#on_error and also see the error logs). + return return_code; +} + +} // namespace iorap::prefetcher + +int main(int argc, char** argv) { + return ::iorap::prefetcher::Main(argc, argv); +} + +#endif // IORAP_PREFETCHER_MAIN diff --git a/src/prefetcher/main_client.cc b/src/prefetcher/main_client.cc new file mode 100644 index 0000000..ff7ee6d --- /dev/null +++ b/src/prefetcher/main_client.cc @@ -0,0 +1,160 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "common/debug.h" +#include "prefetcher/read_ahead.h" +#include "prefetcher/task_id.h" + +#include <android-base/parseint.h> +#include <android-base/logging.h> + +#include <iostream> +#include <optional> +#include <string_view> +#include <string> +#include <vector> + +#include <signal.h> +#include <unistd.h> + + +namespace iorap::prefetcher { + +static void UsageClient(char** argv) { + std::cerr << "UsageClient: " << argv[0] << " <path-to-compiled-trace.pb> [... pathN]" << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Run the readahead daemon which can prefetch files given a command." << std::endl; + std::cerr << "" << std::endl; + std::cerr << " Optional flags:" << std::endl; + std::cerr << " --help,-h Print this UsageClient." << std::endl; + std::cerr << " --verbose,-v Set verbosity (default off)." << std::endl; + std::cerr << " --task-duration-ms,-tdm Set task duration (default: 0ms)." << std::endl; + std::cerr << " --use-sockets,-us Use AF_UNIX sockets (default: off)" << std::endl; + std::cerr << " --wait,-w Wait for key stroke before continuing (default off)." << std::endl; + exit(1); +} + +int MainClient(int argc, char** argv) { + android::base::InitLogging(argv); + android::base::SetLogger(android::base::StderrLogger); + + bool wait_for_keystroke = false; + bool enable_verbose = false; + + bool command_format_text = false; // false = binary. + + unsigned int arg_task_duration_ms = 10000; + std::vector<std::string> arg_input_filenames; + bool arg_use_sockets = false; + + LOG(VERBOSE) << "argparse: argc=" << argc; + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + bool has_arg_next = (arg+1)<argc; + std::string arg_next = has_arg_next ? argv[arg+1] : ""; + + LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; + + if (argstr == "--help" || argstr == "-h") { + UsageClient(argv); + } else if (argstr == "--use-sockets" || argstr == "-us") { + arg_use_sockets = true; + } else if (argstr == "--verbose" || argstr == "-v") { + enable_verbose = true; + } else if (argstr == "--wait" || argstr == "-w") { + wait_for_keystroke = true; + } else if (argstr == "--task-duration-ms" || argstr == "-tdm") { + if (!has_arg_next) { + LOG(ERROR) << "--task-duration-ms: requires uint parameter"; + UsageClient(argv); + } else if (!::android::base::ParseUint(arg_next, &arg_task_duration_ms)) { + LOG(ERROR) << "--task-duration-ms: requires non-negative parameter"; + UsageClient(argv); + } + } else { + arg_input_filenames.push_back(argstr); + } + } + + if (enable_verbose) { + android::base::SetMinimumLogSeverity(android::base::VERBOSE); + + LOG(VERBOSE) << "Verbose check"; + LOG(VERBOSE) << "Debug check: " << ::iorap::kIsDebugBuild; + } else { + android::base::SetMinimumLogSeverity(android::base::DEBUG); + } + + LOG(VERBOSE) << "argparse: argc=" << argc; + + for (int arg = 1; arg < argc; ++arg) { + std::string argstr = argv[arg]; + + LOG(VERBOSE) << "argparse: argv[" << arg << "]=" << argstr; + } + + // Useful to attach a debugger... + // 1) $> iorap.cmd.readahead -w <args> + // 2) $> gdbclient <pid> + if (wait_for_keystroke) { + LOG(INFO) << "Self pid: " << getpid(); + + raise(SIGSTOP); + // LOG(INFO) << "Press any key to continue..."; + // std::cin >> wait_for_keystroke; + } + + // auto system_call = std::make_unique<SystemCallImpl>(); + // TODO: mock readahead calls? + // + // Uncomment this if we want to leave the process around to inspect it from adb shell. + // sleep(100000); + + int return_code = 0; + + LOG(VERBOSE) << "Hello world"; + + ReadAhead read_ahead{arg_use_sockets}; // Don't count the time it takes to fork+exec. + + size_t task_id_counter = 0; + for (const std::string& compiled_trace_path : arg_input_filenames) { + TaskId task_id{task_id_counter++, compiled_trace_path}; + + LOG(DEBUG) << "main: ReadAhead BeginTask: " + << "task_duration_ms=" << arg_task_duration_ms << "," + << task_id; + + read_ahead.BeginTask(task_id); + usleep(arg_task_duration_ms*1000); + + LOG(DEBUG) << "main: ReadAhead FinishTask: " << task_id; + + read_ahead.FinishTask(task_id); + } + LOG(VERBOSE) << "main: Terminating"; + + // 0 -> successfully executed all commands. + // 1 -> failed along the way (#on_error and also see the error logs). + return return_code; +} + +} // namespace iorap::prefetcher + +#if defined(IORAP_PREFETCHER_MAIN_CLIENT) +int main(int argc, char** argv) { + return ::iorap::prefetcher::MainClient(argc, argv); +} + +#endif // IORAP_PREFETCHER_MAIN_CLIENT diff --git a/src/prefetcher/minijail.cc b/src/prefetcher/minijail.cc new file mode 100644 index 0000000..df4b11b --- /dev/null +++ b/src/prefetcher/minijail.cc @@ -0,0 +1,49 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "prefetcher/minijail.h" + +#include <android-base/logging.h> +#include <libminijail.h> + +namespace iorap::prefetcher { + +static const char kSeccompFilePath[] = "/system/etc/seccomp_policy/iorap.prefetcherd.policy"; + +bool MiniJail() { + /* no seccomp policy for this architecture */ + if (access(kSeccompFilePath, R_OK) == -1) { + LOG(WARNING) << "No seccomp filter defined for this architecture."; + return true; + } + + struct minijail* jail = minijail_new(); + if (jail == NULL) { + LOG(WARNING) << "Failed to create minijail."; + return false; + } + + minijail_no_new_privs(jail); + minijail_log_seccomp_filter_failures(jail); + minijail_use_seccomp_filter(jail); + minijail_parse_seccomp_filters(jail, kSeccompFilePath); + minijail_enter(jail); + minijail_destroy(jail); + + LOG(DEBUG) << "minijail installed."; + + return true; +} + +} diff --git a/src/prefetcher/minijail.h b/src/prefetcher/minijail.h new file mode 100644 index 0000000..9caefe6 --- /dev/null +++ b/src/prefetcher/minijail.h @@ -0,0 +1,23 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SRC_PREFETCHER_MINIJAIL_H +#define SRC_PREFETCHER_MINIJAIL_H + +namespace iorap::prefetcher { +// Install a minijail using seccomp_policy/prefetcherd.{ARCH}.policy file. +bool MiniJail(); +} + +#endif // SRC_PREFETCHER_MINIJAIL_H diff --git a/src/prefetcher/prefetcher_daemon.cc b/src/prefetcher/prefetcher_daemon.cc new file mode 100644 index 0000000..f4b9087 --- /dev/null +++ b/src/prefetcher/prefetcher_daemon.cc @@ -0,0 +1,1367 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "prefetcher/minijail.h" +#include "common/cmd_utils.h" +#include "prefetcher/prefetcher_daemon.h" +#include "prefetcher/session_manager.h" +#include "prefetcher/session.h" + +#include <android-base/logging.h> +#include <android-base/properties.h> + +#include <deque> +#include <iomanip> +#include <string> +#include <sstream> +#include <vector> + +#include <fcntl.h> +#include <string.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <sys/un.h> +#include <unistd.h> + +namespace iorap::prefetcher { + +// Gate super-spammy IPC logging behind a property. +// This is beyond merely annoying, enabling this logging causes prefetching to be about 1000x slower. +static bool LogVerboseIpc() { + static bool initialized = false; + static bool verbose_ipc; + + if (initialized == false) { + initialized = true; + + verbose_ipc = + ::android::base::GetBoolProperty("iorapd.readahead.verbose_ipc", /*default*/false); + } + + return verbose_ipc; +} + +static const bool kInstallMiniJail = + ::android::base::GetBoolProperty("iorapd.readahead.minijail", /*default*/true); + +static constexpr const char kCommandFileName[] = "/system/bin/iorap.prefetcherd"; + +static constexpr size_t kPipeBufferSize = 1024 * 1024; // matches /proc/sys/fs/pipe-max-size + +using ArgString = const char*; + +std::ostream& operator<<(std::ostream& os, ReadAheadKind ps) { + switch (ps) { + case ReadAheadKind::kFadvise: + os << "fadvise"; + break; + case ReadAheadKind::kMmapLocked: + os << "mmap"; + break; + case ReadAheadKind::kMlock: + os << "mlock"; + break; + default: + os << "<invalid>"; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, CommandChoice choice) { + switch (choice) { + case CommandChoice::kRegisterFilePath: + os << "kRegisterFilePath"; + break; + case CommandChoice::kUnregisterFilePath: + os << "kUnregisterFilePath"; + break; + case CommandChoice::kReadAhead: + os << "kReadAhead"; + break; + case CommandChoice::kExit: + os << "kExit"; + break; + case CommandChoice::kCreateSession: + os << "kCreateSession"; + break; + case CommandChoice::kDestroySession: + os << "kDestroySession"; + break; + case CommandChoice::kDumpSession: + os << "kDumpSession"; + break; + case CommandChoice::kDumpEverything: + os << "kDumpEverything"; + break; + case CommandChoice::kCreateFdSession: + os << "kCreateFdSession"; + break; + default: + CHECK(false) << "forgot to handle this choice"; + break; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, const Command& command) { + os << "Command{"; + os << "choice=" << command.choice << ","; + + bool has_session_id = true; + bool has_id = true; + switch (command.choice) { + case CommandChoice::kDumpEverything: + case CommandChoice::kExit: + has_session_id = false; + FALLTHROUGH_INTENDED; + case CommandChoice::kCreateFdSession: + case CommandChoice::kCreateSession: + case CommandChoice::kDestroySession: + case CommandChoice::kDumpSession: + has_id = false; + break; + default: + break; + } + + if (has_session_id) { + os << "sid=" << command.session_id << ","; + } + + if (has_id) { + os << "id=" << command.id << ","; + } + + switch (command.choice) { + case CommandChoice::kRegisterFilePath: + os << "file_path="; + + if (command.file_path) { + os << *(command.file_path); + } else { + os << "(nullopt)"; + } + break; + case CommandChoice::kUnregisterFilePath: + break; + case CommandChoice::kReadAhead: + os << "read_ahead_kind=" << command.read_ahead_kind << ","; + os << "length=" << command.length << ","; + os << "offset=" << command.offset << ","; + break; + case CommandChoice::kExit: + break; + case CommandChoice::kCreateFdSession: + os << "fd="; + if (command.fd.has_value()) { + os << command.fd.value(); + } else { + os << "(nullopt)"; + } + os << ","; + FALLTHROUGH_INTENDED; + case CommandChoice::kCreateSession: + os << "description="; + if (command.file_path) { + os << "'" << *(command.file_path) << "'"; + } else { + os << "(nullopt)"; + } + break; + case CommandChoice::kDestroySession: + break; + case CommandChoice::kDumpSession: + break; + case CommandChoice::kDumpEverything: + break; + default: + CHECK(false) << "forgot to handle this choice"; + break; + } + + os << "}"; + + return os; +} + +template <typename T> +struct ParseResult { + T value; + char* next_token; + size_t stream_size; + + ParseResult() : value{}, next_token{nullptr}, stream_size{} { + } + + constexpr operator bool() const { + return next_token != nullptr; + } +}; + +// Very spammy: Keep it off by default. Set to true if changing this code. +static constexpr bool kDebugParsingRead = false; + +#define DEBUG_PREAD if (kDebugParsingRead) LOG(VERBOSE) << "ParsingRead " + + + +// Parse a strong type T from a buffer stream. +// If there's insufficient space left to parse the value, an empty ParseResult is returned. +template <typename T> +ParseResult<T> ParsingRead(char* stream, size_t stream_size) { + if (stream == nullptr) { + DEBUG_PREAD << "stream was null"; + return {}; + } + + if constexpr (std::is_same_v<T, std::string>) { + ParseResult<uint32_t> length = ParsingRead<uint32_t>(stream, stream_size); + + if (!length) { + DEBUG_PREAD << "could not find length"; + // Not enough bytes left? + return {}; + } + + ParseResult<std::string> string_result; + string_result.value.reserve(length); + + stream = length.next_token; + stream_size = length.stream_size; + + for (size_t i = 0; i < length.value; ++i) { + ParseResult<char> char_result = ParsingRead<char>(stream, stream_size); + + stream = char_result.next_token; + stream_size = char_result.stream_size; + + if (!char_result) { + DEBUG_PREAD << "too few chars in stream, expected length: " << length.value; + // Not enough bytes left? + return {}; + } + + string_result.value += char_result.value; + + DEBUG_PREAD << "string preliminary is : " << string_result.value; + } + + DEBUG_PREAD << "parsed string to: " << string_result.value; + string_result.next_token = stream; + return string_result; + } else { + if (sizeof(T) > stream_size) { + return {}; + } + + ParseResult<T> result; + result.next_token = stream + sizeof(T); + result.stream_size = stream_size - sizeof(T); + + memcpy(&result.value, stream, sizeof(T)); + + return result; + } +} + +// Convenience overload to chain multiple ParsingRead together. +template <typename T, typename U> +ParseResult<T> ParsingRead(ParseResult<U> result) { + return ParsingRead<T>(result.next_token, result.stream_size); +} + +class CommandParser { + public: + CommandParser(PrefetcherForkParameters params) { + params_ = params; + } + + std::vector<Command> ParseSocketCommands(bool& eof) { + eof = false; + + std::vector<Command> commands_vec; + + std::vector<char> buf_vector; + buf_vector.resize(1024*1024); // 1MB. + char* buf = &buf_vector[0]; + + // Binary only parsing. The higher level code can parse text + // with ifstream if it really wants to. + char* stream = &buf[0]; + size_t stream_size = buf_vector.size(); + + while (true) { + if (stream_size == 0) { + // TODO: reply with an overflow command. + LOG(WARNING) << "prefetcher_daemon command overflow, dropping all commands."; + stream = &buf[0]; + stream_size = buf_vector.size(); + memset(&buf[0], /*c*/0, buf_vector.size()); + } + + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon block recvmsg for commands (fd=" << params_.input_fd << ")"; + } + + ssize_t count; + struct msghdr hdr; + memset(&hdr, 0, sizeof(hdr)); + + { + union { + struct cmsghdr cmh; + char control[CMSG_SPACE(sizeof(int))]; + } control_un; + memset(&control_un, 0, sizeof(control_un)); + + /* Set 'control_un' to describe ancillary data that we want to receive */ + control_un.cmh.cmsg_len = CMSG_LEN(sizeof(int)); /* fd is sizeof(int) */ + control_un.cmh.cmsg_level = SOL_SOCKET; + control_un.cmh.cmsg_type = SCM_CREDENTIALS; + + // the regular message data will be read into stream + struct iovec iov; + memset(&iov, 0, sizeof(iov)); + iov.iov_base = stream; + iov.iov_len = stream_size; + + /* Set hdr fields to describe 'control_un' */ + hdr.msg_control = control_un.control; + hdr.msg_controllen = sizeof(control_un.control); + hdr.msg_iov = &iov; + hdr.msg_iovlen = 1; + hdr.msg_name = nullptr; /* no peer address */ + hdr.msg_namelen = 0; + + count = TEMP_FAILURE_RETRY(recvmsg(params_.input_fd, &hdr, /*flags*/0)); + } + + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon recvmsg " << count << " for stream size:" << stream_size; + } + + if (count < 0) { + PLOG(ERROR) << "failed to recvmsg from input fd"; + break; + // TODO: let the daemon be restarted by higher level code? + } else if (count == 0) { + LOG(WARNING) << "prefetcher_daemon input_fd end-of-file; terminating"; + eof = true; + break; + // TODO: let the daemon be restarted by higher level code? + } + + { + /* Extract fd from ancillary data if present */ + struct cmsghdr* hp; + hp = CMSG_FIRSTHDR(&hdr); + if (hp && + // FIXME: hp->cmsg_len returns an absurdly large value. is it overflowing? + // (hp->cmsg_len == CMSG_LEN(sizeof(int))) && + (hp->cmsg_level == SOL_SOCKET) && + (hp->cmsg_type == SCM_RIGHTS)) { + + int passed_fd = *(int*) CMSG_DATA(hp); + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon received FD " << passed_fd; + } + + // tack the FD into our dequeue. + // we assume the FDs are sent in-order same as the regular iov are sent in-order. + longbuf_fds_.insert(longbuf_fds_.end(), passed_fd); + } else if (hp != nullptr) { + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon::read got CMSG but it wasn't matching SCM_RIGHTS," + << "cmsg_len=" << hp->cmsg_len << "," + << "cmsg_level=" << hp->cmsg_level << "," + << "cmsg_type=" << hp->cmsg_type; + } + } + } + + longbuf_.insert(longbuf_.end(), stream, stream + count); + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon updated longbuf size: " << longbuf_.size(); + } + + // reconstruct a stream of [iov_Command chdr_fd?]* back into [Command]* + { + if (longbuf_.size() == 0) { + break; + } + + std::vector<char> v(longbuf_.begin(), + longbuf_.end()); + + std::vector<int> v_fds{longbuf_fds_.begin(), longbuf_fds_.end()}; + + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon longbuf_ size: " << v.size(); + if (WOULD_LOG(VERBOSE)) { + std::stringstream dump; + dump << std::hex << std::setfill('0'); + for (size_t i = 0; i < v.size(); ++i) { + dump << std::setw(2) << static_cast<unsigned>(v[i]); + } + + LOG(VERBOSE) << "PrefetcherDaemon longbuf_ dump: " << dump.str(); + } + LOG(VERBOSE) << "PrefetcherDaemon longbuf_fds_ size: " << v_fds.size(); + if (WOULD_LOG(VERBOSE)) { + std::stringstream dump; + for (size_t i = 0; i < v_fds.size(); ++i) { + dump << v_fds[i] << ", "; + } + + LOG(VERBOSE) << "PrefetcherDaemon longbuf_fds_ dump: " << dump.str(); + } + + } + + size_t v_fds_off = 0; + size_t consumed_fds_total = 0; + + size_t v_off = 0; + size_t consumed_bytes = std::numeric_limits<size_t>::max(); + size_t consumed_total = 0; + + while (true) { + std::optional<Command> maybe_command; + maybe_command = Command::Read(&v[v_off], v.size() - v_off, &consumed_bytes); + consumed_total += consumed_bytes; + // Normal every time we get to the end of a buffer. + if (!maybe_command) { + if (LogVerboseIpc()) { + LOG(VERBOSE) << "failed to read command, v_off=" << v_off << ",v_size:" << v.size(); + } + break; + } + + if (maybe_command->RequiresFd()) { + if (v_fds_off < v_fds.size()) { + maybe_command->fd = v_fds[v_fds_off++]; + consumed_fds_total++; + if (LogVerboseIpc()) { + LOG(VERBOSE) << "Append the FD to " << *maybe_command; + } + } else { + LOG(WARNING) << "Failed to acquire FD for " << *maybe_command; + } + } + + // in the next pass ignore what we already consumed. + v_off += consumed_bytes; + + // true as long we don't hit the 'break' above. + DCHECK_EQ(v_off, consumed_total); + if (LogVerboseIpc()) { + LOG(VERBOSE) << "success to read command, v_off=" << v_off << ",v_size:" << v.size() + << "," << *maybe_command; + + // Pretty-print a single command for debugging/testing. + LOG(VERBOSE) << *maybe_command; + } + + // add to the commands we parsed. + commands_vec.push_back(*maybe_command); + } + + // erase however many were consumed + longbuf_.erase(longbuf_.begin(), longbuf_.begin() + consumed_total); + + // erase however many FDs were consumed. + longbuf_fds_.erase(longbuf_fds_.begin(), longbuf_fds_.begin() + consumed_fds_total); + } + break; + } + + return commands_vec; + } + + std::vector<Command> ParseCommands(bool& eof) { + eof = false; + + std::vector<Command> commands_vec; + + std::vector<char> buf_vector; + buf_vector.resize(kPipeBufferSize); + char* buf = &buf_vector[0]; + + // Binary only parsing. The higher level code can parse text + // with ifstream if it really wants to. + char* stream = &buf[0]; + size_t stream_size = buf_vector.size(); + + while (true) { + if (stream_size == 0) { + // TODO: reply with an overflow command. + LOG(WARNING) << "prefetcher_daemon command overflow, dropping all commands."; + stream = &buf[0]; + stream_size = buf_vector.size(); + memset(&buf[0], /*c*/0, buf_vector.size()); + } + + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon block read for commands (fd=" << params_.input_fd << ")"; + } + ssize_t count = TEMP_FAILURE_RETRY(read(params_.input_fd, stream, stream_size)); + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon::read " << count << " for stream size:" << stream_size; + } + + if (count < 0) { + PLOG(ERROR) << "failed to read from input fd"; + break; + // TODO: let the daemon be restarted by higher level code? + } else if (count == 0) { + LOG(WARNING) << "prefetcher_daemon input_fd end-of-file; terminating"; + eof = true; + break; + // TODO: let the daemon be restarted by higher level code? + } + + longbuf_.insert(longbuf_.end(), stream, stream + count); + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon updated longbuf size: " << longbuf_.size(); + } + + std::optional<Command> maybe_command; + { + if (longbuf_.size() == 0) { + break; + } + + std::vector<char> v(longbuf_.begin(), + longbuf_.end()); + + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon longbuf_ size: " << v.size(); + if (WOULD_LOG(VERBOSE)) { + std::stringstream dump; + dump << std::hex << std::setfill('0'); + for (size_t i = 0; i < v.size(); ++i) { + dump << std::setw(2) << static_cast<unsigned>(v[i]); + } + + LOG(VERBOSE) << "PrefetcherDaemon longbuf_ dump: " << dump.str(); + } + } + + size_t v_off = 0; + size_t consumed_bytes = std::numeric_limits<size_t>::max(); + size_t consumed_total = 0; + + while (true) { + maybe_command = Command::Read(&v[v_off], v.size() - v_off, &consumed_bytes); + consumed_total += consumed_bytes; + // Normal every time we get to the end of a buffer. + if (!maybe_command) { + if (LogVerboseIpc()) { + LOG(VERBOSE) << "failed to read command, v_off=" << v_off << ",v_size:" << v.size(); + } + break; + } + + // in the next pass ignore what we already consumed. + v_off += consumed_bytes; + + // true as long we don't hit the 'break' above. + DCHECK_EQ(v_off, consumed_total); + if (LogVerboseIpc()) { + LOG(VERBOSE) << "success to read command, v_off=" << v_off << ",v_size:" << v.size() + << "," << *maybe_command; + + // Pretty-print a single command for debugging/testing. + LOG(VERBOSE) << *maybe_command; + } + + // add to the commands we parsed. + commands_vec.push_back(*maybe_command); + } + + // erase however many were consumed + longbuf_.erase(longbuf_.begin(), longbuf_.begin() + consumed_total); + } + break; + } + + return commands_vec; + } + + private: + bool IsTextMode() const { + return params_.format_text; + } + + PrefetcherForkParameters params_; + + // A buffer long enough to contain a lot of buffers. + // This handles reads that only contain a partial command. + std::deque<char> longbuf_; + + // File descriptor buffers. + std::deque<int> longbuf_fds_; +}; + +static constexpr bool kDebugCommandRead = true; + +#define DEBUG_READ if (kDebugCommandRead) LOG(VERBOSE) << "Command::Read " + +std::optional<Command> Command::Read(char* buf, size_t buf_size, /*out*/size_t* consumed_bytes) { + *consumed_bytes = 0; + if (buf == nullptr) { + return std::nullopt; + } + + Command cmd{}; // zero-initialize any unused fields + ParseResult<CommandChoice> parsed_choice = ParsingRead<CommandChoice>(buf, buf_size); + cmd.choice = parsed_choice.value; + + if (!parsed_choice) { + DEBUG_READ << "no choice"; + return std::nullopt; + } + + switch (parsed_choice.value) { + case CommandChoice::kRegisterFilePath: { + ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); + if (!parsed_session_id) { + DEBUG_READ << "no parsed session id"; + return std::nullopt; + } + + ParseResult<uint32_t> parsed_id = ParsingRead<uint32_t>(parsed_session_id); + if (!parsed_id) { + DEBUG_READ << "no parsed id"; + return std::nullopt; + } + + ParseResult<std::string> parsed_file_path = ParsingRead<std::string>(parsed_id); + + if (!parsed_file_path) { + DEBUG_READ << "no file path"; + return std::nullopt; + } + *consumed_bytes = parsed_file_path.next_token - buf; + + cmd.session_id = parsed_session_id.value; + cmd.id = parsed_id.value; + cmd.file_path = parsed_file_path.value; + + break; + } + case CommandChoice::kUnregisterFilePath: { + ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); + if (!parsed_session_id) { + DEBUG_READ << "no parsed session id"; + return std::nullopt; + } + + ParseResult<uint32_t> parsed_id = ParsingRead<uint32_t>(parsed_session_id); + if (!parsed_id) { + DEBUG_READ << "no parsed id"; + return std::nullopt; + } + *consumed_bytes = parsed_id.next_token - buf; + + cmd.session_id = parsed_session_id.value; + cmd.id = parsed_id.value; + + break; + } + case CommandChoice::kReadAhead: { + ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); + if (!parsed_session_id) { + DEBUG_READ << "no parsed session id"; + return std::nullopt; + } + + ParseResult<uint32_t> parsed_id = ParsingRead<uint32_t>(parsed_session_id); + if (!parsed_id) { + DEBUG_READ << "no parsed id"; + return std::nullopt; + } + + ParseResult<ReadAheadKind> parsed_kind = ParsingRead<ReadAheadKind>(parsed_id); + if (!parsed_kind) { + DEBUG_READ << "no parsed kind"; + return std::nullopt; + } + ParseResult<uint64_t> parsed_length = ParsingRead<uint64_t>(parsed_kind); + if (!parsed_length) { + DEBUG_READ << "no parsed length"; + return std::nullopt; + } + ParseResult<uint64_t> parsed_offset = ParsingRead<uint64_t>(parsed_length); + if (!parsed_offset) { + DEBUG_READ << "no parsed offset"; + return std::nullopt; + } + *consumed_bytes = parsed_offset.next_token - buf; + + cmd.session_id = parsed_session_id.value; + cmd.id = parsed_id.value; + cmd.read_ahead_kind = parsed_kind.value; + cmd.length = parsed_length.value; + cmd.offset = parsed_offset.value; + + break; + } + case CommandChoice::kCreateSession: + case CommandChoice::kCreateFdSession: { + ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); + if (!parsed_session_id) { + DEBUG_READ << "no parsed session id"; + return std::nullopt; + } + + ParseResult<std::string> parsed_description = ParsingRead<std::string>(parsed_session_id); + + if (!parsed_description) { + DEBUG_READ << "no description"; + return std::nullopt; + } + *consumed_bytes = parsed_description.next_token - buf; + + cmd.session_id = parsed_session_id.value; + cmd.file_path = parsed_description.value; + + break; + } + case CommandChoice::kDestroySession: + case CommandChoice::kDumpSession: { + ParseResult<uint32_t> parsed_session_id = ParsingRead<uint32_t>(parsed_choice); + if (!parsed_session_id) { + DEBUG_READ << "no parsed session id"; + return std::nullopt; + } + + *consumed_bytes = parsed_session_id.next_token - buf; + + cmd.session_id = parsed_session_id.value; + + break; + } + case CommandChoice::kExit: + case CommandChoice::kDumpEverything: + *consumed_bytes = parsed_choice.next_token - buf; + // Only need to parse the choice. + break; + default: + LOG(FATAL) << "unrecognized command number " << static_cast<uint32_t>(parsed_choice.value); + break; + } + + return cmd; +} + +bool Command::Write(char* buf, size_t buf_size, /*out*/size_t* produced_bytes) const { + *produced_bytes = 0; + if (buf == nullptr) { + LOG(WARNING) << "null buf, is this expected?"; + return false; + } + + bool has_enough_space = false; + size_t space_requirement = std::numeric_limits<size_t>::max(); + + space_requirement = sizeof(choice); + + switch (choice) { + case CommandChoice::kRegisterFilePath: + space_requirement += sizeof(session_id); + space_requirement += sizeof(id); + space_requirement += sizeof(uint32_t); // string length + + if (!file_path) { + LOG(WARNING) << "Missing file path for kRegisterFilePath"; + return false; + } + + space_requirement += file_path->size(); // string contents + break; + case CommandChoice::kUnregisterFilePath: + space_requirement += sizeof(session_id); + space_requirement += sizeof(id); + break; + case CommandChoice::kReadAhead: + space_requirement += sizeof(session_id); + space_requirement += sizeof(id); + space_requirement += sizeof(read_ahead_kind); + space_requirement += sizeof(length); + space_requirement += sizeof(offset); + break; + case CommandChoice::kCreateSession: + case CommandChoice::kCreateFdSession: + space_requirement += sizeof(session_id); + space_requirement += sizeof(uint32_t); // string length + + if (!file_path) { + LOG(WARNING) << "Missing file path for kCreateSession"; + return false; + } + + space_requirement += file_path->size(); // string contents + break; + case CommandChoice::kDestroySession: + case CommandChoice::kDumpSession: + space_requirement += sizeof(session_id); + break; + case CommandChoice::kExit: + case CommandChoice::kDumpEverything: + // Only need space for the choice. + break; + default: + LOG(FATAL) << "unrecognized command number " << static_cast<uint32_t>(choice); + break; + } + + if (buf_size < space_requirement) { + return false; + } + + *produced_bytes = space_requirement; + + // Always write out the choice. + size_t buf_offset = 0; + + memcpy(&buf[buf_offset], &choice, sizeof(choice)); + buf_offset += sizeof(choice); + + switch (choice) { + case CommandChoice::kRegisterFilePath: + memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); + buf_offset += sizeof(session_id); + memcpy(&buf[buf_offset], &id, sizeof(id)); + buf_offset += sizeof(id); + + { + uint32_t string_length = static_cast<uint32_t>(file_path->size()); + memcpy(&buf[buf_offset], &string_length, sizeof(string_length)); + buf_offset += sizeof(string_length); + } + + DCHECK(file_path.has_value()); + + memcpy(&buf[buf_offset], file_path->c_str(), file_path->size()); + buf_offset += file_path->size(); + break; + case CommandChoice::kUnregisterFilePath: + memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); + buf_offset += sizeof(session_id); + memcpy(&buf[buf_offset], &id, sizeof(id)); + buf_offset += sizeof(id); + break; + case CommandChoice::kReadAhead: + memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); + buf_offset += sizeof(session_id); + memcpy(&buf[buf_offset], &id, sizeof(id)); + buf_offset += sizeof(id); + memcpy(&buf[buf_offset], &read_ahead_kind, sizeof(read_ahead_kind)); + buf_offset += sizeof(read_ahead_kind); + memcpy(&buf[buf_offset], &length, sizeof(length)); + buf_offset += sizeof(length); + memcpy(&buf[buf_offset], &offset, sizeof(offset)); + buf_offset += sizeof(offset); + break; + case CommandChoice::kCreateSession: + case CommandChoice::kCreateFdSession: + memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); + buf_offset += sizeof(session_id); + + { + uint32_t string_length = static_cast<uint32_t>(file_path->size()); + memcpy(&buf[buf_offset], &string_length, sizeof(string_length)); + buf_offset += sizeof(string_length); + } + + DCHECK(file_path.has_value()); + + memcpy(&buf[buf_offset], file_path->c_str(), file_path->size()); + buf_offset += file_path->size(); + + DCHECK_EQ(buf_offset, space_requirement) << *this << ",file_path_size:" << file_path->size(); + DCHECK_EQ(buf_offset, *produced_bytes) << *this; + + break; + case CommandChoice::kDestroySession: + case CommandChoice::kDumpSession: + memcpy(&buf[buf_offset], &session_id, sizeof(session_id)); + buf_offset += sizeof(session_id); + break; + case CommandChoice::kExit: + case CommandChoice::kDumpEverything: + // Only need to write out the choice. + break; + default: + LOG(FATAL) << "should have fallen out in the above switch" + << static_cast<uint32_t>(choice); + break; + } + + DCHECK_EQ(buf_offset, space_requirement) << *this; + DCHECK_EQ(buf_offset, *produced_bytes) << *this; + + return true; +} + +class PrefetcherDaemon::Impl { + public: + std::optional<PrefetcherForkParameters> StartPipesViaFork() { + int pipefds[2]; + if (pipe(&pipefds[0]) != 0) { + PLOG(FATAL) << "Failed to create read/write pipes"; + } + + if (WOULD_LOG(VERBOSE)) { + long pipe_size = static_cast<long>(fcntl(pipefds[0], F_GETPIPE_SZ)); + if (pipe_size < 0) { + PLOG(ERROR) << "Failed to F_GETPIPE_SZ:"; + } + LOG(VERBOSE) << "StartPipesViaFork: default pipe size: " << pipe_size; + } + + for (int i = 0; i < 2; ++i) { + // Default pipe size is usually 64KB. + // Increase to 1MB so that iorapd has to rarely run during prefetching. + if (fcntl(pipefds[i], F_SETPIPE_SZ, kPipeBufferSize) < 0) { + PLOG(FATAL) << "Failed to increase pipe size to max"; + } + } + + pipefd_read_ = pipefds[0]; + pipefd_write_ = pipefds[1]; + + PrefetcherForkParameters params; + params.input_fd = pipefd_read_; + params.output_fd = pipefd_write_; + params.format_text = false; + params.use_sockets = false; + + bool res = StartViaFork(params); + if (res) { + return params; + } else { + return std::nullopt; + } + } + +std::optional<PrefetcherForkParameters> StartSocketViaFork() { + int socket_fds[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, /*protocol*/0, &socket_fds[0]) != 0) { + PLOG(FATAL) << "Failed to create read/write socketpair"; + } + + pipefd_read_ = socket_fds[0]; // iorapd writer, iorap.prefetcherd reader + pipefd_write_ = socket_fds[1]; // iorapd reader, iorap.prefetcherd writer + + PrefetcherForkParameters params; + params.input_fd = pipefd_read_; + params.output_fd = pipefd_write_; + params.format_text = false; + params.use_sockets = true; + + bool res = StartViaFork(params); + if (res) { + return params; + } else { + return std::nullopt; + } + } + + bool StartViaFork(PrefetcherForkParameters params) { + params_ = params; + + forked_ = true; + child_ = fork(); + + if (child_ == -1) { + LOG(FATAL) << "Failed to fork PrefetcherDaemon"; + } else if (child_ > 0) { // we are the caller of this function + LOG(DEBUG) << "forked into iorap.prefetcherd, pid = " << child_; + + return true; + } else { + // we are the child that was forked. + std::stringstream argv; // for logging + std::vector<std::string> argv_vec; + + { + std::stringstream s; + s << "--input-fd"; + argv_vec.push_back(s.str()); + + std::stringstream s2; + s2 << params.input_fd; + argv_vec.push_back(s2.str()); + + argv << " --input-fd" << " " << params.input_fd; + } + + { + std::stringstream s; + s << "--output-fd"; + argv_vec.push_back(s.str()); + + std::stringstream s2; + s2 << params.output_fd; + argv_vec.push_back(s2.str()); + + argv << " --output-fd" << " " << params.output_fd; + } + + + if (params.use_sockets) { + std::stringstream s; + s << "--use-sockets"; + argv_vec.push_back(s.str()); + + argv << " --use-sockets"; + } + + if (WOULD_LOG(VERBOSE)) { + std::stringstream s; + s << "--verbose"; + argv_vec.push_back(s.str()); + + argv << " --verbose"; + } + + std::unique_ptr<ArgString[]> argv_ptr = common::VecToArgv(kCommandFileName, argv_vec); + + LOG(DEBUG) << "fork+exec: " << kCommandFileName << " " + << argv.str(); + execve(kCommandFileName, (char **)argv_ptr.get(), /*envp*/nullptr); + // This should never return. + _exit(EXIT_FAILURE); + } + + DCHECK(false); + return false; + } + + // TODO: Not very useful since this can never return 'true' + // -> in the child we would've already execd which loses all this code. + bool IsDaemon() { + // In the child the pid is always 0. + return child_ > 0; + } + + bool Main(PrefetcherForkParameters params) { + LOG(VERBOSE) << "PrefetcherDaemon::Main " << params; + + CommandParser command_parser{params}; + + Command next_command{}; + + std::vector<Command> many_commands; + + // Ensure alogd is pre-initialized before installing minijail. + LOG(DEBUG) << "Installing minijail"; + + // Install seccomp filter using libminijail. + if (kInstallMiniJail) { + MiniJail(); + } + + while (true) { + bool eof = false; + + if (params.use_sockets) { + // use recvmsg(2). supports receiving FDs. + many_commands = command_parser.ParseSocketCommands(/*out*/eof); + } else { + // use read(2). does not support receiving FDs. + many_commands = command_parser.ParseCommands(/*out*/eof); + } + + if (eof) { + LOG(WARNING) << "PrefetcherDaemon got EOF, terminating"; + return true; + } + + for (auto& command : many_commands) { + if (LogVerboseIpc()) { + LOG(VERBOSE) << "PrefetcherDaemon got command: " << command; + } + + if (command.choice == CommandChoice::kExit) { + LOG(DEBUG) << "PrefetcherDaemon got kExit command, terminating"; + return true; + } + + if (!ReceiveCommand(command)) { + // LOG(WARNING) << "PrefetcherDaemon command processing failure: " << command; + } + + // ReceiveCommand should dup to keep the FD. Avoid leaks. + if (command.fd.has_value()) { + close(*command.fd); + } + } + } + + LOG(VERBOSE) << "PrefetcherDaemon::Main got exit, terminating"; + + return true; + // Terminate. + } + + Impl(PrefetcherDaemon* daemon) { + session_manager_ = SessionManager::CreateManager(SessionKind::kInProcessDirect); + DCHECK(session_manager_ != nullptr); + }; + + ~Impl() { + // Don't do anything if we never called 'StartViaFork' + if (forked_) { + if (!IsDaemon()) { + int status; + waitpid(child_, /*out*/&status, /*options*/0); + } else { + LOG(WARNING) << "execve should have avoided this path"; + // DCHECK(false) << "not possible because the execve would avoid this path"; + } + } + } + + bool SendCommand(const Command& command) { + // Only parent is the sender. + DCHECK(forked_); + //DCHECK(!IsDaemon()); + + char buf[1024]; + size_t stream_size; + if (!command.Write(buf, sizeof(buf), /*out*/&stream_size)) { + PLOG(ERROR) << "Failed to serialize command: " << command; + return false; + } + + if (LogVerboseIpc()) { + LOG(VERBOSE) << "pre-write(fd=" << pipefd_write_ << ", buf=" << buf + << ", size=" << stream_size<< ")"; + } + + if (params_.use_sockets) { + /* iov contains the normal message (Command) */ + struct iovec iov; + memset(&iov, 0, sizeof(iov)); + iov.iov_base = &buf[0]; + iov.iov_len = stream_size; + + struct msghdr msg; + memset(&msg, 0, sizeof(msg)); + + /* point to iov to transmit */ + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + /* no dest address; socket is connected */ + msg.msg_name = nullptr; + msg.msg_namelen = 0; + + // append a CMSG with SCM_RIGHTS if we have an FD. + if (command.fd.has_value()) { + union { + struct cmsghdr cmh; + char control[CMSG_SPACE(sizeof(int))]; /* sized to hold an fd (int) */ + } control_un; + memset(&control_un, 0, sizeof(control_un)); + + msg.msg_control = &control_un.control[0]; + msg.msg_controllen = sizeof(control_un.control); + + struct cmsghdr *hp; + hp = CMSG_FIRSTHDR(&msg); + hp->cmsg_len = CMSG_LEN(sizeof(int)); + hp->cmsg_level = SOL_SOCKET; + hp->cmsg_type = SCM_RIGHTS; + *((int *) CMSG_DATA(hp)) = *(command.fd); + + DCHECK(command.RequiresFd()) << command; + + if (LogVerboseIpc()) { + LOG(VERBOSE) << "append FD to sendmsg: " << *(command.fd); + } + } + + // TODO: add CMSG for the FD passage. + + if (TEMP_FAILURE_RETRY(sendmsg(pipefd_write_, &msg, /*flags*/0)) < 0) { + PLOG(ERROR) << "Failed to sendmsg command: " << command; + return false; + } + } else { + if (TEMP_FAILURE_RETRY(write(pipefd_write_, buf, stream_size)) < 0) { + PLOG(ERROR) << "Failed to write command: " << command; + return false; + } + } + + if (LogVerboseIpc()) { + LOG(VERBOSE) << "write(fd=" << pipefd_write_ << ", buf=" << buf + << ", size=" << stream_size<< ")"; + } + + // TODO: also read the reply? + return true; + } + + bool ReceiveCommand(const Command& command) { + // Only child is the command receiver. + // DCHECK(IsDaemon()); + + switch (command.choice) { + case CommandChoice::kRegisterFilePath: { + std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); + + if (!session) { + LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command; + return false; + } + + CHECK(command.file_path.has_value()) << command; + return session->RegisterFilePath(command.id, *command.file_path); + } + case CommandChoice::kUnregisterFilePath: { + std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); + + if (!session) { + LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command; + return false; + } + + return session->UnregisterFilePath(command.id); + } + case CommandChoice::kReadAhead: { + std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); + + if (!session) { + LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command; + return false; + } + + return session->ReadAhead(command.id, command.read_ahead_kind, command.length, command.offset); + } + // TODO: unreadahead + case CommandChoice::kExit: { + LOG(WARNING) << "kExit should be handled earlier."; + return true; + } + case CommandChoice::kCreateSession: { + std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); + if (session != nullptr) { + LOG(ERROR) << "ReceiveCommand: session for ID already exists: " << command; + return false; + } + CHECK(command.file_path.has_value()) << command; + if (session_manager_->CreateSession(command.session_id, /*description*/*command.file_path) + == nullptr) { + LOG(ERROR) << "ReceiveCommand: Failure to kCreateSession: " << command; + return false; + } + return true; + } + case CommandChoice::kDestroySession: { + if (!session_manager_->DestroySession(command.session_id)) { + LOG(ERROR) << "ReceiveCommand: Failure to kDestroySession: " << command; + return false; + } + return true; + } + case CommandChoice::kDumpSession: { + std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); + + if (!session) { + LOG(ERROR) << "ReceiveCommand: Could not find session for command: " << command; + return false; + } + + // TODO: Consider doing dumpsys support somehow? + session->Dump(LOG_STREAM(DEBUG), /*multiline*/true); + return true; + } + case CommandChoice::kDumpEverything: { + session_manager_->Dump(LOG_STREAM(DEBUG), /*multiline*/true); + break; + } + case CommandChoice::kCreateFdSession: { + std::shared_ptr<Session> session = session_manager_->FindSession(command.session_id); + if (session != nullptr) { + LOG(ERROR) << "ReceiveCommand: session for ID already exists: " << command; + return false; + } + CHECK(command.file_path.has_value()) << command; + CHECK(command.fd.has_value()) << command; + + LOG(VERBOSE) << "ReceiveCommand: kCreateFdSession fd=" << *(command.fd); + + // TODO: Maybe use CreateFdSession instead? + session = + session_manager_->CreateSession(command.session_id, + /*description*/*command.file_path, + command.fd.value()); + if (session == nullptr) { + LOG(ERROR) << "ReceiveCommand: Failure to kCreateFdSession: " << command; + return false; + } + + return session->ProcessFd(*command.fd); + } + } + + return true; + } + + pid_t child_; + bool forked_; + int pipefd_read_; + int pipefd_write_; + PrefetcherForkParameters params_; + // do not ever use an indirect session manager here, as it would cause a lifetime cycle. + std::unique_ptr<SessionManager> session_manager_; // direct only. +}; + +PrefetcherDaemon::PrefetcherDaemon() + : impl_{new Impl{this}} { + LOG(VERBOSE) << "PrefetcherDaemon() constructor"; +} + +bool PrefetcherDaemon::StartViaFork(PrefetcherForkParameters params) { + return impl_->StartViaFork(std::move(params)); +} + + +std::optional<PrefetcherForkParameters> PrefetcherDaemon::StartPipesViaFork() { + return impl_->StartPipesViaFork(); +} + +std::optional<PrefetcherForkParameters> PrefetcherDaemon::StartSocketViaFork() { + return impl_->StartSocketViaFork(); +} + +bool PrefetcherDaemon::Main(PrefetcherForkParameters params) { + return impl_->Main(params); +} + +bool PrefetcherDaemon::SendCommand(const Command& command) { + return impl_->SendCommand(command); +} + +PrefetcherDaemon::~PrefetcherDaemon() { + // required for unique_ptr for incomplete types. +} + +} // namespace iorap::prefetcher diff --git a/src/prefetcher/prefetcher_daemon.h b/src/prefetcher/prefetcher_daemon.h new file mode 100644 index 0000000..693f871 --- /dev/null +++ b/src/prefetcher/prefetcher_daemon.h @@ -0,0 +1,130 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PREFETCHER_DAEMON_H_ +#define PREFETCHER_DAEMON_H_ + +#include "prefetcher/session_manager.h" + +#include <memory> +#include <optional> +#include <ostream> + +namespace iorap { +namespace prefetcher { + +struct PrefetcherForkParameters { + int input_fd; + int output_fd; + bool use_sockets; // use the socket path instead of simpler read/write path. + bool format_text; // true=>text, false=>binary +}; + +inline std::ostream& operator<<(std::ostream& os, const PrefetcherForkParameters& p) { + os << "PrefetcherForkParameters{"; + os << "input_fd=" << p.input_fd << ","; + os << "output_fd=" << p.output_fd << ","; + os << "format_text=" << p.format_text << ","; + os << "use_sockets=" << p.use_sockets << ","; + os << "}"; + return os; +} + + +#ifndef READ_AHEAD_KIND +enum class ReadAheadKind : uint32_t { + kFadvise = 0, + kMmapLocked = 1, + kMlock = 2, +}; +#define READ_AHEAD_KIND 1 +#endif + +std::ostream& operator<<(std::ostream& os, ReadAheadKind k); + +enum class CommandChoice : uint32_t { + kRegisterFilePath, // kRegisterFilePath <sid:uint32> <id:uint32> <path:c-string> + kUnregisterFilePath, // kUnregisterFilePath <sid:uint32> <id:uint32> + kReadAhead, // kReadAhead <sid:uint32> <id:uint32> <kind:uint32_t> <length:uint64> <offset:uint64> + kExit, // kExit + kCreateSession, // kCreateSession <sid:uint32> <description:c-string> + kDestroySession, // kDestroySession <sid:uint32> + kDumpSession, // kDumpSession <sid:uint32> + kDumpEverything, // kDumpEverything + kCreateFdSession, // kCreateFdSession $CMSG{<fd:int>} <sid:uint32> <description:c-string> +}; + +struct Command { + CommandChoice choice; + uint32_t session_id; + uint32_t id; // file_path_id + std::optional<std::string> file_path; // required for choice=kRegisterFilePath. + // also serves as the description for choice=kCreateSession + + // choice=kReadAhead + ReadAheadKind read_ahead_kind; + uint64_t length; + uint64_t offset; + + std::optional<int> fd; // only valid in kCreateFdSession. + + // Deserialize from a char buffer. + // This can only fail if buf_size is too small. + static std::optional<Command> Read(char* buf, size_t buf_size, /*out*/size_t* consumed_bytes); + // Serialize to a char buffer. + // This can only fail if the buf_size is too small. + bool Write(char* buf, size_t buf_size, /*out*/size_t* produced_bytes) const; + + bool RequiresFd() const { + return choice == CommandChoice::kCreateFdSession; + } +}; + +std::ostream& operator<<(std::ostream& os, const Command& command); + +class PrefetcherDaemon { + public: + PrefetcherDaemon(); + ~PrefetcherDaemon(); + + // Asynchronously launch a new fork. + // + // The destructor will waitpid automatically on the child process. + bool StartViaFork(PrefetcherForkParameters params); + + // Launch a new fork , returning the pipes as input/output fds. + std::optional<PrefetcherForkParameters> StartPipesViaFork(); + + // Launch a new fork , returning the socket pair as input/output fds. + std::optional<PrefetcherForkParameters> StartSocketViaFork(); + + // Execute the main code in-process. + // + // Intended as the execve target. + bool Main(PrefetcherForkParameters params); + + // Send a command via IPC. + // The caller must be the parent process after using StartViaFork. + bool SendCommand(const Command& command); + + private: + class Impl; + std::unique_ptr<PrefetcherDaemon::Impl> impl_; +}; + +} // namespace prefetcher +} // namespace iorap + +#endif + diff --git a/src/prefetcher/read_ahead.cc b/src/prefetcher/read_ahead.cc new file mode 100644 index 0000000..8759f32 --- /dev/null +++ b/src/prefetcher/read_ahead.cc @@ -0,0 +1,447 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "read_ahead.h" + +#include "common/trace.h" +#include "prefetcher/session_manager.h" +#include "prefetcher/session.h" +#include "prefetcher/task_id.h" +#include "serialize/arena_ptr.h" +#include "serialize/protobuf_io.h" + +#include <android-base/chrono_utils.h> +#include <android-base/logging.h> +#include <android-base/scopeguard.h> +#include <android-base/properties.h> +#include <android-base/unique_fd.h> +#include <cutils/trace.h> +#include <deque> +#include <fcntl.h> +#include <functional> +#include <stdint.h> +#include <sys/mman.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unordered_map> +#include <utils/Printer.h> + +namespace iorap { +namespace prefetcher { + +enum class PrefetchStrategy { + kFadvise = 0, + kMmapLocked = 1, + kMlock = 2, +}; + +std::ostream& operator<<(std::ostream& os, PrefetchStrategy ps) { + switch (ps) { + case PrefetchStrategy::kFadvise: + os << "fadvise"; + break; + case PrefetchStrategy::kMmapLocked: + os << "mmap"; + break; + case PrefetchStrategy::kMlock: + os << "mlock"; + break; + default: + os << "<invalid>"; + } + return os; +} + +static constexpr PrefetchStrategy kPrefetchStrategy = PrefetchStrategy::kFadvise; +static uint64_t kMaxPrefetchBytes = ::android::base::GetUintProperty<uint64_t>( + "iorapd.max_prefetch_bytes", /*default*/100 * 1024 * 1024); // 100MB by default + +static PrefetchStrategy GetPrefetchStrategy() { + PrefetchStrategy strat = PrefetchStrategy::kFadvise; + + std::string prefetch_env = + ::android::base::GetProperty("iorapd.readahead.strategy", /*default*/""); + + if (prefetch_env == "") { + LOG(VERBOSE) + << "ReadAhead strategy defaulted. Did you want to set iorapd.readahead.strategy ?"; + } else if (prefetch_env == "mmap") { + strat = PrefetchStrategy::kMmapLocked; + LOG(VERBOSE) << "ReadAhead strategy: kMmapLocked"; + } else if (prefetch_env == "mlock") { + strat = PrefetchStrategy::kMlock; + LOG(VERBOSE) << "ReadAhead strategy: kMlock"; + } else if (prefetch_env == "fadvise") { + strat = PrefetchStrategy::kFadvise; + LOG(VERBOSE) << "ReadAhead strategy: kFadvise"; + } else { + LOG(WARNING) << "Unknown iorapd.readahead.strategy: " << prefetch_env << ", ignoring"; + } + + return strat; +} + +struct TaskData { + TaskId task_id; // also the session ID. + + size_t SessionId() const { + if (session != nullptr) { + DCHECK_EQ(session->SessionId(), task_id.id); + } + + // good enough to be used as the session ID. Task IDs are always monotonically increasing. + return task_id.id; + } + + std::shared_ptr<Session> session; + int32_t trace_cookie; // async trace cookie in BeginTask/FinishTask. +}; + +// Remember the last 5 files being prefetched. +static constexpr size_t kRecentDataCount = 5; + +struct RecentData { + TaskId task_id; + size_t file_lengths_sum; +}; + +struct RecentDataKeeper { + std::deque<RecentData> recents_; + std::mutex mutex_; + + void RecordRecent(TaskId task_id, size_t file_lengths_sum) { + std::lock_guard<std::mutex> guard{mutex_}; + + while (recents_.size() > kRecentDataCount) { + recents_.pop_front(); + } + recents_.push_back(RecentData{std::move(task_id), file_lengths_sum}); + } + + void Dump(/*borrow*/::android::Printer& printer) { + bool locked = mutex_.try_lock(); + + printer.printFormatLine("Recent prefetches:"); + if (!locked) { + printer.printLine(""""" (possible deadlock)"); + } + + for (const RecentData& data : recents_) { + printer.printFormatLine(" %s", data.task_id.path.c_str()); + printer.printFormatLine(" Task ID: %zu", data.task_id.id); + printer.printFormatLine(" Bytes count: %zu", data.file_lengths_sum); + } + + if (recents_.empty()) { + printer.printFormatLine(" (None)"); + } + + printer.printLine(""); + + if (locked) { + mutex_.unlock(); + } + } +}; + +struct ReadAhead::Impl { + Impl(bool use_sockets) { + // Flip this property to test in-process vs out-of-process for the prefetcher code. + bool out_of_process = + ::android::base::GetBoolProperty("iorapd.readahead.out_of_process", /*default*/true); + + SessionKind session_kind = + out_of_process ? SessionKind::kOutOfProcessIpc : SessionKind::kInProcessDirect; + + if (use_sockets) { + session_kind = SessionKind::kOutOfProcessSocket; + } + + session_manager_ = SessionManager::CreateManager(session_kind); + session_kind_ = session_kind; + } + + std::unique_ptr<SessionManager> session_manager_; + SessionKind session_kind_; + std::unordered_map<size_t /*task index*/, TaskData> read_ahead_file_map_; + static RecentDataKeeper recent_data_keeper_; + int32_t trace_cookie_{0}; + + bool UseSockets() const { + return session_kind_ == SessionKind::kOutOfProcessSocket; + } +}; + +RecentDataKeeper ReadAhead::Impl::recent_data_keeper_{}; + +ReadAhead::ReadAhead() : ReadAhead(/*use_sockets*/false) { +} + +ReadAhead::ReadAhead(bool use_sockets) : impl_(new Impl(use_sockets)) { +} + +ReadAhead::~ReadAhead() {} + +static bool PerformReadAhead(std::shared_ptr<Session> session, size_t path_id, ReadAheadKind kind, size_t length, size_t offset) { + return session->ReadAhead(path_id, kind, length, offset); +} + +void ReadAhead::FinishTask(const TaskId& id) { + auto it = impl_->read_ahead_file_map_.find(id.id); + if (it == impl_->read_ahead_file_map_.end()) { + LOG(DEBUG) << "Could not find any TaskData for " << id; + return; + } + + TaskData& task_data = it->second; + atrace_async_end(ATRACE_TAG_ACTIVITY_MANAGER, + "ReadAhead Task Scope (for File Descriptors)", + task_data.trace_cookie); + + auto deleter = [&]() { + impl_->read_ahead_file_map_.erase(it); + }; + auto scope_guard = android::base::make_scope_guard(deleter); + (void)scope_guard; + + LOG(VERBOSE) << "ReadAhead (Finish)"; + + if (!impl_->session_manager_->DestroySession(task_data.SessionId())) { + LOG(WARNING) << "ReadAhead: Failed to destroy Session " << task_data.SessionId(); + } +} + +void ReadAhead::BeginTaskForSockets(const TaskId& id, int32_t trace_cookie) { + LOG(VERBOSE) << "BeginTaskForSockets: " << id; + + // TODO: atrace. + android::base::Timer timer{}; + android::base::Timer open_timer{}; + + int trace_file_fd_raw = + TEMP_FAILURE_RETRY(open(id.path.c_str(), /*flags*/O_RDONLY)); + + android::base::unique_fd trace_file_fd{trace_file_fd_raw}; + + if (!trace_file_fd.ok()) { + PLOG(ERROR) << "ReadAhead failed to open trace file: " << id.path; + return; + } + + TaskData task_data; + task_data.task_id = id; + task_data.trace_cookie = trace_cookie; + + std::shared_ptr<Session> session = + impl_->session_manager_->CreateSession(task_data.SessionId(), + /*description*/id.path, + trace_file_fd.get()); + task_data.session = session; + CHECK(session != nullptr); + + task_data.trace_cookie = ++trace_cookie; + + // TODO: maybe getprop and a single line by default? + session->Dump(LOG_STREAM(INFO), /*multiline*/true); + + impl_->read_ahead_file_map_[id.id] = std::move(task_data); + // FinishTask is identical, as it just destroys the session. +} + +void ReadAhead::BeginTask(const TaskId& id) { + { + struct timeval now; + gettimeofday(&now, nullptr); + + uint64_t now_usec = (now.tv_sec * 1000000LL + now.tv_usec); + LOG(DEBUG) << "BeginTask: beginning usec: " << now_usec; + } + + int32_t trace_cookie = ++impl_->trace_cookie_; + atrace_async_begin(ATRACE_TAG_ACTIVITY_MANAGER, + "ReadAhead Task Scope (for File Descriptors)", + trace_cookie); + + if (impl_->UseSockets()) { + BeginTaskForSockets(id, trace_cookie); + return; + } + + LOG(VERBOSE) << "BeginTask: " << id; + + // TODO: atrace. + android::base::Timer timer{}; + + // TODO: refactor this code with SessionDirect::ProcessFd ? + TaskData task_data; + task_data.task_id = id; + task_data.trace_cookie = trace_cookie; + + ScopedFormatTrace atrace_begin_task(ATRACE_TAG_ACTIVITY_MANAGER, + "ReadAhead::BeginTask %s", + id.path.c_str()); + + // Include CreateSession above the Protobuf deserialization so that we can include + // the 'total_duration' as part of the Session dump (relevant when we use IPC mode only). + std::shared_ptr<Session> session = + impl_->session_manager_->CreateSession(task_data.SessionId(), + /*description*/id.path); + + android::base::Timer open_timer{}; + + // XX: Should we rename all the 'Create' to 'Make', or rename the 'Make' to 'Create' ? + // Unfortunately make_unique, make_shared, etc is the standard C++ terminology. + serialize::ArenaPtr<serialize::proto::TraceFile> trace_file_ptr = + serialize::ProtobufIO::Open(id.path); + + if (trace_file_ptr == nullptr) { + // TODO: distinguish between missing trace (this is OK, most apps wont have one) + // and a bad error. + LOG(DEBUG) << "ReadAhead could not start, missing trace file? " << id.path; + return; + } + + task_data.session = session; + CHECK(session != nullptr); + + ReadAheadKind kind = static_cast<ReadAheadKind>(GetPrefetchStrategy()); + + // TODO: The "Task[Id]" should probably be the one owning the trace file. + // When the task is fully complete, the task can be deleted and the + // associated arenas can go with them. + + // TODO: we should probably have the file entries all be relative + // to the package path? + + // Open every file in the trace index. + const serialize::proto::TraceFileIndex& index = trace_file_ptr->index(); + + size_t count_entries = 0; + { + ScopedFormatTrace atrace_register_file_paths(ATRACE_TAG_ACTIVITY_MANAGER, + "ReadAhead::RegisterFilePaths %s", + id.path.c_str()); + for (const serialize::proto::TraceFileIndexEntry& index_entry : index.entries()) { + LOG(VERBOSE) << "ReadAhead: found file entry: " << index_entry.file_name(); + + if (index_entry.id() < 0) { + LOG(WARNING) << "ReadAhead: Skip bad TraceFileIndexEntry, negative ID not allowed: " + << index_entry.id(); + continue; + } + + size_t path_id = index_entry.id(); + const auto& path_file_name = index_entry.file_name(); + + if (!session->RegisterFilePath(path_id, path_file_name)) { + LOG(WARNING) << "ReadAhead: Failed to register file path: " << path_file_name; + } else { + ++count_entries; + } + } + } + LOG(VERBOSE) << "ReadAhead: Registered " << count_entries << " file paths"; + std::chrono::milliseconds open_duration_ms = open_timer.duration(); + + LOG(DEBUG) << "ReadAhead: Opened file&headers in " << open_duration_ms.count() << "ms"; + + size_t length_sum = 0; + size_t prefetch_bytes = 0; + size_t entry_offset = 0; + { + ScopedFormatTrace atrace_perform_read_ahead(ATRACE_TAG_ACTIVITY_MANAGER, + "ReadAhead::PerformReadAhead entries=%zu, path=%s", + count_entries, + id.path.c_str()); + + // Go through every trace entry and readahead every (file,offset,len) tuple. + const serialize::proto::TraceFileList& file_list = trace_file_ptr->list(); + for (const serialize::proto::TraceFileEntry& file_entry : file_list.entries()) { + ++entry_offset; + + if (file_entry.file_length() < 0 || file_entry.file_offset() < 0) { + LOG(WARNING) << "ReadAhead entry negative file length or offset, illegal: " + << "index_id=" << file_entry.index_id() << ", skipping"; + continue; + } + + // Attempt to perform readahead. This can generate more warnings dynamically. + if (!PerformReadAhead(session, file_entry.index_id(), kind, file_entry.file_length(), file_entry.file_offset())) { + // TODO: Do we need below at all? The always-on Dump already prints a % of failed entries. + // LOG(WARNING) << "Failed readahead, bad file length/offset in entry @ " << (entry_offset - 1); + } else { + prefetch_bytes += static_cast<size_t>(file_entry.file_length()); + if (prefetch_bytes >= kMaxPrefetchBytes) { + LOG(WARNING) << "The prefetching size is " + << prefetch_bytes + << " and it exceeds the threshold " + << kMaxPrefetchBytes; + break; + } + } + + length_sum += static_cast<size_t>(file_entry.file_length()); + } + } + + { + ScopedFormatTrace atrace_session_dump(ATRACE_TAG_ACTIVITY_MANAGER, + "ReadAhead Session Dump entries=%zu", + entry_offset); + // TODO: maybe getprop and a single line by default? + session->Dump(LOG_STREAM(INFO), /*multiline*/true); + } + + atrace_int(ATRACE_TAG_ACTIVITY_MANAGER, + "ReadAhead Bytes Length", + static_cast<int32_t>(length_sum)); + + impl_->read_ahead_file_map_[id.id] = std::move(task_data); + + ReadAhead::Impl::recent_data_keeper_.RecordRecent(id, length_sum); +} + +void ReadAhead::Dump(::android::Printer& printer) { + ReadAhead::Impl::recent_data_keeper_.Dump(printer); +} + +std::optional<size_t> ReadAhead::PrefetchSizeInBytes(const std::string& file_path) { + serialize::ArenaPtr<serialize::proto::TraceFile> trace_file_ptr = + serialize::ProtobufIO::Open(file_path); + + if (trace_file_ptr == nullptr) { + LOG(WARNING) << "PrefetchSizeInBytes: bad file at " << file_path; + return std::nullopt; + } + + size_t length_sum = 0; + const serialize::proto::TraceFileList& file_list = trace_file_ptr->list(); + for (const serialize::proto::TraceFileEntry& file_entry : file_list.entries()) { + + if (file_entry.file_length() < 0 || file_entry.file_offset() < 0) { + LOG(WARNING) << "ReadAhead entry negative file length or offset, illegal: " + << "index_id=" << file_entry.index_id() << ", skipping"; + continue; + } + + length_sum += static_cast<size_t>(file_entry.file_length()); + } + + return length_sum; +} + +} // namespace prefetcher +} // namespace iorap + diff --git a/src/prefetcher/read_ahead.h b/src/prefetcher/read_ahead.h new file mode 100644 index 0000000..afc6ec8 --- /dev/null +++ b/src/prefetcher/read_ahead.h @@ -0,0 +1,63 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PREFETCHER_READAHEAD_H_ +#define PREFETCHER_READAHEAD_H_ + +#include <memory> +#include <optional> + +namespace android { +class Printer; +} // namespace android + +namespace iorap { +namespace prefetcher { + +struct TaskId; +struct ReadAheadFileEntry; + +// Manage I/O readahead for a task. +class ReadAhead { + struct Impl; + public: + // Process a task *now*. Currently will block until all readaheads have been + // issued for all entries in that task. + // + // Any memory mapped or file descriptors opened as a side effect must be + // cleaned up with #FinishTask. + void BeginTask(const TaskId& id); + // Complete a task, releasing any memory/file descriptors associated with it. + void FinishTask(const TaskId& id); + + static void Dump(/*borrow*/::android::Printer& printer); + + // Calculate the sum of file_lengths. Returns nullopt if the file path does not + // point to a valid compiled TraceFile. + static std::optional<size_t> PrefetchSizeInBytes(const std::string& file_path); + + ReadAhead(bool use_sockets); + + ReadAhead(); + ~ReadAhead(); + private: + void BeginTaskForSockets(const TaskId& id, int32_t trace_cookie); + std::unique_ptr<Impl> impl_; +}; + +} // namespace prefetcher +} // namespace iorap + +#endif + diff --git a/src/prefetcher/session.cc b/src/prefetcher/session.cc new file mode 100644 index 0000000..8a24b24 --- /dev/null +++ b/src/prefetcher/session.cc @@ -0,0 +1,724 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "session.h" + +#include "prefetcher/prefetcher_daemon.h" +#include "prefetcher/task_id.h" +#include "serialize/arena_ptr.h" +#include "serialize/protobuf_io.h" + +#include <android-base/logging.h> +#include <android-base/properties.h> +#include <android-base/unique_fd.h> +#include <fcntl.h> +#include <functional> +#include <stdint.h> +#include <sys/mman.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unordered_map> + +namespace iorap { +namespace prefetcher { + +// Print per-entry details even if successful. Default-off, too spammy. +static constexpr bool kLogVerboseReadAhead = false; + +std::ostream& operator<<(std::ostream& os, const Session& session) { + session.Dump(os, /*multiline*/false); + return os; +} + +Session::Session() { +} + +void SessionBase::Dump(std::ostream& os, bool multiline) const { + if (!multiline) { + os << "Session{"; + os << "session_id=" << SessionId(); + os << "}"; + return; + } else { + os << "Session (id=" << SessionId() << ")" << std::endl; + return; + } +} + +SessionBase::SessionBase(size_t session_id, std::string description) + : session_id_{session_id}, description_{description} { +} + +std::optional<std::string_view> SessionBase::GetFilePath(size_t path_id) const { + auto it = path_map_.find(path_id); + if (it != path_map_.end()) { + return {it->second}; + } else { + return std::nullopt; + } +} + +bool SessionBase::RemoveFilePath(size_t path_id) { + auto it = path_map_.find(path_id); + if (it != path_map_.end()) { + path_map_.erase(it); + return true; + } else { + return false; + } +} + +bool SessionBase::InsertFilePath(size_t path_id, std::string file_path) { + path_map_.insert({path_id, std::move(file_path)}); + return true; +} + +bool SessionBase::ProcessFd(int fd) { + // Only SessionDirect has an implementation of this. + // TODO: Maybe add a CommandChoice::kProcessFd ? instead of kCreateFdSession? + LOG(FATAL) << "SessionBase::ProcessFd is not implemented"; + + return false; +} + +// +// Direct +// + +std::ostream& operator<<(std::ostream& os, const SessionDirect::Entry& entry) { + os << "Entry{"; + os << "path_id=" << entry.path_id << ","; + os << "kind=" << static_cast<int>(entry.kind) << ","; + os << "length=" << entry.length << ","; + os << "offset=" << entry.offset << ","; + os << "}"; + + return os; +} + +// Just in case the failures are slowing down performance, turn them off. +// static constexpr bool kLogFailures = true; +static constexpr bool kLogFailures = false; + +bool SessionDirect::RegisterFilePath(size_t path_id, std::string_view file_path) { + std::string file_path_str{file_path}; // no c_str for string_view. + + auto fd = TEMP_FAILURE_RETRY(open(file_path_str.c_str(), O_RDONLY)); + if (fd < 0) { + if (kLogFailures) { + PLOG(ERROR) << "Failed to register file path: " << file_path << ", id=" << path_id + << ", open(2) failed: "; + } + fd = android::base::unique_fd{}; // mark as 'bad' descriptor. + } + + LOG(VERBOSE) << "RegisterFilePath path_id=" << path_id << ", file_path=" << file_path_str; + + if (!InsertFilePath(path_id, std::move(file_path_str))) { + return false; + } + + path_fd_map_.insert(std::make_pair(path_id, std::move(fd))); + DCHECK(entry_list_map_[path_id].empty()); + + return true; +} + + +bool SessionDirect::UnregisterFilePath(size_t path_id) { + if (!RemoveFilePath(path_id)) { + return false; + } + + { // Scoped FD reference lifetime. + auto maybe_fd = GetFdForPath(path_id); + + DCHECK(*maybe_fd != nullptr); + const android::base::unique_fd& entry_fd = **maybe_fd; + + auto list = entry_list_map_[path_id]; + + for (const EntryMapping& entry_mapping : list) { + ReadAheadKind kind = entry_mapping.entry.kind; + + switch (kind) { + case ReadAheadKind::kFadvise: + // Nothing to do. + break; + case ReadAheadKind::kMmapLocked: + FALLTHROUGH_INTENDED; + case ReadAheadKind::kMlock: + // Don't do any erases in the unregister file path to avoid paying O(n^2) erase cost. + UnmapWithoutErase(entry_mapping); + break; + } + } + } + + auto it = entry_list_map_.find(path_id); + auto end = entry_list_map_.end(); + DCHECK(it != end); + entry_list_map_.erase(it); + + // Close the FD for this file path. + auto fd_it = path_fd_map_.find(path_id); + DCHECK(fd_it != path_fd_map_.end()); + path_fd_map_.erase(fd_it); + + return true; +} + +// Note: return a pointer because optional doesn't hold references directly. +std::optional<android::base::unique_fd*> SessionDirect::GetFdForPath(size_t path_id) { + auto it = path_fd_map_.find(path_id); + if (it == path_fd_map_.end()) { + return std::nullopt; + } else { + return &it->second; + } +} + +bool SessionDirect::ReadAhead(size_t path_id, + ReadAheadKind kind, + size_t length, + size_t offset) { + // Take by-reference so we can mutate list at the end. + auto& list = entry_list_map_[path_id]; + + Entry entry{path_id, kind, length, offset}; + EntryMapping entry_mapping{entry, /*address*/nullptr, /*success*/false}; + + bool success = true; + + auto maybe_fd = GetFdForPath(path_id); + if (!maybe_fd) { + LOG(ERROR) << "SessionDirect: Failed to find FD for path_id=" << path_id; + return false; + } + + DCHECK(*maybe_fd != nullptr); + const android::base::unique_fd& entry_fd = **maybe_fd; + + std::optional<std::string_view> file_name_opt = GetFilePath(path_id); + DCHECK(file_name_opt.has_value()); // if one map has it, all maps have it. + std::string_view file_name = *file_name_opt; + + if (!entry_fd.ok()) { + LOG(VERBOSE) << "SessionDirect: No file descriptor for (path_id=" << path_id << ") " + << "path '" << file_name << "', failed to readahead entry."; + // Even failures get kept with success=false. + list.push_back(entry_mapping); + return false; + } + + switch (kind) { + case ReadAheadKind::kFadvise: + if (posix_fadvise(entry_fd, offset, length, POSIX_FADV_WILLNEED) != 0) { + PLOG(ERROR) << "SessionDirect: Failed to fadvise entry " << file_name + << ", offset=" << offset << ", length=" << length; + success = false; + } + break; + case ReadAheadKind::kMmapLocked: + FALLTHROUGH_INTENDED; + case ReadAheadKind::kMlock: { + const bool need_mlock = kind == ReadAheadKind::kMlock; + + int flags = MAP_SHARED; + if (!need_mlock) { + // MAP_LOCKED is a best-effort to lock the page. it could still be + // paged in later at a fault. + flags |= MAP_LOCKED; + } + + entry_mapping.address = + mmap(/*addr*/nullptr, length, PROT_READ, flags, entry_fd, offset); + + if (entry_mapping.address == nullptr) { + PLOG(ERROR) << "SessionDirect: Failed to mmap entry " << file_name + << ", offset=" << offset << ", length=" << length; + success = false; + break; + } + + // Strong guarantee that page will be locked if mlock returns successfully. + if (need_mlock && mlock(entry_mapping.address, length) < 0) { + PLOG(ERROR) << "SessionDirect: Failed to mlock entry " << file_name + << ", offset=" << offset << ", length=" << length; + // We already have a mapping address, so we should add it to the list. + // However this didn't succeed 100% because the lock failed, so return false later. + success = false; + } + } + } + + // Keep track of success so we know in Dump() what the number of failed entry mappings were. + entry_mapping.success = success; + + // Keep track of this so that we can clean it up later in UnreadAhead. + list.push_back(entry_mapping); + + if (entry_mapping.success) { + if (kLogVerboseReadAhead) { + LOG(VERBOSE) << "SessionDirect: ReadAhead for " << entry_mapping.entry; + } + } // else one of the errors above already did print. + + return success; +} + +bool SessionDirect::UnreadAhead(size_t path_id, + ReadAheadKind kind, + size_t length, + size_t offset) { + Entry entry{path_id, kind, length, offset}; + + auto list = entry_list_map_[path_id]; + if (list.empty()) { + return false; + } + + std::optional<EntryMapping> entry_mapping; + size_t idx = 0; + + for (size_t i = 0; i < list.size(); ++i) { + if (entry == list[i].entry) { + entry_mapping = list[i]; + idx = 0; + break; + } + } + + if (!entry_mapping) { + return false; + } + + switch (kind) { + case ReadAheadKind::kFadvise: + // Nothing to do. + // TODO: maybe fadvise(RANDOM)? + return true; + case ReadAheadKind::kMmapLocked: + FALLTHROUGH_INTENDED; + case ReadAheadKind::kMlock: + UnmapWithoutErase(*entry_mapping); + return true; + } + + list.erase(list.begin() + idx); + + // FDs close only with UnregisterFilePath. + return true; +} + +void SessionDirect::UnmapWithoutErase(const EntryMapping& entry_mapping) { + void* address = entry_mapping.address; + size_t length = entry_mapping.entry.length; + + // munmap also unlocks. Do not need explicit munlock. + if (munmap(address, length) < 0) { + PLOG(WARNING) << "ReadAhead (Finish): Failed to munmap address: " + << address << ", length: " << length; + } + +} + +bool SessionDirect::ProcessFd(int fd) { + // TODO: the path is advisory, but it would still be cleaner to pass it separately + const char* fd_path = SessionDescription().c_str(); + + android::base::Timer open_timer{}; + android::base::Timer total_timer{}; + + serialize::ArenaPtr<serialize::proto::TraceFile> trace_file_ptr = + serialize::ProtobufIO::Open(fd, fd_path); + + if (trace_file_ptr == nullptr) { + LOG(ERROR) << "SessionDirect::ProcessFd failed, corrupted protobuf format? " << fd_path; + return false; + } + + // TODO: maybe make it part of a kProcessFd type of command? + ReadAheadKind kind = ReadAheadKind::kFadvise; + + // TODO: The "Task[Id]" should probably be the one owning the trace file. + // When the task is fully complete, the task can be deleted and the + // associated arenas can go with them. + + // TODO: we should probably have the file entries all be relative + // to the package path? + + // Open every file in the trace index. + const serialize::proto::TraceFileIndex& index = trace_file_ptr->index(); + + size_t count_entries = 0; + for (const serialize::proto::TraceFileIndexEntry& index_entry : index.entries()) { + LOG(VERBOSE) << "ReadAhead: found file entry: " << index_entry.file_name(); + + if (index_entry.id() < 0) { + LOG(WARNING) << "ReadAhead: Skip bad TraceFileIndexEntry, negative ID not allowed: " + << index_entry.id(); + continue; + } + + size_t path_id = index_entry.id(); + const auto& path_file_name = index_entry.file_name(); + + if (!this->RegisterFilePath(path_id, path_file_name)) { + LOG(WARNING) << "ReadAhead: Failed to register file path: " << path_file_name; + ++count_entries; + } + } + LOG(VERBOSE) << "ReadAhead: Registered " << count_entries << " file paths"; + std::chrono::milliseconds open_duration_ms = open_timer.duration(); + + LOG(DEBUG) << "ProcessFd: open+parsed headers in " << open_duration_ms.count() << "ms"; + + // Go through every trace entry and readahead every (file,offset,len) tuple. + size_t entry_offset = 0; + const serialize::proto::TraceFileList& file_list = trace_file_ptr->list(); + for (const serialize::proto::TraceFileEntry& file_entry : file_list.entries()) { + ++entry_offset; + + if (file_entry.file_length() < 0 || file_entry.file_offset() < 0) { + LOG(WARNING) << "ProcessFd entry negative file length or offset, illegal: " + << "index_id=" << file_entry.index_id() << ", skipping"; + continue; + } + + // Attempt to perform readahead. This can generate more warnings dynamically. + if (!this->ReadAhead(file_entry.index_id(), + kind, + file_entry.file_length(), + file_entry.file_offset())) { + if (kLogFailures) { + LOG(WARNING) << "Failed readahead, bad file length/offset in entry @ " + << (entry_offset - 1); + } + } + } + + std::chrono::milliseconds total_duration_ms = total_timer.duration(); + LOG(DEBUG) << "ProcessFd: total duration " << total_duration_ms.count() << "ms"; + + { + struct timeval now; + gettimeofday(&now, nullptr); + + uint64_t now_usec = (now.tv_sec * 1000000LL + now.tv_usec); + LOG(DEBUG) << "ProcessFd: finishing usec: " << now_usec; + } + + return true; +} + + +static bool IsDumpEveryEntry() { + // Set to 'true' to dump every single entry for debugging (multiline). + // Otherwise it only prints per-file-path summaries. + return ::android::base::GetBoolProperty("iorapd.readahead.dump_all", /*default*/false); +} + +static bool IsDumpEveryPath() { + // Dump per-file-path (entry) stats. Defaults to on if the above property is on. + return ::android::base::GetBoolProperty("iorapd.readahead.dump_paths", /*default*/false); +} + +void SessionDirect::Dump(std::ostream& os, bool multiline) const { + { + struct timeval now; + gettimeofday(&now, nullptr); + + uint64_t now_usec = (now.tv_sec * 1000000LL + now.tv_usec); + LOG(DEBUG) << "SessionDirect::Dump: beginning usec: " << now_usec; + } + + size_t path_count = entry_list_map_.size(); + + size_t read_ahead_entries = 0; + size_t read_ahead_bytes = 0; + + size_t overall_entry_count = 0; + size_t overall_byte_count = 0; + for (auto it = entry_list_map_.begin(); it != entry_list_map_.end(); ++it) { + const auto& entry_mapping_list = it->second; + + for (size_t j = 0; j < entry_mapping_list.size(); ++j) { + const EntryMapping& entry_mapping = entry_mapping_list[j]; + const Entry& entry = entry_mapping.entry; + + ++overall_entry_count; + overall_byte_count += entry.length; + + if (entry_mapping.success) { + ++read_ahead_entries; + read_ahead_bytes += entry.length; + } + } + } + + double overall_success_entry_rate = + read_ahead_entries * 100.0 / overall_entry_count; + double overall_success_byte_rate = + read_ahead_bytes * 100.0 / overall_byte_count; + + size_t fd_count = path_fd_map_.size(); + size_t good_fd_count = 0; + for (auto it = path_fd_map_.begin(); it != path_fd_map_.end(); ++it) { + if (it->second.ok()) { + ++good_fd_count; + } + } + double good_fd_rate = good_fd_count * 100.0 / fd_count; + // double bad_fd_rate = (fd_count - good_fd_count) * 1.0 / fd_count; + + if (!multiline) { + os << "SessionDirect{"; + os << "session_id=" << SessionId() << ","; + + os << "file_paths=" << path_count << " (good: " << good_fd_rate << "),"; + os << "read_ahead_entries=" << read_ahead_entries; + os << "(" << overall_success_entry_rate << "%),"; + os << "read_ahead_bytes=" << read_ahead_bytes << ""; + os << "(" << overall_success_byte_rate << "%),"; + os << "timer=" << timer_.duration().count() << ","; + + os << "}"; + return; + } else { + // Always try to pay attention to these stats below. + // They can be signs of potential performance problems. + os << "Session Direct (id=" << SessionId() << ")" << std::endl; + + os << " Summary: " << std::endl; + os << " Description = " << SessionDescription() << std::endl; + os << " Duration = " << timer_.duration().count() << "ms" << std::endl; + os << " Total File Paths=" << path_count << " (good: " << good_fd_rate << "%)" << std::endl; + os << " Total Entries=" << overall_entry_count; + os << " (good: " << overall_success_entry_rate << "%)" << std::endl; + os << " Total Bytes=" << overall_byte_count << ""; + os << " (good: " << overall_success_byte_rate << "%)" << std::endl; + os << std::endl; + + // Probably too spammy, but they could narrow down the issue for a problem in above stats. + if (!IsDumpEveryPath() && !IsDumpEveryEntry()) { + return; + } + + for (auto it = entry_list_map_.begin(); it != entry_list_map_.end(); ++it) { + size_t path_id = it->first; + const auto& entry_mapping_list = it->second; + + std::optional<std::string_view> file_path = GetFilePath(path_id); + os << " File Path (id=" << path_id << "): "; + if (file_path.has_value()) { + os << "'" << *file_path << "'"; + } else { + os << "(nullopt)"; + } + + auto fd_it = path_fd_map_.find(path_id); + os << ", FD="; + if (fd_it != path_fd_map_.end()) { + const android::base::unique_fd& fd = fd_it->second; + os << fd.get(); // -1 for failed fd. + } else { + os << "(none)"; + } + os << std::endl; + + size_t total_entries = entry_mapping_list.size(); + size_t total_bytes = 0; + + size_t local_read_ahead_entries = 0; + size_t local_read_ahead_bytes = 0; + for (size_t j = 0; j < entry_mapping_list.size(); ++j) { + const EntryMapping& entry_mapping = entry_mapping_list[j]; + const Entry& entry = entry_mapping.entry; + + total_bytes += entry.length; + + // Sidenote: Bad FDs will have 100% failed mappings. + // Good FDs may sometimes have failed mappings. + if (entry_mapping.success) { + ++local_read_ahead_entries; + local_read_ahead_bytes += entry.length; + } + + if (IsDumpEveryEntry()) { + os << " Entry " << j << " details:" << std::endl; + os << " " << entry << std::endl; + os << " Mapping " << (entry_mapping.success ? "Succeeded" : "Failed") + << ", Address " << entry_mapping.address << std::endl; + } + } + + double entry_success_rate = local_read_ahead_entries * 100.0 / total_entries; + double bytes_success_rate = local_read_ahead_bytes * 100.0 / total_bytes; + + double entry_failure_rate = (total_entries - local_read_ahead_entries) * 100.0 / total_entries; + double bytes_failure_rate = (total_bytes - local_read_ahead_bytes) * 100.0 / total_bytes; + + os << " Successful: Entries=" << local_read_ahead_entries + << " (" << entry_success_rate << "%)" + << ", Bytes=" << local_read_ahead_bytes + << " (" << bytes_success_rate << "%)" + << std::endl; + os << " Failed: Entries=" << (total_entries - local_read_ahead_entries) + << " (" << entry_failure_rate << "%)" + << ", Bytes=" << (total_bytes - local_read_ahead_bytes) + << " (" << bytes_failure_rate << "%)" + << std::endl; + os << " Total: Entries=" << total_entries + << ", Bytes=" << total_bytes + << std::endl; + } + + return; + } +} + +SessionDirect::~SessionDirect() { + for (auto it = entry_list_map_.begin(); it != entry_list_map_.end();) { + size_t path_id = it->first; + + ++it; // the iterator is removed in the following Unregister method. + UnregisterFilePath(path_id); + } +} + +// +// Indirect +// + +SessionIndirect::SessionIndirect(size_t session_id, + std::string description, + std::shared_ptr<PrefetcherDaemon> daemon, + bool send_command) + : SessionBase{session_id, description}, + daemon_{daemon} { + // Don't do anything in e.g. subclasses. + if (!send_command) { + return; + } + + Command cmd{}; + cmd.choice = CommandChoice::kCreateSession; + cmd.session_id = session_id; + cmd.file_path = description; + + LOG(VERBOSE) << "SessionIndirect: " << cmd; + + if (!daemon_->SendCommand(cmd)) { + LOG(FATAL) << "SessionIndirect: Failure to create session " << session_id + << ", description: " << description; + } +} + +SessionIndirect::~SessionIndirect() { + Command cmd{}; + cmd.choice = CommandChoice::kDestroySession; + cmd.session_id = SessionId(); + + if (!daemon_->SendCommand(cmd)) { + LOG(WARNING) << "SessionIndirect: Failure to destroy session " << SessionId() + << ", description: " << SessionDescription(); + } +} + +void SessionIndirect::Dump(std::ostream& os, bool multiline) const { + // SessionBase::Dump(os, multiline); + // TODO: does having the local dump do anything for us? + + Command cmd{}; + cmd.choice = CommandChoice::kDumpSession; + cmd.session_id = SessionId(); + + daemon_->SendCommand(cmd); +} + +bool SessionIndirect::RegisterFilePath(size_t path_id, std::string_view file_path) { + Command cmd{}; + cmd.choice = CommandChoice::kRegisterFilePath; + cmd.session_id = SessionId(); + cmd.id = path_id; + cmd.file_path = file_path; + + return daemon_->SendCommand(cmd); +} + +bool SessionIndirect::UnregisterFilePath(size_t path_id) { + Command cmd{}; + cmd.choice = CommandChoice::kUnregisterFilePath; + cmd.session_id = SessionId(); + cmd.id = path_id; + + return daemon_->SendCommand(cmd); +} +bool SessionIndirect::ReadAhead(size_t path_id, + ReadAheadKind kind, + size_t length, + size_t offset) { + Command cmd{}; + cmd.choice = CommandChoice::kReadAhead; + cmd.session_id = SessionId(); + cmd.id = path_id; + cmd.read_ahead_kind = kind; + cmd.length = length; + cmd.offset = offset; + + return daemon_->SendCommand(cmd); +} + +bool SessionIndirect::UnreadAhead(size_t path_id, + ReadAheadKind kind, + size_t length, + size_t offset) { + LOG(WARNING) << "UnreadAhead: command not implemented yet"; + return true; +} + +// +// IndirectSocket +// + +SessionIndirectSocket::SessionIndirectSocket(size_t session_id, + int fd, + std::string description, + std::shared_ptr<PrefetcherDaemon> daemon) + : SessionIndirect{session_id, description, daemon, /*send_command*/false} { + // TODO: all of the WriteCommand etc in the daemon. + Command cmd{}; + cmd.choice = CommandChoice::kCreateFdSession; + cmd.fd = fd; + cmd.session_id = session_id; + cmd.file_path = description; + + LOG(VERBOSE) << "SessionIndirectSocket: " << cmd; + + if (!daemon_->SendCommand(cmd)) { + LOG(FATAL) << "SessionIndirectSocket: Failure to create session " << session_id + << ", description: " << description; + } + + // This goes into the SessionDirect ctor + SessionDirect::ProcessFd + // as implemented in PrefetcherDaemon::ReceiveCommand +} + +SessionIndirectSocket::~SessionIndirectSocket() { +} + +} // namespace prefetcher +} // namespace iorap diff --git a/src/prefetcher/session.h b/src/prefetcher/session.h new file mode 100644 index 0000000..a4a9e6b --- /dev/null +++ b/src/prefetcher/session.h @@ -0,0 +1,236 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PREFETCHER_SESSION_H_ +#define PREFETCHER_SESSION_H_ + +#include <android-base/chrono_utils.h> +#include <android-base/unique_fd.h> + +#include <optional> +#include <memory> +#include <string> +#include <string_view> +#include <unordered_map> +#include <vector> + +namespace iorap { +namespace prefetcher { + +#ifndef READ_AHEAD_KIND +#define READ_AHEAD_KIND 1 +enum class ReadAheadKind : uint32_t { + kFadvise = 0, + kMmapLocked = 1, + kMlock = 2, +}; +#endif + +class Session { + public: + virtual bool RegisterFilePath(size_t path_id, std::string_view file_path) = 0; + virtual bool UnregisterFilePath(size_t path_id) = 0; + + // Immediately perform a readahead now. + // Fadvise: the readahead will have been queued by the kernel. + // MmapLocked/Mlock: the memory is pinned by the requested process. + virtual bool ReadAhead(size_t path_id, ReadAheadKind kind, size_t length, size_t offset) = 0; + + // Cancels a readahead previously done. + // The length/offset should match the call of ReadAhead. + virtual bool UnreadAhead(size_t path_id, ReadAheadKind kind, size_t length, size_t offset) = 0; + + // Multi-line detailed dump, e.g. for dumpsys. + // Single-line summary dump, e.g. for logcat. + virtual void Dump(std::ostream& os, bool multiline) const = 0; + + // Process the FD for kCreateFdSession. + // Assumes there's a compiled_trace.pb at the fd, calling this function + // will immediately process it and execute any read-aheads. + // + // FD is borrowed only for the duration of the function call. + virtual bool ProcessFd(int fd) = 0; + + // Get the session ID associated with this session. + // Session IDs are distinct, they are not used for new sessions. + virtual size_t SessionId() const = 0; + + // Get this session's description. + // Only useful for logging/dumping. + virtual const std::string& SessionDescription() const = 0; + + // Implicitly unregister any remaining file paths. + // All read-aheads are also cancelled. + virtual ~Session() {} + + protected: + Session(); +}; + +// Single-line summary dump of Session. +std::ostream& operator<<(std::ostream&os, const Session& session); + +class SessionBase : public Session { + public: + virtual void Dump(std::ostream& os, bool multiline) const override; + virtual ~SessionBase() {} + + virtual size_t SessionId() const override { + return session_id_; + } + + virtual const std::string& SessionDescription() const override { + return description_; + } + + virtual bool ProcessFd(int fd) override; + + protected: + SessionBase(size_t session_id, std::string description); + std::optional<std::string_view> GetFilePath(size_t path_id) const; + bool RemoveFilePath(size_t path_id); + bool InsertFilePath(size_t path_id, std::string file_path); + + android::base::Timer timer_{}; + private: + // Note: Store filename for easier debugging and for dumping. + std::unordered_map</*path_id*/size_t, std::string> path_map_; + size_t session_id_; + std::string description_; +}; + +// In-process session. +class SessionDirect : public SessionBase { + public: + virtual bool RegisterFilePath(size_t path_id, std::string_view file_path) override; + + virtual bool UnregisterFilePath(size_t path_id) override; + virtual bool ReadAhead(size_t path_id, + ReadAheadKind kind, + size_t length, + size_t offset); + + virtual bool UnreadAhead(size_t path_id, + ReadAheadKind kind, + size_t length, + size_t offset) override; + + virtual bool ProcessFd(int fd) override; + + virtual void Dump(std::ostream& os, bool multiline) const override; + + virtual ~SessionDirect(); + + SessionDirect(size_t session_id, std::string description) + : SessionBase{session_id, std::move(description)} { + } + protected: + struct Entry { + size_t path_id; + ReadAheadKind kind; + size_t length; + size_t offset; + + constexpr bool operator==(const Entry& other) const { + return path_id == other.path_id && + kind == other.kind && + length == other.length && + offset == other.offset; + } + constexpr bool operator!=(const Entry& other) const { + return !(*this == other); + } + + friend std::ostream& operator<<(std::ostream& os, const Entry& entry); + }; + + struct EntryMapping { + Entry entry; + void* address; + bool success; + }; + + // bool EntryReadAhead(const Entry& entry) const; + // bool EntryUnReadAhead(const Entry& entry) const; + + void UnmapWithoutErase(const EntryMapping& entry_mapping); + std::optional<android::base::unique_fd*> GetFdForPath(size_t path_id); + + private: + std::unordered_map</*path_id*/size_t, std::vector<EntryMapping>> entry_list_map_; + std::unordered_map</*path_id*/size_t, android::base::unique_fd> path_fd_map_; + + public: + friend std::ostream& operator<<(std::ostream& os, const SessionDirect::Entry& entry); +}; + +std::ostream& operator<<(std::ostream& os, const SessionDirect::Entry& entry); + +class PrefetcherDaemon; + +// Out-of-process session. Requires prefetcher daemon. +class SessionIndirect : public SessionBase { + public: + virtual bool RegisterFilePath(size_t path_id, std::string_view file_path) override; + + virtual bool UnregisterFilePath(size_t path_id) override; + virtual bool ReadAhead(size_t path_id, + ReadAheadKind kind, + size_t length, + size_t offset) override; + + virtual bool UnreadAhead(size_t path_id, + ReadAheadKind kind, + size_t length, + size_t offset) override; + + virtual void Dump(std::ostream& os, bool multiline) const override; + + // Creates a new session indirectly. + // Writes to daemon the new session command. + SessionIndirect(size_t session_id, + std::string description, + std::shared_ptr<PrefetcherDaemon> daemon, + bool send_command = true); + + // Destroys the current session. + // Writes to daemon that the session is to be destroyed. + virtual ~SessionIndirect(); + + protected: + std::shared_ptr<PrefetcherDaemon> daemon_; +}; + +// Out-of-process session. Requires prefetcher daemon. +class SessionIndirectSocket : public SessionIndirect { + public: + // Creates a new session indirectly. + // Writes to daemon the new session command. + SessionIndirectSocket(size_t session_id, + int fd, + std::string description, + std::shared_ptr<PrefetcherDaemon> daemon); + // Destroys the current session. + // Writes to daemon that the session is to be destroyed. + virtual ~SessionIndirectSocket(); + + private: +}; + + +} // namespace prefetcher +} // namespace iorap + +#endif + diff --git a/src/prefetcher/session_manager.cc b/src/prefetcher/session_manager.cc new file mode 100644 index 0000000..d6fab1f --- /dev/null +++ b/src/prefetcher/session_manager.cc @@ -0,0 +1,281 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "session_manager.h" + +#include "prefetcher/prefetcher_daemon.h" +#include "prefetcher/session.h" +#include "prefetcher/task_id.h" +#include "serialize/arena_ptr.h" +#include "serialize/protobuf_io.h" + +#include <android-base/logging.h> +#include <android-base/chrono_utils.h> +#include <android-base/unique_fd.h> +#include <fcntl.h> +#include <functional> +#include <stdint.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unordered_map> + +namespace iorap { +namespace prefetcher { + +std::ostream& operator<<(std::ostream& os, const SessionManager& manager) { + manager.Dump(os, /*multiline*/false); + return os; +} + +SessionManager::SessionManager() { +} + +class SessionManagerBase : public SessionManager { + public: + virtual void Dump(std::ostream& os, bool multiline) const { + if (!multiline) { + os << "SessionManager{"; + + os << "sessions=["; + for (auto it = sessions_map_.begin(); + it != sessions_map_.end(); + ++it) { + os << "(" << it->second.description << ") "; + it->second.session->Dump(os, /*multiline*/false); + } + os << "]"; + return; + } + + os << "SessionManager (session count = " << sessions_map_.size() << "):" << std::endl; + os << std::endl; + + for (auto it = sessions_map_.begin(); + it != sessions_map_.end(); + ++it) { + os << "Description: " << it->second.description << std::endl; + it->second.session->Dump(os, /*multiline*/true); + } + + // TODO: indentations? Use this pseudo line break for the time being. + os << "--------------------------------" << std::endl; + } + + virtual ~SessionManagerBase() {} + + virtual std::shared_ptr<Session> FindSession(size_t session_id) const override { + auto it = sessions_map_.find(session_id); + if (it != sessions_map_.end()) { + DCHECK_EQ(session_id, it->second.SessionId()); + return it->second.session; + } else { + return nullptr; + } + } + + virtual bool DestroySession(size_t session_id) override { + auto it = sessions_map_.find(session_id); + if (it != sessions_map_.end()) { + sessions_map_.erase(it); + return true; + } else { + return false; + } + } + + protected: + void InsertNewSession(std::shared_ptr<Session> session, std::string description) { + DCHECK(!FindSession(session->SessionId())) << "session cannot already exist"; + + size_t session_id = session->SessionId(); + + SessionData data; + data.session = std::move(session); + data.description = std::move(description); + + sessions_map_.insert({session_id, std::move(data)}); + } + + private: + struct SessionData { + std::shared_ptr<Session> session; + std::string description; + + size_t SessionId() const { + return session->SessionId(); + } + }; + + std::unordered_map</*session_id*/size_t, SessionData> sessions_map_; +}; + +class SessionManagerDirect : public SessionManagerBase { + public: + virtual std::shared_ptr<Session> CreateSession(size_t session_id, + std::string description) override { + LOG(VERBOSE) << "CreateSessionDirect id=" << session_id << ", description=" << description; + + std::shared_ptr<Session> session = + std::static_pointer_cast<Session>(std::make_shared<SessionDirect>(session_id, + description)); + DCHECK(FindSession(session_id) == nullptr); + InsertNewSession(session, std::move(description)); + return session; + } + + SessionManagerDirect() { + // Intentionally left empty. + } + + private: +}; + + +class SessionManagerIndirect : public SessionManagerBase { + public: + virtual std::shared_ptr<Session> CreateSession(size_t session_id, + std::string description) override { + LOG(VERBOSE) << "CreateSessionIndirect id=" << session_id << ", description=" << description; + + std::shared_ptr<Session> session = + std::static_pointer_cast<Session>(std::make_shared<SessionIndirect>(session_id, + description, + daemon_)); + InsertNewSession(session, description); + return session; + } + + SessionManagerIndirect() : daemon_{std::make_shared<PrefetcherDaemon>()} { + //StartViaFork etc. + // TODO: also expose a 'MainLoop(...) -> daemon::Main(..)' somehow in the base interface. + auto params = daemon_->StartPipesViaFork(); + if (!params) { + LOG(FATAL) << "Failed to fork+exec iorap.prefetcherd"; + } + } + + virtual ~SessionManagerIndirect() { + Command cmd{}; + cmd.choice = CommandChoice::kExit; + + if (!daemon_->SendCommand(cmd)) { + LOG(FATAL) << "Failed to nicely exit iorap.prefetcherd"; + } + } + + virtual void Dump(std::ostream& os, bool multiline) const override { + Command cmd{}; + cmd.choice = CommandChoice::kDumpEverything; + + if (!daemon_->SendCommand(cmd)) { + LOG(ERROR) << "Failed to transmit kDumpEverything to iorap.prefetcherd"; + } + } + + + private: + // No lifetime cycle: PrefetcherDaemon only has a SessionManagerDirect in it. + std::shared_ptr<PrefetcherDaemon> daemon_; +}; + +class SessionManagerIndirectSocket : public SessionManagerBase { + public: + virtual std::shared_ptr<Session> CreateSession(size_t session_id, + std::string description) override { + DCHECK(false) << "not supposed to create a regular session for Socket"; + + LOG(VERBOSE) << "CreateSessionIndirect id=" << session_id << ", description=" << description; + + std::shared_ptr<Session> session = + std::static_pointer_cast<Session>(std::make_shared<SessionIndirect>(session_id, + description, + daemon_)); + InsertNewSession(session, description); + return session; + } + + virtual std::shared_ptr<Session> CreateSession(size_t session_id, + std::string description, + std::optional<int> fd) override { + CHECK(fd.has_value()); + LOG(VERBOSE) << "CreateSessionIndirectSocket id=" << session_id + << ", description=" << description + << ", fd=" << *fd; + + std::shared_ptr<Session> session = + std::static_pointer_cast<Session>(std::make_shared<SessionIndirectSocket>(session_id, + *fd, + description, + daemon_)); + InsertNewSession(session, description); + return session; + } + + SessionManagerIndirectSocket() : daemon_{std::make_shared<PrefetcherDaemon>()} { + auto params = daemon_->StartSocketViaFork(); + if (!params) { + LOG(FATAL) << "Failed to fork+exec iorap.prefetcherd"; + } + } + + virtual ~SessionManagerIndirectSocket() { + Command cmd{}; + cmd.choice = CommandChoice::kExit; + + if (!daemon_->SendCommand(cmd)) { + LOG(FATAL) << "Failed to nicely exit iorap.prefetcherd"; + } + } + + virtual void Dump(std::ostream& os, bool multiline) const override { + Command cmd{}; + cmd.choice = CommandChoice::kDumpEverything; + + if (!daemon_->SendCommand(cmd)) { + LOG(ERROR) << "Failed to transmit kDumpEverything to iorap.prefetcherd"; + } + } + + + private: + // No lifetime cycle: PrefetcherDaemon only has a SessionManagerDirect in it. + std::shared_ptr<PrefetcherDaemon> daemon_; +}; + +std::unique_ptr<SessionManager> SessionManager::CreateManager(SessionKind kind) { + LOG(VERBOSE) << "SessionManager::CreateManager kind=" << kind; + + switch (kind) { + case SessionKind::kInProcessDirect: { + SessionManager* ptr = new SessionManagerDirect(); + return std::unique_ptr<SessionManager>{ptr}; + } + case SessionKind::kOutOfProcessIpc: { + SessionManager* ptr = new SessionManagerIndirect(); + return std::unique_ptr<SessionManager>{ptr}; + } + case SessionKind::kOutOfProcessSocket: { + SessionManager* ptr = new SessionManagerIndirectSocket(); + return std::unique_ptr<SessionManager>{ptr}; + } + default: { + LOG(FATAL) << "Invalid session kind: " << static_cast<int>(kind); + break; + } + } +} + +} // namespace prefetcher +} // namespace iorap diff --git a/src/prefetcher/session_manager.h b/src/prefetcher/session_manager.h new file mode 100644 index 0000000..45120d5 --- /dev/null +++ b/src/prefetcher/session_manager.h @@ -0,0 +1,91 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PREFETCHER_SESSION_MANAGER_H_ +#define PREFETCHER_SESSION_MANAGER_H_ + +#include <optional> +#include <ostream> +#include <memory> + +namespace iorap { +namespace prefetcher { + +class Session; + +enum class SessionKind : uint32_t { + kInProcessDirect, + kOutOfProcessIpc, + kOutOfProcessSocket, +}; + +inline std::ostream& operator<<(std::ostream& os, SessionKind kind) { + if (kind == SessionKind::kInProcessDirect) { + os << "kInProcessDirect"; + } else if (kind == SessionKind::kOutOfProcessIpc) { + os << "kOutOfProcessIpc"; + } else if (kind == SessionKind::kOutOfProcessSocket) { + os << "kOutOfProcessSocket"; + } else { + os << "(invalid)"; + } + return os; +} + +class SessionManager { + public: + static std::unique_ptr<SessionManager> CreateManager(SessionKind kind); + + // Create a new session. The description is used by Dump. + // Manager maintains a strong ref to this session, so DestroySession must also + // be called prior to all refs dropping to 0. + virtual std::shared_ptr<Session> CreateSession(size_t session_id, + std::string description) = 0; + + // Create a new session. The description is used by Dump. + // Manager maintains a strong ref to this session, so DestroySession must also + // be called prior to all refs dropping to 0. + virtual std::shared_ptr<Session> CreateSession(size_t session_id, + std::string description, + std::optional<int> fd) { + return CreateSession(session_id, description); + } + + // Look up an existing session that was already created. + // Returns null if there is no such session. + virtual std::shared_ptr<Session> FindSession(size_t session_id) const = 0; + + // Drop all manager references to an existing session. + // Returns false if the session does not exist already. + virtual bool DestroySession(size_t session_id) = 0; + + // Multi-line detailed dump, e.g. for dumpsys. + // Single-line summary dump, e.g. for logcat. + virtual void Dump(std::ostream& os, bool multiline) const = 0; + + // Note: session lifetime is tied to manager. The manager has strong pointers to sessions. + virtual ~SessionManager() {} + + protected: + SessionManager(); +}; + +// Single-line summary dump of Session. +std::ostream& operator<<(std::ostream&os, const SessionManager& session); + +} // namespace prefetcher +} // namespace iorap + +#endif + diff --git a/src/prefetcher/task_id.h b/src/prefetcher/task_id.h new file mode 100644 index 0000000..dc26954 --- /dev/null +++ b/src/prefetcher/task_id.h @@ -0,0 +1,39 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef PREFETCHER_TASK_ID_H_ +#define PREFETCHER_TASK_ID_H_ + +#include <ostream> +#include <string> + +namespace iorap { +namespace prefetcher { + +struct TaskId { + size_t id; // Unique monotonically increasing ID. + std::string path; // File path to the trace file. + + friend std::ostream& operator<<(std::ostream& os, const TaskId& task_id) { + os << "TaskId { id: " << task_id.id << ", path: " << task_id.path << "}"; + return os; + } + +}; + +} // namespace prefetcher +} // namespace iorap + +#endif + diff --git a/src/serialize/TraceFile.proto b/src/serialize/TraceFile.proto new file mode 100644 index 0000000..fc72d0d --- /dev/null +++ b/src/serialize/TraceFile.proto @@ -0,0 +1,47 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; // use required fields, which aren't in proto3. + +package iorap.serialize.proto; // C++ namespace iorap::serialize::proto +option java_package = "com.google.android.iorap"; +option optimize_for = LITE_RUNTIME; + +// TODO: should these fields be 'packed' for "smaller encoding" ? + +message TraceFile { + required TraceFileIndex index = 1; + required TraceFileList list = 2; +} + +message TraceFileIndex { + repeated TraceFileIndexEntry entries = 1; +} + +message TraceFileIndexEntry { + required int64 id = 1; + required string file_name = 2; +} + +message TraceFileList { + repeated TraceFileEntry entries = 1; +} + +message TraceFileEntry { + required int64 index_id = 1; + required int64 file_offset = 2; + required int64 file_length = 3; +} + +// XX: use nested messages?
\ No newline at end of file diff --git a/src/serialize/arena_ptr.h b/src/serialize/arena_ptr.h new file mode 100644 index 0000000..d8bea26 --- /dev/null +++ b/src/serialize/arena_ptr.h @@ -0,0 +1,76 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERIALIZE_ARENA_PTR_H_ +#define SERIALIZE_ARENA_PTR_H_ + +#include <google/protobuf/arena.h> +#include <memory> + +namespace iorap { +namespace serialize { + +/** + * @file + * + * Helpers for protobuf arena allocators. We use smart pointers + * with an arena embedded inside of them to avoid caring about the + * arena in other parts of libiorap. + */ + +// Arena-managed objects must not be deleted manually. +// When the Arena goes out of scope, it cleans everything up itself. +template <typename T> +void DoNotDelete(T*) {} + +template <typename T, typename Base = std::unique_ptr<T, decltype(&DoNotDelete<T>)>> +struct ArenaPtr : public Base { + template <typename... Args> + static ArenaPtr<T> Make(Args&& ... args) { + ArenaPtr<T> arena_ptr(nullptr); + arena_ptr.reset(google::protobuf::Arena::Create<T>(arena_ptr.arena_.get(), + std::forward<Args>(args)...)); + return arena_ptr; + } + + ArenaPtr(std::nullptr_t) : Base(nullptr, &DoNotDelete<T>) {} // NOLINT explicit. + + private: + // Use a unique_ptr because Arena doesn't support move semantics. + std::unique_ptr<google::protobuf::Arena> arena_{new google::protobuf::Arena{}}; +}; + +template <typename T, typename Base = std::shared_ptr<T>> +struct ArenaSharedPtr : public Base { + template <typename... Args> + static ArenaSharedPtr<T> Make(Args&& ... args) { + ArenaSharedPtr<T> arena_ptr(nullptr, &DoNotDelete<T>); + arena_ptr.reset(google::protobuf::Arena::Create<T>(arena_ptr.arena_.get(), + std::forward<Args>(args)...)); + return arena_ptr; + } + + ArenaSharedPtr() = default; + template <typename Deleter> + ArenaSharedPtr(std::nullptr_t, Deleter d) : Base(nullptr, d) {} // NOLINT explicit. + + private: + std::shared_ptr<google::protobuf::Arena> arena_{new google::protobuf::Arena{}}; +}; + +} // namespace serialize +} // namespace iorap + +#endif + diff --git a/src/serialize/protobuf_io.cc b/src/serialize/protobuf_io.cc new file mode 100644 index 0000000..1b6420f --- /dev/null +++ b/src/serialize/protobuf_io.cc @@ -0,0 +1,173 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "protobuf_io.h" + +#include "common/trace.h" +#include "serialize/arena_ptr.h" + +#include <android-base/chrono_utils.h> +#include <android-base/logging.h> +#include <android-base/unique_fd.h> +#include <fcntl.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#include <utils/Trace.h> + +#include "google/protobuf/io/zero_copy_stream_impl_lite.h" +#include "system/iorap/src/serialize/TraceFile.pb.h" + +namespace iorap { +namespace serialize { + +ArenaPtr<proto::TraceFile> ProtobufIO::Open(std::string file_path) { + // TODO: file a bug about this. + // Note: can't use {} here, clang think it's narrowing from long->int. + android::base::unique_fd fd(TEMP_FAILURE_RETRY(::open(file_path.c_str(), O_RDONLY))); + if (fd.get() < 0) { + PLOG(DEBUG) << "ProtobufIO: open failed: " << file_path; + return nullptr; + } + + return Open(fd.get(), file_path.c_str()); +} + +ArenaPtr<proto::TraceFile> ProtobufIO::Open(int fd, const char* file_path) { + + ScopedFormatTrace atrace_protobuf_io_open(ATRACE_TAG_ACTIVITY_MANAGER, + "ProtobufIO::Open %s", + file_path); + android::base::Timer timer{}; + + struct stat buf; + if (fstat(fd, /*out*/&buf) < 0) { + PLOG(ERROR) << "ProtobufIO: open error, fstat failed: " << file_path; + return nullptr; + } + // XX: off64_t for stat::st_size ? + + // Using the mmap appears to be the only way to do zero-copy with protobuf lite. + void* data = mmap(/*addr*/nullptr, + buf.st_size, + PROT_READ, MAP_SHARED | MAP_POPULATE, + fd, + /*offset*/0); + if (data == nullptr) { + PLOG(ERROR) << "ProtobufIO: open error, mmap failed: " << file_path; + return nullptr; + } + + ArenaPtr<proto::TraceFile> protobuf_trace_file = ArenaPtr<proto::TraceFile>::Make(); + if (protobuf_trace_file == nullptr) { + LOG(ERROR) << "ProtobufIO: open error, failed to create arena: " << file_path; + return nullptr; + } + + google::protobuf::io::ArrayInputStream protobuf_input_stream{data, static_cast<int>(buf.st_size)}; + if (!protobuf_trace_file->ParseFromZeroCopyStream(/*in*/&protobuf_input_stream)) { + // XX: Does protobuf on android already have the right LogHandler ? + LOG(ERROR) << "ProtobufIO: open error, protobuf parsing failed: " << file_path; + return nullptr; + } + + if (munmap(data, buf.st_size) < 0) { + PLOG(WARNING) << "ProtobufIO: open problem, munmap failed, possibly memory leak? " + << file_path; + } + + LOG(VERBOSE) << "ProtobufIO: open succeeded: " << file_path << ", duration: " << timer; + return protobuf_trace_file; +} + +iorap::expected<size_t /*bytes written*/, int /*errno*/> ProtobufIO::WriteFully( + const ::google::protobuf::MessageLite& message, + std::string_view file_path) { + + std::string str{file_path}; + android::base::unique_fd fd(TEMP_FAILURE_RETRY( + ::open(str.c_str(), + O_CREAT | O_TRUNC | O_RDWR, + S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP))); // ugo: rw-rw---- + if (fd.get() < 0) { + int err = errno; + PLOG(ERROR) << "ProtobufIO: open failed: " << file_path; + return unexpected{err}; + } + + return WriteFully(message, fd.get(), file_path); +} + +iorap::expected<size_t /*bytes written*/, int /*errno*/> ProtobufIO::WriteFully( + const ::google::protobuf::MessageLite& message, + int fd, + std::string_view file_path) { + + int byte_size = message.ByteSize(); + if (byte_size < 0) { + DCHECK(false) << "Invalid protobuf size: " << byte_size; + LOG(ERROR) << "ProtobufIO: Invalid protobuf size: " << byte_size; + return unexpected{EDOM}; + } + size_t serialized_size = static_cast<size_t>(byte_size); + + // Change the file to be exactly the length of the protobuf. + if (ftruncate(fd, static_cast<off_t>(serialized_size)) < 0) { + int err = errno; + PLOG(ERROR) << "ProtobufIO: ftruncate (size=" << serialized_size << ") failed"; + return unexpected{err}; + } + + // Using the mmap appears to be the only way to do zero-copy with protobuf lite. + void* data = mmap(/*addr*/nullptr, + serialized_size, + PROT_WRITE, + MAP_SHARED, + fd, + /*offset*/0); + if (data == nullptr) { + int err = errno; + PLOG(ERROR) << "ProtobufIO: mmap failed: " << file_path; + return unexpected{err}; + } + + // Zero-copy write from protobuf to file via memory-map. + ::google::protobuf::io::ArrayOutputStream output_stream{data, byte_size}; + if (!message.SerializeToZeroCopyStream(/*inout*/&output_stream)) { + // This should never happen since we pre-allocated the file and memory map to be large + // enough to store the full protobuf. + DCHECK(false) << "ProtobufIO:: SerializeToZeroCopyStream failed despite precalculating size"; + LOG(ERROR) << "ProtobufIO: SerializeToZeroCopyStream failed"; + return unexpected{EXFULL}; + } + + // Guarantee that changes are written back prior to munmap. + if (msync(data, static_cast<size_t>(serialized_size), MS_SYNC) < 0) { + int err = errno; + PLOG(ERROR) << "ProtobufIO: msync failed"; + return unexpected{err}; + } + + if (munmap(data, serialized_size) < 0) { + PLOG(WARNING) << "ProtobufIO: munmap failed, possibly memory leak? " + << file_path; + } + + return serialized_size; +} + +} // namespace serialize +} // namespace iorap + diff --git a/src/serialize/protobuf_io.h b/src/serialize/protobuf_io.h new file mode 100644 index 0000000..1092fd7 --- /dev/null +++ b/src/serialize/protobuf_io.h @@ -0,0 +1,64 @@ +// Copyright (C) 2017 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef SERIALIZE_PROTOBUF_IO_H_ +#define SERIALIZE_PROTOBUF_IO_H_ + +#include "common/expected.h" +#include "serialize/arena_ptr.h" +#include "system/iorap/src/serialize/TraceFile.pb.h" + +#include <string> +#include <string_view> + +namespace iorap { +namespace serialize { + +// XX: either the namespace should be called pb|proto[buf] +// or we should hide the protobuf-ness from the names and call this class "IO" , "Reader", etc? +// although an obvious name might be the "OpenFactory" or "ProtobufFacade" that reads too much +// like a bad joke. + +// Helpers to read a TraceFile protobuf from a file [descriptor]. +class ProtobufIO { + public: + // XX: proto::TraceFile seems annoying, maybe just serialize::TraceFile ? + + // Open the protobuf associated at the filepath. Returns null on failure. + static ArenaPtr<proto::TraceFile> Open(std::string file_path); + // Open the protobuf from the file descriptor. Returns null on failure. + static ArenaPtr<proto::TraceFile> Open(int fd, const char* file_path = "<unknown>"); + + // Save the protobuf by overwriting the file at file_path. + // The file state is indeterminate at failure. + // Returns # of bytes written out on success, otherwise the errno value. + static iorap::expected<size_t /*bytes written*/, int /*errno*/> WriteFully( + const ::google::protobuf::MessageLite& message, + std::string_view file_path); + // Save the protobuf by truncating the file already open at 'fd'. + // The file state is indeterminate at failure. + // Returns # of bytes written out on success, otherwise the errno value. + static iorap::expected<size_t /*bytes written*/, int /*errno*/> WriteFully( + const ::google::protobuf::MessageLite& message, + int fd, + std::string_view file_path = "<unknown>"); + + ProtobufIO() = delete; +}; + +} // namespace serialize +} // namespace iorap + +#endif + |