Trust CompOs-signed artifacts

If the current artifacts are missing or invalid, and if we have
pending CompOs artifacts, then attempt to use them. This includes
verifying the signatures and adding them to fs-verity if need be.

This is largely a proposal in the form of a CL. Note specifically the
definition of what a signature file looks like
(compos_signature.proto, VerityUtils.cpp).

I rationalised the way we handle multiple certificate subjects because
it was starting to get messy & confusing.

Apart from various refactorings, the significant changes remain behind
an if (false). It is currently largely untestable (we don't have
anything to produce signatures) and there's a couple more CLs to come,
but I think this is a big enough CL as it stands.

Bug: 190166662
Test: Create pending directory, see it deleted.
Test: Create valid pending directory, it gets renamed, fails verification
Test: Invalid signature file is rejected
Test: Presubmit
Change-Id: I20ef65f3c382bcfd5db8747e73fc0148a4b978e9

9 files changed, 446 insertions, 137 deletions

diff --git a/ondevice-signing/Android.bp b/ondevice-signing/Android.bp
index 9085d81e..c8ce373d 100644
--- a/ondevice-signing/Android.bp
+++ b/ondevice-signing/Android.bp
@@ -11,8 +11,6 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-// List of clang-tidy checks that are reported as errors.
-// Please keep this list ordered lexicographically.
 
 package {
     // See: http://go/android-license-faq
@@ -23,6 +21,8 @@ package {
     default_applicable_licenses: ["system_security_license"],
 }
 
+// List of clang-tidy checks that are reported as errors.
+// Please keep this list ordered lexicographically.
 tidy_errors = [
   "cert-err34-c",
   "google-default-arguments",
@@ -95,6 +95,7 @@ cc_binary {
   static_libs: [
     "libc++fs",
     "lib_odsign_proto",
+    "lib_compos_proto",
   ],
 
   shared_libs: [
diff --git a/ondevice-signing/CertUtils.cpp b/ondevice-signing/CertUtils.cpp
index ce2b0fd2..acc11e45 100644
--- a/ondevice-signing/CertUtils.cpp
+++ b/ondevice-signing/CertUtils.cpp
@@ -31,8 +31,10 @@
 
 #include "KeyConstants.h"
 
-const char kRootCommonName[] = "ODS";
+// Common properties for all of our certificates.
 constexpr int kCertLifetimeSeconds = 10 * 365 * 24 * 60 * 60;
+const char* const kIssuerCountry = "US";
+const char* const kIssuerOrg = "Android";
 
 using android::base::ErrnoError;
 using android::base::Error;
@@ -105,7 +107,7 @@ Result<void> verifySignature(const std::string& message, const std::string& sign
                               (const uint8_t*)signature.c_str(), signature.length(), rsaKey->get());
 
     if (!success) {
-        return Error() << "Failed to verify signature.";
+        return Error() << "Failed to verify signature";
     }
     return {};
 }
@@ -126,7 +128,7 @@ static Result<bssl::UniquePtr<EVP_PKEY>> toRsaPkey(const std::vector<uint8_t>& p
 }
 
 static Result<void> createCertificate(
-    const char* commonName, const std::vector<uint8_t>& publicKey,
+    const CertSubject& subject, const std::vector<uint8_t>& publicKey,
     const std::function<android::base::Result<std::string>(const std::string&)>& signFunction,
     const std::optional<std::string>& issuerCertPath, const std::string& path) {
 
@@ -141,7 +143,7 @@ static Result<void> createCertificate(
     X509_set_version(x509.get(), 2);
     X509_gmtime_adj(X509_get_notBefore(x509.get()), 0);
     X509_gmtime_adj(X509_get_notAfter(x509.get()), kCertLifetimeSeconds);
-    ASN1_INTEGER_set(X509_get_serialNumber(x509.get()), selfSigned ? 1 : 2);
+    ASN1_INTEGER_set(X509_get_serialNumber(x509.get()), subject.serialNumber);
 
     bssl::UniquePtr<X509_ALGOR> algor(X509_ALGOR_new());
     if (!algor ||
@@ -164,9 +166,9 @@ static Result<void> createCertificate(
     if (!subjectName) {
         return Error() << "Unable to get x509 subject name";
     }
-    addNameEntry(subjectName, "C", "US");
-    addNameEntry(subjectName, "O", "Android");
-    addNameEntry(subjectName, "CN", commonName);
+    addNameEntry(subjectName, "C", kIssuerCountry);
+    addNameEntry(subjectName, "O", kIssuerOrg);
+    addNameEntry(subjectName, "CN", subject.commonName);
 
     if (selfSigned) {
         if (!X509_set_issuer_name(x509.get(), subjectName)) {
@@ -177,9 +179,9 @@ static Result<void> createCertificate(
         if (!issuerName) {
             return Error() << "Unable to get x509 issuer name";
         }
-        addNameEntry(issuerName, "C", "US");
-        addNameEntry(issuerName, "O", "Android");
-        addNameEntry(issuerName, "CN", kRootCommonName);
+        addNameEntry(issuerName, "C", kIssuerCountry);
+        addNameEntry(issuerName, "O", kIssuerOrg);
+        addNameEntry(issuerName, "CN", kRootSubject.commonName);
     }
 
     // Beware: context contains a pointer to issuerCert, so we need to keep it alive.
@@ -239,14 +241,14 @@ Result<void> createSelfSignedCertificate(
     const std::vector<uint8_t>& publicKey,
     const std::function<Result<std::string>(const std::string&)>& signFunction,
     const std::string& path) {
-    return createCertificate(kRootCommonName, publicKey, signFunction, {}, path);
+    return createCertificate(kRootSubject, publicKey, signFunction, {}, path);
 }
 
 android::base::Result<void> createLeafCertificate(
-    const char* commonName, const std::vector<uint8_t>& publicKey,
+    const CertSubject& subject, const std::vector<uint8_t>& publicKey,
     const std::function<android::base::Result<std::string>(const std::string&)>& signFunction,
     const std::string& issuerCertPath, const std::string& path) {
-    return createCertificate(commonName, publicKey, signFunction, issuerCertPath, path);
+    return createCertificate(subject, publicKey, signFunction, issuerCertPath, path);
 }
 
 Result<std::vector<uint8_t>> extractPublicKey(EVP_PKEY* pkey) {
@@ -340,7 +342,8 @@ Result<CertInfo> verifyAndExtractCertInfoFromX509(const std::string& path,
     return cert_info;
 }
 
-Result<std::vector<uint8_t>> createPkcs7(const std::vector<uint8_t>& signed_digest) {
+Result<std::vector<uint8_t>> createPkcs7(const std::vector<uint8_t>& signed_digest,
+                                         const CertSubject& signer) {
     CBB out, outer_seq, wrapped_seq, seq, digest_algos_set, digest_algo, null;
     CBB content_info, issuer_and_serial, signer_infos, signer_info, sign_algo, signature;
     uint8_t *pkcs7_data, *name_der;
@@ -353,13 +356,14 @@ Result<std::vector<uint8_t>> createPkcs7(const std::vector<uint8_t>& signed_dige
         return Error() << "Unable to get x509 subject name";
     }
     X509_NAME_add_entry_by_txt(name, "C", MBSTRING_ASC,
-                               reinterpret_cast<const unsigned char*>("US"), -1, -1, 0);
+                               reinterpret_cast<const unsigned char*>(kIssuerCountry), -1, -1, 0);
     X509_NAME_add_entry_by_txt(name, "O", MBSTRING_ASC,
-                               reinterpret_cast<const unsigned char*>("Android"), -1, -1, 0);
+                               reinterpret_cast<const unsigned char*>(kIssuerOrg), -1, -1, 0);
     X509_NAME_add_entry_by_txt(name, "CN", MBSTRING_ASC,
-                               reinterpret_cast<const unsigned char*>("ODS"), -1, -1, 0);
+                               reinterpret_cast<const unsigned char*>(signer.commonName), -1, -1,
+                               0);
 
-    BN_set_word(serial, 1);
+    BN_set_word(serial, signer.serialNumber);
     name_der_len = i2d_X509_NAME(name, &name_der);
     CBB_init(&out, 1024);
 
diff --git a/ondevice-signing/CertUtils.h b/ondevice-signing/CertUtils.h
index b412d21a..1fa5bbcd 100644
--- a/ondevice-signing/CertUtils.h
+++ b/ondevice-signing/CertUtils.h
@@ -22,22 +22,37 @@
 
 #include <android-base/result.h>
 
+// Information extracted from a certificate.
 struct CertInfo {
     std::string subjectCn;
     std::vector<uint8_t> subjectKey;
 };
 
+// Subjects of certificates we issue.
+struct CertSubject {
+    const char* commonName;
+    unsigned serialNumber;
+};
+
+// These are all the certificates we ever sign (the first one being our
+// self-signed cert).  We shouldn't really re-use serial numbers for different
+// certificates for the same subject but we do; only one should be in use at a
+// time though.
+inline const CertSubject kRootSubject{"ODS", 1};
+inline const CertSubject kCompOsSubject{"CompOs", 2};
+
 android::base::Result<void> createSelfSignedCertificate(
     const std::vector<uint8_t>& publicKey,
     const std::function<android::base::Result<std::string>(const std::string&)>& signFunction,
     const std::string& path);
 
 android::base::Result<void> createLeafCertificate(
-    const char* commonName, const std::vector<uint8_t>& publicKey,
+    const CertSubject& subject, const std::vector<uint8_t>& publicKey,
     const std::function<android::base::Result<std::string>(const std::string&)>& signFunction,
     const std::string& issuerCertPath, const std::string& outPath);
 
-android::base::Result<std::vector<uint8_t>> createPkcs7(const std::vector<uint8_t>& signedData);
+android::base::Result<std::vector<uint8_t>> createPkcs7(const std::vector<uint8_t>& signedData,
+                                                        const CertSubject& signer);
 
 android::base::Result<std::vector<uint8_t>>
 extractPublicKeyFromX509(const std::vector<uint8_t>& x509);
diff --git a/ondevice-signing/KeystoreKey.cpp b/ondevice-signing/KeystoreKey.cpp
index 9780787c..03bb6d58 100644
--- a/ondevice-signing/KeystoreKey.cpp
+++ b/ondevice-signing/KeystoreKey.cpp
@@ -122,7 +122,7 @@ Result<std::vector<uint8_t>> KeystoreKey::createKey() {
         return Error() << "Failed to create new key: " << status;
     }
 
-    // Exteact the nublir key from the certificate, HMAC it and store the signature
+    // Extract the public key from the certificate, HMAC it and store the signature
     auto cert = metadata.certificate;
     if (!cert) {
         return Error() << "Key did not have a certificate.";
@@ -178,7 +178,7 @@ bool KeystoreKey::initialize() {
         return false;
     }
     mPublicKey = *key;
-    LOG(ERROR) << "Initialized Keystore key.";
+    LOG(INFO) << "Initialized Keystore key.";
     return true;
 }
 
diff --git a/ondevice-signing/VerityUtils.cpp b/ondevice-signing/VerityUtils.cpp
index 243e7df8..36f85b50 100644
--- a/ondevice-signing/VerityUtils.cpp
+++ b/ondevice-signing/VerityUtils.cpp
@@ -32,6 +32,7 @@
 
 #include "CertUtils.h"
 #include "SigningKey.h"
+#include "compos_signature.pb.h"
 
 #define FS_VERITY_MAX_DIGEST_SIZE 64
 
@@ -40,7 +41,10 @@ using android::base::Error;
 using android::base::Result;
 using android::base::unique_fd;
 
+using compos::proto::Signature;
+
 static const char* kFsVerityInitPath = "/system/bin/fsverity_init";
+static const char* kSignatureExtension = ".signature";
 
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
 #define cpu_to_le16(v) ((__force __le16)(uint16_t)(v))
@@ -50,7 +54,11 @@ static const char* kFsVerityInitPath = "/system/bin/fsverity_init";
 #define le16_to_cpu(v) (__builtin_bswap16((__force uint16_t)(v)))
 #endif
 
-static std::string toHex(std::span<uint8_t> data) {
+static bool isSignatureFile(const std::filesystem::path& path) {
+    return path.extension().native() == kSignatureExtension;
+}
+
+static std::string toHex(std::span<const uint8_t> data) {
     std::stringstream ss;
     for (auto it = data.begin(); it != data.end(); ++it) {
         ss << std::setfill('0') << std::setw(2) << std::hex << static_cast<unsigned>(*it);
@@ -64,16 +72,11 @@ static int read_callback(void* file, void* buf, size_t count) {
     return 0;
 }
 
-Result<std::vector<uint8_t>> createDigest(const std::string& path) {
+Result<std::vector<uint8_t>> createDigest(int fd) {
     struct stat filestat;
-    unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
-    if (fd < 0) {
-        return ErrnoError() << "Failed to open " << path;
-    }
-
-    int ret = stat(path.c_str(), &filestat);
+    int ret = fstat(fd, &filestat);
     if (ret < 0) {
-        return ErrnoError() << "Failed to stat " << path;
+        return ErrnoError() << "Failed to fstat";
     }
     struct libfsverity_merkle_tree_params params = {
         .version = 1,
@@ -85,13 +88,21 @@ Result<std::vector<uint8_t>> createDigest(const std::string& path) {
     struct libfsverity_digest* digest;
     ret = libfsverity_compute_digest(&fd, &read_callback, &params, &digest);
     if (ret < 0) {
-        return ErrnoError() << "Failed to compute fs-verity digest for " << path;
+        return ErrnoError() << "Failed to compute fs-verity digest";
     }
     std::vector<uint8_t> digestVector(&digest->digest[0], &digest->digest[32]);
     free(digest);
     return digestVector;
 }
 
+Result<std::vector<uint8_t>> createDigest(const std::string& path) {
+    unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
+    if (!fd.ok()) {
+        return ErrnoError() << "Unable to open";
+    }
+    return createDigest(fd.get());
+}
+
 namespace {
 template <typename T> struct DeleteAsPODArray {
     void operator()(T* x) {
@@ -129,10 +140,32 @@ static Result<std::vector<uint8_t>> signDigest(const SigningKey& key,
     return std::vector<uint8_t>(signed_digest->begin(), signed_digest->end());
 }
 
+Result<void> enableFsVerity(int fd, std::span<uint8_t> pkcs7) {
+    struct fsverity_enable_arg arg = {.version = 1};
+
+    arg.sig_ptr = reinterpret_cast<uint64_t>(pkcs7.data());
+    arg.sig_size = pkcs7.size();
+    arg.hash_algorithm = FS_VERITY_HASH_ALG_SHA256;
+    arg.block_size = 4096;
+
+    int ret = ioctl(fd, FS_IOC_ENABLE_VERITY, &arg);
+
+    if (ret != 0) {
+        return ErrnoError() << "Failed to call FS_IOC_ENABLE_VERITY";
+    }
+
+    return {};
+}
+
 Result<std::string> enableFsVerity(const std::string& path, const SigningKey& key) {
-    auto digest = createDigest(path);
+    unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
+    if (!fd.ok()) {
+        return ErrnoError() << "Failed to open " << path;
+    }
+
+    auto digest = createDigest(fd.get());
     if (!digest.ok()) {
-        return digest.error();
+        return Error() << digest.error() << ": " << path;
     }
 
     auto signed_digest = signDigest(key, digest.value());
@@ -140,20 +173,14 @@ Result<std::string> enableFsVerity(const std::string& path, const SigningKey& ke
         return signed_digest.error();
     }
 
-    auto pkcs7_data = createPkcs7(signed_digest.value());
-
-    struct fsverity_enable_arg arg = {.version = 1};
-
-    arg.sig_ptr = (uint64_t)pkcs7_data->data();
-    arg.sig_size = pkcs7_data->size();
-    arg.hash_algorithm = FS_VERITY_HASH_ALG_SHA256;
-    arg.block_size = 4096;
-
-    unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
-    int ret = ioctl(fd, FS_IOC_ENABLE_VERITY, &arg);
+    auto pkcs7_data = createPkcs7(signed_digest.value(), kRootSubject);
+    if (!pkcs7_data.ok()) {
+        return pkcs7_data.error();
+    }
 
-    if (ret != 0) {
-        return ErrnoError() << "Failed to call FS_IOC_ENABLE_VERITY on " << path;
+    auto enabled = enableFsVerity(fd.get(), pkcs7_data.value());
+    if (!enabled.ok()) {
+        return Error() << enabled.error() << ": " << path;
     }
 
     // Return the root hash as a hex string
@@ -163,12 +190,10 @@ Result<std::string> enableFsVerity(const std::string& path, const SigningKey& ke
 Result<std::map<std::string, std::string>> addFilesToVerityRecursive(const std::string& path,
                                                                      const SigningKey& key) {
     std::map<std::string, std::string> digests;
-    std::error_code ec;
 
+    std::error_code ec;
     auto it = std::filesystem::recursive_directory_iterator(path, ec);
-    auto end = std::filesystem::recursive_directory_iterator();
-
-    while (!ec && it != end) {
+    for (auto end = std::filesystem::recursive_directory_iterator(); it != end; it.increment(ec)) {
         if (it->is_regular_file()) {
             LOG(INFO) << "Adding " << it->path() << " to fs-verity...";
             auto result = enableFsVerity(it->path(), key);
@@ -177,38 +202,49 @@ Result<std::map<std::string, std::string>> addFilesToVerityRecursive(const std::
             }
             digests[it->path()] = *result;
         }
-        ++it;
     }
     if (ec) {
-        return Error() << "Failed to iterate " << path << ": " << ec;
+        return Error() << "Failed to iterate " << path << ": " << ec.message();
     }
 
     return digests;
 }
 
-Result<std::string> isFileInVerity(const std::string& path) {
-    unsigned int flags;
-
-    unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
-    if (fd < 0) {
-        return ErrnoError() << "Failed to open " << path;
+Result<std::string> readVerityDigest(int fd) {
+    auto d = makeUniqueWithTrailingData<fsverity_digest>(FS_VERITY_MAX_DIGEST_SIZE);
+    d->digest_size = FS_VERITY_MAX_DIGEST_SIZE;
+    auto ret = ioctl(fd, FS_IOC_MEASURE_VERITY, d.get());
+    if (ret < 0) {
+        return ErrnoError() << "Failed to FS_IOC_MEASURE_VERITY";
     }
+    return toHex({&d->digest[0], &d->digest[d->digest_size]});
+}
 
+Result<std::string> isFileInVerity(int fd) {
+    unsigned int flags;
     int ret = ioctl(fd, FS_IOC_GETFLAGS, &flags);
     if (ret < 0) {
-        return ErrnoError() << "Failed to FS_IOC_GETFLAGS for " << path;
+        return ErrnoError() << "Failed to FS_IOC_GETFLAGS";
     }
     if (!(flags & FS_VERITY_FL)) {
-        return Error() << "File is not in fs-verity: " << path;
+        return Error() << "File is not in fs-verity";
     }
 
-    auto d = makeUniqueWithTrailingData<fsverity_digest>(FS_VERITY_MAX_DIGEST_SIZE);
-    d->digest_size = FS_VERITY_MAX_DIGEST_SIZE;
-    ret = ioctl(fd, FS_IOC_MEASURE_VERITY, d.get());
-    if (ret < 0) {
-        return ErrnoError() << "Failed to FS_IOC_MEASURE_VERITY for " << path;
+    return readVerityDigest(fd);
+}
+
+Result<std::string> isFileInVerity(const std::string& path) {
+    unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
+    if (!fd.ok()) {
+        return ErrnoError() << "Failed to open " << path;
     }
-    return toHex({&d->digest[0], &d->digest[d->digest_size]});
+
+    auto digest = isFileInVerity(fd);
+    if (!digest.ok()) {
+        return Error() << digest.error() << ": " << path;
+    }
+
+    return digest;
 }
 
 Result<std::map<std::string, std::string>> verifyAllFilesInVerity(const std::string& path) {
@@ -242,6 +278,113 @@ Result<std::map<std::string, std::string>> verifyAllFilesInVerity(const std::str
     return digests;
 }
 
+Result<Signature> readSignature(const std::filesystem::path& signature_path) {
+    unique_fd fd(TEMP_FAILURE_RETRY(open(signature_path.c_str(), O_RDONLY | O_CLOEXEC)));
+    if (fd == -1) {
+        return ErrnoError();
+    }
+    Signature signature;
+    if (!signature.ParseFromFileDescriptor(fd.get())) {
+        return Error() << "Failed to parse";
+    }
+    return signature;
+}
+
+Result<std::map<std::string, std::string>>
+verifyAllFilesUsingCompOs(const std::string& directory_path,
+                          const std::vector<uint8_t>& compos_key) {
+    std::map<std::string, std::string> new_digests;
+    std::vector<std::filesystem::path> signature_files;
+
+    std::error_code ec;
+    auto it = std::filesystem::recursive_directory_iterator(directory_path, ec);
+    for (auto end = std::filesystem::recursive_directory_iterator(); it != end; it.increment(ec)) {
+        auto& path = it->path();
+        if (it->is_regular_file()) {
+            if (isSignatureFile(path)) {
+                continue;
+            }
+
+            unique_fd fd(TEMP_FAILURE_RETRY(open(path.c_str(), O_RDONLY | O_CLOEXEC)));
+            if (!fd.ok()) {
+                return ErrnoError() << "Can't open " << path;
+            }
+
+            auto signature_path = path;
+            signature_path += kSignatureExtension;
+            auto signature = readSignature(signature_path);
+            if (!signature.ok()) {
+                return Error() << "Invalid signature " << signature_path << ": "
+                               << signature.error();
+            }
+            signature_files.push_back(signature_path);
+
+            // Note that these values are not yet trusted.
+            auto& raw_digest = signature->digest();
+            auto& raw_signature = signature->signature();
+
+            // Make sure the signature matches the CompOs public key, and not some other
+            // fs-verity trusted key.
+            auto verified = verifySignature(raw_digest, raw_signature, compos_key);
+            if (!verified.ok()) {
+                return Error() << verified.error() << ": " << path;
+            }
+
+            std::span<const uint8_t> digest_bytes(
+                reinterpret_cast<const uint8_t*>(raw_digest.data()), raw_digest.size());
+            std::string compos_digest = toHex(digest_bytes);
+
+            auto verity_digest = isFileInVerity(fd);
+            if (verity_digest.ok()) {
+                // The file is already in fs-verity. We need to make sure it was signed
+                // by CompOs, so we just check that it has the digest we expect.
+                if (verity_digest.value() != compos_digest) {
+                    return Error() << "fs-verity digest does not match signature file: " << path;
+                }
+            } else {
+                // Not in fs-verity yet. But we have a valid signature of some
+                // digest. If it's not the correct digest for the file then
+                // enabling fs-verity will fail, so we don't need to check it
+                // explicitly ourselves. Otherwise we should be good.
+                std::vector<uint8_t> signature_bytes(raw_signature.begin(), raw_signature.end());
+                auto pkcs7 = createPkcs7(signature_bytes, kCompOsSubject);
+                if (!pkcs7.ok()) {
+                    return Error() << pkcs7.error() << ": " << path;
+                }
+
+                LOG(INFO) << "Adding " << path << " to fs-verity...";
+                auto enabled = enableFsVerity(fd, pkcs7.value());
+                if (!enabled.ok()) {
+                    return Error() << enabled.error() << ": " << path;
+                }
+            }
+
+            new_digests[path] = compos_digest;
+        } else if (it->is_directory()) {
+            // These are fine to ignore
+        } else if (it->is_symlink()) {
+            return Error() << "Rejecting artifacts, symlink at " << path;
+        } else {
+            return Error() << "Rejecting artifacts, unexpected file type for " << path;
+        }
+    }
+    if (ec) {
+        return Error() << "Failed to iterate " << directory_path << ": " << ec.message();
+    }
+
+    // Delete the signature files now that they have served their purpose.  (ART
+    // has no use for them, and their presence could cause verification to fail
+    // on subsequent boots.)
+    for (auto& signature_path : signature_files) {
+        std::filesystem::remove(signature_path, ec);
+        if (ec) {
+            return Error() << "Failed to delete " << signature_path << ": " << ec.message();
+        }
+    }
+
+    return new_digests;
+}
+
 Result<void> addCertToFsVerityKeyring(const std::string& path, const char* keyName) {
     const char* const argv[] = {kFsVerityInitPath, "--load-extra-key", keyName};
 
diff --git a/ondevice-signing/VerityUtils.h b/ondevice-signing/VerityUtils.h
index dca31843..8d8e62c9 100644
--- a/ondevice-signing/VerityUtils.h
+++ b/ondevice-signing/VerityUtils.h
@@ -24,5 +24,9 @@ android::base::Result<void> addCertToFsVerityKeyring(const std::string& path, co
 android::base::Result<std::vector<uint8_t>> createDigest(const std::string& path);
 android::base::Result<std::map<std::string, std::string>>
 verifyAllFilesInVerity(const std::string& path);
+
 android::base::Result<std::map<std::string, std::string>>
 addFilesToVerityRecursive(const std::string& path, const SigningKey& key);
+
+android::base::Result<std::map<std::string, std::string>>
+verifyAllFilesUsingCompOs(const std::string& path, const std::vector<uint8_t>& compos_key);
diff --git a/ondevice-signing/odsign_main.cpp b/ondevice-signing/odsign_main.cpp
index 5fad7fc1..55d8b1c5 100644
--- a/ondevice-signing/odsign_main.cpp
+++ b/ondevice-signing/odsign_main.cpp
@@ -59,10 +59,10 @@ static const bool kForceCompilation = false;
 static const bool kUseCompOs = false;  // STOPSHIP if true
 
 static const char* kVirtApexPath = "/apex/com.android.virt";
-static const char* kCompOsCommonName = "CompOS";
 const std::string kCompOsCert = "/data/misc/odsign/compos_key.cert";
 const std::string kCompOsPublicKey = "/data/misc/odsign/compos_key.pubkey";
 const std::string kCompOsKeyBlob = "/data/misc/odsign/compos_key.blob";
+const std::string kCompOsPendingDir = "/data/misc/apexdata/com.android.art/compos-pending";
 
 static const char* kOdsignVerificationDoneProp = "odsign.verification.done";
 static const char* kOdsignKeyDoneProp = "odsign.key.done";
@@ -82,6 +82,48 @@ static std::vector<uint8_t> readBytesFromFile(const std::string& path) {
     return std::vector<uint8_t>(str.begin(), str.end());
 }
 
+static int removeDirectory(const std::string& directory) {
+    std::error_code ec;
+    auto num_removed = std::filesystem::remove_all(directory, ec);
+    if (ec) {
+        LOG(ERROR) << "Can't remove " << directory << ": " << ec.message();
+        return 0;
+    } else {
+        if (num_removed > 0) {
+            LOG(INFO) << "Removed " << num_removed << " entries from " << directory;
+        }
+        return num_removed;
+    }
+}
+
+static bool directoryHasContent(const std::string& directory) {
+    std::error_code ec;
+    return std::filesystem::is_directory(directory, ec) &&
+           !std::filesystem::is_empty(directory, ec);
+}
+
+art::odrefresh::ExitCode compileArtifacts(bool force) {
+    const char* const argv[] = {kOdrefreshPath, force ? "--force-compile" : "--compile"};
+    const int exit_code =
+        logwrap_fork_execvp(arraysize(argv), argv, nullptr, false, LOG_ALOG, false, nullptr);
+    return static_cast<art::odrefresh::ExitCode>(exit_code);
+}
+
+art::odrefresh::ExitCode checkArtifacts() {
+    const char* const argv[] = {kOdrefreshPath, "--check"};
+    const int exit_code =
+        logwrap_fork_execvp(arraysize(argv), argv, nullptr, false, LOG_ALOG, false, nullptr);
+    return static_cast<art::odrefresh::ExitCode>(exit_code);
+}
+
+static std::string toHex(const std::vector<uint8_t>& digest) {
+    std::stringstream ss;
+    for (auto it = digest.begin(); it != digest.end(); ++it) {
+        ss << std::setfill('0') << std::setw(2) << std::hex << static_cast<unsigned>(*it);
+    }
+    return ss.str();
+}
+
 bool compOsPresent() {
     return access(kVirtApexPath, F_OK) == 0;
 }
@@ -124,7 +166,7 @@ Result<std::vector<uint8_t>> extractPublicKeyFromLeafCert(const SigningKey& key,
                                                           const std::string& certPath,
                                                           const std::string& expectedCn) {
     if (access(certPath.c_str(), F_OK) < 0) {
-        return ErrnoError() << "Certificate not found: " << kCompOsCert;
+        return ErrnoError() << "Certificate not found: " << certPath;
     }
     auto trustedPublicKey = key.getPublicKey();
     if (!trustedPublicKey.ok()) {
@@ -146,7 +188,7 @@ Result<std::vector<uint8_t>> extractPublicKeyFromLeafCert(const SigningKey& key,
     return existingCertInfo.value().subjectKey;
 }
 
-Result<void> verifyOrGenerateCompOsKey(const SigningKey& signingKey) {
+Result<std::vector<uint8_t>> verifyOrGenerateCompOsKey(const SigningKey& signingKey) {
     auto compOsStatus = FakeCompOs::newInstance();
     if (!compOsStatus.ok()) {
         return Error() << "Failed to start CompOs: " << compOsStatus.error();
@@ -194,28 +236,13 @@ Result<void> verifyOrGenerateCompOsKey(const SigningKey& signingKey) {
     auto signFunction = [&](const std::string& to_be_signed) {
         return signingKey.sign(to_be_signed);
     };
-    auto certStatus = createLeafCertificate(kCompOsCommonName, publicKey, signFunction,
+    auto certStatus = createLeafCertificate(kCompOsSubject, publicKey, signFunction,
                                             kSigningKeyCert, kCompOsCert);
     if (!certStatus.ok()) {
         return Error() << "Failed to create CompOs cert: " << certStatus.error();
     }
 
-    return {};
-}
-
-art::odrefresh::ExitCode compileArtifacts(bool force) {
-    const char* const argv[] = {kOdrefreshPath, force ? "--force-compile" : "--compile"};
-    const int exit_code =
-        logwrap_fork_execvp(arraysize(argv), argv, nullptr, false, LOG_ALOG, false, nullptr);
-    return static_cast<art::odrefresh::ExitCode>(exit_code);
-}
-
-static std::string toHex(const std::vector<uint8_t>& digest) {
-    std::stringstream ss;
-    for (auto it = digest.begin(); it != digest.end(); ++it) {
-        ss << std::setfill('0') << std::setw(2) << std::hex << static_cast<unsigned>(*it);
-    }
-    return ss.str();
+    return publicKey;
 }
 
 Result<std::map<std::string, std::string>> computeDigests(const std::string& path) {
@@ -229,7 +256,8 @@ Result<std::map<std::string, std::string>> computeDigests(const std::string& pat
         if (it->is_regular_file()) {
             auto digest = createDigest(it->path());
             if (!digest.ok()) {
-                return Error() << "Failed to compute digest for " << it->path();
+                return Error() << "Failed to compute digest for " << it->path() << ": "
+                               << digest.error();
             }
             digests[it->path()] = toHex(*digest);
         }
@@ -341,20 +369,6 @@ Result<void> persistDigests(const std::map<std::string, std::string>& digests,
     return {};
 }
 
-static int removeArtifacts() {
-    std::error_code ec;
-    auto num_removed = std::filesystem::remove_all(kArtArtifactsDir, ec);
-    if (ec) {
-        LOG(ERROR) << "Can't remove " << kArtArtifactsDir << ": " << ec.message();
-        return 0;
-    } else {
-        if (num_removed > 0) {
-            LOG(INFO) << "Removed " << num_removed << " entries from " << kArtArtifactsDir;
-        }
-        return num_removed;
-    }
-}
-
 static Result<void> verifyArtifacts(const SigningKey& key, bool supportsFsVerity) {
     auto signInfo = getOdsignInfo(key);
     // Tell init we're done with the key; this is a boot time optimization
@@ -386,10 +400,108 @@ static Result<void> verifyArtifacts(const SigningKey& key, bool supportsFsVerity
     return {};
 }
 
+Result<std::vector<uint8_t>> addCompOsCertToFsVerityKeyring(const SigningKey& signingKey) {
+    std::vector<uint8_t> compos_key;
+    auto existing_key =
+        extractPublicKeyFromLeafCert(signingKey, kCompOsCert, kCompOsSubject.commonName);
+    if (existing_key.ok()) {
+        LOG(INFO) << "Found and verified existing CompOs public key certificate: " << kCompOsCert;
+        compos_key = std::move(existing_key.value());
+    } else {
+        LOG(WARNING) << existing_key.error();
+
+        auto new_key = verifyOrGenerateCompOsKey(signingKey);
+        if (!new_key.ok()) {
+            return new_key.error();
+        } else {
+            LOG(INFO) << "Generated new CompOs public key certificate";
+            compos_key = std::move(new_key.value());
+        }
+    }
+
+    auto cert_add_result = addCertToFsVerityKeyring(kCompOsCert, "fsv_compos");
+    if (!cert_add_result.ok()) {
+        // Best efforts only - nothing we can do if deletion fails.
+        unlink(kCompOsCert.c_str());
+        return Error() << "Failed to add CompOs certificate to fs-verity keyring: "
+                       << cert_add_result.error();
+    }
+
+    return compos_key;
+}
+
+art::odrefresh::ExitCode checkCompOsPendingArtifacts(const std::vector<uint8_t>& compos_key,
+                                                     const SigningKey& signingKey,
+                                                     bool* digests_verified) {
+    if (!directoryHasContent(kCompOsPendingDir)) {
+        return art::odrefresh::ExitCode::kCompilationRequired;
+    }
+
+    // CompOs has generated some artifacts that may, or may not, match the
+    // current state.  But if there are already valid artifacts present the
+    // CompOs ones are redundant.
+    art::odrefresh::ExitCode odrefresh_status = checkArtifacts();
+    if (odrefresh_status != art::odrefresh::ExitCode::kCompilationRequired) {
+        if (odrefresh_status == art::odrefresh::ExitCode::kOkay) {
+            LOG(INFO) << "Current artifacts are OK, deleting pending artifacts";
+            removeDirectory(kCompOsPendingDir);
+        }
+        return odrefresh_status;
+    }
+
+    // No useful current artifacts, lets see if the CompOs ones are ok
+    LOG(INFO) << "Current artifacts are out of date, switching to pending artifacts";
+    removeDirectory(kArtArtifactsDir);
+    std::error_code ec;
+    std::filesystem::rename(kCompOsPendingDir, kArtArtifactsDir, ec);
+    if (ec) {
+        LOG(ERROR) << "Can't rename " << kCompOsPendingDir << " to " << kArtArtifactsDir << ": "
+                   << ec.message();
+        removeDirectory(kCompOsPendingDir);
+        return art::odrefresh::ExitCode::kCompilationRequired;
+    }
+    // TODO: Make sure that we check here that the contents of the artifacts
+    // correspond to their filenames (and extensions) - the CompOs signatures
+    // can't guarantee that.
+    odrefresh_status = checkArtifacts();
+    if (odrefresh_status != art::odrefresh::ExitCode::kOkay) {
+        LOG(WARNING) << "Pending artifacts are not OK";
+        return odrefresh_status;
+    }
+
+    // The artifacts appear to be up to date - but we haven't
+    // verified that they are genuine yet.
+    Result<std::map<std::string, std::string>> digests =
+        verifyAllFilesUsingCompOs(kArtArtifactsDir, compos_key);
+
+    if (digests.ok()) {
+        auto persisted = persistDigests(digests.value(), signingKey);
+
+        // Having signed the digests (or failed to), we're done with the signing key.
+        SetProperty(kOdsignKeyDoneProp, "1");
+
+        if (persisted.ok()) {
+            *digests_verified = true;
+            LOG(INFO) << "Pending artifacts successfully verified.";
+            return art::odrefresh::ExitCode::kOkay;
+        } else {
+            LOG(WARNING) << persisted.error();
+        }
+    } else {
+        LOG(WARNING) << "Pending artifact verification failed: " << digests.error();
+    }
+
+    // We can't use the existing artifacts, so we will need to generate new
+    // ones.
+    removeDirectory(kArtArtifactsDir);
+    return art::odrefresh::ExitCode::kCompilationRequired;
+}
+
 int main(int /* argc */, char** /* argv */) {
     auto errorScopeGuard = []() {
         // In case we hit any error, remove the artifacts and tell Zygote not to use anything
-        removeArtifacts();
+        removeDirectory(kArtArtifactsDir);
+        removeDirectory(kCompOsPendingDir);
         // Tell init we don't need to use our key anymore
         SetProperty(kOdsignKeyDoneProp, "1");
         // Tell init we're done with verification, and that it was an error
@@ -440,46 +552,37 @@ int main(int /* argc */, char** /* argv */) {
         }
     }
 
+    art::odrefresh::ExitCode odrefresh_status = art::odrefresh::ExitCode::kCompilationRequired;
+    bool digests_verified = false;
+
     if (supportsCompOs) {
-        auto compos_key = extractPublicKeyFromLeafCert(*key, kCompOsCert, kCompOsCommonName);
+        auto compos_key = addCompOsCertToFsVerityKeyring(*key);
         if (!compos_key.ok()) {
-            LOG(WARNING) << compos_key.error();
-
-            auto status = verifyOrGenerateCompOsKey(*key);
-            if (!status.ok()) {
-                LOG(ERROR) << status.error();
-            } else {
-                LOG(INFO) << "Generated new CompOs public key certificate";
-            }
+            LOG(ERROR) << compos_key.error();
         } else {
-            LOG(INFO) << "Found and verified existing CompOs public key certificate: "
-                      << kCompOsCert;
-        };
-        auto cert_add_result = addCertToFsVerityKeyring(kCompOsCert, "fsv_compos");
-        if (!cert_add_result.ok()) {
-            LOG(ERROR) << "Failed to add CompOs certificate to fs-verity keyring: "
-                       << cert_add_result.error();
-            // Best efforts only - nothing we can do if deletion fails.
-            unlink(kCompOsCert.c_str());
-            // TODO - what do we do now?
-            // return -1;
+            odrefresh_status =
+                checkCompOsPendingArtifacts(compos_key.value(), *key, &digests_verified);
         }
     }
 
-    art::odrefresh::ExitCode odrefresh_status = compileArtifacts(kForceCompilation);
+    if (odrefresh_status == art::odrefresh::ExitCode::kCompilationRequired) {
+        odrefresh_status = compileArtifacts(kForceCompilation);
+    }
     if (odrefresh_status == art::odrefresh::ExitCode::kOkay) {
         LOG(INFO) << "odrefresh said artifacts are VALID";
-        // A post-condition of validating artifacts is that if the ones on /system
-        // are used, kArtArtifactsDir is removed. Conversely, if kArtArtifactsDir
-        // exists, those are artifacts that will be used, and we should verify them.
-        int err = access(kArtArtifactsDir.c_str(), F_OK);
-        // If we receive any error other than ENOENT, be suspicious
-        bool artifactsPresent = (err == 0) || (err < 0 && errno != ENOENT);
-        if (artifactsPresent) {
-            auto verificationResult = verifyArtifacts(*key, supportsFsVerity);
-            if (!verificationResult.ok()) {
-                LOG(ERROR) << verificationResult.error();
-                return -1;
+        if (!digests_verified) {
+            // A post-condition of validating artifacts is that if the ones on /system
+            // are used, kArtArtifactsDir is removed. Conversely, if kArtArtifactsDir
+            // exists, those are artifacts that will be used, and we should verify them.
+            int err = access(kArtArtifactsDir.c_str(), F_OK);
+            // If we receive any error other than ENOENT, be suspicious
+            bool artifactsPresent = (err == 0) || (err < 0 && errno != ENOENT);
+            if (artifactsPresent) {
+                auto verificationResult = verifyArtifacts(*key, supportsFsVerity);
+                if (!verificationResult.ok()) {
+                    LOG(ERROR) << verificationResult.error();
+                    return -1;
+                }
             }
         }
     } else if (odrefresh_status == art::odrefresh::ExitCode::kCompilationSuccess ||
diff --git a/ondevice-signing/proto/Android.bp b/ondevice-signing/proto/Android.bp
index fd48f314..fc086771 100644
--- a/ondevice-signing/proto/Android.bp
+++ b/ondevice-signing/proto/Android.bp
@@ -27,3 +27,13 @@ cc_library_static {
     },
     srcs: ["odsign_info.proto"],
 }
+
+cc_library_static {
+    name: "lib_compos_proto",
+    host_supported: true,
+    proto: {
+        export_proto_headers: true,
+        type: "full",
+    },
+    srcs: ["compos_signature.proto"],
+}
diff --git a/ondevice-signing/proto/compos_signature.proto b/ondevice-signing/proto/compos_signature.proto
new file mode 100644
index 00000000..0659ade3
--- /dev/null
+++ b/ondevice-signing/proto/compos_signature.proto
@@ -0,0 +1,29 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+syntax = "proto3";
+
+package compos.proto;
+
+// Data provided by CompOS to allow validation of a file it generated.
+message Signature {
+    // The fs-verity digest (root hash of the Merkle tree) of the file
+    // contents.
+    bytes digest = 1;
+
+    // Signature of the digest, signed using CompOS's private key.
+    bytes signature = 2;
+}