Write ninja rules for generating stub.c files

This creates ninja rules in out/api_surfaces.ninja to generate stub.c
files using ndkstubgen. The actual stubs will be created ad-hoc during
ninja execution (Step 4)

Details
1. Create a cc subfolder to prevent pollution of the top-level
   orchestrator module
2. Create a wrapper shell script to run ndkstubgen.py in ninja.
   ndkstubgen.py has a dep on the __init__.py of the symbolfile module.
   The wrapper script fixes the pythonpath
3. Create the api_levels.json file for ndkstubgen. The source of truth
   is in Soong and product_config, but version map is stable enough to
   be hardcoded in build/orchestrator for now

Test: python -m test_ninja_syntax (In build/orchestrator/ninja)
Test: python -m cc.test_stub_generator
Test: multitree_build
out/out/intermediates/api_surfaces/publicapi/current/libc/arm/stub.c
(Using inner_build_soong.py as .inner_build)

Change-Id: I8353a15b1c46ceb228e2b76528908eabbb715958

9 files changed, 361 insertions, 63 deletions

diff --git a/core/api_assembly.py b/core/api_assembly.py
index ccfb5f1..9551cab 100644
--- a/core/api_assembly.py
+++ b/core/api_assembly.py
@@ -19,7 +19,7 @@ import json
 import os
 import sys
 
-import api_assembly_cc
+from cc.api_assembly import CcApiAssemblyContext
 import ninja_tools
 
 
@@ -143,7 +143,7 @@ def assemble_resource_api_library(context, ninja, build_file, stub_library):
 
 
 STUB_LANGUAGE_HANDLERS = {
-    "cc_libraries": api_assembly_cc.assemble_cc_api_library,
+    "cc_libraries": CcApiAssemblyContext().get_cc_api_assembler(),
     "java_libraries": assemble_java_api_library,
     "resource_libraries": assemble_resource_api_library,
 }
diff --git a/core/api_assembly_cc.py b/core/api_assembly_cc.py
deleted file mode 100644
index 2716bd3..0000000
--- a/core/api_assembly_cc.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/python3
-#
-# Copyright (C) 2022 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import os
-import textwrap
-
-ASSEMBLE_PHONY_TARGET = "multitree-sdk"
-
-def assemble_cc_api_library(context, ninja, build_file, stub_library):
-    staging_dir = context.out.api_library_dir(stub_library.api_surface,
-            stub_library.api_surface_version, stub_library.name, base=context.out.Base.OUTER)
-    work_dir = context.out.api_library_work_dir(stub_library.api_surface,
-            stub_library.api_surface_version, stub_library.name, base=context.out.Base.OUTER)
-
-    # Generate rules to copy headers
-    api_deps = []
-    for contrib in stub_library.contributions:
-        for headers in contrib.library_contribution["headers"]:
-            # Each header module gets its own include dir
-            # TODO: Improve the readability of the generated out/ directory
-            include_dir = os.path.join(staging_dir, headers["name"])
-            root = headers["root"]
-            for file in headers["headers"]:
-                # TODO: Deal with collisions of the same name from multiple contributions
-                # Remove the root from the full filepath
-                # e.g. bionic/libc/include/stdio.h --> stdio.h
-                relpath = os.path.relpath(file, root)
-                include = os.path.join(include_dir, relpath)
-                ninja.add_copy_file(include, os.path.join(contrib.inner_tree.root, file))
-                api_deps.append(include)
-
-        api = contrib.library_contribution["api"]
-        api_out = os.path.join(staging_dir, os.path.basename(api))
-        ninja.add_copy_file(api_out, os.path.join(contrib.inner_tree.root, api))
-        api_deps.append(api_out)
-
-        # TODO: Generate Android.bp files
-
-    # Generate phony rule to build the library
-    # TODO: This name probably conflictgs with something
-    phony = "-".join([stub_library.api_surface, str(stub_library.api_surface_version),
-                      stub_library.name])
-    ninja.add_phony(phony, api_deps)
-    # Add a global phony to assemnble all apis
-    ninja.add_global_phony(ASSEMBLE_PHONY_TARGET, [phony])
diff --git a/core/cc/__init__.py b/core/cc/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/core/cc/__init__.py
diff --git a/core/cc/api_assembly.py b/core/cc/api_assembly.py
new file mode 100644
index 0000000..877f9da
--- /dev/null
+++ b/core/cc/api_assembly.py
@@ -0,0 +1,149 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+import textwrap
+from cc.stub_generator import StubGenerator, GenCcStubsInput
+
+ARCHES = ["arm", "arm64", "x86", "x86_64"]
+
+ASSEMBLE_PHONY_TARGET = "multitree-sdk"
+
+class CcApiAssemblyContext(object):
+    """Context object for managing global state of CC API Assembly."""
+    def __init__(self):
+        self._stub_generator = StubGenerator()
+        self._api_levels_file_added = False
+
+    def get_cc_api_assembler(self):
+        """Return a callback to assemble CC APIs.
+
+        The callback is a member of the context object,
+        and therefore has access to its state."""
+        return self.assemble_cc_api_library
+
+    def assemble_cc_api_library(self, context, ninja, build_file, stub_library):
+        staging_dir = context.out.api_library_dir(stub_library.api_surface,
+                stub_library.api_surface_version, stub_library.name)
+        work_dir = context.out.api_library_work_dir(stub_library.api_surface,
+                stub_library.api_surface_version, stub_library.name)
+
+        # Generate rules to copy headers.
+        api_deps = []
+        for contrib in stub_library.contributions:
+            for headers in contrib.library_contribution["headers"]:
+                # Each header module gets its own include dir.
+                # TODO: Improve the readability of the generated out/ directory.
+                include_dir = os.path.join(staging_dir, headers["name"])
+                root = headers["root"]
+                for file in headers["headers"]:
+                    # TODO: Deal with collisions of the same name from multiple
+                    # contributions.
+                    # Remove the root from the full filepath.
+                    # e.g. bionic/libc/include/stdio.h --> stdio.h
+                    relpath = os.path.relpath(file, root)
+                    include = os.path.join(include_dir, relpath)
+                    ninja.add_copy_file(include, os.path.join(contrib.inner_tree.root, file))
+                    api_deps.append(include)
+
+            api = contrib.library_contribution["api"]
+            api_out = os.path.join(staging_dir, os.path.basename(api))
+            ninja.add_copy_file(api_out, os.path.join(contrib.inner_tree.root, api))
+            api_deps.append(api_out)
+
+            # Generate rules to run ndkstubgen.
+            extra_args = self._additional_ndkstubgen_args(stub_library.api_surface)
+            for arch in ARCHES:
+                inputs = GenCcStubsInput(
+                        arch = arch,
+                        version = stub_library.api_surface_version,
+                        version_map = self._api_levels_file(context),
+                        api = api_out,
+                        additional_args = extra_args,
+                )
+                # Generate stub.c files for each arch.
+                # TODO: Compile .so files using stub.c files.
+                stub_outputs = self._stub_generator.add_stubgen_action(ninja, inputs, work_dir)
+
+            # TODO: Generate Android.bp files.
+
+        # Generate rules to build the API levels map.
+        if not self._api_levels_file_added:
+            self._add_api_levels_file(context, ninja)
+            self._api_levels_file_added = True
+
+
+        # Generate phony rule to build the library.
+        # TODO: This name probably conflictgs with something.
+        phony = "-".join([stub_library.api_surface, str(stub_library.api_surface_version),
+                          stub_library.name])
+        ninja.add_phony(phony, api_deps)
+        # Add a global phony to assemnble all apis
+        ninja.add_global_phony(ASSEMBLE_PHONY_TARGET, [phony])
+
+    def _additional_ndkstubgen_args(self, api_surface: str) -> str:
+        if api_surface == "vendorapi":
+            return "--llndk"
+        if api_surface == "systemapi":
+            # The "systemapi" surface has contributions from the following:
+            # 1. Apex, which are annotated as #apex in map.txt
+            # 2. Platform, which are annotated as #sytemapi in map.txt
+            #
+            # Run ndkstubgen with both these annotations.
+            return "--apex --systemapi"
+        return ""
+
+    def _add_api_levels_file(self, context, ninja):
+        # ndkstubgen uses a map for converting Android version codes to a
+        # numeric code. e.g. "R" --> 30
+        # The map contains active_codenames as well, which get mapped to a preview level
+        # (9000+).
+        # TODO: Keep this in sync with build/soong/android/api_levels.go.
+        active_codenames = ["UpsideDownCake"]
+        preview_api_level_base = 9000
+        api_levels = {
+            "G": 9,
+            "I": 14,
+            "J": 16,
+            "J-MR1": 17,
+            "J-MR2": 18,
+            "K":     19,
+            "L":     21,
+            "L-MR1": 22,
+            "M":     23,
+            "N":     24,
+            "N-MR1": 25,
+            "O":     26,
+            "O-MR1": 27,
+            "P":     28,
+            "Q":     29,
+            "R":     30,
+            "S":     31,
+            "S-V2":  32,
+            "Tiramisu": 33,
+        }
+        for index, codename in enumerate(active_codenames):
+            api_levels[codename] = preview_api_level_base + index
+
+        file = self._api_levels_file(context)
+        ninja.add_write_file(file, json.dumps(api_levels))
+
+    def _api_levels_file(self, context) -> str:
+        """Returns a path in to generated api_levels map in the intermediates directory.
+
+        This file is not specific to a single stub library, and can be generated once"""
+        return context.out.api_surfaces_work_dir("api_levels.json")
diff --git a/core/cc/ndkstubgen_runner.sh b/core/cc/ndkstubgen_runner.sh
new file mode 100755
index 0000000..bd92d81
--- /dev/null
+++ b/core/cc/ndkstubgen_runner.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Copyright 2022 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# A  wrapper script that fixes the path for running ndkstubgen
+# This puts the `symbolfile` module on the search path
+# This script must be run from outer tree root
+
+PYTHONPATH="${PYTHONPATH}:orchestrator/build/soong/cc" orchestrator/build/soong/cc/ndkstubgen/__init__.py $@
diff --git a/core/cc/stub_generator.py b/core/cc/stub_generator.py
new file mode 100644
index 0000000..881f912
--- /dev/null
+++ b/core/cc/stub_generator.py
@@ -0,0 +1,100 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A module for writing ninja rules that generate C stubs."""
+
+import os
+
+from typing import NamedTuple
+
+from ninja_tools import Ninja
+from ninja_syntax import Variable, Rule, BuildAction
+
+# This is the path relative to outer tree root
+NDKSTUBGEN = "orchestrator/build/orchestrator/core/cc/ndkstubgen_runner.sh"
+
+class GenCcStubsInput:
+
+    def __init__(self, arch: str, version: str, api: str, version_map: str, additional_args=""):
+        self.arch = arch  # target device arch (e.g. x86)
+        self.version = version  # numeric API level (e.g. 33)
+        self.version_map = version_map # path to API level map (e.g. Q-->29)
+        self.api = api  # path to map.txt
+        self.additional_args = additional_args  # additional args to ndkstubgen (e.g. --llndk)
+
+class GenCcStubsOutput(NamedTuple):
+    stub: str
+    version_script: str
+    symbol_list: str
+
+class StubGenerator:
+
+    def __init__(self):
+        self._stubgen_rule = None
+
+    def _add_stubgen_rule(self, ninja: Ninja):
+        """This adds a ninja rule to run ndkstubgen
+
+        Running ndkstubgen creates C stubs from API .map.txt files
+        """
+        # Create a variable name for the binary.
+        ninja.add_variable(Variable("ndkstubgen", NDKSTUBGEN))
+
+        # Add a rule to the ninja file.
+        rule = Rule("genCcStubsRule")
+        rule.add_variable(
+            "description",
+            "Generate stub .c files from .map.txt API description files")
+        rule.add_variable(
+            "command",
+            "${ndkstubgen} --arch ${arch} --api ${apiLevel} --api-map ${apiMap} ${additionalArgs} ${in} ${out}"
+        )
+        ninja.add_rule(rule)
+        self._stubgen_rule = rule
+
+    def add_stubgen_action(self, ninja: Ninja, stub_input: GenCcStubsInput,
+                           work_dir: str) -> GenCcStubsOutput:
+        """This adds a ninja build action to generate stubs using `genCcStubsRule`"""
+        # Add stubgen rule if it has not been added yet.
+        if not self._stubgen_rule:
+            self._add_stubgen_rule(ninja)
+
+        outputs = GenCcStubsOutput(
+            stub=os.path.join(work_dir, stub_input.arch, "stub.c"),
+            version_script=os.path.join(work_dir, stub_input.arch, "stub.map"),
+            symbol_list=os.path.join(work_dir, stub_input.arch,
+                                     "abi_symbol_list.txt")
+        )
+
+        # Create the ninja build action.
+        stubgen_action = BuildAction(
+            output=list(outputs),
+            inputs=stub_input.api,
+            rule=self._stubgen_rule.name,
+            implicits=[
+                    NDKSTUBGEN,
+                    stub_input.version_map,
+            ],
+        )
+        stubgen_action.add_variable("arch", stub_input.arch)
+        stubgen_action.add_variable("apiLevel", stub_input.version)
+        stubgen_action.add_variable("apiMap", stub_input.version_map)
+        stubgen_action.add_variable("additionalArgs",
+                                           stub_input.additional_args)
+
+        # Add the build action to the ninja file.
+        ninja.add_build_action(stubgen_action)
+        return outputs
diff --git a/core/cc/test_stub_generator.py b/core/cc/test_stub_generator.py
new file mode 100644
index 0000000..578e360
--- /dev/null
+++ b/core/cc/test_stub_generator.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# ibuted under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for stub_generator.py."""
+
+import unittest
+
+import os
+import sys
+
+from ninja_tools import Ninja
+from ninja_syntax import Variable, BuildAction
+from cc.stub_generator import StubGenerator, GenCcStubsInput, NDKSTUBGEN
+from utils import TestContext
+
+class TestStubGenerator(unittest.TestCase):
+
+    def setUp(self):
+        self.ninja_context = TestContext("test_dir", "stubgen_test")
+
+    def _get_stub_inputs(self):
+        return GenCcStubsInput("x86", "33", "libfoo.map.txt", "api_levels.json")
+
+    def test_implicit_deps(self):
+        """Test that all impicit deps of stub generation are added.
+
+        ndkstubgen and api_levels.json are implicit deps.
+        ninja should recomplie stubs if there is a change in either of these.
+        """
+        ninja = Ninja(context=self.ninja_context, file=None)
+        stub_generator = StubGenerator()
+        stub_generator.add_stubgen_action(ninja, self._get_stub_inputs(), "out")
+        build_actions = [node for node in ninja.nodes if isinstance(node,
+                                                                    BuildAction)]
+        self.assertIsNotNone(build_actions)
+        self.assertTrue(all([NDKSTUBGEN in x.implicits for x in build_actions]))
+        self.assertTrue(["api_levels.json" in x.implicits for x in build_actions])
+
+    def test_output_contains_c_stubs(self):
+        ninja = Ninja(context=self.ninja_context, file=None)
+        stub_generator = StubGenerator()
+        outputs = stub_generator.add_stubgen_action(ninja, self._get_stub_inputs(), "out")
+        self.assertGreater(len(outputs), 0)
+        self.assertTrue("stub.c" in outputs.stub)
+        self.assertTrue("stub.map" in outputs.version_script)
+
+if __name__ == "__main__":
+    unittest.main()
+
diff --git a/core/ninja_tools.py b/core/ninja_tools.py
index 0887a7a..36e5382 100644
--- a/core/ninja_tools.py
+++ b/core/ninja_tools.py
@@ -35,6 +35,7 @@ class Ninja(ninja_writer.Writer):
         super().__init__(file, builddir=context.out.root(base=context.out.Base.OUTER), **kwargs)
         self._context = context
         self._did_copy_file = False
+        self._write_rule = None
         self._phonies = {}
 
     def add_copy_file(self, copy_to, copy_from):
@@ -44,7 +45,7 @@ class Ninja(ninja_writer.Writer):
             rule.add_variable("command", "mkdir -p ${out_dir} && " + self._context.tools.acp()
                     + " -f ${in} ${out}")
             self.add_rule(rule)
-        build_action = BuildAction(copy_to, "copy_file", inputs=[copy_from,],
+        build_action = BuildAction(output=copy_to, rule="copy_file", inputs=[copy_from,],
                 implicits=[self._context.tools.acp()])
         build_action.add_variable("out_dir", os.path.dirname(copy_to))
         self.add_build_action(build_action)
@@ -64,3 +65,18 @@ class Ninja(ninja_writer.Writer):
         for phony, deps in self._phonies.items():
             self.add_phony(phony, deps)
         super().write()
+
+    def add_write_file(self, filepath:str , content:str):
+        """Writes the content as a string to filepath
+        The content is written as-is, special characters are not escaped
+        """
+        if not self._write_rule:
+            rule = Rule("write_file")
+            rule.add_variable("description", "Writes content to out")
+            rule.add_variable("command", "printf '${content}' > ${out}")
+            self.add_rule(rule)
+            self._write_rule = rule
+
+        build_action = BuildAction(output=filepath, rule="write_file")
+        build_action.add_variable("content", content)
+        self.add_build_action(build_action)
diff --git a/core/utils.py b/core/utils.py
index 038808d..b8a9cb8 100644
--- a/core/utils.py
+++ b/core/utils.py
@@ -113,13 +113,21 @@ class OutDir(object):
 
     def api_library_work_dir(self, surface, version, library, **kwargs):
         """Intermediate scratch directory for library inside an API surface."""
-        return self._generate_path(_INTERMEDIATES, _API_SURFACES, surface,
-                                   str(version), library, **kwargs)
+        return self._generate_path(self.api_surfaces_work_dir(),
+                                   surface,
+                                   str(version),
+                                   library,
+                                   **kwargs)
 
     def api_surfaces_dir(self, **kwargs):
         """The published api_surfaces directory."""
         return self._generate_path(_API_SURFACES, **kwargs)
 
+    def api_surfaces_work_dir(self, *args):
+        """Intermediates / scratch directory for API surface assembly.
+        Useful for creating artifacts that are expected to be shared between multiple API libraries"""
+        return self._generate_path(_INTERMEDIATES, _API_SURFACES, *args)
+
     def outer_ninja_file(self, **kwargs):
         return self._generate_path("multitree.ninja", **kwargs)