Merge "Remove auto-added java deps from bp2build progress"

5 files changed, 105 insertions, 36 deletions

diff --git a/scripts/bp2build_progress/bp2build_module_dep_infos.py b/scripts/bp2build_progress/bp2build_module_dep_infos.py
index b04fb743..c189a98b 100755
--- a/scripts/bp2build_progress/bp2build_module_dep_infos.py
+++ b/scripts/bp2build_progress/bp2build_module_dep_infos.py
@@ -56,7 +56,9 @@ def _get_java_source_extensions(module):
   return out
 
 
-def module_type_info_from_json(module_graph, module_type, ignored_dep_names):
+def module_type_info_from_json(
+    module_graph, module_type, ignored_dep_names, ignore_java_auto_deps
+):
   """Builds a map of module name to _ModuleTypeInfo for each module of module_type.
 
      Dependency edges pointing to modules in ignored_dep_names are not followed.
@@ -102,7 +104,7 @@ def module_type_info_from_json(module_graph, module_type, ignored_dep_names):
     type_infos[module_name] = info
 
   dependency_analysis.visit_json_module_graph_post_order(
-      module_graph, ignored_dep_names, filter_by_type, update_infos)
+      module_graph, ignored_dep_names, ignore_java_auto_deps, filter_by_type, update_infos)
 
   return {
       name: info for name, info in type_infos.items() if name in modules_of_type
@@ -134,7 +136,16 @@ def main():
   parser.add_argument(
       "--ignore-by-name",
       default="",
-      help="Comma-separated list. When building the tree of transitive dependencies, will not follow dependency edges pointing to module names listed by this flag."
+      help=(
+          "Comma-separated list. When building the tree of transitive"
+          " dependencies, will not follow dependency edges pointing to module"
+          " names listed by this flag."
+      ),
+  )
+  parser.add_argument(
+      "--ignore-java-auto-deps",
+      action="store_true",
+      help="whether to ignore automatically added java deps",
   )
   args = parser.parse_args()
 
@@ -142,10 +153,15 @@ def main():
   ignore_by_name = args.ignore_by_name
 
   module_graph = dependency_analysis.get_json_module_type_info(module_type)
-  type_infos = module_type_info_from_json(module_graph, module_type,
-                                          ignore_by_name.split(","))
+  type_infos = module_type_info_from_json(
+      module_graph,
+      module_type,
+      ignore_by_name.split(","),
+      args.ignore_java_auto_deps,
+  )
 
   _write_output(sys.stdout, type_infos)
 
+
 if __name__ == "__main__":
   main()
diff --git a/scripts/bp2build_progress/bp2build_progress.py b/scripts/bp2build_progress/bp2build_progress.py
index d4d46da5..d6e450b4 100755
--- a/scripts/bp2build_progress/bp2build_progress.py
+++ b/scripts/bp2build_progress/bp2build_progress.py
@@ -83,7 +83,9 @@ class InputModule:
   def __str__(self):
     total = self.num_deps
     converted = self.num_deps - self.num_unconverted_deps
-    percent = converted / self.num_deps * 100
+    percent = 1
+    if self.num_deps > 0:
+      percent = converted / self.num_deps * 100
     return f"{self.module.name}: {percent:.1f}% ({converted}/{total}) converted"
 
 
@@ -288,10 +290,10 @@ def generate_report(report_data):
 def adjacency_list_from_json(
     module_graph: ...,
     ignore_by_name: List[str],
+    ignore_java_auto_deps: bool,
     top_level_modules: List[str],
-    collect_transitive_dependencies: bool = True
+    collect_transitive_dependencies: bool = True,
 ) -> Dict[ModuleInfo, Set[ModuleInfo]]:
-
   def filter_by_name(json):
     return json["Name"] in top_level_modules
 
@@ -327,7 +329,7 @@ def adjacency_list_from_json(
             module_adjacency_list.get(dep_module_info, set()))
 
   dependency_analysis.visit_json_module_graph_post_order(
-      module_graph, ignore_by_name, filter_by_name, collect_dependencies)
+      module_graph, ignore_by_name, ignore_java_auto_deps, filter_by_name, collect_dependencies)
 
   return module_adjacency_list
 
@@ -378,6 +380,7 @@ def get_module_adjacency_list(
     top_level_modules: List[str],
     use_queryview: bool,
     ignore_by_name: List[str],
+    ignore_java_auto_deps: bool = False,
     collect_transitive_dependencies: bool = True,
     banchan_mode: bool = False) -> Dict[ModuleInfo, Set[ModuleInfo]]:
   # The main module graph containing _all_ modules in the Soong build,
@@ -392,8 +395,12 @@ def get_module_adjacency_list(
     else:
       module_graph = dependency_analysis.get_json_module_info(banchan_mode)
       module_adjacency_list = adjacency_list_from_json(
-          module_graph, ignore_by_name, top_level_modules,
-          collect_transitive_dependencies)
+          module_graph,
+          ignore_by_name,
+          ignore_java_auto_deps,
+          top_level_modules,
+          collect_transitive_dependencies,
+      )
   except subprocess.CalledProcessError as err:
     sys.exit(f"""Error running: '{' '.join(err.cmd)}':"
 Stdout:
@@ -446,7 +453,16 @@ def main():
   parser.add_argument(
       "--ignore-by-name",
       default="",
-      help="Comma-separated list. When building the tree of transitive dependencies, will not follow dependency edges pointing to module names listed by this flag."
+      help=(
+          "Comma-separated list. When building the tree of transitive"
+          " dependencies, will not follow dependency edges pointing to module"
+          " names listed by this flag."
+      ),
+  )
+  parser.add_argument(
+      "--ignore-java-auto-deps",
+      action="store_true",
+      help="whether to ignore automatically added java deps",
   )
   parser.add_argument(
       "--banchan",
diff --git a/scripts/bp2build_progress/bp2build_progress_test.py b/scripts/bp2build_progress/bp2build_progress_test.py
index 0ad29a41..7a1e8009 100644
--- a/scripts/bp2build_progress/bp2build_progress_test.py
+++ b/scripts/bp2build_progress/bp2build_progress_test.py
@@ -82,9 +82,9 @@ class Bp2BuildProgressTest(unittest.TestCase):
       autospec=True,
       return_value=_queryview_graph)
   def test_get_module_adjacency_list_queryview_transitive_deps(self, _):
-    adjacency_dict = bp2build_progress.get_module_adjacency_list(['a', 'f'],
-                                                                 True, set(),
-                                                                 True, False)
+    adjacency_dict = bp2build_progress.get_module_adjacency_list(
+        ['a', 'f'], True, set(), False, True, False
+    )
 
     a = bp2build_progress.ModuleInfo(
         name='a', kind='type1', dirname='pkg', num_deps=2, created_by=None)
@@ -112,9 +112,9 @@ class Bp2BuildProgressTest(unittest.TestCase):
       autospec=True,
       return_value=_queryview_graph)
   def test_get_module_adjacency_list_queryview_direct_deps(self, _):
-    adjacency_dict = bp2build_progress.get_module_adjacency_list(['a', 'f'],
-                                                                 True, set(),
-                                                                 False, False)
+    adjacency_dict = bp2build_progress.get_module_adjacency_list(
+        ['a', 'f'], True, set(), False, False
+    )
 
     a = bp2build_progress.ModuleInfo(
         name='a', kind='type1', dirname='pkg', num_deps=2, created_by=None)
@@ -143,9 +143,9 @@ class Bp2BuildProgressTest(unittest.TestCase):
       autospec=True,
       return_value=_soong_module_graph)
   def test_get_module_adjacency_list_soong_module_transitive_deps(self, _):
-    adjacency_dict = bp2build_progress.get_module_adjacency_list(['a', 'f'],
-                                                                 False, set(),
-                                                                 True, False)
+    adjacency_dict = bp2build_progress.get_module_adjacency_list(
+        ['a', 'f'], False, set(), False, True, False
+    )
 
     a = bp2build_progress.ModuleInfo(
         name='a', kind='type1', dirname='pkg', num_deps=2, created_by='')
diff --git a/scripts/bp2build_progress/dependency_analysis.py b/scripts/bp2build_progress/dependency_analysis.py
index 53a235fe..fe2f44f2 100644
--- a/scripts/bp2build_progress/dependency_analysis.py
+++ b/scripts/bp2build_progress/dependency_analysis.py
@@ -175,8 +175,9 @@ def _ignore_json_module(json_module, ignore_by_name):
   return False
 
 
-def visit_json_module_graph_post_order(module_graph, ignore_by_name,
-                                       filter_predicate, visit):
+def visit_json_module_graph_post_order(
+    module_graph, ignore_by_name, ignore_java_auto_deps, filter_predicate, visit
+):
   # The set of ignored modules. These modules (and their dependencies) are not shown
   # in the graph or report.
   ignored = set()
@@ -221,7 +222,9 @@ def visit_json_module_graph_post_order(module_graph, ignore_by_name,
         json_module_graph_post_traversal(key)
 
     for dep in module["Deps"]:
-      if ignore_json_dep(dep, module["Name"], ignored):
+      if ignore_json_dep(
+          dep, module["Name"], ignored, ignore_java_auto_deps
+      ):
         continue
 
       dep_name = dep["Name"]
@@ -414,7 +417,38 @@ def is_prebuilt_to_source_dep(dep):
   return dep["Tag"] == "android.prebuiltDependencyTag {BaseDependencyTag:{}}"
 
 
-def ignore_json_dep(dep, module_name, ignored_keys):
+def _is_java_auto_dep(dep):
+  # Soong adds a number of dependencies automatically for Java deps, making it
+  # difficult to understand the actual dependencies, remove the
+  # non-user-specified deps
+  tag = dep["Tag"]
+  if not tag:
+    return False
+  return (
+      (
+          tag.startswith("java.dependencyTag")
+          and (
+              "name:proguard-raise" in tag
+              or "name:bootclasspath" in tag
+              or "name:system modules" in tag
+              or "name:framework-res" in tag
+              or "name:sdklib" in tag
+              or "name:java9lib" in tag
+          )
+          or (
+              tag.startswith("java.usesLibraryDependencyTag")
+              or tag.startswith("java.hiddenAPIStubsDependencyTag")
+          )
+      )
+      or (
+          tag.startswith("android.sdkMemberDependencyTag")
+          or tag.startswith("java.scopeDependencyTag")
+      )
+      or tag.startswith("dexpreopt.dex2oatDependencyTag")
+  )
+
+
+def ignore_json_dep(dep, module_name, ignored_keys, ignore_java_auto_deps):
   """Whether to ignore a json dependency based on heuristics.
 
   Args:
@@ -424,6 +458,9 @@ def ignore_json_dep(dep, module_name, ignored_keys):
   """
   if is_prebuilt_to_source_dep(dep):
     return True
+  if ignore_java_auto_deps and _is_java_auto_dep(dep):
+    return True
   name = dep["Name"]
-  return _ModuleKey(name,
-                    dep["Variations"]) in ignored_keys or name == module_name
+  return (
+      _ModuleKey(name, dep["Variations"]) in ignored_keys or name == module_name
+  )
diff --git a/scripts/bp2build_progress/dependency_analysis_test.py b/scripts/bp2build_progress/dependency_analysis_test.py
index 9b8c8cbd..a966d683 100755
--- a/scripts/bp2build_progress/dependency_analysis_test.py
+++ b/scripts/bp2build_progress/dependency_analysis_test.py
@@ -52,7 +52,7 @@ class DependencyAnalysisTest(unittest.TestCase):
       visited_modules.append(module['Name'])
 
     dependency_analysis.visit_json_module_graph_post_order(
-        graph, set(), only_a, visit)
+        graph, set(), False, only_a, visit)
 
     expected_visited = ['d', 'b', 'e', 'c', 'a']
     self.assertListEqual(visited_modules, expected_visited)
@@ -83,7 +83,7 @@ class DependencyAnalysisTest(unittest.TestCase):
       visited_modules.append(module['Name'])
 
     dependency_analysis.visit_json_module_graph_post_order(
-        graph, set('b'), only_a, visit)
+        graph, set('b'), False, only_a, visit)
 
     expected_visited = ['e', 'c', 'a']
     self.assertListEqual(visited_modules, expected_visited)
@@ -114,7 +114,7 @@ class DependencyAnalysisTest(unittest.TestCase):
       visited_modules.append(module['Name'])
 
     dependency_analysis.visit_json_module_graph_post_order(
-        graph, set(), only_a, visit)
+        graph, set(), False, only_a, visit)
 
     expected_visited = ['e', 'c', 'a']
     self.assertListEqual(visited_modules, expected_visited)
@@ -151,7 +151,7 @@ class DependencyAnalysisTest(unittest.TestCase):
       visited_modules.append(module['Name'])
 
     dependency_analysis.visit_json_module_graph_post_order(
-        graph, set(), only_a, visit)
+        graph, set(), False, only_a, visit)
 
     expected_visited = ['e', 'c', 'a']
     self.assertListEqual(visited_modules, expected_visited)
@@ -182,7 +182,7 @@ class DependencyAnalysisTest(unittest.TestCase):
       visited_modules.append(module['Name'])
 
     dependency_analysis.visit_json_module_graph_post_order(
-        graph, set(), only_a, visit)
+        graph, set(), False, only_a, visit)
 
     expected_visited = ['e', 'c', 'a']
     self.assertListEqual(visited_modules, expected_visited)
@@ -203,7 +203,7 @@ class DependencyAnalysisTest(unittest.TestCase):
       visited_modules.append(module['Name'])
 
     dependency_analysis.visit_json_module_graph_post_order(
-        graph, set(), only_a, visit)
+        graph, set(), False, only_a, visit)
 
     expected_visited = ['a']
     self.assertListEqual(visited_modules, expected_visited)
@@ -245,7 +245,7 @@ class DependencyAnalysisTest(unittest.TestCase):
       visited_modules.append(module['Name'])
 
     dependency_analysis.visit_json_module_graph_post_order(
-        graph, set(), only_a, visit)
+        graph, set(), False, only_a, visit)
 
     expected_visited = ['d', 'b', 'a', 'e', 'c', 'a']
     self.assertListEqual(visited_modules, expected_visited)
@@ -286,7 +286,7 @@ class DependencyAnalysisTest(unittest.TestCase):
       visited_modules.append(module['Name'])
 
     dependency_analysis.visit_json_module_graph_post_order(
-        graph, set(), only_a, visit)
+        graph, set(), False, only_a, visit)
 
     expected_visited = ['a']
     self.assertListEqual(visited_modules, expected_visited)
@@ -314,7 +314,7 @@ class DependencyAnalysisTest(unittest.TestCase):
       visited_modules.append(module['Name'])
 
     dependency_analysis.visit_json_module_graph_post_order(
-        graph, set(), only_a, visit)
+        graph, set(), False, only_a, visit)
 
     expected_visited = ['d', 'b', 'e', 'c', 'a']
     self.assertListEqual(visited_modules, expected_visited)