Add support for writing 'manifest' protos to turbine

for parity with JavaBuilder. The proto includes a listing of sources,
including information about which ones were generated.

RELNOTES: N/A

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=289528891

8 files changed, 189 insertions, 26 deletions

diff --git a/java/com/google/turbine/binder/Binder.java b/java/com/google/turbine/binder/Binder.java
index 2468a3f..76cbd99 100644
--- a/java/com/google/turbine/binder/Binder.java
+++ b/java/com/google/turbine/binder/Binder.java
@@ -92,7 +92,7 @@ public class Binder {
         bind(
             log,
             units,
-            /* generatedSources= */ ImmutableList.of(),
+            /* generatedSources= */ ImmutableMap.of(),
             /* generatedClasses= */ ImmutableMap.of(),
             classpath,
             bootclasspath,
@@ -109,7 +109,7 @@ public class Binder {
   static BindingResult bind(
       TurbineLog log,
       List<CompUnit> units,
-      ImmutableList<SourceFile> generatedSources,
+      ImmutableMap<String, SourceFile> generatedSources,
       ImmutableMap<String, byte[]> generatedClasses,
       ClassPath classpath,
       ClassPath bootclasspath,
@@ -475,7 +475,7 @@ public class Binder {
     private final ImmutableList<SourceModuleInfo> modules;
     private final CompoundEnv<ClassSymbol, BytecodeBoundClass> classPathEnv;
     private final TopLevelIndex tli;
-    private final ImmutableList<SourceFile> generatedSources;
+    private final ImmutableMap<String, SourceFile> generatedSources;
     private final ImmutableMap<String, byte[]> generatedClasses;
     private final Statistics statistics;
 
@@ -484,7 +484,7 @@ public class Binder {
         ImmutableList<SourceModuleInfo> modules,
         CompoundEnv<ClassSymbol, BytecodeBoundClass> classPathEnv,
         TopLevelIndex tli,
-        ImmutableList<SourceFile> generatedSources,
+        ImmutableMap<String, SourceFile> generatedSources,
         ImmutableMap<String, byte[]> generatedClasses,
         Statistics statistics) {
       this.units = units;
@@ -514,7 +514,7 @@ public class Binder {
       return tli;
     }
 
-    public ImmutableList<SourceFile> generatedSources() {
+    public ImmutableMap<String, SourceFile> generatedSources() {
       return generatedSources;
     }
 
@@ -531,7 +531,7 @@ public class Binder {
           units, modules, classPathEnv, tli, generatedSources, generatedClasses, statistics);
     }
 
-    public BindingResult withGeneratedSources(ImmutableList<SourceFile> generatedSources) {
+    public BindingResult withGeneratedSources(ImmutableMap<String, SourceFile> generatedSources) {
       return new BindingResult(
           units, modules, classPathEnv, tli, generatedSources, generatedClasses, statistics);
     }
diff --git a/java/com/google/turbine/main/Main.java b/java/com/google/turbine/main/Main.java
index 66d3c1b..0e77c65 100644
--- a/java/com/google/turbine/main/Main.java
+++ b/java/com/google/turbine/main/Main.java
@@ -32,6 +32,8 @@ import com.google.turbine.binder.ClassPathBinder;
 import com.google.turbine.binder.CtSymClassBinder;
 import com.google.turbine.binder.JimageClassBinder;
 import com.google.turbine.binder.Processing;
+import com.google.turbine.binder.bound.SourceTypeBoundClass;
+import com.google.turbine.binder.sym.ClassSymbol;
 import com.google.turbine.deps.Dependencies;
 import com.google.turbine.deps.Transitive;
 import com.google.turbine.diag.SourceFile;
@@ -43,6 +45,8 @@ import com.google.turbine.options.TurbineOptions.ReducedClasspathMode;
 import com.google.turbine.options.TurbineOptionsParser;
 import com.google.turbine.parse.Parser;
 import com.google.turbine.proto.DepsProto;
+import com.google.turbine.proto.ManifestProto;
+import com.google.turbine.proto.ManifestProto.CompilationUnit;
 import com.google.turbine.tree.Tree.CompUnit;
 import com.google.turbine.zip.Zip;
 import java.io.BufferedOutputStream;
@@ -56,7 +60,6 @@ import java.time.LocalDateTime;
 import java.time.ZoneId;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.jar.Attributes;
@@ -183,7 +186,9 @@ public class Main {
         throw new AssertionError(reducedClasspathMode);
     }
 
-    if (options.outputDeps().isPresent() || options.output().isPresent()) {
+    if (options.outputDeps().isPresent()
+        || options.output().isPresent()
+        || options.outputManifest().isPresent()) {
       // TODO(cushon): parallelize
       Lowered lowered = Lower.lowerAll(bound.units(), bound.modules(), bound.classPathEnv());
 
@@ -200,6 +205,9 @@ public class Main {
         Map<String, byte[]> transitive = Transitive.collectDeps(bootclasspath, bound);
         writeOutput(options, bound.generatedClasses(), lowered.bytes(), transitive);
       }
+      if (options.outputManifest().isPresent()) {
+        writeManifestProto(options, bound.units(), bound.generatedSources());
+      }
     }
 
     writeSources(options, bound.generatedSources());
@@ -317,7 +325,8 @@ public class Main {
   }
 
   /** Writes source files generated by annotation processors. */
-  private static void writeSources(TurbineOptions options, List<SourceFile> generatedSources)
+  private static void writeSources(
+      TurbineOptions options, ImmutableMap<String, SourceFile> generatedSources)
       throws IOException {
     if (!options.gensrcOutput().isPresent()) {
       return;
@@ -326,7 +335,7 @@ public class Main {
     try (OutputStream os = Files.newOutputStream(path);
         BufferedOutputStream bos = new BufferedOutputStream(os, BUFFER_SIZE);
         JarOutputStream jos = new JarOutputStream(bos)) {
-      for (SourceFile source : generatedSources) {
+      for (SourceFile source : generatedSources.values()) {
         addEntry(jos, source.path(), source.source().getBytes(UTF_8));
       }
       writeManifest(jos, manifest());
@@ -378,6 +387,29 @@ public class Main {
     }
   }
 
+  private static void writeManifestProto(
+      TurbineOptions options,
+      ImmutableMap<ClassSymbol, SourceTypeBoundClass> units,
+      ImmutableMap<String, SourceFile> generatedSources)
+      throws IOException {
+    ManifestProto.Manifest.Builder manifest = ManifestProto.Manifest.newBuilder();
+    for (Map.Entry<ClassSymbol, SourceTypeBoundClass> e : units.entrySet()) {
+      manifest.addCompilationUnit(
+          CompilationUnit.newBuilder()
+              .setPath(e.getValue().source().path())
+              .setPkg(e.getKey().packageName())
+              .addTopLevel(e.getKey().simpleName())
+              .setGeneratedByAnnotationProcessor(
+                  generatedSources.containsKey(e.getValue().source().path()))
+              .build());
+    }
+    try (OutputStream os =
+        new BufferedOutputStream(
+            Files.newOutputStream(Paths.get(options.outputManifest().get())))) {
+      manifest.build().writeTo(os);
+    }
+  }
+
   /** Normalize timestamps. */
   static final long DEFAULT_TIMESTAMP =
       LocalDateTime.of(2010, 1, 1, 0, 0, 0)
diff --git a/java/com/google/turbine/options/TurbineOptions.java b/java/com/google/turbine/options/TurbineOptions.java
index d3dabc8..4dcc408 100644
--- a/java/com/google/turbine/options/TurbineOptions.java
+++ b/java/com/google/turbine/options/TurbineOptions.java
@@ -96,6 +96,9 @@ public abstract class TurbineOptions {
   /** Output jdeps file. */
   public abstract Optional<String> outputDeps();
 
+  /** Output manifest file. */
+  public abstract Optional<String> outputManifest();
+
   /** The direct dependencies. */
   public abstract ImmutableSet<String> directJars();
 
@@ -213,6 +216,8 @@ public abstract class TurbineOptions {
 
     public abstract Builder setOutputDeps(String outputDeps);
 
+    public abstract Builder setOutputManifest(String outputManifest);
+
     public abstract Builder setTargetLabel(String targetLabel);
 
     public abstract Builder setInjectingRuleKind(String injectingRuleKind);
diff --git a/java/com/google/turbine/options/TurbineOptionsParser.java b/java/com/google/turbine/options/TurbineOptionsParser.java
index 693e2ad..17d4bf6 100644
--- a/java/com/google/turbine/options/TurbineOptionsParser.java
+++ b/java/com/google/turbine/options/TurbineOptionsParser.java
@@ -103,6 +103,9 @@ public class TurbineOptionsParser {
         case "--output_deps":
           builder.setOutputDeps(readOne(argumentDeque));
           break;
+        case "--output_manifest_proto":
+          builder.setOutputManifest(readOne(argumentDeque));
+          break;
         case "--direct_dependencies":
           builder.setDirectJars(readList(argumentDeque));
           break;
diff --git a/java/com/google/turbine/processing/TurbineFiler.java b/java/com/google/turbine/processing/TurbineFiler.java
index 6fe2de9..186eb7f 100644
--- a/java/com/google/turbine/processing/TurbineFiler.java
+++ b/java/com/google/turbine/processing/TurbineFiler.java
@@ -22,7 +22,6 @@ import static java.nio.charset.StandardCharsets.UTF_8;
 import com.google.common.base.Function;
 import com.google.common.base.Supplier;
 import com.google.common.base.Suppliers;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteStreams;
 import com.google.turbine.diag.SourceFile;
@@ -81,8 +80,8 @@ public class TurbineFiler implements Filer {
   private final Map<String, byte[]> generatedClasses = new LinkedHashMap<>();
 
   /** Generated source file objects from all rounds. */
-  public ImmutableList<SourceFile> generatedSources() {
-    return ImmutableList.copyOf(generatedSources.values());
+  public ImmutableMap<String, SourceFile> generatedSources() {
+    return ImmutableMap.copyOf(generatedSources);
   }
 
   /** Generated class file objects from all rounds. */
diff --git a/javatests/com/google/turbine/main/MainTest.java b/javatests/com/google/turbine/main/MainTest.java
index b37cc8c..be38d97 100644
--- a/javatests/com/google/turbine/main/MainTest.java
+++ b/javatests/com/google/turbine/main/MainTest.java
@@ -20,17 +20,25 @@ import static com.google.common.base.StandardSystemProperty.JAVA_CLASS_VERSION;
 import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth8.assertThat;
+import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat;
 import static com.google.turbine.testing.TestClassPaths.optionsWithBootclasspath;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.fail;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.io.ByteStreams;
 import com.google.common.io.MoreFiles;
+import com.google.protobuf.ExtensionRegistry;
 import com.google.turbine.diag.TurbineError;
 import com.google.turbine.options.TurbineOptions;
+import com.google.turbine.proto.ManifestProto;
+import java.io.BufferedInputStream;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.io.Writer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.LocalDateTime;
@@ -38,12 +46,18 @@ import java.time.ZoneId;
 import java.util.Enumeration;
 import java.util.LinkedHashMap;
 import java.util.Map;
+import java.util.Set;
 import java.util.jar.Attributes;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
 import java.util.stream.Stream;
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.TypeElement;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -187,9 +201,10 @@ public class MainTest {
       }
       Manifest manifest = jarFile.getManifest();
       Attributes attributes = manifest.getMainAttributes();
-      assertThat(
-              attributes.entrySet().stream()
-                  .collect(toImmutableMap(e -> e.getKey().toString(), Map.Entry::getValue)))
+      ImmutableMap<String, ?> entries =
+          attributes.entrySet().stream()
+              .collect(toImmutableMap(e -> e.getKey().toString(), Map.Entry::getValue));
+      assertThat(entries)
           .containsExactly(
               "Created-By", "bazel",
               "Manifest-Version", "1.0",
@@ -202,9 +217,10 @@ public class MainTest {
     try (JarFile jarFile = new JarFile(gensrcOutput.toFile())) {
       Manifest manifest = jarFile.getManifest();
       Attributes attributes = manifest.getMainAttributes();
-      assertThat(
-              attributes.entrySet().stream()
-                  .collect(toImmutableMap(e -> e.getKey().toString(), Map.Entry::getValue)))
+      ImmutableMap<String, ?> entries =
+          attributes.entrySet().stream()
+              .collect(toImmutableMap(e -> e.getKey().toString(), Map.Entry::getValue));
+      assertThat(entries)
           .containsExactly(
               "Created-By", "bazel",
               "Manifest-Version", "1.0");
@@ -276,4 +292,77 @@ public class MainTest {
           .containsExactly("META-INF/", "META-INF/MANIFEST.MF");
     }
   }
+
+  @SupportedAnnotationTypes("*")
+  public static class SourceGeneratingProcessor extends AbstractProcessor {
+
+    @Override
+    public SourceVersion getSupportedSourceVersion() {
+      return SourceVersion.latestSupported();
+    }
+
+    private boolean first = true;
+
+    @Override
+    public boolean process(Set<? extends TypeElement> annotations, RoundEnvironment roundEnv) {
+      if (first) {
+        try (Writer writer = processingEnv.getFiler().createSourceFile("g.Gen").openWriter()) {
+          writer.write("package g; class Gen {}");
+        } catch (IOException e) {
+          throw new UncheckedIOException(e);
+        }
+        first = false;
+      }
+      return false;
+    }
+  }
+
+  @Test
+  public void testManifestProto() throws IOException {
+    Path src = temporaryFolder.newFile("Foo.java").toPath();
+    MoreFiles.asCharSink(src, UTF_8).write("package f; @Deprecated class Foo {}");
+
+    Path output = temporaryFolder.newFile("output.jar").toPath();
+    Path gensrcOutput = temporaryFolder.newFile("gensrcOutput.jar").toPath();
+    Path manifestProtoOutput = temporaryFolder.newFile("manifest.proto").toPath();
+
+    Main.compile(
+        optionsWithBootclasspath()
+            .setSources(ImmutableList.of(src.toString()))
+            .setTargetLabel("//foo:foo")
+            .setInjectingRuleKind("foo_library")
+            .setOutput(output.toString())
+            .setGensrcOutput(gensrcOutput.toString())
+            .setOutputManifest(manifestProtoOutput.toString())
+            .setProcessors(ImmutableList.of(SourceGeneratingProcessor.class.getName()))
+            .build());
+
+    assertThat(readManifestProto(manifestProtoOutput))
+        .isEqualTo(
+            ManifestProto.Manifest.newBuilder()
+                .addCompilationUnit(
+                    ManifestProto.CompilationUnit.newBuilder()
+                        .setPkg("f")
+                        .addTopLevel("Foo")
+                        .setPath(src.toString())
+                        .setGeneratedByAnnotationProcessor(false)
+                        .build())
+                .addCompilationUnit(
+                    ManifestProto.CompilationUnit.newBuilder()
+                        .setPkg("g")
+                        .addTopLevel("Gen")
+                        .setPath("g/Gen.java")
+                        .setGeneratedByAnnotationProcessor(true)
+                        .build())
+                .build());
+  }
+
+  private static ManifestProto.Manifest readManifestProto(Path manifestProtoOutput)
+      throws IOException {
+    ManifestProto.Manifest.Builder manifest = ManifestProto.Manifest.newBuilder();
+    try (InputStream is = new BufferedInputStream(Files.newInputStream(manifestProtoOutput))) {
+      manifest.mergeFrom(is, ExtensionRegistry.getEmptyRegistry());
+    }
+    return manifest.build();
+  }
 }
diff --git a/javatests/com/google/turbine/processing/ProcessingIntegrationTest.java b/javatests/com/google/turbine/processing/ProcessingIntegrationTest.java
index 57b4f0d..ed5af6a 100644
--- a/javatests/com/google/turbine/processing/ProcessingIntegrationTest.java
+++ b/javatests/com/google/turbine/processing/ProcessingIntegrationTest.java
@@ -244,15 +244,10 @@ public class ProcessingIntegrationTest {
             TestClassPaths.TURBINE_BOOTCLASSPATH,
             Optional.empty());
 
-    assertThat(bound.generatedSources().stream().map(s -> s.path()).collect(toImmutableList()))
-        .containsExactly("Gen.java", "source.txt");
+    assertThat(bound.generatedSources().keySet()).containsExactly("Gen.java", "source.txt");
     assertThat(bound.generatedClasses().keySet()).containsExactly("class.txt");
 
-    assertThat(
-            bound.generatedSources().stream()
-                .filter(s -> s.path().equals("source.txt"))
-                .collect(onlyElement())
-                .source())
+    assertThat(bound.generatedSources().get("source.txt").source())
         .isEqualTo("hello source output");
     assertThat(new String(bound.generatedClasses().get("class.txt"), UTF_8))
         .isEqualTo("hello class output");
diff --git a/proto/manifest.proto b/proto/manifest.proto
new file mode 100644
index 0000000..20e85ef
--- /dev/null
+++ b/proto/manifest.proto
@@ -0,0 +1,40 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Definitions for dependency reports.
+
+syntax = "proto2";
+
+option java_package = "com.google.turbine.proto";
+option java_outer_classname = "ManifestProto";
+
+// Information about a single compilation unit (.java file)
+message CompilationUnit {
+  // The path to the compilation unit
+  optional string path = 1;
+
+  // The package of the source file
+  optional string pkg = 2;
+
+  // Whether the source was generated by an annotation processor
+  optional bool generated_by_annotation_processor = 3;
+
+  // The list of top-level types in the compilation unit
+  repeated string top_level = 4;
+}
+
+// Top-level message found in .manifest artifacts
+message Manifest {
+  repeated CompilationUnit compilation_unit = 1;
+}
\ No newline at end of file