diff options
author | Colin Cross <ccross@android.com> | 2023-07-18 15:57:09 -0700 |
---|---|---|
committer | Colin Cross <ccross@android.com> | 2023-07-26 23:39:46 +0000 |
commit | 47efcdcf6bc845608f137df06d3b1d4426088839 (patch) | |
tree | 27f6d50d15f0aac34de380af06a50f837d38e490 | |
parent | d3b1b77184b1c002136039e70074899a35b1ed77 (diff) | |
download | soong-47efcdcf6bc845608f137df06d3b1d4426088839.tar.gz |
Merge META-INF/services/* files in merge_zips -jar
kotlinx_coroutines_test and kotlinx_coroutine_android each provide a
META-INF/services/kotlinx.coroutines.CoroutineExceptionHandler with
different contents, and the final contents needs to be the combination
of the two files. Implement service merging in merge_zips when the
-jar argument is provided.
Bug: 290933559
Test: TestMergeZips
Change-Id: I69f80d1265c64c671d308ef4cdccfa1564abe056
Merged-In: I69f80d1265c64c671d308ef4cdccfa1564abe056
(cherry picked from commit 7592d5a0bdec6848b1679eb29a28eb8dddfe4c87)
-rw-r--r-- | cmd/merge_zips/merge_zips.go | 22 | ||||
-rw-r--r-- | cmd/merge_zips/merge_zips_test.go | 68 | ||||
-rw-r--r-- | jar/Android.bp | 1 | ||||
-rw-r--r-- | jar/services.go | 128 | ||||
-rw-r--r-- | third_party/zip/android.go | 2 |
5 files changed, 198 insertions, 23 deletions
diff --git a/cmd/merge_zips/merge_zips.go b/cmd/merge_zips/merge_zips.go index e3d1179b4..a70a9d158 100644 --- a/cmd/merge_zips/merge_zips.go +++ b/cmd/merge_zips/merge_zips.go @@ -122,7 +122,7 @@ func (be ZipEntryFromBuffer) Size() uint64 { } func (be ZipEntryFromBuffer) WriteToZip(dest string, zw *zip.Writer) error { - w, err := zw.CreateHeader(be.fh) + w, err := zw.CreateHeaderAndroid(be.fh) if err != nil { return err } @@ -562,6 +562,8 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string } } + var jarServices jar.Services + // Finally, add entries from all the input zips. for _, inputZip := range inputZips { _, copyFully := zipsToNotStrip[inputZip.Name()] @@ -570,6 +572,14 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string } for i, entry := range inputZip.Entries() { + if emulateJar && jarServices.IsServiceFile(entry) { + // If this is a jar, collect service files to combine instead of adding them to the zip. + err := jarServices.AddServiceFile(entry) + if err != nil { + return err + } + continue + } if copyFully || !out.isEntryExcluded(entry.Name) { if err := out.copyEntry(inputZip, i); err != nil { return err @@ -585,6 +595,16 @@ func mergeZips(inputZips []InputZip, writer *zip.Writer, manifest, pyMain string } if emulateJar { + // Combine all the service files into a single list of combined service files and add them to the zip. + for _, serviceFile := range jarServices.ServiceFiles() { + _, err := out.addZipEntry(serviceFile.Name, ZipEntryFromBuffer{ + fh: serviceFile.FileHeader, + content: serviceFile.Contents, + }) + if err != nil { + return err + } + } return out.writeEntries(out.jarSorted()) } else if sortEntries { return out.writeEntries(out.alphanumericSorted()) diff --git a/cmd/merge_zips/merge_zips_test.go b/cmd/merge_zips/merge_zips_test.go index cb5843607..767d4e61f 100644 --- a/cmd/merge_zips/merge_zips_test.go +++ b/cmd/merge_zips/merge_zips_test.go @@ -17,6 +17,7 @@ package main import ( "bytes" "fmt" + "hash/crc32" "os" "strconv" "strings" @@ -27,28 +28,34 @@ import ( ) type testZipEntry struct { - name string - mode os.FileMode - data []byte + name string + mode os.FileMode + data []byte + method uint16 } var ( - A = testZipEntry{"A", 0755, []byte("foo")} - a = testZipEntry{"a", 0755, []byte("foo")} - a2 = testZipEntry{"a", 0755, []byte("FOO2")} - a3 = testZipEntry{"a", 0755, []byte("Foo3")} - bDir = testZipEntry{"b/", os.ModeDir | 0755, nil} - bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil} - bbb = testZipEntry{"b/b/b", 0755, nil} - ba = testZipEntry{"b/a", 0755, []byte("foob")} - bc = testZipEntry{"b/c", 0755, []byte("bar")} - bd = testZipEntry{"b/d", 0700, []byte("baz")} - be = testZipEntry{"b/e", 0700, []byte("")} - - metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil} - manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest")} - manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2")} - moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info")} + A = testZipEntry{"A", 0755, []byte("foo"), zip.Deflate} + a = testZipEntry{"a", 0755, []byte("foo"), zip.Deflate} + a2 = testZipEntry{"a", 0755, []byte("FOO2"), zip.Deflate} + a3 = testZipEntry{"a", 0755, []byte("Foo3"), zip.Deflate} + bDir = testZipEntry{"b/", os.ModeDir | 0755, nil, zip.Deflate} + bbDir = testZipEntry{"b/b/", os.ModeDir | 0755, nil, zip.Deflate} + bbb = testZipEntry{"b/b/b", 0755, nil, zip.Deflate} + ba = testZipEntry{"b/a", 0755, []byte("foo"), zip.Deflate} + bc = testZipEntry{"b/c", 0755, []byte("bar"), zip.Deflate} + bd = testZipEntry{"b/d", 0700, []byte("baz"), zip.Deflate} + be = testZipEntry{"b/e", 0700, []byte(""), zip.Deflate} + + service1a = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\n"), zip.Store} + service1b = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass3\n"), zip.Deflate} + service1combined = testZipEntry{"META-INF/services/service1", 0755, []byte("class1\nclass2\nclass3\n"), zip.Store} + service2 = testZipEntry{"META-INF/services/service2", 0755, []byte("class1\nclass2\n"), zip.Deflate} + + metainfDir = testZipEntry{jar.MetaDir, os.ModeDir | 0755, nil, zip.Deflate} + manifestFile = testZipEntry{jar.ManifestFile, 0755, []byte("manifest"), zip.Deflate} + manifestFile2 = testZipEntry{jar.ManifestFile, 0755, []byte("manifest2"), zip.Deflate} + moduleInfoFile = testZipEntry{jar.ModuleInfoClass, 0755, []byte("module-info"), zip.Deflate} ) type testInputZip struct { @@ -236,6 +243,15 @@ func TestMergeZips(t *testing.T) { "in1": true, }, }, + { + name: "services", + in: [][]testZipEntry{ + {service1a, service2}, + {service1b}, + }, + jar: true, + out: []testZipEntry{service1combined, service2}, + }, } for _, test := range testCases { @@ -256,7 +272,7 @@ func TestMergeZips(t *testing.T) { closeErr := writer.Close() if closeErr != nil { - t.Fatal(err) + t.Fatal(closeErr) } if test.err != "" { @@ -266,12 +282,16 @@ func TestMergeZips(t *testing.T) { t.Fatal("incorrect err, want:", test.err, "got:", err) } return + } else if err != nil { + t.Fatal("unexpected err: ", err) } if !bytes.Equal(want, out.Bytes()) { t.Error("incorrect zip output") t.Errorf("want:\n%s", dumpZip(want)) t.Errorf("got:\n%s", dumpZip(out.Bytes())) + os.WriteFile("/tmp/got.zip", out.Bytes(), 0755) + os.WriteFile("/tmp/want.zip", want, 0755) } }) } @@ -286,8 +306,14 @@ func testZipEntriesToBuf(entries []testZipEntry) []byte { Name: e.name, } fh.SetMode(e.mode) + fh.Method = e.method + fh.UncompressedSize64 = uint64(len(e.data)) + fh.CRC32 = crc32.ChecksumIEEE(e.data) + if fh.Method == zip.Store { + fh.CompressedSize64 = fh.UncompressedSize64 + } - w, err := zw.CreateHeader(&fh) + w, err := zw.CreateHeaderAndroid(&fh) if err != nil { panic(err) } diff --git a/jar/Android.bp b/jar/Android.bp index 46113d877..c03e49174 100644 --- a/jar/Android.bp +++ b/jar/Android.bp @@ -21,6 +21,7 @@ bootstrap_go_package { pkgPath: "android/soong/jar", srcs: [ "jar.go", + "services.go", ], testSrcs: [ "jar_test.go", diff --git a/jar/services.go b/jar/services.go new file mode 100644 index 000000000..d06a6dc99 --- /dev/null +++ b/jar/services.go @@ -0,0 +1,128 @@ +// Copyright 2023 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jar + +import ( + "android/soong/third_party/zip" + "bufio" + "hash/crc32" + "sort" + "strings" +) + +const servicesPrefix = "META-INF/services/" + +// Services is used to collect service files from multiple zip files and produce a list of ServiceFiles containing +// the unique lines from all the input zip entries with the same name. +type Services struct { + services map[string]*ServiceFile +} + +// ServiceFile contains the combined contents of all input zip entries with a single name. +type ServiceFile struct { + Name string + FileHeader *zip.FileHeader + Contents []byte + Lines []string +} + +// IsServiceFile returns true if the zip entry is in the META-INF/services/ directory. +func (Services) IsServiceFile(entry *zip.File) bool { + return strings.HasPrefix(entry.Name, servicesPrefix) +} + +// AddServiceFile adds a zip entry in the META-INF/services/ directory to the list of service files that need +// to be combined. +func (j *Services) AddServiceFile(entry *zip.File) error { + if j.services == nil { + j.services = map[string]*ServiceFile{} + } + + service := entry.Name + serviceFile := j.services[service] + fh := entry.FileHeader + if serviceFile == nil { + serviceFile = &ServiceFile{ + Name: service, + FileHeader: &fh, + } + j.services[service] = serviceFile + } + + f, err := entry.Open() + if err != nil { + return err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + if line != "" { + serviceFile.Lines = append(serviceFile.Lines, line) + } + } + + if err := scanner.Err(); err != nil { + return err + } + + return nil +} + +// ServiceFiles returns the list of combined service files, each containing all the unique lines from the +// corresponding service files in the input zip entries. +func (j *Services) ServiceFiles() []ServiceFile { + services := make([]ServiceFile, 0, len(j.services)) + + for _, serviceFile := range j.services { + serviceFile.Lines = dedupServicesLines(serviceFile.Lines) + serviceFile.Lines = append(serviceFile.Lines, "") + serviceFile.Contents = []byte(strings.Join(serviceFile.Lines, "\n")) + + serviceFile.FileHeader.UncompressedSize64 = uint64(len(serviceFile.Contents)) + serviceFile.FileHeader.CRC32 = crc32.ChecksumIEEE(serviceFile.Contents) + if serviceFile.FileHeader.Method == zip.Store { + serviceFile.FileHeader.CompressedSize64 = serviceFile.FileHeader.UncompressedSize64 + } + + services = append(services, *serviceFile) + } + + sort.Slice(services, func(i, j int) bool { + return services[i].Name < services[j].Name + }) + + return services +} + +func dedupServicesLines(in []string) []string { + writeIndex := 0 +outer: + for readIndex := 0; readIndex < len(in); readIndex++ { + for compareIndex := 0; compareIndex < writeIndex; compareIndex++ { + if interface{}(in[readIndex]) == interface{}(in[compareIndex]) { + // The value at readIndex already exists somewhere in the output region + // of the slice before writeIndex, skip it. + continue outer + } + } + if readIndex != writeIndex { + in[writeIndex] = in[readIndex] + } + writeIndex++ + } + return in[0:writeIndex] +} diff --git a/third_party/zip/android.go b/third_party/zip/android.go index f8e45c56d..0f41f6200 100644 --- a/third_party/zip/android.go +++ b/third_party/zip/android.go @@ -170,7 +170,7 @@ func (w *Writer) CreateCompressedHeader(fh *FileHeader) (io.WriteCloser, error) func (w *Writer) CreateHeaderAndroid(fh *FileHeader) (io.Writer, error) { writeDataDescriptor := fh.Method != Store if writeDataDescriptor { - fh.Flags &= DataDescriptorFlag + fh.Flags |= DataDescriptorFlag } else { fh.Flags &= ^uint16(DataDescriptorFlag) } |