1 files changed, 436 insertions, 0 deletions
diff --git a/src/tools/ak/liteparse/liteparse.go b/src/tools/ak/liteparse/liteparse.go
new file mode 100644
index 0000000..9ab50d8
--- /dev/null
+++ b/src/tools/ak/liteparse/liteparse.go
@@ -0,0 +1,436 @@
+// Copyright 2018 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package liteparse does a light parsing of android resources files that can be used at a later
+// stage to generate R.java files.
+package liteparse
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"src/common/golang/flags"
+	"src/common/golang/walk"
+	rdpb "src/tools/ak/res/proto/res_data_go_proto"
+	"src/tools/ak/res/res"
+	"src/tools/ak/res/respipe/respipe"
+	"src/tools/ak/res/resxml/resxml"
+	"src/tools/ak/types"
+	"google.golang.org/protobuf/proto"
+)
+
+var (
+	// Cmd defines the command to run the res parser.
+	Cmd = types.Command{
+		Init:  Init,
+		Run:   Run,
+		Desc:  desc,
+		Flags: []string{"resourceFiles", "rPbOutput"},
+	}
+
+	resourceFiles flags.StringList
+	rPbOutput     string
+	pkg           string
+
+	initOnce sync.Once
+)
+
+const (
+	numParsers = 25
+)
+
+// Init initializes parse. Flags here need to match flags in AndroidResourceParsingAction.
+func Init() {
+	initOnce.Do(func() {
+		flag.Var(&resourceFiles, "res_files", "Resource files and asset directories to parse.")
+		flag.StringVar(&rPbOutput, "out", "", "Path to the output proto file.")
+		flag.StringVar(&pkg, "pkg", "", "Java package name.")
+	})
+}
+
+func desc() string {
+	return "Lite parses the resource files to generate an R.pb."
+}
+
+// Run runs the parser.
+func Run() {
+	rscs := ParseAll(context.Background(), resourceFiles, pkg)
+	b, err := proto.Marshal(rscs)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if err = ioutil.WriteFile(rPbOutput, b, 0644); err != nil {
+		log.Fatal(err)
+	}
+}
+
+type resourceFile struct {
+	pathInfo *res.PathInfo
+	contents []byte
+}
+
+// ParseAll parses all the files in resPaths, which can contain both files and directories,
+// and returns pb.
+func ParseAll(ctx context.Context, resPaths []string, packageName string) *rdpb.Resources {
+	resFiles, err := walk.Files(resPaths)
+	if err != nil {
+		log.Fatal(err)
+	}
+	pifs, rscs, err := initializeFileParse(resFiles, packageName)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if len(pifs) == 0 {
+		return rscs
+	}
+
+	piC := make(chan *res.PathInfo, len(pifs))
+	for _, pi := range pifs {
+		piC <- pi
+	}
+	close(piC)
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	resC, errC := ResParse(ctx, piC)
+	rscs.Resource, err = processResAndErr(resC, errC)
+	if err != nil {
+		cancel()
+		log.Fatal(err)
+	}
+	return rscs
+}
+
+// ResParse consumes a stream of resource paths and converts them into resource protos. These
+// protos will only have the minimal name/type info set.
+func ResParse(ctx context.Context, piC <-chan *res.PathInfo) (<-chan *rdpb.Resource, <-chan error) {
+	parserC := make(chan *res.PathInfo)
+	var parsedResCs []<-chan *rdpb.Resource
+	var parsedErrCs []<-chan error
+
+	for i := 0; i < numParsers; i++ {
+		parsedResC, parsedErrC := xmlParser(ctx, parserC)
+		parsedResCs = append(parsedResCs, parsedResC)
+		parsedErrCs = append(parsedErrCs, parsedErrC)
+	}
+	pathResC := make(chan *rdpb.Resource)
+	pathErrC := make(chan error)
+	go func() {
+		defer close(pathResC)
+		defer close(pathErrC)
+		defer close(parserC)
+
+		for pi := range piC {
+			np, err := needsParse(pi)
+			if err != nil {
+				pathErrC <- err
+				return
+			} else if np {
+				parserC <- pi
+			}
+			if !parsePathInfo(ctx, pi, pathResC, pathErrC) {
+				return
+			}
+		}
+	}()
+	parsedResCs = append(parsedResCs, pathResC)
+	parsedErrCs = append(parsedErrCs, pathErrC)
+	resC := respipe.MergeResStreams(ctx, parsedResCs)
+	errC := respipe.MergeErrStreams(ctx, parsedErrCs)
+
+	return resC, errC
+}
+
+// ParseAllContents parses all resource files with paths and contents and returns pb representing
+// the R class that is generated from the files with the package packageName.
+// paths and contents must have the same length, and a file with paths[i] file path
+// has file contents contents[i].
+func ParseAllContents(ctx context.Context, paths []string, contents [][]byte, packageName string) (*rdpb.Resources, error) {
+	if len(paths) != len(contents) {
+		return nil, fmt.Errorf("length of paths (%v) and contents (%v) are not equal", len(paths), len(contents))
+	}
+	pifs, rscs, err := initializeFileParse(paths, packageName)
+	if err != nil {
+		return nil, err
+	}
+	if len(pifs) == 0 {
+		return rscs, nil
+	}
+
+	var rfC []*resourceFile
+	for i, pi := range pifs {
+		rfC = append(rfC, &resourceFile{
+			pathInfo: pi,
+			contents: contents[i],
+		})
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	resC, errC := resParseContents(ctx, rfC)
+	rscs.Resource, err = processResAndErr(resC, errC)
+	if err != nil {
+		return nil, err
+	}
+	return rscs, nil
+}
+
+// resParseContents consumes resource files and converts them into resource protos.
+// These protos will only have the minimal name/type info set.
+// The returned channels will be consumed by processRessAndErr.
+func resParseContents(ctx context.Context, rfC []*resourceFile) (<-chan *rdpb.Resource, <-chan error) {
+	parserC := make(chan *resourceFile)
+	var parsedResCs []<-chan *rdpb.Resource
+	var parsedErrCs []<-chan error
+
+	for i := 0; i < numParsers; i++ {
+		parsedResC, parsedErrC := xmlParserContents(ctx, parserC)
+		parsedResCs = append(parsedResCs, parsedResC)
+		parsedErrCs = append(parsedErrCs, parsedErrC)
+	}
+	pathResC := make(chan *rdpb.Resource)
+	pathErrC := make(chan error)
+	go func() {
+		defer close(pathResC)
+		defer close(pathErrC)
+		defer close(parserC)
+
+		for _, rf := range rfC {
+			if needsParseContents(rf.pathInfo, bytes.NewReader(rf.contents)) {
+				parserC <- rf
+			}
+			if !parsePathInfo(ctx, rf.pathInfo, pathResC, pathErrC) {
+				return
+			}
+		}
+	}()
+	parsedResCs = append(parsedResCs, pathResC)
+	parsedErrCs = append(parsedErrCs, pathErrC)
+	resC := respipe.MergeResStreams(ctx, parsedResCs)
+	errC := respipe.MergeErrStreams(ctx, parsedErrCs)
+
+	return resC, errC
+}
+
+// initializeFileParse returns a slice of all PathInfos of files contained in each file path,
+// which must be a file (not a directory). It also returns Resources with packageName.
+func initializeFileParse(filePaths []string, packageName string) ([]*res.PathInfo, *rdpb.Resources, error) {
+	rscs := &rdpb.Resources{
+		Pkg: packageName,
+	}
+
+	pifs, err := res.MakePathInfos(filePaths)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return pifs, rscs, nil
+}
+
+// parsePathInfo attempts to parse the PathInfo and send the provided Resource and error to the
+// provided chan. If the context is canceled, returns false, and otherwise, returns true.
+func parsePathInfo(ctx context.Context, pi *res.PathInfo, pathResC chan<- *rdpb.Resource, pathErrC chan<- error) bool {
+	if rawName, ok := pathAsRes(pi); ok {
+		fqn, err := res.ParseName(rawName, pi.Type)
+		if err != nil {
+			return respipe.SendErr(ctx, pathErrC, respipe.Errorf(ctx, "%s: name parse failed: %v", pi.Path, err))
+		}
+		r := new(rdpb.Resource)
+		if err := fqn.SetResource(r); err != nil {
+			return respipe.SendErr(ctx, pathErrC, respipe.Errorf(ctx, "%s: name->proto failed: %v", fqn, err))
+		}
+		return respipe.SendRes(ctx, pathResC, r)
+	}
+	return true
+}
+
+// processResAndErr processes the res and err channels and returns the resources if successful
+// or the first encountered error.
+func processResAndErr(resC <-chan *rdpb.Resource, errC <-chan error) ([]*rdpb.Resource, error) {
+	parseErrChan := make(chan error, 1)
+	go func() {
+		for err := range errC {
+			if err != nil {
+				parseErrChan <- err
+				return
+			}
+		}
+	}()
+
+	doneChan := make(chan struct{}, 1)
+	var res []*rdpb.Resource
+	go func() {
+		for r := range resC {
+			res = append(res, r)
+		}
+		doneChan <- struct{}{}
+	}()
+
+	select {
+	case err := <-parseErrChan:
+		return nil, err
+	case <-doneChan:
+	}
+
+	return res, nil
+}
+
+// xmlParser consumes a stream of paths that need to have their xml contents parsed into resource
+// protos. We only need to get names and types - so the parsing is very quick.
+func xmlParser(ctx context.Context, piC <-chan *res.PathInfo) (<-chan *rdpb.Resource, <-chan error) {
+	resC := make(chan *rdpb.Resource)
+	errC := make(chan error)
+	go func() {
+		defer close(resC)
+		defer close(errC)
+		for p := range piC {
+			if !syncParse(respipe.PrefixErr(ctx, fmt.Sprintf("%s xml-parse: ", p.Path)), p, resC, errC) {
+				// ctx must have been canceled - exit.
+				return
+			}
+		}
+	}()
+	return resC, errC
+}
+
+// xmlParserContents consumes a stream of resource files that need to have their xml contents
+// parsed into resource protos. We only need to get names and types - so the parsing is very quick.
+func xmlParserContents(ctx context.Context, rfC <-chan *resourceFile) (<-chan *rdpb.Resource, <-chan error) {
+	resC := make(chan *rdpb.Resource)
+	errC := make(chan error)
+	go func() {
+		defer close(resC)
+		defer close(errC)
+		for rf := range rfC {
+			if !syncParseContents(respipe.PrefixErr(ctx, fmt.Sprintf("%s xml-parse: ", rf.pathInfo.Path)), rf.pathInfo, bytes.NewReader(rf.contents), resC, errC) {
+				// ctx must have been canceled - exit.
+				return
+			}
+		}
+	}()
+	return resC, errC
+}
+
+func syncParse(ctx context.Context, p *res.PathInfo, resC chan<- *rdpb.Resource, errC chan<- error) bool {
+	f, err := os.Open(p.Path)
+	if err != nil {
+		return respipe.SendErr(ctx, errC, respipe.Errorf(ctx, "open failed: %v", err))
+	}
+	defer f.Close()
+	return syncParseContents(ctx, p, f, resC, errC)
+}
+
+func syncParseContents(ctx context.Context, p *res.PathInfo, fileReader io.Reader, resC chan<- *rdpb.Resource, errC chan<- error) bool {
+	parsedResC, mergedErrC := parseContents(ctx, p, fileReader)
+	for parsedResC != nil || mergedErrC != nil {
+		select {
+		case r, ok := <-parsedResC:
+			if !ok {
+				parsedResC = nil
+				continue
+			}
+			if !respipe.SendRes(ctx, resC, r) {
+				return false
+			}
+		case e, ok := <-mergedErrC:
+			if !ok {
+				mergedErrC = nil
+				continue
+			}
+			if !respipe.SendErr(ctx, errC, e) {
+				return false
+			}
+		}
+
+	}
+	return true
+}
+
+func parseContents(ctx context.Context, filePathInfo *res.PathInfo, fileReader io.Reader) (resC <-chan *rdpb.Resource, errC <-chan error) {
+	xmlC, xmlErrC := resxml.StreamDoc(ctx, fileReader)
+	var parsedErrC <-chan error
+	if filePathInfo.Type == res.ValueType {
+		ctx := respipe.PrefixErr(ctx, "mini-values-parse: ")
+		resC, parsedErrC = valuesParse(ctx, xmlC)
+	} else {
+		ctx := respipe.PrefixErr(ctx, "mini-non-values-parse: ")
+		resC, parsedErrC = nonValuesParse(ctx, xmlC)
+	}
+	errC = respipe.MergeErrStreams(ctx, []<-chan error{parsedErrC, xmlErrC})
+	return resC, errC
+}
+
+// needsParse determines if a path needs to have a values / nonvalues xml parser run to extract
+// resource information.
+func needsParse(pi *res.PathInfo) (bool, error) {
+	r, err := os.Open(pi.Path)
+	if err != nil {
+		return false, fmt.Errorf("Unable to open file %s: %s", pi.Path, err)
+	}
+	defer r.Close()
+
+	return needsParseContents(pi, r), nil
+}
+
+// needsParseContents determines if a path with the corresponding reader for contents needs to have a
+// values / nonvalues xml parser run to extract resource information.
+func needsParseContents(pi *res.PathInfo, r io.Reader) bool {
+	if pi.Type == res.Raw {
+		return false
+	}
+	if filepath.Ext(pi.Path) == ".xml" {
+		return true
+	}
+	if filepath.Ext(pi.Path) == "" {
+		var header [5]byte
+		_, err := io.ReadFull(r, header[:])
+		if err != nil && err != io.EOF {
+			log.Fatal("Unable to read file %s: %s", pi.Path, err)
+		}
+		if string(header[:]) == "<?xml" {
+			return true
+		}
+	}
+	return false
+}
+
+// pathAsRes determines if a particular res.PathInfo is also a standalone resource.
+func pathAsRes(pi *res.PathInfo) (string, bool) {
+	if pi.Type.Kind() == res.Value || (pi.Type.Kind() == res.Both && strings.HasPrefix(pi.TypeDir, "values")) {
+		return "", false
+	}
+	p := path.Base(pi.Path)
+	// Only split on last index of dot when the resource is of RAW type.
+	// Some drawable resources (Nine-Patch files) ends with .9.png which should not
+	// be included in the resource name.
+	if dot := strings.LastIndex(p, "."); dot >= 0 && pi.Type == res.Raw {
+		return p[:dot], true
+	}
+	if dot := strings.Index(p, "."); dot >= 0 {
+		return p[:dot], true
+	}
+	return p, true
+}