diff options
Diffstat (limited to 'attribute_stats/src/Analyzer.java')
| -rw-r--r-- | attribute_stats/src/Analyzer.java | 663 |
1 files changed, 663 insertions, 0 deletions
diff --git a/attribute_stats/src/Analyzer.java b/attribute_stats/src/Analyzer.java new file mode 100644 index 000000000..8da53ea38 --- /dev/null +++ b/attribute_stats/src/Analyzer.java @@ -0,0 +1,663 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Eclipse Public License, Version 1.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.eclipse.org/org/documents/epl-v10.php + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; + +/** + * Gathers statistics about attribute usage in layout files. This is how the "topAttrs" + * attributes listed in ADT's extra-view-metadata.xml (which drives the common attributes + * listed in the top of the context menu) is determined by running this script on a body + * of sample layout code. + * <p> + * This program takes one or more directory paths, and then it searches all of them recursively + * for layout files that are not in folders containing the string "test", and computes and + * prints frequency statistics. + */ +public class Analyzer { + /** Number of attributes to print for each view */ + public static final int ATTRIBUTE_COUNT = 6; + /** Separate out any attributes that constitute less than N percent of the total */ + public static final int THRESHOLD = 10; // percent + + private List<File> mDirectories; + private File mCurrentFile; + private boolean mListAdvanced; + + /** Map from view id to map from attribute to frequency count */ + private Map<String, Map<String, Usage>> mFrequencies = + new HashMap<String, Map<String, Usage>>(100); + + private Map<String, Map<String, Usage>> mLayoutAttributeFrequencies = + new HashMap<String, Map<String, Usage>>(100); + + private Map<String, String> mTopAttributes = new HashMap<String, String>(100); + private Map<String, String> mTopLayoutAttributes = new HashMap<String, String>(100); + + private int mFileVisitCount; + private int mLayoutFileCount; + private File mXmlMetadataFile; + + private Analyzer(List<File> directories, File xmlMetadataFile, boolean listAdvanced) { + mDirectories = directories; + mXmlMetadataFile = xmlMetadataFile; + mListAdvanced = listAdvanced; + } + + public static void main(String[] args) { + if (args.length < 1) { + System.err.println("Usage: " + Analyzer.class.getSimpleName() + + " <directory1> [directory2 [directory3 ...]]\n"); + System.err.println("Recursively scans for layouts in the given directory and"); + System.err.println("computes statistics about attribute frequencies."); + System.exit(-1); + } + + File metadataFile = null; + List<File> directories = new ArrayList<File>(); + boolean listAdvanced = false; + for (int i = 0, n = args.length; i < n; i++) { + String arg = args[i]; + + if (arg.equals("--list")) { + // List ALL encountered attributes + listAdvanced = true; + continue; + } + + // The -metadata flag takes a pointer to an ADT extra-view-metadata.xml file + // and attempts to insert topAttrs attributes into it (and saves it as same + // file +.mod as an extension). This isn't listed on the usage flag because + // it's pretty brittle and requires some manual fixups to the file afterwards. + if (arg.equals("--metadata")) { + i++; + File file = new File(args[i]); + if (!file.exists()) { + System.err.println(file.getName() + " does not exist"); + System.exit(-5); + } + if (!file.isFile() || !file.getName().endsWith(".xml")) { + System.err.println(file.getName() + " must be an XML file"); + System.exit(-4); + } + metadataFile = file; + continue; + } + File directory = new File(arg); + if (!directory.exists()) { + System.err.println(directory.getName() + " does not exist"); + System.exit(-2); + } + + if (!directory.isDirectory()) { + System.err.println(directory.getName() + " is not a directory"); + System.exit(-3); + } + + directories.add(directory); + } + + new Analyzer(directories, metadataFile, listAdvanced).analyze(); + } + + private void analyze() { + for (File directory : mDirectories) { + scanDirectory(directory); + } + + if (mListAdvanced) { + listAdvanced(); + } + + printStatistics(); + + if (mXmlMetadataFile != null) { + printMergedMetadata(); + } + } + + private void scanDirectory(File directory) { + File[] files = directory.listFiles(); + if (files == null) { + return; + } + + for (File file : files) { + mFileVisitCount++; + if (mFileVisitCount % 50000 == 0) { + System.out.println("Analyzed " + mFileVisitCount + " files..."); + } + + if (file.isFile()) { + scanFile(file); + } else if (file.isDirectory()) { + // Skip stuff related to tests + if (file.getName().contains("test")) { + continue; + } + + // Recurse over subdirectories + scanDirectory(file); + } + } + } + + private void scanFile(File file) { + if (file.getName().endsWith(".xml")) { + File parent = file.getParentFile(); + if (parent.getName().startsWith("layout")) { + analyzeLayout(file); + } + } + + } + + private void analyzeLayout(File file) { + mCurrentFile = file; + mLayoutFileCount++; + Document document = null; + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + InputSource is = new InputSource(new StringReader(readFile(file))); + try { + factory.setNamespaceAware(true); + factory.setValidating(false); + DocumentBuilder builder = factory.newDocumentBuilder(); + document = builder.parse(is); + + analyzeDocument(document); + + } catch (ParserConfigurationException e) { + // pass -- ignore files we can't parse + } catch (SAXException e) { + // pass -- ignore files we can't parse + } catch (IOException e) { + // pass -- ignore files we can't parse + } + } + + + private void analyzeDocument(Document document) { + analyzeElement(document.getDocumentElement()); + } + + private void analyzeElement(Element element) { + if (element.getTagName().equals("item")) { + // Resource files shouldn't be in the layout/ folder but I came across + // some cases + System.out.println("Warning: found <item> tag in a layout file in " + + mCurrentFile.getPath()); + return; + } + + countAttributes(element); + countLayoutAttributes(element); + + // Recurse over children + NodeList childNodes = element.getChildNodes(); + for (int i = 0, n = childNodes.getLength(); i < n; i++) { + Node child = childNodes.item(i); + if (child.getNodeType() == Node.ELEMENT_NODE) { + analyzeElement((Element) child); + } + } + } + + private void countAttributes(Element element) { + String tag = element.getTagName(); + Map<String, Usage> attributeMap = mFrequencies.get(tag); + if (attributeMap == null) { + attributeMap = new HashMap<String, Usage>(70); + mFrequencies.put(tag, attributeMap); + } + + NamedNodeMap attributes = element.getAttributes(); + for (int i = 0, n = attributes.getLength(); i < n; i++) { + Node attribute = attributes.item(i); + String name = attribute.getNodeName(); + + if (name.startsWith("android:layout_")) { + // Skip layout attributes; they are a function of the parent layout that this + // view is embedded within, not the view itself. + // TODO: Consider whether we should incorporate this info or make statistics + // about that as well? + continue; + } + + if (name.equals("android:id")) { + // Skip ids: they are (mostly) unrelated to the view type and the tool + // already offers id editing prominently + continue; + } + + if (name.startsWith("xmlns:")) { + // Unrelated to frequency counts + continue; + } + + Usage usage = attributeMap.get(name); + if (usage == null) { + usage = new Usage(name); + } else { + usage.incrementCount(); + } + attributeMap.put(name, usage); + } + } + + private void countLayoutAttributes(Element element) { + String parentTag = element.getParentNode().getNodeName(); + Map<String, Usage> attributeMap = mLayoutAttributeFrequencies.get(parentTag); + if (attributeMap == null) { + attributeMap = new HashMap<String, Usage>(70); + mLayoutAttributeFrequencies.put(parentTag, attributeMap); + } + + NamedNodeMap attributes = element.getAttributes(); + for (int i = 0, n = attributes.getLength(); i < n; i++) { + Node attribute = attributes.item(i); + String name = attribute.getNodeName(); + + if (!name.startsWith("android:layout_")) { + continue; + } + + // Skip layout_width and layout_height; they are mandatory in all but GridLayout so not + // very interesting + if (name.equals("android:layout_width") || name.equals("android:layout_height")) { + continue; + } + + Usage usage = attributeMap.get(name); + if (usage == null) { + usage = new Usage(name); + } else { + usage.incrementCount(); + } + attributeMap.put(name, usage); + } + } + + // Copied from AdtUtils + private static String readFile(File file) { + try { + return readFile(new FileReader(file)); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + + return null; + } + + private static String readFile(Reader inputStream) { + BufferedReader reader = null; + try { + reader = new BufferedReader(inputStream); + StringBuilder sb = new StringBuilder(2000); + while (true) { + int c = reader.read(); + if (c == -1) { + return sb.toString(); + } else { + sb.append((char)c); + } + } + } catch (IOException e) { + // pass -- ignore files we can't read + } finally { + try { + if (reader != null) { + reader.close(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + return null; + } + + private void printStatistics() { + System.out.println("Analyzed " + mLayoutFileCount + + " layouts (in a directory trees containing " + mFileVisitCount + " files)"); + System.out.println("Top " + ATTRIBUTE_COUNT + + " for each view (excluding layout_ attributes) :"); + System.out.println("\n"); + System.out.println(" Rank Count Share Attribute"); + System.out.println("========================================================="); + List<String> views = new ArrayList<String>(mFrequencies.keySet()); + Collections.sort(views); + for (String view : views) { + String top = processUageMap(view, mFrequencies.get(view)); + if (top != null) { + mTopAttributes.put(view, top); + } + } + + System.out.println("\n\n\nTop " + ATTRIBUTE_COUNT + " layout attributes (excluding " + + "mandatory layout_width and layout_height):"); + System.out.println("\n"); + System.out.println(" Rank Count Share Attribute"); + System.out.println("========================================================="); + views = new ArrayList<String>(mLayoutAttributeFrequencies.keySet()); + Collections.sort(views); + for (String view : views) { + String top = processUageMap(view, mLayoutAttributeFrequencies.get(view)); + if (top != null) { + mTopLayoutAttributes.put(view, top); + } + } + } + + private static String processUageMap(String view, Map<String, Usage> map) { + if (map == null) { + return null; + } + + if (view.indexOf('.') != -1 && !view.startsWith("android.")) { + // Skip custom views + return null; + } + + List<Usage> values = new ArrayList<Usage>(map.values()); + if (values.size() == 0) { + return null; + } + + Collections.sort(values); + int totalCount = 0; + for (Usage usage : values) { + totalCount += usage.count; + } + + System.out.println("\n<" + view + ">:"); + if (view.equals("#document")) { + System.out.println("(Set on root tag, probably intended for included context)"); + } + + int place = 1; + int count = 0; + int prevCount = -1; + float prevPercentage = 0f; + StringBuilder sb = new StringBuilder(); + for (Usage usage : values) { + if (count++ >= ATTRIBUTE_COUNT && usage.count < prevCount) { + break; + } + + float percentage = 100 * usage.count/(float)totalCount; + if (percentage < THRESHOLD && prevPercentage >= THRESHOLD) { + System.out.println(" -----Less than 10%-------------------------------------"); + } + System.out.printf(" %1d. %5d %5.1f%% %s\n", place, usage.count, + percentage, usage.attribute); + + prevPercentage = percentage; + if (prevCount != usage.count) { + prevCount = usage.count; + place++; + } + + if (percentage >= THRESHOLD /*&& usage.count > 1*/) { // 1:Ignore when not enough data? + if (sb.length() > 0) { + sb.append(','); + } + String name = usage.attribute; + if (name.startsWith("android:")) { + name = name.substring("android:".length()); + } + sb.append(name); + } + } + + return sb.length() > 0 ? sb.toString() : null; + } + + private void printMergedMetadata() { + assert mXmlMetadataFile != null; + String metadata = readFile(mXmlMetadataFile); + if (metadata == null || metadata.length() == 0) { + System.err.println("Invalid metadata file"); + System.exit(-6); + } + + System.err.flush(); + System.out.println("\n\nUpdating layout metadata file..."); + System.out.flush(); + + StringBuilder sb = new StringBuilder((int) (2 * mXmlMetadataFile.length())); + String[] lines = metadata.split("\n"); + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + sb.append(line).append('\n'); + int classIndex = line.indexOf("class=\""); + if (classIndex != -1) { + int start = classIndex + "class=\"".length(); + int end = line.indexOf('"', start + 1); + if (end != -1) { + String view = line.substring(start, end); + if (view.startsWith("android.widget.")) { + view = view.substring("android.widget.".length()); + } else if (view.startsWith("android.view.")) { + view = view.substring("android.view.".length()); + } else if (view.startsWith("android.webkit.")) { + view = view.substring("android.webkit.".length()); + } + String top = mTopAttributes.get(view); + if (top == null) { + System.err.println("Warning: No frequency data for view " + view); + } else { + sb.append(line.substring(0, classIndex)); // Indentation + + sb.append("topAttrs=\""); + sb.append(top); + sb.append("\"\n"); + } + + top = mTopLayoutAttributes.get(view); + if (top != null) { + // It's a layout attribute + sb.append(line.substring(0, classIndex)); // Indentation + + sb.append("topLayoutAttrs=\""); + sb.append(top); + sb.append("\"\n"); + } + } + } + } + + System.out.println("\nTop attributes:"); + System.out.println("--------------------------"); + List<String> views = new ArrayList<String>(mTopAttributes.keySet()); + Collections.sort(views); + for (String view : views) { + String top = mTopAttributes.get(view); + System.out.println(view + ": " + top); + } + + System.out.println("\nTop layout attributes:"); + System.out.println("--------------------------"); + views = new ArrayList<String>(mTopLayoutAttributes.keySet()); + Collections.sort(views); + for (String view : views) { + String top = mTopLayoutAttributes.get(view); + System.out.println(view + ": " + top); + } + + System.out.println("\nModified XML metadata file:\n"); + String newContent = sb.toString(); + File output = new File(mXmlMetadataFile.getParentFile(), mXmlMetadataFile.getName() + ".mod"); + if (output.exists()) { + output.delete(); + } + try { + BufferedWriter writer = new BufferedWriter(new FileWriter(output)); + writer.write(newContent); + writer.close(); + } catch (IOException e) { + e.printStackTrace(); + } + System.out.println("Done - wrote " + output.getPath()); + } + + //private File mPublicFile = new File(location, "data/res/values/public.xml"); + private File mPublicFile = new File("/Volumes/AndroidWork/git/frameworks/base/core/res/res/values/public.xml"); + + private void listAdvanced() { + Set<String> keys = new HashSet<String>(1000); + + // Merged usages across view types + Map<String, Usage> mergedUsages = new HashMap<String, Usage>(100); + + for (Entry<String,Map<String,Usage>> entry : mFrequencies.entrySet()) { + String view = entry.getKey(); + if (view.indexOf('.') != -1 && !view.startsWith("android.")) { + // Skip custom views etc + continue; + } + Map<String, Usage> map = entry.getValue(); + for (Usage usage : map.values()) { +// if (usage.count == 1) { +// System.out.println("Only found *one* usage of " + usage.attribute); +// } +// if (usage.count < 4) { +// System.out.println("Only found " + usage.count + " usage of " + usage.attribute); +// } + + String attribute = usage.attribute; + int index = attribute.indexOf(':'); + if (index == -1 || attribute.startsWith("android:")) { + Usage merged = mergedUsages.get(attribute); + if (merged == null) { + merged = new Usage(attribute); + merged.count = usage.count; + mergedUsages.put(attribute, merged); + } else { + merged.count += usage.count; + } + } + } + } + + for (Usage usage : mergedUsages.values()) { + String attribute = usage.attribute; + if (usage.count < 4) { + System.out.println("Only found " + usage.count + " usage of " + usage.attribute); + continue; + } + int index = attribute.indexOf(':'); + if (index != -1) { + attribute = attribute.substring(index + 1); // +1: skip ':' + } + keys.add(attribute); + } + + List<String> sorted = new ArrayList<String>(keys); + Collections.sort(sorted); + System.out.println("\nEncountered Attributes"); + System.out.println("-----------------------------"); + for (String attribute : sorted) { + System.out.println(attribute); + } + + System.out.println(); + } + + private static class Usage implements Comparable<Usage> { + public String attribute; + public int count; + + + public Usage(String attribute) { + super(); + this.attribute = attribute; + + count = 1; + } + + public void incrementCount() { + count++; + } + + @Override + public int compareTo(Usage o) { + // Sort by decreasing frequency, then sort alphabetically + int frequencyDelta = o.count - count; + if (frequencyDelta != 0) { + return frequencyDelta; + } else { + return attribute.compareTo(o.attribute); + } + } + + @Override + public String toString() { + return attribute + ": " + count; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((attribute == null) ? 0 : attribute.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Usage other = (Usage) obj; + if (attribute == null) { + if (other.attribute != null) + return false; + } else if (!attribute.equals(other.attribute)) + return false; + return true; + } + } +} |