Fix error processing XML when german character is used.

Bug 24507894

The character set was not being properly detected for the XML
file when it was being loaded, so it was always using the default
character set. For Windows, this turned out to be something other
than UTF-8. Now, a library is used to detect the character set
before parsing.

Change-Id: I2ea041701de59f0e72fd18288b0f62cb2beb74e6

3 files changed, 50 insertions, 15 deletions

diff --git a/compilerCommon/build.gradle b/compilerCommon/build.gradle
index 61477dad..f2d1e2cc 100644
--- a/compilerCommon/build.gradle
+++ b/compilerCommon/build.gradle
@@ -42,6 +42,7 @@ dependencies {
     compile 'org.apache.commons:commons-lang3:3.3.2'
     compile 'com.tunnelvisionlabs:antlr4:4.5'
     compile 'commons-io:commons-io:2.4'
+    compile 'com.googlecode.juniversalchardet:juniversalchardet:1.0.3'
 }
 
 project.tasks.create(name : "generateXmlParser", type : JavaExec) {
diff --git a/compilerCommon/src/main/java/android/databinding/tool/store/LayoutFileParser.java b/compilerCommon/src/main/java/android/databinding/tool/store/LayoutFileParser.java
index 238f61ab..89a52137 100644
--- a/compilerCommon/src/main/java/android/databinding/tool/store/LayoutFileParser.java
+++ b/compilerCommon/src/main/java/android/databinding/tool/store/LayoutFileParser.java
@@ -20,6 +20,7 @@ import org.antlr.v4.runtime.misc.NotNull;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.commons.lang3.StringUtils;
+import org.mozilla.universalchardet.UniversalDetector;
 import org.w3c.dom.Document;
 import org.w3c.dom.Node;
 import org.w3c.dom.NodeList;
@@ -38,8 +39,10 @@ import android.databinding.tool.util.Preconditions;
 import android.databinding.tool.util.XmlEditor;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
@@ -81,19 +84,20 @@ public class LayoutFileParser {
             });
             final String xmlNoExtension = ParserHelper.stripExtension(xml.getName());
             final String newTag = xml.getParentFile().getName() + '/' + xmlNoExtension;
-            File original = stripFileAndGetOriginal(xml, newTag, originalFileLookup);
+            final String encoding = findEncoding(xml);
+            File original = stripFileAndGetOriginal(xml, newTag, originalFileLookup, encoding);
             if (original == null) {
                 L.d("assuming the file is the original for %s", xml.getAbsoluteFile());
                 original = xml;
             }
-            return parseOriginalXml(original, pkg);
+            return parseOriginalXml(original, pkg, encoding);
         } finally {
             Scope.exit();
         }
     }
 
-    private ResourceBundle.LayoutFileBundle parseOriginalXml(final File original, String pkg)
-            throws IOException {
+    private ResourceBundle.LayoutFileBundle parseOriginalXml(final File original, String pkg,
+            String encoding) throws IOException {
         try {
             Scope.enter(new FileScopeProvider() {
                 @Override
@@ -102,7 +106,9 @@ public class LayoutFileParser {
                 }
             });
             final String xmlNoExtension = ParserHelper.stripExtension(original.getName());
-            ANTLRInputStream inputStream = new ANTLRInputStream(new FileReader(original));
+            FileInputStream fin = new FileInputStream(original);
+            InputStreamReader reader = new InputStreamReader(fin, encoding);
+            ANTLRInputStream inputStream = new ANTLRInputStream(reader);
             XMLLexer lexer = new XMLLexer(inputStream);
             CommonTokenStream tokenStream = new CommonTokenStream(lexer);
             XMLParser parser = new XMLParser(tokenStream);
@@ -376,7 +382,7 @@ public class LayoutFileParser {
     }
 
     private File stripFileAndGetOriginal(File xml, String binderId,
-            LayoutXmlProcessor.OriginalFileLookup originalFileLookup)
+            LayoutXmlProcessor.OriginalFileLookup originalFileLookup, String encoding)
             throws ParserConfigurationException, IOException, SAXException,
             XPathExpressionException {
         L.d("parsing resource file %s", xml.getAbsolutePath());
@@ -396,7 +402,7 @@ public class LayoutFileParser {
         // now if file has any binding expressions, find and delete them
         boolean changed = isBindingLayout(doc, xPath);
         if (changed) {
-            stripBindingTags(xml, binderId);
+            stripBindingTags(xml, binderId, encoding);
         }
         return actualFile;
     }
@@ -441,11 +447,34 @@ public class LayoutFileParser {
         return result;
     }
 
-    private void stripBindingTags(File xml, String newTag) throws IOException {
-        String res = XmlEditor.strip(xml, newTag);
+    private void stripBindingTags(File xml, String newTag, String encoding) throws IOException {
+        String res = XmlEditor.strip(xml, newTag, encoding);
         if (res != null) {
             L.d("file %s has changed, overwriting %s", xml.getName(), xml.getAbsolutePath());
-            FileUtils.writeStringToFile(xml, res);
+            FileUtils.writeStringToFile(xml, res, encoding);
+        }
+    }
+
+    private static String findEncoding(File f) throws IOException {
+        FileInputStream fin = new FileInputStream(f);
+        try {
+            UniversalDetector universalDetector = new UniversalDetector(null);
+
+            byte[] buf = new byte[4096];
+            int nread;
+            while ((nread = fin.read(buf)) > 0 && !universalDetector.isDone()) {
+                universalDetector.handleData(buf, 0, nread);
+            }
+
+            universalDetector.dataEnd();
+
+            String encoding = universalDetector.getDetectedCharset();
+            if (encoding == null) {
+                encoding = "utf-8";
+            }
+            return encoding;
+        } finally {
+            fin.close();
         }
     }
 
diff --git a/compilerCommon/src/main/java/android/databinding/tool/util/XmlEditor.java b/compilerCommon/src/main/java/android/databinding/tool/util/XmlEditor.java
index 795b433c..168a022c 100644
--- a/compilerCommon/src/main/java/android/databinding/tool/util/XmlEditor.java
+++ b/compilerCommon/src/main/java/android/databinding/tool/util/XmlEditor.java
@@ -20,11 +20,13 @@ import org.antlr.v4.runtime.ANTLRInputStream;
 import org.antlr.v4.runtime.CommonTokenStream;
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.tree.TerminalNode;
+import org.apache.commons.io.Charsets;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringEscapeUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
+import org.mozilla.universalchardet.UniversalDetector;
 
 import android.databinding.parser.BindingExpressionLexer;
 import android.databinding.parser.BindingExpressionParser;
@@ -34,8 +36,10 @@ import android.databinding.parser.XMLParser.AttributeContext;
 import android.databinding.parser.XMLParser.ElementContext;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
@@ -47,8 +51,10 @@ import java.util.List;
  */
 public class XmlEditor {
 
-    public static String strip(File f, String newTag) throws IOException {
-        ANTLRInputStream inputStream = new ANTLRInputStream(new FileReader(f));
+    public static String strip(File f, String newTag, String encoding) throws IOException {
+        FileInputStream fin = new FileInputStream(f);
+        InputStreamReader reader = new InputStreamReader(fin, encoding);
+        ANTLRInputStream inputStream = new ANTLRInputStream(reader);
         XMLLexer lexer = new XMLLexer(inputStream);
         CommonTokenStream tokenStream = new CommonTokenStream(lexer);
         XMLParser parser = new XMLParser(tokenStream);
@@ -205,12 +211,11 @@ public class XmlEditor {
 
     private static Position endTagPosition(XMLParser.ElementContext context) {
         if (context.content() == null) {
-            // no content, so just subtract from the "/>"
-            Position endTag = toEndPosition(context.getStop());
+            // no content, so just choose the start of the "/>"
+            Position endTag = toPosition(context.getStop());
             if (endTag.charIndex <= 0) {
                 L.e("invalid input in %s", context);
             }
-            endTag.charIndex -= 2;
             return endTag;
         } else {
             // tag with no attributes, but with content