am ee454371: Merge "Change the default character encoding for JSON responses to UTF-8"

* commit 'ee454371157d9a3fb74b70ee8ba93cf96559acf3':
  Change the default character encoding for JSON responses to UTF-8

6 files changed, 153 insertions, 10 deletions

diff --git a/src/main/java/com/android/volley/toolbox/HttpHeaderParser.java b/src/main/java/com/android/volley/toolbox/HttpHeaderParser.java
index da04490..7306052 100644
--- a/src/main/java/com/android/volley/toolbox/HttpHeaderParser.java
+++ b/src/main/java/com/android/volley/toolbox/HttpHeaderParser.java
@@ -131,10 +131,14 @@ public class HttpHeaderParser {
     }
 
     /**
-     * Returns the charset specified in the Content-Type of this header,
-     * or the HTTP default (ISO-8859-1) if none can be found.
+     * Retrieve a charset from headers
+     *
+     * @param headers An {@link java.util.Map} of headers
+     * @param defaultCharset Charset to return if none can be found
+     * @return Returns the charset specified in the Content-Type of this header,
+     * or the defaultCharset if none can be found.
      */
-    public static String parseCharset(Map<String, String> headers) {
+    public static String parseCharset(Map<String, String> headers, String defaultCharset) {
         String contentType = headers.get(HTTP.CONTENT_TYPE);
         if (contentType != null) {
             String[] params = contentType.split(";");
@@ -148,6 +152,14 @@ public class HttpHeaderParser {
             }
         }
 
-        return HTTP.DEFAULT_CONTENT_CHARSET;
+        return defaultCharset;
+    }
+
+    /**
+     * Returns the charset specified in the Content-Type of this header,
+     * or the HTTP default (ISO-8859-1) if none can be found.
+     */
+    public static String parseCharset(Map<String, String> headers) {
+        return parseCharset(headers, HTTP.DEFAULT_CONTENT_CHARSET);
     }
 }
diff --git a/src/main/java/com/android/volley/toolbox/JsonArrayRequest.java b/src/main/java/com/android/volley/toolbox/JsonArrayRequest.java
index b1eae80..e932a61 100644
--- a/src/main/java/com/android/volley/toolbox/JsonArrayRequest.java
+++ b/src/main/java/com/android/volley/toolbox/JsonArrayRequest.java
@@ -45,8 +45,8 @@ public class JsonArrayRequest extends JsonRequest<JSONArray> {
     @Override
     protected Response<JSONArray> parseNetworkResponse(NetworkResponse response) {
         try {
-            String jsonString =
-                new String(response.data, HttpHeaderParser.parseCharset(response.headers));
+            String jsonString = new String(response.data,
+                    HttpHeaderParser.parseCharset(response.headers, PROTOCOL_CHARSET));
             return Response.success(new JSONArray(jsonString),
                     HttpHeaderParser.parseCacheHeaders(response));
         } catch (UnsupportedEncodingException e) {
diff --git a/src/main/java/com/android/volley/toolbox/JsonObjectRequest.java b/src/main/java/com/android/volley/toolbox/JsonObjectRequest.java
index 74821cb..2991898 100644
--- a/src/main/java/com/android/volley/toolbox/JsonObjectRequest.java
+++ b/src/main/java/com/android/volley/toolbox/JsonObjectRequest.java
@@ -63,8 +63,8 @@ public class JsonObjectRequest extends JsonRequest<JSONObject> {
     @Override
     protected Response<JSONObject> parseNetworkResponse(NetworkResponse response) {
         try {
-            String jsonString =
-                new String(response.data, HttpHeaderParser.parseCharset(response.headers));
+            String jsonString = new String(response.data,
+                    HttpHeaderParser.parseCharset(response.headers, PROTOCOL_CHARSET));
             return Response.success(new JSONObject(jsonString),
                     HttpHeaderParser.parseCacheHeaders(response));
         } catch (UnsupportedEncodingException e) {
diff --git a/src/main/java/com/android/volley/toolbox/JsonRequest.java b/src/main/java/com/android/volley/toolbox/JsonRequest.java
index f11ac14..95f4ecb 100644
--- a/src/main/java/com/android/volley/toolbox/JsonRequest.java
+++ b/src/main/java/com/android/volley/toolbox/JsonRequest.java
@@ -32,8 +32,8 @@ import java.io.UnsupportedEncodingException;
  * @param <T> JSON type of response expected
  */
 public abstract class JsonRequest<T> extends Request<T> {
-    /** Charset for request. */
-    private static final String PROTOCOL_CHARSET = "utf-8";
+    /** Default charset for JSON request. */
+    protected static final String PROTOCOL_CHARSET = "utf-8";
 
     /** Content type for request. */
     private static final String PROTOCOL_CONTENT_TYPE =
diff --git a/src/test/java/com/android/volley/toolbox/HttpHeaderParserTest.java b/src/test/java/com/android/volley/toolbox/HttpHeaderParserTest.java
index 60c2727..f9230c6 100644
--- a/src/test/java/com/android/volley/toolbox/HttpHeaderParserTest.java
+++ b/src/test/java/com/android/volley/toolbox/HttpHeaderParserTest.java
@@ -199,6 +199,10 @@ public class HttpHeaderParserTest {
         headers.put("Content-Type", "text/plain; charset=utf-8");
         assertEquals("utf-8", HttpHeaderParser.parseCharset(headers));
 
+        // Charset specified, ignore default charset
+        headers.put("Content-Type", "text/plain; charset=utf-8");
+        assertEquals("utf-8", HttpHeaderParser.parseCharset(headers, "ISO-8859-1"));
+
         // Extra whitespace
         headers.put("Content-Type", "text/plain;    charset=utf-8 ");
         assertEquals("utf-8", HttpHeaderParser.parseCharset(headers));
@@ -211,6 +215,10 @@ public class HttpHeaderParserTest {
         headers.clear();
         assertEquals("ISO-8859-1", HttpHeaderParser.parseCharset(headers));
 
+        // No Content-Type header, use default charset
+        headers.clear();
+        assertEquals("utf-8", HttpHeaderParser.parseCharset(headers, "utf-8"));
+
         // Empty value
         headers.put("Content-Type", "text/plain; charset=");
         assertEquals("ISO-8859-1", HttpHeaderParser.parseCharset(headers));
@@ -219,6 +227,10 @@ public class HttpHeaderParserTest {
         headers.put("Content-Type", "text/plain");
         assertEquals("ISO-8859-1", HttpHeaderParser.parseCharset(headers));
 
+        // None charset specified, use default charset
+        headers.put("Content-Type", "application/json");
+        assertEquals("utf-8", HttpHeaderParser.parseCharset(headers, "utf-8"));
+
         // None specified, extra semicolon
         headers.put("Content-Type", "text/plain;");
         assertEquals("ISO-8859-1", HttpHeaderParser.parseCharset(headers));
diff --git a/src/test/java/com/android/volley/toolbox/JsonRequestCharsetTest.java b/src/test/java/com/android/volley/toolbox/JsonRequestCharsetTest.java
new file mode 100644
index 0000000..db6f648
--- /dev/null
+++ b/src/test/java/com/android/volley/toolbox/JsonRequestCharsetTest.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright (C) 2011 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.volley.toolbox;
+
+import com.android.volley.NetworkResponse;
+import com.android.volley.Response;
+import com.android.volley.toolbox.JsonArrayRequest;
+import com.android.volley.toolbox.JsonObjectRequest;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+
+import java.lang.Exception;
+import java.lang.String;
+import java.nio.charset.Charset;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.*;
+
+@RunWith(RobolectricTestRunner.class)
+public class JsonRequestCharsetTest {
+
+    /**
+     * String in Czech - "Retezec v cestine."
+     */
+    private static final String TEXT_VALUE = "\u0158et\u011bzec v \u010de\u0161tin\u011b.";
+    private static final String TEXT_NAME = "text";
+    private static final int TEXT_INDEX = 0;
+
+    /**
+     * Copyright symbol has different encoding in utf-8 and ISO-8859-1,
+     * and it doesn't exists in ISO-8859-2
+     */
+    private static final String COPY_VALUE = "\u00a9";
+    private static final String COPY_NAME = "copyright";
+    private static final int COPY_INDEX = 1;
+
+    @Test public void defaultCharsetJsonObject() throws Exception {
+        // UTF-8 is default charset for JSON
+        byte[] data = jsonObjectString().getBytes(Charset.forName("UTF-8"));
+        NetworkResponse network = new NetworkResponse(data);
+        JsonObjectRequest objectRequest = new JsonObjectRequest("", null, null, null);
+        Response<JSONObject> objectResponse = objectRequest.parseNetworkResponse(network);
+
+        assertNotNull(objectResponse);
+        assertTrue(objectResponse.isSuccess());
+        assertEquals(TEXT_VALUE, objectResponse.result.getString(TEXT_NAME));
+        assertEquals(COPY_VALUE, objectResponse.result.getString(COPY_NAME));
+    }
+
+    @Test public void defaultCharsetJsonArray() throws Exception {
+        // UTF-8 is default charset for JSON
+        byte[] data = jsonArrayString().getBytes(Charset.forName("UTF-8"));
+        NetworkResponse network = new NetworkResponse(data);
+        JsonArrayRequest arrayRequest = new JsonArrayRequest("", null, null);
+        Response<JSONArray> arrayResponse = arrayRequest.parseNetworkResponse(network);
+
+        assertNotNull(arrayResponse);
+        assertTrue(arrayResponse.isSuccess());
+        assertEquals(TEXT_VALUE, arrayResponse.result.getString(TEXT_INDEX));
+        assertEquals(COPY_VALUE, arrayResponse.result.getString(COPY_INDEX));
+    }
+
+    @Test public void specifiedCharsetJsonObject() throws Exception {
+        byte[] data = jsonObjectString().getBytes(Charset.forName("ISO-8859-1"));
+        Map<String, String> headers = new HashMap<String, String>();
+        headers.put("Content-Type", "application/json; charset=iso-8859-1");
+        NetworkResponse network = new NetworkResponse(data, headers);
+        JsonObjectRequest objectRequest = new JsonObjectRequest("", null, null, null);
+        Response<JSONObject> objectResponse = objectRequest.parseNetworkResponse(network);
+
+        assertNotNull(objectResponse);
+        assertTrue(objectResponse.isSuccess());
+        //don't check the text in Czech, ISO-8859-1 doesn't support some Czech characters
+        assertEquals(COPY_VALUE, objectResponse.result.getString(COPY_NAME));
+    }
+
+    @Test public void specifiedCharsetJsonArray() throws Exception {
+        byte[] data = jsonArrayString().getBytes(Charset.forName("ISO-8859-2"));
+        Map<String, String> headers = new HashMap<String, String>();
+        headers.put("Content-Type", "application/json; charset=iso-8859-2");
+        NetworkResponse network = new NetworkResponse(data, headers);
+        JsonArrayRequest arrayRequest = new JsonArrayRequest("", null, null);
+        Response<JSONArray> arrayResponse = arrayRequest.parseNetworkResponse(network);
+
+        assertNotNull(arrayResponse);
+        assertTrue(arrayResponse.isSuccess());
+        assertEquals(TEXT_VALUE, arrayResponse.result.getString(TEXT_INDEX));
+        // don't check the copyright symbol, ISO-8859-2 doesn't have it, but it has Czech characters
+    }
+
+    private static String jsonObjectString() throws Exception {
+        JSONObject json = new JSONObject().put(TEXT_NAME, TEXT_VALUE).put(COPY_NAME, COPY_VALUE);
+        return json.toString();
+    }
+
+    private static String jsonArrayString() throws Exception {
+        JSONArray json = new JSONArray().put(TEXT_INDEX, TEXT_VALUE).put(COPY_INDEX, COPY_VALUE);
+        return json.toString();
+    }
+}