CarrierSettings data conversion tools

These tools convert Android carrier configs and APNs from
XML format to protobuf format to be consumed by CarrierSettings.

Bug: 144283419
Test: make
Change-Id: Id40690bedff74f80e12fe03f287d3a81a61d5a08

14 files changed, 2774 insertions, 0 deletions

diff --git a/Android.bp b/Android.bp
new file mode 100644
index 0000000..2ae42ad
--- /dev/null
+++ b/Android.bp
@@ -0,0 +1,111 @@
+// Copyright (C) 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+python_binary_host {
+    name: "update_apn",
+    main: "python/update_apn.py",
+    srcs: [
+        "python/update_apn.py",
+        "proto/*.proto",
+    ],
+    proto: {
+        canonical_path_from_root: false
+    },
+    libs: [
+        "libprotobuf-python",
+        "py-six",
+    ],
+}
+
+python_binary_host {
+    name: "update_carrier_data",
+    main: "python/update_carrier_data.py",
+    srcs: [
+        "python/compare.py",
+        "python/update_carrier_data.py",
+        "proto/*.proto",
+    ],
+    proto: {
+        canonical_path_from_root: false
+    },
+    libs: [
+        "libprotobuf-python",
+        "py-six",
+    ],
+}
+
+java_binary_host {
+    name: "CarrierConfigConverterV2",
+    srcs: [
+        "java/CarrierConfigConverterV2.java",
+        "java/CarrierProtoUtils.java",
+        "proto/*.proto",
+        ":telephonyprovider-proto-sources",
+    ],
+    java_resources: [
+      ":telephonyprovider-assets-carrierlist",
+    ],
+    main_class: "com.google.carrier.CarrierConfigConverterV2",
+    proto: {
+        type: "full",
+        canonical_path_from_root: false,
+    },
+    plugins: [
+        "dagger2-auto-value",
+    ],
+    static_libs: [
+        "dagger2-auto-value",
+        "guava-21.0",
+        "jcommander",
+        "libprotobuf-java-full",
+    ],
+}
+
+java_binary_host {
+    name: "GenCarrierList",
+    srcs: [
+        "java/CarrierProtoUtils.java",
+        "java/GenCarrierList.java",
+        "proto/*.proto",
+    ],
+    main_class: "com.google.carrier.GenCarrierList",
+    proto: {
+        type: "full",
+        canonical_path_from_root: false,
+    },
+    static_libs: [
+        "guava-21.0",
+        "jcommander",
+        "libprotobuf-java-full",
+    ],
+}
+
+java_binary_host {
+    name: "GenDeviceSettings",
+    srcs: [
+        "java/CarrierProtoUtils.java",
+        "java/GenDeviceSettings.java",
+        "proto/*.proto",
+    ],
+    main_class: "com.google.carrier.GenDeviceSettings",
+    proto: {
+        type: "full",
+        canonical_path_from_root: false,
+    },
+    static_libs: [
+        "guava-21.0",
+        "jcommander",
+        "libprotobuf-java-full",
+    ],
+}
diff --git a/OWNERS b/OWNERS
new file mode 100644
index 0000000..b5cab7d
--- /dev/null
+++ b/OWNERS
@@ -0,0 +1,4 @@
+mewan@google.com
+mberionne@google.com
+amruthr@google.com
+satk@google.com
diff --git a/README b/README
new file mode 100644
index 0000000..a189be7
--- /dev/null
+++ b/README
@@ -0,0 +1,16 @@
+This tool converts carrier config and APNs from XML format to protobuf format.
+
+AOSP default carrier configs can be found in packages/apps/CarrierConfig,
+and APNs in device/sample/etc/apns-full-conf.xml.
+
+Usage:
+
+$ source build/envsetup.sh
+$ lunch foo_bar
+$ croot
+$ source <path>/main.sh
+
+The commands above build the tool from source code and run them.
+
+This tool is best supported on Android 11 code base. See additional steps
+in bin/README.md run it on Android 10.
diff --git a/bin/README.md b/bin/README.md
new file mode 100644
index 0000000..8a538db
--- /dev/null
+++ b/bin/README.md
@@ -0,0 +1,20 @@
+To run this tool in Android 10 code base, the python tools need to be prebuilt
+on Android 11 code base and dropped to this directory. This is to workaround
+a bug in the older version of protobuf.
+
+How to:
+
+1. From Android 11 code base, build the python tools:
+```
+croot
+m update_apn
+m update_carrier_data
+```
+
+2. Copy the generated binary to this directory:
+```
+cp out/host/linux-x86/bin/update_apn <path/to/this/directory>
+cp out/host/linux-x86/bin/update_carrier_data <path/to/this/directory>
+```
+
+3. Run the main.sh as usual
diff --git a/java/CarrierConfigConverterV2.java b/java/CarrierConfigConverterV2.java
new file mode 100644
index 0000000..8268522
--- /dev/null
+++ b/java/CarrierConfigConverterV2.java
@@ -0,0 +1,878 @@
+/*
+ * Copyright (C) 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.carrier;
+
+import static com.google.common.collect.Multimaps.flatteningToMultimap;
+import static com.google.common.collect.Multimaps.toMultimap;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.Comparator.comparing;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.auto.value.AutoValue;
+import com.google.common.base.Ascii;
+import com.google.common.base.CharMatcher;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.MultimapBuilder;
+import com.google.protobuf.Descriptors;
+import com.google.protobuf.TextFormat;
+import com.google.carrier.CarrierConfig;
+import com.google.carrier.CarrierId;
+import com.google.carrier.CarrierList;
+import com.google.carrier.CarrierMap;
+import com.google.carrier.CarrierSettings;
+import com.google.carrier.IntArray;
+import com.google.carrier.MultiCarrierSettings;
+import com.google.carrier.TextArray;
+import com.android.providers.telephony.CarrierIdProto.CarrierAttribute;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NamedNodeMap;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+import org.xml.sax.SAXException;
+
+/**
+ * This command converts carrier config XML into text protobuf.
+ *
+ * <ul>
+ *   <li>input: the assets/ from AOSP CarrierConfig app
+ *   <li>input: vendor.xml file(s) which override(s) assets
+ *   <li>input: a tier-1 carrier list in text protobuf (in --output_dir)
+ *   <li>input: the version number for output files
+ *   <li>output: an other_carriers.textpb - a list of other (non-tier-1) carriers
+ *   <li>output: an others.textpb containing carrier configs for non tier-1 carriers
+ *   <li>output: a .textpb for every single tier-1 carrier
+ * </ul>
+ */
+@Parameters(separators = "=")
+public final class CarrierConfigConverterV2 {
+  @Parameter(names = "--assets", description = "The source AOSP assets/ directory.")
+  private String assetsDirName = "/tmp/carrierconfig/assets";
+
+  @Parameter(
+      names = "--vendor_xml",
+      description =
+          "The source vendor.xml file(s). If multiple files provided, the order decides config"
+              + " precedence, ie. configs in a file are overwritten by configs in files AFTER it.")
+  private List<String> vendorXmlFiles = ImmutableList.of("/tmp/carrierconfig/vendor.xml");
+
+  @Parameter(
+      names = "--output_dir",
+      description = "The destination data directory, with tier1_carriers.textpb in it.")
+  private String outputDir = "/tmp/carrierconfig/out";
+
+  @Parameter(names = "--version", description = "The version number for all output textpb.")
+  private int version = 1;
+
+  // For configs in vendor.xml w/o mcc/mnc, they are the default config values for all carriers.
+  // In CarrierSettings, a special mcc/mnc "000000" is used to look up default config.
+  private static final String MCCMNC_FOR_DEFAULT_SETTINGS = "000000";
+
+  // Resource file path to the AOSP carrier list file
+  private static final String RESOURCE_CARRIER_LIST =
+      "/assets/carrier_list.textpb";
+
+  // Constants used in parsing XMLs.
+  private static final String XML_SUFFIX = ".xml";
+  private static final String CARRIER_CONFIG_MCCMNC_XML_PREFIX = "carrier_config_mccmnc_";
+  private static final String CARRIER_CONFIG_CID_XML_PREFIX = "carrier_config_carrierid_";
+  private static final String KEY_MCCMNC_PREFIX = "mccmnc_";
+  private static final String KEY_CID_PREFIX = "cid_";
+  private static final String TAG_CARRIER_CONFIG = "carrier_config";
+
+  /** Entry point when invoked from command line. */
+  public static void main(String[] args) throws IOException {
+    CarrierConfigConverterV2 converter = new CarrierConfigConverterV2();
+    new JCommander(converter, args);
+    converter.convert();
+  }
+
+  /** Entry point when invoked from other Java code, eg. the server side conversion tool. */
+  public static void convert(
+      String vendorXmlFile, String assetsDirName, String outputDir, int version)
+      throws IOException {
+    CarrierConfigConverterV2 converter = new CarrierConfigConverterV2();
+    converter.vendorXmlFiles = ImmutableList.of(vendorXmlFile);
+    converter.assetsDirName = assetsDirName;
+    converter.outputDir = outputDir;
+    converter.version = version;
+    converter.convert();
+  }
+
+  private void convert() throws IOException {
+    String carriersTextpbFile = getPathAsString(outputDir, "tier1_carriers.textpb");
+    String settingsTextpbDir = getPathAsString(outputDir, "setting");
+    CarrierList tier1Carriers;
+    ArrayList<CarrierMap> otherCarriers = new ArrayList<>();
+    ArrayList<String> outFiles = new ArrayList<>();
+    HashMap<CarrierId, Map<String, CarrierConfig.Config>> rawConfigs = new HashMap<>();
+    TreeMap<String, CarrierConfig> tier1Configs = new TreeMap<>();
+    TreeMap<String, CarrierConfig> othersConfigs = new TreeMap<>();
+    DocumentBuilder xmlDocBuilder = getDocumentBuilder();
+    Multimap<Integer, CarrierId> aospCarrierList = loadAospCarrierList();
+    Multimap<CarrierId, Integer> reverseAospCarrierList = reverseAospCarrierList(aospCarrierList);
+
+    /*
+     * High-level flow:
+     * 1. Parse all input XMLs into memory
+     * 2. Collect a list of interested carriers from input, represented by CarrierId.
+     * 2. For each CarrierId, build its carreir configs, following AOSP DefaultCarrierConfigService.
+     * 3. Merge CarrierId's as per tier1_carriers.textpb
+     */
+
+    // 1. Parse all input XMLs into memory
+    Map<String, Document> assetsXmls = new HashMap<>();
+    List<Document> vendorXmls = new ArrayList<>();
+    // Parse assets/carrier_config_*.xml
+    for (File childFile : new File(assetsDirName).listFiles()) {
+      String childFileName = childFile.getName();
+      String fullChildName = childFile.getCanonicalPath();
+      if (childFileName.startsWith(CARRIER_CONFIG_MCCMNC_XML_PREFIX)) {
+        String mccMnc =
+            childFileName.substring(
+                CARRIER_CONFIG_MCCMNC_XML_PREFIX.length(), childFileName.indexOf(XML_SUFFIX));
+        if (!mccMnc.matches("\\d{5,6}")) {
+          throw new IOException("Invalid mcc/mnc " + mccMnc + " found in " + childFileName);
+        }
+        try {
+          assetsXmls.put(KEY_MCCMNC_PREFIX + mccMnc, parseXmlDoc(fullChildName, xmlDocBuilder));
+        } catch (SAXException | IOException e) {
+          throw new IOException("Failed to parse " + childFileName, e);
+        }
+      } else if (childFileName.startsWith(CARRIER_CONFIG_CID_XML_PREFIX)) {
+        String cidAndCarrierName =
+            childFileName.substring(
+                CARRIER_CONFIG_CID_XML_PREFIX.length(), childFileName.indexOf(XML_SUFFIX));
+        int cid = -1;
+        try {
+          cid = Integer.parseInt(cidAndCarrierName.split("_", -1)[0]);
+        } catch (NumberFormatException e) {
+          throw new IOException("Invalid carrierid found in " + childFileName, e);
+        }
+        try {
+          assetsXmls.put(KEY_CID_PREFIX + cid, parseXmlDoc(fullChildName, xmlDocBuilder));
+        } catch (SAXException | IOException e) {
+          throw new IOException("Failed to parse " + childFileName, e);
+        }
+      }
+      // ignore other malformatted files.
+    }
+    // Parse vendor.xml files
+    for (String vendorXmlFile : vendorXmlFiles) {
+      try {
+        vendorXmls.add(parseXmlDoc(vendorXmlFile, xmlDocBuilder));
+      } catch (SAXException | IOException e) {
+        throw new IOException("Failed to parse " + vendorXmlFile, e);
+      }
+    }
+
+    // 2. Collect all carriers from input, represented by CarrierId.
+    List<CarrierId> carriers = new ArrayList<>();
+    // Traverse <carrier_config /> labels in each file.
+    for (Map.Entry<String, Document> xml : assetsXmls.entrySet()) {
+      if (xml.getKey().startsWith(KEY_MCCMNC_PREFIX)) {
+        String mccMnc = xml.getKey().substring(KEY_MCCMNC_PREFIX.length());
+        for (Element element : getElementsByTagName(xml.getValue(), TAG_CARRIER_CONFIG)) {
+          try {
+            CarrierId id = parseCarrierId(element).setMccMnc(mccMnc).build();
+            carriers.add(id);
+          } catch (UnsupportedOperationException e) {
+            throw new IOException("Unsupported syntax in assets/ for " + mccMnc, e);
+          }
+        }
+      } else if (xml.getKey().startsWith(KEY_CID_PREFIX)) {
+        int cid = Integer.parseInt(xml.getKey().substring(KEY_CID_PREFIX.length()));
+        if (aospCarrierList.containsKey(cid)) {
+          carriers.addAll(aospCarrierList.get(cid));
+        } else {
+          System.err.printf("Undefined cid %d in assets/. Ignore.\n", cid);
+        }
+      }
+    }
+    for (Document vendorXml : vendorXmls) {
+      for (Element element : getElementsByTagName(vendorXml, TAG_CARRIER_CONFIG)) {
+        // First, try to parse cid
+        if (element.hasAttribute("cid")) {
+          String cidAsString = element.getAttribute("cid");
+          int cid = Integer.parseInt(cidAsString);
+          if (aospCarrierList.containsKey(cid)) {
+            carriers.addAll(aospCarrierList.get(cid));
+          } else {
+            System.err.printf("Undefined cid %d in vendor.xml. Ignore.\n", cid);
+          }
+        } else {
+          // Then, try to parse CarrierId
+          CarrierId.Builder id = parseCarrierId(element);
+          // A valid mccmnc is 5- or 6-digit. But vendor.xml see special cases below:
+          // Case 1: a <carrier_config> element may have neither "mcc" nor "mnc".
+          // Such a tag provides a base config value for all carriers. CarrierSettings uses
+          // mcc/mnc 000/000 to identify that, and does the merge at run time instead of
+          // build time (which is now).
+          // Case 2: a <carrier_config> element may have just "mcc" and not "mnc" for
+          // country-wise config. Such a element doesn't make a carrier; but still keep it so
+          // can be used if a mccmnc appears in APNs later.
+          if (id.getMccMnc().isEmpty()) {
+            // special case 1
+            carriers.add(id.setMccMnc(MCCMNC_FOR_DEFAULT_SETTINGS).build());
+          } else if (id.getMccMnc().length() == 3) {
+            // special case 2
+            carriers.add(id.build());
+          } else if (id.getMccMnc().length() == 5 || id.getMccMnc().length() == 6) {
+            // Normal mcc+mnc
+            carriers.add(id.build());
+          } else {
+            System.err.printf("Invalid mcc/mnc: %s. Ignore.\n", id.getMccMnc());
+          }
+        }
+      }
+    }
+
+    // 3. For each CarrierId, build its carrier configs, following AOSP DefaultCarrierConfigService.
+    for (CarrierId carrier : carriers) {
+      Map<String, CarrierConfig.Config> config = ImmutableMap.of();
+
+      CarrierIdentifier id = getCid(carrier, reverseAospCarrierList);
+      if (id.getCarrierId() != -1) {
+        HashMap<String, CarrierConfig.Config> configBySpecificCarrierId =
+            parseCarrierConfigFromXml(
+                assetsXmls.get(KEY_CID_PREFIX + id.getSpecificCarrierId()), id);
+        HashMap<String, CarrierConfig.Config> configByCarrierId =
+            parseCarrierConfigFromXml(assetsXmls.get(KEY_CID_PREFIX + id.getCarrierId()), id);
+        HashMap<String, CarrierConfig.Config> configByMccMncFallBackCarrierId =
+            parseCarrierConfigFromXml(assetsXmls.get(KEY_CID_PREFIX + id.getMccmncCarrierId()), id);
+        // priority: specific carrier id > carrier id > mccmnc fallback carrier id
+        if (!configBySpecificCarrierId.isEmpty()) {
+          config = configBySpecificCarrierId;
+        } else if (!configByCarrierId.isEmpty()) {
+          config = configByCarrierId;
+        } else if (!configByMccMncFallBackCarrierId.isEmpty()) {
+          config = configByMccMncFallBackCarrierId;
+        }
+      }
+      if (config.isEmpty()) {
+        // fallback to use mccmnc.xml when there is no carrier id named configuration found.
+        config =
+            parseCarrierConfigFromXml(assetsXmls.get(KEY_MCCMNC_PREFIX + carrier.getMccMnc()), id);
+      }
+      // Treat vendor.xml files as if they were appended to the carrier configs read from assets.
+      for (Document vendorXml : vendorXmls) {
+        HashMap<String, CarrierConfig.Config> vendorConfig =
+            parseCarrierConfigFromVendorXml(vendorXml, id);
+        config.putAll(vendorConfig);
+      }
+
+      rawConfigs.put(carrier, config);
+    }
+
+    // Read tier1_carriers.textpb
+    try (InputStream carriersTextpb = new FileInputStream(new File(carriersTextpbFile));
+        BufferedReader br = new BufferedReader(new InputStreamReader(carriersTextpb, UTF_8))) {
+      CarrierList.Builder builder = CarrierList.newBuilder();
+      TextFormat.getParser().merge(br, builder);
+      tier1Carriers = builder.build();
+    }
+
+    // Compose tier1Configs and othersConfigs from rawConfigs
+    rawConfigs.forEach(
+        (carrierId, configs) -> {
+          String cname = getCanonicalName(tier1Carriers, carrierId);
+          CarrierConfig.Builder ccb = toCarrierConfigBuilder(configs);
+          if (cname != null) { // tier-1 carrier
+            if (tier1Configs.containsKey(cname)) {
+              tier1Configs.put(
+                  cname, CarrierProtoUtils.mergeCarrierConfig(tier1Configs.get(cname), ccb));
+            } else {
+              tier1Configs.put(cname, ccb.build());
+            }
+          } else { // other carrier
+            cname = generateCanonicalNameForOthers(carrierId);
+            otherCarriers.add(
+                CarrierMap.newBuilder().addCarrierId(carrierId).setCanonicalName(cname).build());
+            othersConfigs.put(cname, ccb.build());
+          }
+        });
+
+    // output tier1 carrier settings
+    for (int i = 0; i < tier1Carriers.getEntryCount(); i++) {
+      CarrierMap cm = tier1Carriers.getEntry(i);
+      String cname = cm.getCanonicalName();
+      String fileName = getPathAsString(settingsTextpbDir, cname + ".textpb");
+
+      outFiles.add(fileName);
+
+      try (OutputStream os = new FileOutputStream(new File(fileName));
+          BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, UTF_8))) {
+        CarrierSettings.Builder cs = CarrierSettings.newBuilder().setCanonicalName(cname);
+        if (tier1Configs.containsKey(cname)) {
+          cs.setConfigs(sortConfig(tier1Configs.get(cname)).toBuilder().build());
+        }
+        cs.setVersion(version);
+        TextFormat.printUnicode(cs.build(), bw);
+      }
+    }
+
+    // Output other carriers list
+    String otherCarriersFile = getPathAsString(outputDir, "other_carriers.textpb");
+    outFiles.add(otherCarriersFile);
+    CarrierProtoUtils.sortCarrierMapEntries(otherCarriers);
+    try (OutputStream os = new FileOutputStream(new File(otherCarriersFile));
+        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, UTF_8))) {
+      CarrierList cl =
+          CarrierList.newBuilder().addAllEntry(otherCarriers).setVersion(version).build();
+      TextFormat.printUnicode(cl, bw);
+    }
+
+    // Output other carriers settings
+    String othersFileName = getPathAsString(settingsTextpbDir, "others.textpb");
+    outFiles.add(othersFileName);
+    try (OutputStream os = new FileOutputStream(new File(othersFileName));
+        BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(os, UTF_8))) {
+      MultiCarrierSettings.Builder mcs = MultiCarrierSettings.newBuilder().setVersion(version);
+      othersConfigs.forEach(
+          (cname, cc) -> {
+            mcs.addSetting(
+                CarrierSettings.newBuilder()
+                    .setCanonicalName(cname)
+                    .setConfigs(sortConfig(cc).toBuilder().build())
+                    .build());
+          });
+      TextFormat.printUnicode(mcs.build(), bw);
+    }
+
+    // Print out the list of all output file names
+    System.out.println("SUCCESS! Files generated:");
+    for (String fileName : outFiles) {
+      System.out.println(fileName);
+    }
+  }
+
+  private static DocumentBuilder getDocumentBuilder() {
+    try {
+      DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+      return dbFactory.newDocumentBuilder();
+    } catch (ParserConfigurationException e) {
+      throw new IllegalStateException(e);
+    }
+  }
+
+  private static Multimap<Integer, CarrierId> loadAospCarrierList() throws IOException {
+    com.android.providers.telephony.CarrierIdProto.CarrierList.Builder aospCarrierList =
+        com.android.providers.telephony.CarrierIdProto.CarrierList.newBuilder();
+    try (InputStream textpb =
+            CarrierConfigConverterV2.class.getResourceAsStream(RESOURCE_CARRIER_LIST);
+        BufferedReader textpbReader = new BufferedReader(new InputStreamReader(textpb, UTF_8))) {
+      TextFormat.getParser().merge(textpbReader, aospCarrierList);
+    }
+    return aospCarrierList.getCarrierIdList().stream()
+        .collect(
+            flatteningToMultimap(
+                cid -> cid.getCanonicalId(),
+                cid -> carrierAttributeToCarrierId(cid.getCarrierAttributeList()).stream(),
+                MultimapBuilder.linkedHashKeys().arrayListValues()::build));
+  }
+
+  // Convert `CarrierAttribute`s to `CarrierId`s.
+  // A CarrierAttribute message with fields not supported by CarrierSettings, like preferred_apn,
+  // is ignored.
+  private static ImmutableList<CarrierId> carrierAttributeToCarrierId(
+      List<CarrierAttribute> carrierAttributes) {
+    List<CarrierId> result = new ArrayList<>();
+    ImmutableSet<Descriptors.FieldDescriptor> supportedFields =
+        ImmutableSet.of(
+            CarrierAttribute.getDescriptor().findFieldByName("mccmnc_tuple"),
+            CarrierAttribute.getDescriptor().findFieldByName("imsi_prefix_xpattern"),
+            CarrierAttribute.getDescriptor().findFieldByName("spn"),
+            CarrierAttribute.getDescriptor().findFieldByName("gid1"));
+    for (CarrierAttribute carrierAttribute : carrierAttributes) {
+      if (!carrierAttribute.getAllFields().keySet().stream().allMatch(supportedFields::contains)) {
+        // This `CarrierAttribute` contains unsupported fields; skip.
+        continue;
+      }
+      for (String mccmnc : carrierAttribute.getMccmncTupleList()) {
+        CarrierId.Builder carrierId = CarrierId.newBuilder().setMccMnc(mccmnc);
+        if (carrierAttribute.getImsiPrefixXpatternCount() > 0) {
+          for (String imsi : carrierAttribute.getImsiPrefixXpatternList()) {
+            result.add(carrierId.setImsi(imsi).build());
+          }
+        } else if (carrierAttribute.getGid1Count() > 0) {
+          for (String gid1 : carrierAttribute.getGid1List()) {
+            result.add(carrierId.setGid1(gid1).build());
+          }
+        } else if (carrierAttribute.getSpnCount() > 0) {
+          for (String spn : carrierAttribute.getSpnList()) {
+            // Some SPN has trailng space character \r, messing up textpb. Remove them.
+            // It won't affect CarrierSettings which uses prefix matching for SPN.
+            result.add(carrierId.setSpn(CharMatcher.whitespace().trimTrailingFrom(spn)).build());
+          }
+        } else { // Ignore other attributes not supported by CarrierSettings
+          result.add(carrierId.build());
+        }
+      }
+    }
+    // Dedup
+    return ImmutableSet.copyOf(result).asList();
+  }
+
+  private static Multimap<CarrierId, Integer> reverseAospCarrierList(
+      Multimap<Integer, CarrierId> aospCarrierList) {
+    return aospCarrierList.entries().stream()
+        .collect(
+            toMultimap(
+                entry -> entry.getValue(),
+                entry -> entry.getKey(),
+                MultimapBuilder.linkedHashKeys().arrayListValues()::build));
+  }
+
+  private static Document parseXmlDoc(String fileName, DocumentBuilder xmlDocBuilder)
+      throws SAXException, IOException {
+    try (InputStream configXml = new FileInputStream(new File(fileName))) {
+      Document xmlDoc = xmlDocBuilder.parse(configXml);
+      xmlDoc.getDocumentElement().normalize();
+      return xmlDoc;
+    }
+  }
+
+  private static ImmutableList<Element> getElementsByTagName(Document xmlDoc, String tagName) {
+    if (xmlDoc == null) {
+      return ImmutableList.of();
+    }
+    ImmutableList.Builder<Element> result = new ImmutableList.Builder<>();
+    xmlDoc.getDocumentElement().normalize();
+    NodeList nodeList = xmlDoc.getElementsByTagName(tagName);
+    for (int i = 0; i < nodeList.getLength(); i++) {
+      Node node = nodeList.item(i);
+      if (node.getNodeType() == Node.ELEMENT_NODE) {
+        result.add((Element) node);
+      }
+    }
+    return result.build();
+  }
+
+  static CarrierConfig sortConfig(CarrierConfig in) {
+    final CarrierConfig.Builder result = in.toBuilder().clearConfig();
+    in.getConfigList().stream()
+        .sorted(comparing(CarrierConfig.Config::getKey))
+        .forEachOrdered((c) -> result.addConfig(c));
+    return result.build();
+  }
+
+  static String getCanonicalName(CarrierList pList, CarrierId pId) {
+    for (int i = 0; i < pList.getEntryCount(); i++) {
+      CarrierMap cm = pList.getEntry(i);
+      for (int j = 0; j < cm.getCarrierIdCount(); j++) {
+        CarrierId cid = cm.getCarrierId(j);
+        if (cid.equals(pId)) {
+          return cm.getCanonicalName();
+        }
+      }
+    }
+    return null;
+  }
+
+  static String generateCanonicalNameForOthers(CarrierId pId) {
+    // Not a tier-1 carrier: generate name
+    StringBuilder genName = new StringBuilder(pId.getMccMnc());
+    switch (pId.getMvnoDataCase()) {
+      case GID1:
+        genName.append("GID1=");
+        genName.append(Ascii.toUpperCase(pId.getGid1()));
+        break;
+      case SPN:
+        genName.append("SPN=");
+        genName.append(Ascii.toUpperCase(pId.getSpn()));
+        break;
+      case IMSI:
+        genName.append("IMSI=");
+        genName.append(Ascii.toUpperCase(pId.getImsi()));
+        break;
+      default: // MVNODATA_NOT_SET
+        // Do nothing
+    }
+    return genName.toString();
+  }
+
+  /**
+   * Converts a map with carrier configs to a {@link CarrierConfig.Builder}.
+   *
+   * @see #parseCarrierConfigToMap
+   */
+  private static CarrierConfig.Builder toCarrierConfigBuilder(
+      Map<String, CarrierConfig.Config> configs) {
+    CarrierConfig.Builder builder = CarrierConfig.newBuilder();
+    configs.forEach(
+        (key, value) -> {
+          builder.addConfig(value.toBuilder().setKey(key));
+        });
+    return builder;
+  }
+
+  /**
+   * Returns a map with carrier configs parsed from a assets/*.xml.
+   *
+   * @return a map, key being the carrier config key, value being a {@link CarrierConfig.Config}
+   *     with one of the value set.
+   */
+  private static HashMap<String, CarrierConfig.Config> parseCarrierConfigFromXml(
+      Document xmlDoc, CarrierIdentifier carrier) throws IOException {
+    HashMap<String, CarrierConfig.Config> configMap = new HashMap<>();
+    for (Element element : getElementsByTagName(xmlDoc, TAG_CARRIER_CONFIG)) {
+      if (carrier != null && !checkFilters(false, element, carrier)) {
+        continue;
+      }
+      configMap.putAll(parseCarrierConfigToMap(element));
+    }
+    return configMap;
+  }
+
+  /**
+   * Returns a map with carrier configs parsed from the vendor.xml.
+   *
+   * @return a map, key being the carrier config key, value being a {@link CarrierConfig.Config}
+   *     with one of the value set.
+   */
+  private static HashMap<String, CarrierConfig.Config> parseCarrierConfigFromVendorXml(
+      Document xmlDoc, CarrierIdentifier carrier) throws IOException {
+    HashMap<String, CarrierConfig.Config> configMap = new HashMap<>();
+    for (Element element : getElementsByTagName(xmlDoc, TAG_CARRIER_CONFIG)) {
+      if (carrier != null && !checkFilters(true, element, carrier)) {
+        continue;
+      }
+      configMap.putAll(parseCarrierConfigToMap(element));
+    }
+    return configMap;
+  }
+
+  /**
+   * Returns a map with carrier configs parsed from the XML element.
+   *
+   * @return a map, key being the carrier config key, value being a {@link CarrierConfig.Config}
+   *     with one of the value set.
+   */
+  private static HashMap<String, CarrierConfig.Config> parseCarrierConfigToMap(Element element)
+      throws IOException {
+    HashMap<String, CarrierConfig.Config> configMap = new HashMap<>();
+    NodeList nList;
+    // bool value
+    nList = element.getElementsByTagName("boolean");
+    for (int i = 0; i < nList.getLength(); i++) {
+      Node nNode = nList.item(i);
+      if (nNode.getNodeType() != Node.ELEMENT_NODE) {
+        continue;
+      }
+      Element eElement = (Element) nNode;
+      String key = eElement.getAttribute("name");
+      boolean value = Boolean.parseBoolean(eElement.getAttribute("value"));
+      configMap.put(key, CarrierConfig.Config.newBuilder().setBoolValue(value).build());
+    }
+    // int value
+    nList = element.getElementsByTagName("int");
+    for (int i = 0; i < nList.getLength(); i++) {
+      Node nNode = nList.item(i);
+      if (nNode.getNodeType() != Node.ELEMENT_NODE) {
+        continue;
+      }
+      Element eElement = (Element) nNode;
+      String key = eElement.getAttribute("name");
+      int value = Integer.parseInt(eElement.getAttribute("value"));
+      configMap.put(key, CarrierConfig.Config.newBuilder().setIntValue(value).build());
+    }
+    // long value
+    nList = element.getElementsByTagName("long");
+    for (int i = 0; i < nList.getLength(); i++) {
+      Node nNode = nList.item(i);
+      if (nNode.getNodeType() != Node.ELEMENT_NODE) {
+        continue;
+      }
+      Element eElement = (Element) nNode;
+      String key = eElement.getAttribute("name");
+      long value = Long.parseLong(eElement.getAttribute("value"));
+      configMap.put(key, CarrierConfig.Config.newBuilder().setLongValue(value).build());
+    }
+    // text value
+    nList = element.getElementsByTagName("string");
+    for (int i = 0; i < nList.getLength(); i++) {
+      Node nNode = nList.item(i);
+      if (nNode.getNodeType() != Node.ELEMENT_NODE) {
+        continue;
+      }
+      Element eElement = (Element) nNode;
+      String key = eElement.getAttribute("name");
+      String value = String.valueOf(eElement.getTextContent());
+      configMap.put(key, CarrierConfig.Config.newBuilder().setTextValue(value).build());
+    }
+    // text array
+    nList = element.getElementsByTagName("string-array");
+    for (int i = 0; i < nList.getLength(); i++) {
+      Node nNode = nList.item(i);
+      if (nNode.getNodeType() != Node.ELEMENT_NODE) {
+        continue;
+      }
+      Element eElement = (Element) nNode;
+      String key = eElement.getAttribute("name");
+      CarrierConfig.Config.Builder cccb = CarrierConfig.Config.newBuilder();
+      TextArray.Builder cctb = TextArray.newBuilder();
+      NodeList subList = eElement.getElementsByTagName("item");
+      for (int j = 0; j < subList.getLength(); j++) {
+        Node subNode = subList.item(j);
+        if (subNode.getNodeType() != Node.ELEMENT_NODE) {
+          continue;
+        }
+        Element subElement = (Element) subNode;
+        String value = String.valueOf(subElement.getAttribute("value"));
+        cctb.addItem(value);
+      }
+      configMap.put(key, cccb.setTextArray(cctb.build()).build());
+    }
+    // bool array
+    nList = element.getElementsByTagName("int-array");
+    for (int i = 0; i < nList.getLength(); i++) {
+      Node nNode = nList.item(i);
+      if (nNode.getNodeType() != Node.ELEMENT_NODE) {
+        continue;
+      }
+      Element eElement = (Element) nNode;
+      String key = eElement.getAttribute("name");
+      CarrierConfig.Config.Builder cccb = CarrierConfig.Config.newBuilder();
+      IntArray.Builder ccib = IntArray.newBuilder();
+      NodeList subList = eElement.getElementsByTagName("item");
+      for (int j = 0; j < subList.getLength(); j++) {
+        Node subNode = subList.item(j);
+        if (subNode.getNodeType() != Node.ELEMENT_NODE) {
+          continue;
+        }
+        Element subElement = (Element) subNode;
+        int value = Integer.parseInt(subElement.getAttribute("value"));
+        ccib.addItem(value);
+      }
+      configMap.put(key, cccb.setIntArray(ccib.build()).build());
+    }
+    return configMap;
+  }
+
+  /**
+   * Returns {@code true} if a <carrier_config ...> element matches the carrier identifier.
+   *
+   * <p>Copied from AOSP DefaultCarrierConfigService.
+   */
+  private static boolean checkFilters(boolean isVendorXml, Element element, CarrierIdentifier id) {
+    // Special case: in vendor.xml, the <carrier_config> element w/o mcc/mnc provides a base config
+    // value for all carriers. CarrierSettings uses mcc/mnc 000/000 to identify that, and does the
+    // merge at run time instead of build time (which is now).
+    // Hence, such an element should only match 000/000.
+    if (isVendorXml
+        && !element.hasAttribute("mcc")
+        && !element.hasAttribute("mnc")
+        && !element.hasAttribute("cid")) {
+      return MCCMNC_FOR_DEFAULT_SETTINGS.equals(id.getMcc() + id.getMnc());
+    }
+    boolean result = true;
+    NamedNodeMap attributes = element.getAttributes();
+    for (int i = 0; i < attributes.getLength(); i++) {
+      String attribute = attributes.item(i).getNodeName();
+      String value = attributes.item(i).getNodeValue();
+      switch (attribute) {
+        case "mcc":
+          result = result && value.equals(id.getMcc());
+          break;
+        case "mnc":
+          result = result && value.equals(id.getMnc());
+          break;
+        case "gid1":
+          result = result && Ascii.equalsIgnoreCase(value, id.getGid1());
+          break;
+        case "spn":
+          result = result && matchOnSP(value, id);
+          break;
+        case "imsi":
+          result = result && matchOnImsi(value, id);
+          break;
+        case "cid":
+          result =
+              result
+                  && ((Integer.parseInt(value) == id.getCarrierId())
+                      || (Integer.parseInt(value) == id.getSpecificCarrierId()));
+          break;
+        case "name":
+          // name is used together with cid for readability. ignore for filter.
+          break;
+        default:
+          System.err.println("Unsupported attribute " + attribute + "=" + value);
+          result = false;
+      }
+    }
+    return result;
+  }
+
+  /**
+   * Returns {@code true} if an "spn" attribute in <carrier_config ...> element matches the carrier
+   * identifier.
+   *
+   * <p>Copied from AOSP DefaultCarrierConfigService.
+   */
+  private static boolean matchOnSP(String xmlSP, CarrierIdentifier id) {
+    boolean matchFound = false;
+
+    String currentSP = id.getSpn();
+    // <carrier_config ... spn="null"> means expecting SIM SPN empty in AOSP convention.
+    if (Ascii.equalsIgnoreCase("null", xmlSP)) {
+      if (currentSP.isEmpty()) {
+        matchFound = true;
+      }
+    } else if (currentSP != null) {
+      Pattern spPattern = Pattern.compile(xmlSP, Pattern.CASE_INSENSITIVE);
+      Matcher matcher = spPattern.matcher(currentSP);
+      matchFound = matcher.matches();
+    }
+    return matchFound;
+  }
+
+  /**
+   * Returns {@code true} if an "imsi" attribute in <carrier_config ...> element matches the carrier
+   * identifier.
+   *
+   * <p>Copied from AOSP DefaultCarrierConfigService.
+   */
+  private static boolean matchOnImsi(String xmlImsi, CarrierIdentifier id) {
+    boolean matchFound = false;
+
+    String currentImsi = id.getImsi();
+    // If we were able to retrieve current IMSI, see if it matches.
+    if (currentImsi != null) {
+      Pattern imsiPattern = Pattern.compile(xmlImsi, Pattern.CASE_INSENSITIVE);
+      Matcher matcher = imsiPattern.matcher(currentImsi);
+      matchFound = matcher.matches();
+    }
+    return matchFound;
+  }
+
+  /**
+   * Parses a {@link CarrierId} out of a <carrier_config ...> tag.
+   *
+   * <p>This is purely used for discover potential carriers expressed by this tag, the return value
+   * may not reflect all attributes of the tag.
+   */
+  private static CarrierId.Builder parseCarrierId(Element element) {
+    CarrierId.Builder builder = CarrierId.newBuilder();
+    String mccMnc = element.getAttribute("mcc") + element.getAttribute("mnc");
+    builder.setMccMnc(mccMnc);
+    if (element.hasAttribute("imsi")) {
+      builder.setImsi(element.getAttribute("imsi"));
+    } else if (element.hasAttribute("gid1")) {
+      builder.setGid1(element.getAttribute("gid1"));
+    } else if (element.hasAttribute("gid2")) {
+      throw new UnsupportedOperationException(
+          "Not support attribute `gid2`: " + element.getAttribute("gid2"));
+    } else if (element.hasAttribute("spn")) {
+      builder.setSpn(element.getAttribute("spn"));
+    }
+    return builder;
+  }
+
+  // Same as {@link java.nio.file.Paths#get} but returns a String
+  private static String getPathAsString(String first, String... more) {
+    return java.nio.file.Paths.get(first, more).toString();
+  }
+
+  /** Mirror of Android CarrierIdentifier class. Default value of a carrier id is -1. */
+  @AutoValue
+  abstract static class CarrierIdentifier {
+    abstract String getMcc();
+
+    abstract String getMnc();
+
+    abstract String getImsi();
+
+    abstract String getGid1();
+
+    abstract String getSpn();
+
+    abstract int getCarrierId();
+
+    abstract int getSpecificCarrierId();
+
+    abstract int getMccmncCarrierId();
+
+    static CarrierIdentifier create(
+        CarrierId carrier, int carrierId, int specificCarrierId, int mccmncCarrierId) {
+      String mcc = carrier.getMccMnc().substring(0, 3);
+      String mnc = carrier.getMccMnc().length() > 3 ? carrier.getMccMnc().substring(3) : "";
+      return new AutoValue_CarrierConfigConverterV2_CarrierIdentifier(
+          mcc,
+          mnc,
+          carrier.getImsi(),
+          carrier.getGid1(),
+          carrier.getSpn(),
+          carrierId,
+          specificCarrierId,
+          mccmncCarrierId);
+    }
+  }
+
+  private static CarrierIdentifier getCid(
+      CarrierId carrierId, Multimap<CarrierId, Integer> reverseAospCarrierList) {
+    // Mimic TelephonyManager#getCarrierIdFromMccMnc, which is implemented by
+    // CarrierResolver#getCarrierIdFromMccMnc.
+    CarrierId mccMnc = CarrierId.newBuilder().setMccMnc(carrierId.getMccMnc()).build();
+    int mccMncCarrierId = reverseAospCarrierList.get(mccMnc).stream().findFirst().orElse(-1);
+
+    List<Integer> cids = ImmutableList.copyOf(reverseAospCarrierList.get(carrierId));
+    // No match: use -1
+    if (cids.isEmpty()) {
+      return CarrierIdentifier.create(carrierId, -1, -1, mccMncCarrierId);
+    }
+    // One match: use as both carrierId and specificCarrierId
+    if (cids.size() == 1) {
+      return CarrierIdentifier.create(carrierId, cids.get(0), cids.get(0), mccMncCarrierId);
+    }
+    // Two matches:  specificCarrierId is always bigger than carrierId
+    if (cids.size() == 2) {
+      return CarrierIdentifier.create(
+          carrierId,
+          Math.min(cids.get(0), cids.get(1)),
+          Math.max(cids.get(0), cids.get(1)),
+          mccMncCarrierId);
+    }
+    // Cannot be more than 2 matches.
+    throw new IllegalStateException("More than two cid's found for " + carrierId + ": " + cids);
+  }
+
+  private CarrierConfigConverterV2() {}
+}
diff --git a/java/CarrierProtoUtils.java b/java/CarrierProtoUtils.java
new file mode 100644
index 0000000..0280499
--- /dev/null
+++ b/java/CarrierProtoUtils.java
@@ -0,0 +1,242 @@
+/*
+ * Copyright (C) 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.carrier;
+
+import com.google.common.base.Preconditions;
+import com.google.carrier.CarrierConfig;
+import com.google.carrier.CarrierId;
+import com.google.carrier.CarrierMap;
+import com.google.carrier.CarrierSettings;
+import com.google.carrier.MultiCarrierSettings;
+import com.google.carrier.VendorConfigClient;
+import com.google.carrier.VendorConfigs;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+/** Utility methods */
+public class CarrierProtoUtils {
+
+  /**
+   * The base of release version.
+   *
+   * A valid release version must be a multiple of the base.
+   * A file version must be smaller than the base - otherwise the version schema is broken.
+   */
+  public static final long RELEASE_VERSION_BASE = 1000000000L;
+
+  /**
+   * Bump the version by an offset, to differentiate release/branch.
+   *
+   * The input version must be smaller than RELEASE_VERSION_BASE, and the offset must be a
+   * multiple of RELEASE_VERSION_BASE.
+   */
+  public static long addVersionOffset(long version, long offset) {
+
+    Preconditions.checkArgument(version < RELEASE_VERSION_BASE,
+        "OMG, by design the file version should be smaller than %s, but is %s.",
+        RELEASE_VERSION_BASE, version);
+    Preconditions.checkArgument((offset % RELEASE_VERSION_BASE) == 0,
+        "The offset %s is not a multiple of %s",
+        offset, RELEASE_VERSION_BASE);
+
+    return version + offset;
+  }
+
+  /**
+   * Merge configs fields in {@code patch} into {@code base}; configs sorted by key.
+   * See {@link mergeCarrierConfig(CarrierConfig, CarrierConfig.Builder)}.
+   */
+  public static CarrierConfig mergeCarrierConfig(CarrierConfig base, CarrierConfig patch) {
+    Preconditions.checkNotNull(patch);
+
+    return mergeCarrierConfig(base, patch.toBuilder());
+  }
+
+  /**
+   * Merge configs fields in {@code patch} into {@code base}; configs sorted by key.
+   *
+   * <p>Sorting is desired because:
+   *
+   * <ul>
+   *   <li>1. The order doesn't matter for the consumer so sorting will not cause behavior change;
+   *   <li>2. The output proto is serilized to text for human review; undeterministic order can
+   *       confuse reviewers as they cannot easily tell if it's re-ordering or actual data change.
+   * </ul>
+   */
+  public static CarrierConfig mergeCarrierConfig(CarrierConfig base, CarrierConfig.Builder patch) {
+    Preconditions.checkNotNull(base);
+    Preconditions.checkNotNull(patch);
+
+    CarrierConfig.Builder baseBuilder = base.toBuilder();
+
+    // Traverse each config in patch
+    for (int i = 0; i < patch.getConfigCount(); i++) {
+      CarrierConfig.Config patchConfig = patch.getConfig(i);
+
+      // Try to find an config in base with the same key as the config from patch
+      int j = 0;
+      for (j = 0; j < baseBuilder.getConfigCount(); j++) {
+        if (baseBuilder.getConfig(j).getKey().equals(patchConfig.getKey())) {
+          break;
+        }
+      }
+
+      // If match found, replace base with patch; otherwise append patch into base.
+      if (j < baseBuilder.getConfigCount()) {
+        baseBuilder.setConfig(j, patchConfig);
+      } else {
+        baseBuilder.addConfig(patchConfig);
+      }
+    }
+
+    // Sort configs in baseBuilder by key
+    List<CarrierConfig.Config> configs = new ArrayList<>(baseBuilder.getConfigList());
+    Collections.sort(configs, Comparator.comparing(CarrierConfig.Config::getKey));
+    baseBuilder.clearConfig();
+    baseBuilder.addAllConfig(configs);
+
+    return baseBuilder.build();
+  }
+
+  /**
+   * Find a carrier's CarrierSettings by canonical_name from a MultiCarrierSettings.
+   *
+   * <p>Return null if not found.
+   */
+  public static CarrierSettings findCarrierSettingsByCanonicalName(
+      MultiCarrierSettings mcs, String name) {
+
+    Preconditions.checkNotNull(mcs);
+    Preconditions.checkNotNull(name);
+
+    for (int i = 0; i < mcs.getSettingCount(); i++) {
+      CarrierSettings cs = mcs.getSetting(i);
+      if (cs.getCanonicalName().equals(name)) {
+        return cs;
+      }
+    }
+
+    return null;
+  }
+
+  /** Apply device overly to a carrier setting */
+  public static CarrierSettings.Builder applyDeviceOverlayToCarrierSettings(
+      MultiCarrierSettings allOverlay, CarrierSettings.Builder base) {
+    return applyDeviceOverlayToCarrierSettings(allOverlay, base.build());
+  }
+
+  /** Apply device overly to a carrier setting */
+  public static CarrierSettings.Builder applyDeviceOverlayToCarrierSettings(
+      MultiCarrierSettings allOverlay, CarrierSettings base) {
+
+    Preconditions.checkNotNull(allOverlay);
+    Preconditions.checkNotNull(base);
+
+    // Find overlay of the base carrier. If not found, just return base.
+    CarrierSettings overlay =
+        findCarrierSettingsByCanonicalName(allOverlay, base.getCanonicalName());
+    if (overlay == null) {
+      return base.toBuilder();
+    }
+
+    CarrierSettings.Builder resultBuilder = base.toBuilder();
+    // Add version number of base settings and overlay, so the result version number
+    // monotonically increases.
+    resultBuilder.setVersion(base.getVersion() + overlay.getVersion());
+    // Merge overlay settings into base settings
+    resultBuilder.setConfigs(mergeCarrierConfig(resultBuilder.getConfigs(), overlay.getConfigs()));
+    // Replace base apns with overlay apns (if not empty)
+    if (overlay.getApns().getApnCount() > 0) {
+      resultBuilder.setApns(overlay.getApns());
+    }
+    // Merge the overlay vendor configuration into base vendor configuration
+    // Can be cutomized
+    return resultBuilder;
+  }
+
+  /** Apply device overly to multiple carriers setting */
+  public static MultiCarrierSettings applyDeviceOverlayToMultiCarrierSettings(
+      MultiCarrierSettings overlay, MultiCarrierSettings base) {
+
+    Preconditions.checkNotNull(overlay);
+    Preconditions.checkNotNull(base);
+
+    MultiCarrierSettings.Builder resultBuilder = base.toBuilder().clearSetting();
+    long version = 0L;
+
+    for (CarrierSettings cs : base.getSettingList()) {
+      // Apply overlay and put overlayed carrier setting back
+      CarrierSettings.Builder merged = applyDeviceOverlayToCarrierSettings(overlay, cs);
+      resultBuilder.addSetting(merged);
+      // The top-level version number is the sum of all version numbers of settings
+      version += merged.getVersion();
+    }
+    resultBuilder.setVersion(version);
+
+    return resultBuilder.build();
+  }
+
+  /**
+   * Sort a list of CarrierMap with single CarrierId.
+   *
+   * <p>Precondition: no duplication in input list
+   *
+   * <p>Order by:
+   *
+   * <ul>
+   *   <li>mcc_mnc
+   *   <li>(for the same mcc_mnc) any mvno_data comes before MVNODATA_NOT_SET (Preconditon: there is
+   *       only one entry with MVNODATA_NOT_SET per mcc_mnc otherwise they're duplicated)
+   *   <li>(for MVNOs of the same mcc_mnc) mvno_data case + value as string
+   * </ul>
+   */
+  public static void sortCarrierMapEntries(List<CarrierMap> list) {
+    final Comparator<CarrierMap> byMccMnc =
+        Comparator.comparing(
+            (cm) -> {
+              return cm.getCarrierId(0).getMccMnc();
+            });
+    final Comparator<CarrierMap> mvnoFirst =
+        Comparator.comparingInt(
+            (cm) -> {
+              switch (cm.getCarrierId(0).getMvnoDataCase()) {
+                case MVNODATA_NOT_SET:
+                  return 1;
+                default:
+                  return 0;
+              }
+            });
+    final Comparator<CarrierMap> byMvnoDataCaseValue =
+        Comparator.comparing(
+            (cm) -> {
+              final CarrierId cid = cm.getCarrierId(0);
+              switch (cid.getMvnoDataCase()) {
+                case GID1:
+                  return "GID1=" + cid.getGid1();
+                case IMSI:
+                  return "IMSI=" + cid.getImsi();
+                case SPN:
+                  return "SPN=" + cid.getSpn();
+                case MVNODATA_NOT_SET:
+                  throw new AssertionError("MNO should not be compared here but in `mvnoFirst`");
+              }
+              throw new AssertionError("uncaught case " + cid.getMvnoDataCase());
+            });
+    Collections.sort(list, byMccMnc.thenComparing(mvnoFirst).thenComparing(byMvnoDataCaseValue));
+  }
+}
diff --git a/java/GenCarrierList.java b/java/GenCarrierList.java
new file mode 100644
index 0000000..d63f2ae
--- /dev/null
+++ b/java/GenCarrierList.java
@@ -0,0 +1,128 @@
+/*
+ * Copyright (C) 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.carrier;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.carrier.CarrierId;
+import com.google.carrier.CarrierList;
+import com.google.carrier.CarrierMap;
+import com.google.common.base.Verify;
+import com.google.protobuf.TextFormat;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * This command line tool generate a single carrier_list.pb. The output pb is used as a reverse-map:
+ * each entry is a CarrierMap with a canonical_name and only one CarrierId; entries are sorted by
+ * mcc_mnc, and MVNO comes before MNO. So that when matching a CarrierId against this list, the
+ * first match is always the best guess of carrier name.
+ */
+@Parameters(separators = "=")
+public final class GenCarrierList {
+  @Parameter(
+      names = "--version_offset",
+      description =
+          "The value to be added to file version, used to differentiate releases/branches.")
+  private long versionOffset = 0L;
+
+  @Parameter(
+      names = "--in_textpbs",
+      description = "A comma-separated list of input CarrierList textpb files")
+  private List<String> textpbFileNames = Arrays.asList();
+
+  @Parameter(names = "--out_pb", description = "The output CarrierList pb file")
+  private String outFileName = "/tmp/carrier_list.pb";
+
+  @Parameter(
+      names = "--with_version_number",
+      description = "Encode version number into output pb filename.")
+  private boolean versionInFileName = false;
+
+  public static final String BINARY_PB_SUFFIX = ".pb";
+  public static final String TEXT_PB_SUFFIX = ".textpb";
+
+  public static void main(String[] args) throws IOException {
+    GenCarrierList generator = new GenCarrierList();
+    new JCommander(generator, args);
+    generator.generate();
+  }
+
+  private void generate() throws IOException {
+    long version = 0;
+
+    Verify.verify(
+        outFileName.endsWith(BINARY_PB_SUFFIX),
+        "Output filename must end with %s.",
+        BINARY_PB_SUFFIX);
+
+    List<CarrierMap> list = new ArrayList<>();
+    for (String textpbFileName : textpbFileNames) {
+      // Load textpbFileName
+      CarrierList carrierList = null;
+      try (BufferedReader br = Files.newBufferedReader(Paths.get(textpbFileName), UTF_8)) {
+        CarrierList.Builder builder = CarrierList.newBuilder();
+        TextFormat.getParser().merge(br, builder);
+        carrierList = builder.build();
+      }
+
+      // Add carrierList into list
+      for (CarrierMap cm : carrierList.getEntryList()) {
+        for (CarrierId cid : cm.getCarrierIdList()) {
+          list.add(cm.toBuilder().clearCarrierId().addCarrierId(cid).build());
+        }
+      }
+
+      // Add up version number
+      version += carrierList.getVersion();
+    }
+
+    // Bump version according to the release
+    version = CarrierProtoUtils.addVersionOffset(version, versionOffset);
+
+    // Sort the list as per carrier identification algorithm requirement
+    CarrierProtoUtils.sortCarrierMapEntries(list);
+
+    // Convert list to CarrierList
+    CarrierList clist = CarrierList.newBuilder().setVersion(version).addAllEntry(list).build();
+
+    // Output
+    String outFileMainName = outFileName.replace(BINARY_PB_SUFFIX, "");
+    try (BufferedWriter bw =
+        Files.newBufferedWriter(Paths.get(outFileMainName + TEXT_PB_SUFFIX), UTF_8)) {
+      TextFormat.printUnicode(clist, bw);
+    }
+
+    if (versionInFileName) {
+      outFileMainName += "-" + clist.getVersion();
+    }
+    try (OutputStream os = Files.newOutputStream(Paths.get(outFileMainName + BINARY_PB_SUFFIX))) {
+      clist.writeTo(os);
+    }
+  }
+
+  private GenCarrierList() {}
+}
diff --git a/java/GenDeviceSettings.java b/java/GenDeviceSettings.java
new file mode 100644
index 0000000..33d3a73
--- /dev/null
+++ b/java/GenDeviceSettings.java
@@ -0,0 +1,181 @@
+/*
+ * Copyright (C) 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.carrier;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import com.beust.jcommander.JCommander;
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+import com.google.carrier.CarrierSettings;
+import com.google.carrier.MultiCarrierSettings;
+import com.google.protobuf.ExtensionRegistry;
+import com.google.protobuf.Message;
+import com.google.protobuf.TextFormat;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+/**
+ * This command line tool generates device-specific settings from device overlay and base settings.
+ */
+@Parameters(separators = "=")
+public class GenDeviceSettings {
+  @Parameter(
+      names = "--version_offset",
+      description =
+          "The value to be added to file version, used to differentiate releases/branches.")
+  private long versionOffset = 0L;
+
+  @Parameter(names = "--device_overlay", description = "The input device override textpb file.")
+  private String deviceFileName = "/tmp/device/muskie.textpb";
+
+  @Parameter(names = "--base_setting_dir", description = "The path to input settings directory.")
+  private String baseSettingDirName = "/tmp/setting";
+
+  @Parameter(
+      names = "--others_file",
+      description = "The file name of others carrier settings in the input directory.")
+  private String othersFileName = "others.textpb";
+
+  @Parameter(
+      names = "--device_setting_dir",
+      description = "The path to putput <deice>_settings directory.")
+  private String deviceSettingDirName = "/tmp/muskie_setting";
+
+  @Parameter(
+      names = "--with_version_number",
+      description = "Encode version number into output pb filename.")
+  private boolean versionInFileName = false;
+
+  @Parameter(names = "--with_device_name", description = "Encode device name into output filename.")
+  private String deviceInFileName = "";
+
+  private static final String PB_SUFFIX = ".pb";
+  private static final String TEXT_PB_SUFFIX = ".textpb";
+
+  private static final ExtensionRegistry registry = ExtensionRegistry.newInstance();
+
+  public static void main(String[] args) throws IOException {
+    GenDeviceSettings generator = new GenDeviceSettings();
+    new JCommander(generator, args);
+    generator.generate();
+  }
+
+  private void generate() throws IOException {
+    // Load device overlay
+    MultiCarrierSettings deviceOverlay = null;
+    try (BufferedReader br = Files.newBufferedReader(Paths.get(deviceFileName), UTF_8)) {
+      MultiCarrierSettings.Builder builder = MultiCarrierSettings.newBuilder();
+      TextFormat.getParser().merge(br, registry, builder);
+      deviceOverlay = builder.build();
+    }
+
+    // Create output settings directory if not exist.
+    File deviceSettingDir = new File(deviceSettingDirName);
+    if (!deviceSettingDir.exists()) {
+      deviceSettingDir.mkdirs();
+    }
+
+    // For each carrier (and others) in baseSettingDir, find its overlay and apply.
+    File baseSettingDir = new File(baseSettingDirName);
+    for (String childName : baseSettingDir.list((dir, name) -> name.endsWith(TEXT_PB_SUFFIX))) {
+      System.out.println("Processing " + childName);
+
+      File baseSettingFile = new File(baseSettingDir, childName);
+
+      Message generatedMessage = null;
+      long version = 0L;
+
+      if (othersFileName.equals(childName)) {
+
+        // Load others setting
+        MultiCarrierSettings.Builder othersSetting = null;
+        try (BufferedReader br = Files.newBufferedReader(baseSettingFile.toPath(), UTF_8)) {
+          MultiCarrierSettings.Builder builder = MultiCarrierSettings.newBuilder();
+          TextFormat.getParser().merge(br, registry, builder);
+          othersSetting = builder;
+        }
+
+        /*
+         * For non-tier1 carriers, DO NOT allow device overlay for now.
+         * There is no easy way to generate a mononical increasing version number with overlay.
+         * And if we do device overlay for a carrier, it should probobaly be tier-1.
+         */
+
+        // Bump version according to the release
+        othersSetting.setVersion(
+            CarrierProtoUtils.addVersionOffset(othersSetting.getVersion(), versionOffset));
+
+        // Convert vendor specific data into binary format
+        // Can be customized
+
+        generatedMessage = othersSetting.build();
+        version = othersSetting.getVersion();
+
+      } else { // a tier-1 carrier's setting
+
+        // Load carrier setting
+        CarrierSettings.Builder carrierSetting = null;
+        try (BufferedReader br = Files.newBufferedReader(baseSettingFile.toPath(), UTF_8)) {
+          CarrierSettings.Builder builder = CarrierSettings.newBuilder();
+          TextFormat.getParser().merge(br, registry, builder);
+          carrierSetting = builder;
+        }
+
+        // Apply device overlay
+        carrierSetting =
+            CarrierProtoUtils.applyDeviceOverlayToCarrierSettings(deviceOverlay, carrierSetting);
+
+        // Bump version according to the release
+        carrierSetting.setVersion(
+            CarrierProtoUtils.addVersionOffset(carrierSetting.getVersion(), versionOffset));
+
+        // Convert vendor specific data into binary format
+        // Can be customized
+
+        generatedMessage = carrierSetting.build();
+        version = carrierSetting.getVersion();
+
+      }
+
+      // Output
+      String outFileMainName = childName.replace(TEXT_PB_SUFFIX, "");
+
+      File deviceSettingTextFile = new File(deviceSettingDir, outFileMainName + TEXT_PB_SUFFIX);
+      try (BufferedWriter bw = Files.newBufferedWriter(deviceSettingTextFile.toPath(), UTF_8)) {
+        TextFormat.printUnicode(generatedMessage, bw);
+      }
+
+      if (!deviceInFileName.isEmpty()) {
+        outFileMainName = deviceInFileName + "-" + outFileMainName;
+      }
+      if (versionInFileName) {
+        outFileMainName += "-" + version;
+      }
+      File deviceSettingFile = new File(deviceSettingDir, outFileMainName + PB_SUFFIX);
+      try (OutputStream os = Files.newOutputStream(deviceSettingFile.toPath())) {
+        generatedMessage.writeTo(os);
+      }
+    }
+  }
+
+  private GenDeviceSettings() {}
+}
diff --git a/main.sh b/main.sh
new file mode 100644
index 0000000..ed68108
--- /dev/null
+++ b/main.sh
@@ -0,0 +1,127 @@
+#!/bin/bash
+#
+# Copyright (C) 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+###############################################################################
+# Input files and output directory: can be customized.
+INPUT_CARRIERCONFIG_XML_FILE="packages/apps/CarrierConfig/res/xml/vendor.xml"
+INPUT_CARRIERCONFIG_ASSETS_DIR="packages/apps/CarrierConfig/assets"
+INPUT_APNS_XML_FILE="device/sample/etc/apns-full-conf.xml"
+OUTPUT_DIR="/tmp/CarrierSettings/etc"
+###############################################################################
+
+( # start sub-shell so can use 'set -e' to abort on any failure
+set -e
+
+# 1. Build tools
+echo 'step 1. Building tools ...'
+m CarrierConfigConverterV2 update_apn update_carrier_data GenCarrierList GenDeviceSettings
+echo 'Done.'
+
+echo 'step 2. Converting config files ...'
+# 2a. Create a temp directory as workspace
+TMP_DIR=$(mktemp -d -t cs-XXXXXXX)
+DATA_SETTING_DIR=$TMP_DIR/data/setting
+DATA_DEVICE_DIR=$TMP_DIR/data/device
+ASSETS_DIR=$TMP_DIR/assets
+INNER_TMP_DIR=$TMP_DIR/tmp
+TIER1_CARRIERS_FILE=$TMP_DIR/data/tier1_carriers.textpb
+DEVICE_FILE=$DATA_DEVICE_DIR/device.textpb
+mkdir -p "$DATA_SETTING_DIR" > /dev/null
+mkdir -p "$DATA_DEVICE_DIR" > /dev/null
+mkdir -p "$ASSETS_DIR" > /dev/null
+mkdir -p "$INNER_TMP_DIR" > /dev/null
+touch "$TIER1_CARRIERS_FILE" > /dev/null
+touch "$DEVICE_FILE" > /dev/null
+
+# 2b. Copy input files to workspace
+cp $INPUT_CARRIERCONFIG_XML_FILE "$TMP_DIR"/vendor.xml > /dev/null
+cp $INPUT_CARRIERCONFIG_ASSETS_DIR/* "$ASSETS_DIR"/ > /dev/null
+cp $INPUT_APNS_XML_FILE "$TMP_DIR"/apns-full-conf.xml > /dev/null
+
+# 2c. Convert XMLs to TEXTPB
+
+# DO NOT change the EPOCH date. It's used by CarrierSettings server.
+EPOCH=$(date -d '2018-06-01T00:00:00Z' +%s)
+NOW=$(date +%s)
+TIMESTAMP="$((NOW-EPOCH))"
+
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+UPDATE_APN=$SCRIPT_DIR/bin/update_apn
+[ -x "$UPDATE_APN" ] || UPDATE_APN=out/host/linux-x86/bin/update_apn
+UPDATE_CARRIER_DATA=$SCRIPT_DIR/bin/update_carrier_data
+[ -x "$UPDATE_CARRIER_DATA" ] || UPDATE_CARRIER_DATA=out/host/linux-x86/bin/update_carrier_data
+
+# To use multiple vendor.xml files, just provide multiple `--vendor_xml=___.xml`
+# lines in the command below. The order decides config precedence: a file is
+# overwritten by files AFTER it.
+out/host/linux-x86/bin/CarrierConfigConverterV2 \
+  --output_dir="$TMP_DIR"/data \
+  --vendor_xml="$TMP_DIR"/vendor.xml \
+  --assets="$ASSETS_DIR"/ \
+  --version=$TIMESTAMP > /dev/null
+"$UPDATE_APN" \
+  --apn_file="$TMP_DIR"/apns-full-conf.xml \
+  --data_dir="$TMP_DIR"/data \
+  --out_file="$INNER_TMP_DIR"/apns.textpb
+"$UPDATE_CARRIER_DATA" \
+  --data_dir="$TMP_DIR"/data \
+  --in_file="$INNER_TMP_DIR"/apns.textpb
+
+# 2d. Convert TEXTPB to PB
+mkdir -p "$INNER_TMP_DIR"/pb > /dev/null
+mkdir -p "$INNER_TMP_DIR"/textpb > /dev/null
+
+out/host/linux-x86/bin/GenCarrierList \
+  --version_offset=0 \
+  --with_version_number \
+  --out_pb="$INNER_TMP_DIR"/carrier_list.pb \
+  --in_textpbs="$TIER1_CARRIERS_FILE","$TMP_DIR"/data/other_carriers.textpb \
+  > /dev/null
+mv "$INNER_TMP_DIR"/carrier_list*.pb "$INNER_TMP_DIR"/pb > /dev/null
+mv "$INNER_TMP_DIR"/carrier_list.textpb "$INNER_TMP_DIR"/textpb > /dev/null
+
+for device in "$DATA_DEVICE_DIR"/*.textpb; do
+  [[ -e "$device" ]] || break
+  device=${device%.*} && device=${device##*/} \
+  && device_dir="${INNER_TMP_DIR}/${device}" && mkdir -p "${INNER_TMP_DIR}" \
+  && mkdir -p "${INNER_TMP_DIR}/textpb/${device}" > /dev/null \
+  && out/host/linux-x86/bin/GenDeviceSettings \
+  --device_overlay="$DATA_DEVICE_DIR/${device}.textpb" \
+  --base_setting_dir="$DATA_SETTING_DIR" \
+  --device_setting_dir="${device_dir}" \
+  --version_offset=0 \
+  --with_device_name="${device}" \
+  --with_version_number > /dev/null \
+  && mv "${device_dir}"/*.pb "${INNER_TMP_DIR}/pb" > /dev/null \
+  && mv "${device_dir}"/*.textpb "${INNER_TMP_DIR}/textpb/${device}" > /dev/null \
+  && rmdir "${device_dir}"
+done
+
+echo 'Done.'
+
+echo 'step 3. Copy generated files to output directory ...'
+
+mkdir -p $OUTPUT_DIR > /dev/null
+rm -rf "${OUTPUT_DIR:?}"/* > /dev/null
+cp -r "$INNER_TMP_DIR"/pb  $OUTPUT_DIR > /dev/null
+cp -r "$INNER_TMP_DIR"/textpb  $OUTPUT_DIR > /dev/null
+rm -rf "${TMP_DIR:?}" > /dev/null
+
+echo 'Generated files:'
+find $OUTPUT_DIR -type f
+
+echo 'Done.'
+) # end sub-shell
diff --git a/proto/carrier_list.proto b/proto/carrier_list.proto
new file mode 100644
index 0000000..37d4055
--- /dev/null
+++ b/proto/carrier_list.proto
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+syntax = "proto2";
+
+package com.google.carrier;
+
+option java_multiple_files = true;
+option java_outer_classname = "CarrierListProtos";
+
+// The carrier ID is matched against SIM data to determine carrier
+message CarrierId {
+  // Mobile Country Code (MCC) & Mobile Network Code (MNC)
+  optional string mcc_mnc = 1;
+
+  // Additional data to identify MVNO
+  oneof mvno_data {
+    // SPN (Service Provider Name)
+    string spn = 2;
+
+    // IMSI prefix pattern
+    string imsi = 3;
+
+    // Group identifier (level 1) prefix
+    string gid1 = 4;
+  }
+
+  reserved 5;
+}
+
+// Maps CarrierIds to an internal unique carrier name
+message CarrierMap {
+  // A unique canonical carrier name
+  // This name is the primary key to identify a carrier
+  // Typically a canonical_name looks like <carrier_name>_<iso_country_code>
+  optional string canonical_name = 1;
+
+  // A collection of network IDs owned by this carrier
+  repeated CarrierId carrier_id = 2;
+
+  reserved 3;
+}
+
+// Maps CarrierId to internal unique carrier name
+message CarrierList {
+  // A collection of carrier maps; one entry for one carrier
+  repeated CarrierMap entry = 1;
+
+  // The version number of this CarrierList file
+  optional int64 version = 2;
+}
diff --git a/proto/carrier_settings.proto b/proto/carrier_settings.proto
new file mode 100644
index 0000000..e9f81cd
--- /dev/null
+++ b/proto/carrier_settings.proto
@@ -0,0 +1,214 @@
+/*
+ * Copyright (C) 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+syntax = "proto2";
+
+package com.google.carrier;
+
+option java_multiple_files = true;
+option java_outer_classname = "CarrierSettingsProtos";
+
+// Settings of one carrier, including apns and configs
+// This is the payload to be delivered from server
+message CarrierSettings {
+  // A unique canonical carrier name
+  optional string canonical_name = 1;
+
+  // Version number of current carrier’s settings
+  optional int64 version = 2;
+
+  // Carrier APNs
+  optional CarrierApns apns = 3;
+
+  // Carrier configs
+  optional CarrierConfig configs = 4;
+
+  reserved 5;
+
+  // Vendor carrier configs
+  optional VendorConfigs vendor_configs = 6;
+}
+
+// A collection of multiple carriers’ settings
+message MultiCarrierSettings {
+  // Version number
+  optional int64 version = 1;
+
+  // List of CarrierSettings
+  repeated CarrierSettings setting = 2;
+}
+
+// An access point name (aka. APN) entry
+message ApnItem {
+  // The name of APN, map to xml apn "carrier" attribute
+  // eg. Verizon Internet, may visible to user in Settings
+  optional string name = 1;
+  // The value of APN, eg. send to modem for data call. map to xml
+  // "apn" attribute, eg. vzwinternet
+  optional string value = 2;
+
+  // Next two fields type and bearer_bitmask affect how APN is selected by
+  // platform. eg. type means APN capability and bearer_bitmask specifies
+  // which RATs apply.
+  // Note mcc/mnc and mvno data doesn't belong to this proto because they
+  // define a carrier.
+  // APN types as defined in Android code PhoneConstants.java
+  enum ApnType {
+    ALL = 0;      // this APN can serve all kinds of data connections
+    DEFAULT = 1;  // internet data
+    MMS = 2;
+    SUPL = 3;
+    DUN = 4;
+    HIPRI = 5;
+    FOTA = 6;
+    IMS = 7;
+    CBS = 8;
+    IA = 9;  // Initial attach
+    EMERGENCY = 10;
+    XCAP = 11;
+    UT = 12;
+  }
+  repeated ApnType type = 3;
+
+  // Network types that this APN applies to, separated by "|". A network type
+  // is represented as an integer defined in TelephonyManager.NETWORK_TYPE_*.
+  // Default value "0" means all network types.
+  optional string bearer_bitmask = 4 [default = "0"];
+
+  // Below are all parameters for the APN
+  // APN server / auth parameters.
+  optional string server = 5;
+  optional string proxy = 6;
+  optional string port = 7;
+  optional string user = 8;
+  optional string password = 9;
+  optional int32 authtype = 10 [default = -1];
+
+  // MMS configuration.
+  optional string mmsc = 11;
+  optional string mmsc_proxy = 12;
+  optional string mmsc_proxy_port = 13;
+
+  // Protocols allowed to connect to the APN.
+  enum Protocol {
+    IP = 0;
+    IPV6 = 1;
+    IPV4V6 = 2;
+    PPP = 3;
+  }
+  optional Protocol protocol = 14 [default = IP];
+  optional Protocol roaming_protocol = 15 [default = IP];
+
+  // MTU for the connections.
+  optional int32 mtu = 16 [default = 0];
+  // An ID used to sync the APN in modem.
+  optional int32 profile_id = 17;
+  // Max connections.
+  optional int32 max_conns = 18 [default = 0];
+  // The wait time required between disconnecting and connecting, in seconds.
+  optional int32 wait_time = 19 [default = 0];
+  // The time to limit max connection, in seconds.
+  optional int32 max_conns_time = 20 [default = 0];
+  reserved 21;
+  // Whether to be persisted to modem.
+  optional bool modem_cognitive = 22 [default = false];
+  // Whether visible in APN settings.
+  optional bool user_visible = 23 [default = true];
+  // Whether editable in APN settings.
+  optional bool user_editable = 24 [default = true];
+
+  // If > 0: when an APN becomes a preferred APN on user/framework
+  // selection, other APNs with the same apn_set_id will also be preferred
+  // by framework when selecting APNs.
+  optional int32 apn_set_id = 25 [default = 0];
+
+  // The skip 464xlat flag. Flag works as follows.
+  // SKIP_464XLAT_DEFAULT: the APN will skip 464xlat only if the APN has type
+  //                       IMS and does not support INTERNET which has type
+  //                       DEFAULT or HIPRI.
+  // SKIP_464XLAT_DISABLE: the APN will NOT skip 464xlat
+  // SKIP_464XLAT_ENABLE: the APN will skip 464xlat
+  enum Xlat {
+    SKIP_464XLAT_DEFAULT = 0;
+    SKIP_464XLAT_DISABLE = 1;
+    SKIP_464XLAT_ENABLE = 2;
+  }
+  optional Xlat skip_464xlat = 26 [default = SKIP_464XLAT_DEFAULT];
+}
+
+// A collection of all APNs for a carrier
+message CarrierApns {
+  reserved 1;
+
+  // APNs belong to this carrier
+  repeated ApnItem apn = 2;
+}
+
+// An array of text
+message TextArray {
+  repeated string item = 1;
+}
+
+// An array of int
+message IntArray {
+  repeated int32 item = 1;
+}
+
+// Carrier configs
+message CarrierConfig {
+  reserved 1, 3;
+
+  // Key-Value pair as a config entry
+  message Config {
+    optional string key = 1;
+
+    oneof value {
+      string text_value = 2;
+      int32 int_value = 3;
+      int64 long_value = 4;
+      bool bool_value = 5;
+      TextArray text_array = 6;
+      IntArray int_array = 7;
+    }
+  }
+
+  // Key-value pairs, holding all config entries
+  repeated Config config = 2;
+}
+
+// The configs of one vendor client.
+message VendorConfigClient {
+  // Name of the client for which the configuration items need to
+  // be stored
+  required string name = 1;
+
+  // Binary blob containing the configuration. The format
+  // of the configuration depends on the specific client.
+  // For some clients, the proto representation of {@link VendorConfigData}
+  // defined in vendorconfigdata.proto is used.
+  optional bytes value = 2;
+
+  // Range of extensions. The extensions from 100 to 1000 are reserved for
+  // Google's internal usage.
+  extensions 100 to 5000;
+}
+
+// A collection of configs from vendor clients.
+message VendorConfigs {
+  reserved 1;
+
+  // Configuration
+  repeated VendorConfigClient client = 2;
+}
diff --git a/python/compare.py b/python/compare.py
new file mode 100644
index 0000000..bcf89a7
--- /dev/null
+++ b/python/compare.py
@@ -0,0 +1,58 @@
+# Copyright (C) 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Utility class for handling protobuf message."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from google.protobuf import descriptor
+from six.moves import xrange  # pylint:disable=redefined-builtin
+
+
+def NormalizeRepeatedFields(protobuf_message):
+  """Sorts all repeated fields and removes duplicates.
+
+  Modifies pb in place.
+
+  Args:
+    protobuf_message: The Message object to normalize.
+
+  Returns:
+    protobuf_message, modified in place.
+  """
+  for desc, values in protobuf_message.ListFields():
+    if desc.label is descriptor.FieldDescriptor.LABEL_REPEATED:
+      # Sort then de-dup
+      values.sort()
+      # De-dupe in place. Can't use set, etc. because messages aren't
+      # hashable.
+      for i in xrange(len(values) - 1, 0, -1):
+        if values[i] == values[i - 1]:
+          del values[i]
+
+  return protobuf_message
+
+
+def Proto2Equals(a, b):
+  """Tests if two proto2 objects are equal.
+
+  Recurses into nested messages. Uses list (not set) semantics for comparing
+  repeated fields, ie duplicates and order matter.
+
+  Returns:
+    boolean
+  """
+  return (a.SerializeToString(deterministic=True)
+          == b.SerializeToString(deterministic=True))
diff --git a/python/update_apn.py b/python/update_apn.py
new file mode 100644
index 0000000..67e6e0f
--- /dev/null
+++ b/python/update_apn.py
@@ -0,0 +1,259 @@
+# Copyright (C) 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Read APN conf xml file and output an textpb.
+
+How to run:
+
+update_apn.par --apn_file=./apns-full-conf.xml \
+--data_dir=./data --out_file=/tmpapns.textpb
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import argparse
+import collections
+from xml.dom import minidom
+from google.protobuf import text_format
+
+import carrier_list_pb2
+import carrier_settings_pb2
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    '--apn_file', default='./apns-full-conf.xml', help='Path to APN xml file')
+parser.add_argument(
+    '--data_dir', default='./data', help='Folder path for CarrierSettings data')
+parser.add_argument(
+    '--out_file', default='./tmpapns.textpb', help='Temp APN file')
+FLAGS = parser.parse_args()
+
+CARRIER_LISTS = ['tier1_carriers.textpb', 'other_carriers.textpb']
+
+
+def to_string(cid):
+  """Return a string for CarrierId."""
+  ret = cid.mcc_mnc
+  if cid.HasField('spn'):
+    ret += 'SPN=' + cid.spn.upper()
+  if cid.HasField('imsi'):
+    ret += 'IMSI=' + cid.imsi.upper()
+  if cid.HasField('gid1'):
+    ret += 'GID1=' + cid.gid1.upper()
+  return ret
+
+
+def get_cname(cid, known_carriers):
+  """Return a canonical name based on cid and known_carriers.
+
+  If found a match in known_carriers, return it. Otherwise generate a new one
+  by concating the values.
+
+  Args:
+    cid: proto of CarrierId
+    known_carriers: mapping from mccmnc and possible mvno data to canonical name
+
+  Returns:
+    string for canonical name, like verizon_us or 27402
+  """
+  name = to_string(cid)
+  if name in known_carriers:
+    return known_carriers[name]
+  else:
+    return name
+
+
+def get_knowncarriers(files):
+  """Create a mapping from mccmnc and possible mvno data to canonical name.
+
+  Args:
+    files: list of paths to carrier list textpb files
+
+  Returns:
+    A dict, key is to_string(carrier_id), value is cname.
+  """
+  ret = dict()
+  for path in files:
+    with open(path, 'r', encoding='utf-8') as f:
+      carriers = text_format.Parse(f.read(), carrier_list_pb2.CarrierList())
+      for carriermap in carriers.entry:
+        # could print error if already exist
+        for cid in carriermap.carrier_id:
+          ret[to_string(cid)] = carriermap.canonical_name
+
+  return ret
+
+
+def gen_cid(apn_node):
+  """Generate carrier id proto from APN node.
+
+  Args:
+    apn_node: DOM node from getElementsByTag
+
+  Returns:
+    CarrierId proto
+  """
+  ret = carrier_list_pb2.CarrierId()
+  ret.mcc_mnc = (apn_node.getAttribute('mcc') + apn_node.getAttribute('mnc'))
+  mvno_type = apn_node.getAttribute('mvno_type')
+  mvno_data = apn_node.getAttribute('mvno_match_data')
+  if mvno_type.lower() == 'spn':
+    ret.spn = mvno_data
+  if mvno_type.lower() == 'imsi':
+    ret.imsi = mvno_data
+  # in apn xml, gid means gid1, and no gid2
+  if mvno_type.lower() == 'gid':
+    ret.gid1 = mvno_data
+  return ret
+
+
+APN_TYPE_MAP = {
+    '*': carrier_settings_pb2.ApnItem.ALL,
+    'default': carrier_settings_pb2.ApnItem.DEFAULT,
+    'internet': carrier_settings_pb2.ApnItem.DEFAULT,
+    'vzw800': carrier_settings_pb2.ApnItem.DEFAULT,
+    'mms': carrier_settings_pb2.ApnItem.MMS,
+    'sup': carrier_settings_pb2.ApnItem.SUPL,
+    'supl': carrier_settings_pb2.ApnItem.SUPL,
+    'agps': carrier_settings_pb2.ApnItem.SUPL,
+    'pam': carrier_settings_pb2.ApnItem.DUN,
+    'dun': carrier_settings_pb2.ApnItem.DUN,
+    'hipri': carrier_settings_pb2.ApnItem.HIPRI,
+    'ota': carrier_settings_pb2.ApnItem.FOTA,
+    'fota': carrier_settings_pb2.ApnItem.FOTA,
+    'admin': carrier_settings_pb2.ApnItem.FOTA,
+    'ims': carrier_settings_pb2.ApnItem.IMS,
+    'cbs': carrier_settings_pb2.ApnItem.CBS,
+    'ia': carrier_settings_pb2.ApnItem.IA,
+    'emergency': carrier_settings_pb2.ApnItem.EMERGENCY,
+    'xcap': carrier_settings_pb2.ApnItem.XCAP,
+    'ut': carrier_settings_pb2.ApnItem.UT,
+}
+
+
+def map_apntype(typestr):
+  """Map from APN type string to list of ApnType enums.
+
+  Args:
+    typestr: APN type string in apn conf xml, comma separated
+  Returns:
+    List of ApnType values in ApnItem proto.
+  """
+  typelist = [apn.strip().lower() for apn in typestr.split(',')]
+  return list(set([APN_TYPE_MAP[t] for t in typelist if t]))
+
+
+APN_PROTOCOL_MAP = {
+    'ip': carrier_settings_pb2.ApnItem.IP,
+    'ipv4': carrier_settings_pb2.ApnItem.IP,
+    'ipv6': carrier_settings_pb2.ApnItem.IPV6,
+    'ipv4v6': carrier_settings_pb2.ApnItem.IPV4V6,
+    'ppp': carrier_settings_pb2.ApnItem.PPP
+}
+
+BOOL_MAP = {'true': True, 'false': False, '1': True, '0': False}
+
+APN_SKIPXLAT_MAP = {
+    -1: carrier_settings_pb2.ApnItem.SKIP_464XLAT_DEFAULT,
+    0: carrier_settings_pb2.ApnItem.SKIP_464XLAT_DISABLE,
+    1: carrier_settings_pb2.ApnItem.SKIP_464XLAT_ENABLE
+}
+
+# not include already handeld string keys like mcc, protocol
+APN_STRING_KEYS = [
+    'bearer_bitmask', 'server', 'proxy', 'port', 'user', 'password', 'mmsc',
+    'mmsc_proxy', 'mmsc_proxy_port'
+]
+
+# keys that are different between apn.xml and apn.proto
+APN_REMAP_KEYS = {
+    'mmsproxy': 'mmsc_proxy',
+    'mmsport': 'mmsc_proxy_port'
+}
+
+APN_INT_KEYS = [
+    'authtype', 'mtu', 'profile_id', 'max_conns', 'wait_time', 'max_conns_time'
+]
+
+APN_BOOL_KEYS = [
+    'modem_cognitive', 'user_visible', 'user_editable'
+]
+
+
+def gen_apnitem(node):
+  """Create ApnItem proto based on APN node from xml file.
+
+  Args:
+    node: xml dom node from apn conf xml file.
+
+  Returns:
+    An ApnItem proto.
+  """
+  apn = carrier_settings_pb2.ApnItem()
+  apn.name = node.getAttribute('carrier')
+  apn.value = node.getAttribute('apn')
+  apn.type.extend(map_apntype(node.getAttribute('type')))
+  for key in ['protocol', 'roaming_protocol']:
+    if node.hasAttribute(key):
+      setattr(apn, key, APN_PROTOCOL_MAP[node.getAttribute(key).lower()])
+
+  for key in node.attributes.keys():
+    # Treat bearer as bearer_bitmask if no bearer_bitmask specified
+    if key == 'bearer' and not node.hasAttribute('bearer_bitmask'):
+      setattr(apn, 'bearer_bitmask', node.getAttribute(key))
+      continue
+    if key == 'skip_464xlat':
+      setattr(apn, key, APN_SKIPXLAT_MAP[int(node.getAttribute(key))])
+      continue
+    if key in APN_STRING_KEYS:
+      setattr(apn, key, node.getAttribute(key))
+      continue
+    if key in APN_REMAP_KEYS:
+      setattr(apn, APN_REMAP_KEYS[key], node.getAttribute(key))
+      continue
+    if key in APN_INT_KEYS:
+      setattr(apn, key, int(node.getAttribute(key)))
+      continue
+    if key in APN_BOOL_KEYS:
+      setattr(apn, key, BOOL_MAP[node.getAttribute(key).lower()])
+      continue
+
+  return apn
+
+
+def main():
+  known = get_knowncarriers([FLAGS.data_dir + '/' + f for f in CARRIER_LISTS])
+
+  with open(FLAGS.apn_file, 'r', encoding='utf-8') as apnfile:
+    dom = minidom.parse(apnfile)
+
+  apn_map = collections.defaultdict(list)
+  for apn_node in dom.getElementsByTagName('apn'):
+    cname = get_cname(gen_cid(apn_node), known)
+    apn = gen_apnitem(apn_node)
+    apn_map[cname].append(apn)
+
+  mcs = carrier_settings_pb2.MultiCarrierSettings()
+  for c in apn_map:
+    carriersettings = mcs.setting.add()
+    carriersettings.canonical_name = c
+    carriersettings.apns.apn.extend(apn_map[c])
+
+  with open(FLAGS.out_file, 'w', encoding='utf-8') as apnout:
+    apnout.write(text_format.MessageToString(mcs, as_utf8=True))
+
+
+if __name__ == '__main__':
+  main()
diff --git a/python/update_carrier_data.py b/python/update_carrier_data.py
new file mode 100644
index 0000000..74eee78
--- /dev/null
+++ b/python/update_carrier_data.py
@@ -0,0 +1,473 @@
+# Copyright (C) 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Read a MultiCarrierSettings file and update CarrierSettings data.
+
+For APNs in the input file, they are simply appended to the apn list of the
+corresponding carrier in CarrierSettings data. If a new carrier (identified by
+canonical_name) appears in input, the other_carriers.textpb will be updated.
+
+How to run:
+
+update_carrier_data.par \
+--in_file=/tmp/tmpapns.textpb \
+--data_dir=/tmp/carrier/data
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import argparse
+import copy
+import os
+import compare
+from google.protobuf import text_format
+import carrier_list_pb2
+import carrier_settings_pb2
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    '--data_dir', default='./data', help='Folder path for CarrierSettings data')
+parser.add_argument(
+    '--in_file', default='./tmpapns.textpb', help='Temp APN file')
+FLAGS = parser.parse_args()
+
+TIER1_CARRIERS_TEXTPB = os.path.join(FLAGS.data_dir, 'tier1_carriers.textpb')
+OTHER_CARRIERS_TEXTPB = os.path.join(FLAGS.data_dir, 'other_carriers.textpb')
+
+
+def equals_apn(a, b):
+  """Tell if two ApnItem proto are the same."""
+  a = compare.NormalizeRepeatedFields(copy.deepcopy(a))
+  b = compare.NormalizeRepeatedFields(copy.deepcopy(b))
+  # ignore 'name' field
+  a.ClearField('name')
+  b.ClearField('name')
+  return compare.Proto2Equals(a, b)
+
+
+def find_apn(apn, apns):
+  """Tell if apn is in apns."""
+  for a in apns:
+    if equals_apn(apn, a):
+      return True
+  return False
+
+
+def merge_mms_apn(a, b):
+  """Try to merge mmsc fields of b into a, if that's the only diff."""
+  aa = compare.NormalizeRepeatedFields(copy.deepcopy(a))
+  bb = compare.NormalizeRepeatedFields(copy.deepcopy(b))
+  # check if any fields other than mms are different
+  for field in ['name', 'mmsc_proxy', 'mmsc_proxy_port']:
+    aa.ClearField(field)
+    bb.ClearField(field)
+  if compare.Proto2Equals(aa, bb):
+    for field in ['mmsc_proxy', 'mmsc_proxy_port']:
+      if b.HasField(field):
+        setattr(a, field, getattr(b, field))
+
+
+def clean_apn(setting):
+  """Remove duplicated ApnItems from a CarrierSettings proto.
+
+  Args:
+    setting: a CarrierSettings proto
+
+  Returns:
+    None
+  """
+  if not setting.HasField('apns') or len(setting.apns.apn) <= 1:
+    return
+  apns = setting.apns.apn[:]
+  cleaned_apns = [a for n, a in enumerate(apns) if not find_apn(a, apns[:n])]
+  del setting.apns.apn[:]
+  setting.apns.apn.extend(cleaned_apns)
+
+
+def merge_apns(dest_apns, source_apns):
+  """Merge source_apns into dest_apns."""
+  for apn in dest_apns:
+    for source in source_apns:
+      merge_mms_apn(apn, source)
+  ret = list(dest_apns)
+  for source in source_apns:
+    if not find_apn(source, ret):
+      ret.append(source)
+  return ret
+
+
+def to_string(cid):
+  """Return the string representation of a CarrierId."""
+  ret = cid.mcc_mnc
+  if cid.HasField('spn'):
+    ret += 'SPN=' + cid.spn.upper()
+  if cid.HasField('imsi'):
+    ret += 'IMSI=' + cid.imsi.upper()
+  if cid.HasField('gid1'):
+    ret += 'GID1=' + cid.gid1.upper()
+  return ret
+
+
+def to_carrier_id(cid_string):
+  """Return the CarrierId from its string representation."""
+  cid = carrier_list_pb2.CarrierId()
+  if 'SPN=' in cid_string:
+    ind = cid_string.find('SPN=')
+    cid.mcc_mnc = cid_string[:ind]
+    cid.spn = cid_string[ind + len('SPN='):]
+  elif 'IMSI=' in cid_string:
+    ind = cid_string.find('IMSI=')
+    cid.mcc_mnc = cid_string[:ind]
+    cid.imsi = cid_string[ind + len('IMSI='):]
+  elif 'GID1=' in cid_string:
+    ind = cid_string.find('GID1=')
+    cid.mcc_mnc = cid_string[:ind]
+    cid.gid1 = cid_string[ind + len('GID1='):]
+  else:
+    cid.mcc_mnc = cid_string
+  return cid
+
+
+def get_input(path):
+  """Read input MultiCarrierSettings textpb file.
+
+  Args:
+    path: the path to input MultiCarrierSettings textpb file
+
+  Returns:
+    A MultiCarrierSettings. None when failed.
+  """
+  mcs = None
+  with open(path, 'r', encoding='utf-8') as f:
+    mcs = carrier_settings_pb2.MultiCarrierSettings()
+    text_format.Merge(f.read(), mcs)
+
+  return mcs
+
+
+def get_knowncarriers(files):
+  """Create a mapping from mccmnc and possible mvno data to canonical name.
+
+  Args:
+    files: list of paths to carrier list textpb files
+
+  Returns:
+    A dict, key is to_string(carrier_id), value is cname.
+  """
+  ret = dict()
+  for path in files:
+    with open(path, 'r', encoding='utf-8') as f:
+      carriers = carrier_list_pb2.CarrierList()
+      text_format.Merge(f.read(), carriers)
+      for carriermap in carriers.entry:
+        for cid in carriermap.carrier_id:
+          ret[to_string(cid)] = carriermap.canonical_name
+
+  return ret
+
+
+def clear_apn_fields_in_default_value(carrier_settings):
+
+  def clean(apn):
+    if apn.HasField('bearer_bitmask') and apn.bearer_bitmask == '0':
+      apn.ClearField('bearer_bitmask')
+    return apn
+
+  for apn in carrier_settings.apns.apn:
+    clean(apn)
+  return carrier_settings
+
+
+def merge_carrier_settings(patch, carrier_file):
+  """Merge a CarrierSettings into a base CarrierSettings in textpb file.
+
+  This function merge apns only. It assumes that the patch and base have the
+  same canonical_name.
+
+  Args:
+    patch: the carrier_settings to be merged
+    carrier_file: the path to the base carrier_settings file
+  """
+  # Load base
+  with open(carrier_file, 'r', encoding='utf-8') as f:
+    base_setting = text_format.ParseLines(f,
+                                          carrier_settings_pb2.CarrierSettings())
+
+  clean_apn(patch)
+  clean_apn(base_setting)
+
+  # Merge apns
+  apns = base_setting.apns.apn[:]
+  apns = merge_apns(apns, patch.apns.apn[:])
+  del base_setting.apns.apn[:]
+  base_setting.apns.apn.extend(apns)
+
+  # Write back
+  with open(carrier_file, 'w', encoding='utf-8') as f:
+    text_format.PrintMessage(base_setting, f, as_utf8=True)
+
+
+def merge_multi_carrier_settings(patch_list, carrier_file):
+  """Merge CarrierSettings into a base MultiCarrierSettings in textpb file.
+
+  This function merge apns only. The base may or may not contains an entry with
+  the same canonical_name as the patch.
+
+  Args:
+    patch_list: a list of CarrierSettings to be merged
+    carrier_file: the path to the base MultiCarrierSettings file
+  """
+  # Load base
+  with open(carrier_file, 'r', encoding='utf-8') as f:
+    base_settings = text_format.ParseLines(
+        f, carrier_settings_pb2.MultiCarrierSettings())
+
+  for patch in patch_list:
+    clean_apn(patch)
+    # find the (first and only) entry with patch.canonical_name and update it.
+    for setting in base_settings.setting:
+      if setting.canonical_name == patch.canonical_name:
+        clean_apn(setting)
+        apns = setting.apns.apn[:]
+        apns = merge_apns(apns, patch.apns.apn[:])
+        del setting.apns.apn[:]
+        setting.apns.apn.extend(apns)
+        break
+    # Or if no match, append it to base_settings
+    else:
+      base_settings.setting.extend([patch])
+
+  # Write back
+  with open(carrier_file, 'w', encoding='utf-8') as f:
+    text_format.PrintMessage(base_settings, f, as_utf8=True)
+
+
+def add_new_carriers(cnames, carrier_list_file):
+  """Add a new carrier into a CarrierList in textpb file.
+
+  The carrier_id of the new carrier is induced from the cname, assuming
+  that the cname is constructed by to_string.
+
+  Args:
+    cnames: a list of canonical_name of new carriers
+    carrier_list_file: the path to the CarrierList textpb file
+
+  Returns:
+    None
+  """
+  with open(carrier_list_file, 'r', encoding='utf-8') as f:
+    carriers = text_format.ParseLines(f, carrier_list_pb2.CarrierList())
+
+  for cname in cnames:
+    # Append the new carrier
+    new_carrier = carriers.entry.add()
+    new_carrier.canonical_name = cname
+    new_carrier.carrier_id.extend([to_carrier_id(cname)])
+
+  tmp = sorted(carriers.entry, key=lambda c: c.canonical_name)
+  del carriers.entry[:]
+  carriers.entry.extend(tmp)
+
+  with open(carrier_list_file, 'w', encoding='utf-8') as f:
+    text_format.PrintMessage(carriers, f, as_utf8=True)
+
+
+def add_apns_for_other_carriers_by_mccmnc(apns, tier1_carriers, other_carriers):
+  """Add APNs for carriers in others.textpb that doesn't have APNs, by mccmnc.
+
+  If a carrier defined as mccmnc + mvno_data doesn't hava APNs, it should use
+  APNs from the carrier defined as mccmnc only.
+
+  Modifies others.textpb file in-place.
+
+  Args:
+    apns: a list of CarrierSettings message with APNs only.
+    tier1_carriers: parsed tier-1 carriers list; must not contain new carriers.
+      A dict, key is to_string(carrier_id), value is cname.
+    other_carriers: parsed other carriers list; must not contain new carriers. A
+      dict, key is to_string(carrier_id), value is cname.
+  """
+  # Convert apns from a list to a map, key being the canonical_name
+  apns_dict = {
+      carrier_settings.canonical_name: carrier_settings
+      for carrier_settings in apns
+  }
+
+  others_textpb = '%s/setting/others.textpb' % FLAGS.data_dir
+  with open(others_textpb, 'r', encoding='utf-8') as others_textpb_file:
+    others = text_format.ParseLines(others_textpb_file,
+                                    carrier_settings_pb2.MultiCarrierSettings())
+
+  for setting in others.setting:
+    if not setting.HasField('apns'):
+      carrier_id = to_carrier_id(setting.canonical_name)
+      if carrier_id.HasField('mvno_data'):
+        carrier_id.ClearField('mvno_data')
+        carrier_id_str_of_mccmnc = to_string(carrier_id)
+        cname_of_mccmnc = tier1_carriers.get(
+            carrier_id_str_of_mccmnc) or other_carriers.get(
+                carrier_id_str_of_mccmnc)
+        if cname_of_mccmnc:
+          apn = apns_dict.get(cname_of_mccmnc)
+          if apn:
+            setting.apns.CopyFrom(apn.apns)
+
+  with open(others_textpb, 'w', encoding='utf-8') as others_textpb_file:
+    text_format.PrintMessage(others, others_textpb_file, as_utf8=True)
+
+
+def add_carrierconfig_for_new_carriers(cnames, tier1_carriers, other_carriers):
+  """Add carrier configs for new (non-tier-1) carriers.
+
+   For new carriers, ie. carriers existing in APN but not CarrierConfig:
+      - for <mccmnc>: copy carrier config of <mcc>.
+      - for <mccmnc>(GID1|SPN|IMSI)=<mvnodata>: copy carrier config of <mccmnc>,
+      or <mcc>.
+
+  Modifies others.textpb file in-place.
+
+  Args:
+    cnames: a list of canonical_name of new carriers.
+    tier1_carriers: parsed tier-1 carriers list; must not contain new carriers.
+      A dict, key is to_string(carrier_id), value is cname.
+    other_carriers: parsed other carriers list; must not contain new carriers. A
+      dict, key is to_string(carrier_id), value is cname.
+  """
+  carrier_configs_map = {}
+
+  others_textpb = '%s/setting/others.textpb' % FLAGS.data_dir
+  with open(others_textpb, 'r', encoding='utf-8') as others_textpb_file:
+    others = text_format.ParseLines(others_textpb_file,
+                                    carrier_settings_pb2.MultiCarrierSettings())
+    for setting in others.setting:
+      if setting.canonical_name in other_carriers:
+        carrier_configs_map[setting.canonical_name] = setting.configs
+  for cid_str, cname in tier1_carriers.items():
+    tier1_textpb = '%s/setting/%s.textpb' % (FLAGS.data_dir, cname)
+    with open(tier1_textpb, 'r', encoding='utf-8') as tier1_textpb_file:
+      tier1 = text_format.ParseLines(tier1_textpb_file,
+                                     carrier_settings_pb2.CarrierSettings())
+      carrier_configs_map[cid_str] = tier1.configs
+
+  for setting in others.setting:
+    if setting.canonical_name in cnames:
+      carrier_id = to_carrier_id(setting.canonical_name)
+      mccmnc = carrier_id.mcc_mnc
+      mcc = mccmnc[:3]
+      if mccmnc in carrier_configs_map:
+        setting.configs.config.extend(carrier_configs_map[mccmnc].config[:])
+      elif mcc in carrier_configs_map:
+        setting.configs.config.extend(carrier_configs_map[mcc].config[:])
+
+  with open(others_textpb, 'w', encoding='utf-8') as others_textpb_file:
+    text_format.PrintMessage(others, others_textpb_file, as_utf8=True)
+
+
+def cleanup_mcc_only_carriers():
+  """Removes mcc-only carriers from other_carriers.textpb & others.textpb.
+
+  Modifies other_carriers.textpb file & others.textpb file in-place.
+  """
+  mcc_only_carriers = set()
+
+  with open(
+      OTHER_CARRIERS_TEXTPB, 'r',
+      encoding='utf-8') as other_carriers_textpb_file:
+    other_carriers = text_format.ParseLines(other_carriers_textpb_file,
+                                            carrier_list_pb2.CarrierList())
+
+  other_carriers_entry_with_mccmnc = []
+  for carrier in other_carriers.entry:
+    for carrier_id in carrier.carrier_id:
+      if len(carrier_id.mcc_mnc) == 3:
+        mcc_only_carriers.add(carrier.canonical_name)
+      else:
+        other_carriers_entry_with_mccmnc.append(carrier)
+  del other_carriers.entry[:]
+  other_carriers.entry.extend(other_carriers_entry_with_mccmnc)
+
+  # Finish early if no mcc_only_carriers; that means no file modification
+  # required.
+  if not mcc_only_carriers:
+    return
+
+  with open(
+      OTHER_CARRIERS_TEXTPB, 'w',
+      encoding='utf-8') as other_carriers_textpb_file:
+    text_format.PrintMessage(
+        other_carriers, other_carriers_textpb_file, as_utf8=True)
+
+  others_textpb = os.path.join(FLAGS.data_dir, 'setting', 'others.textpb')
+  with open(others_textpb, 'r', encoding='utf-8') as others_textpb_file:
+    others = text_format.ParseLines(others_textpb_file,
+                                    carrier_settings_pb2.MultiCarrierSettings())
+  copy_others_setting = others.setting[:]
+  del others.setting[:]
+  others.setting.extend([
+      setting for setting in copy_others_setting
+      if setting.canonical_name not in mcc_only_carriers
+  ])
+
+  with open(others_textpb, 'w', encoding='utf-8') as others_textpb_file:
+    text_format.PrintMessage(others, others_textpb_file, as_utf8=True)
+
+
+def main():
+  apns = get_input(FLAGS.in_file).setting
+  tier1_carriers = get_knowncarriers([TIER1_CARRIERS_TEXTPB])
+  other_carriers = get_knowncarriers([OTHER_CARRIERS_TEXTPB])
+  new_carriers = []
+
+  # Step 1a: merge APNs into CarrierConfigs by canonical name.
+  # Also find out "new carriers" existing in APNs but not in CarrierConfigs.
+  other_carriers_patch = []
+  for carrier_settings in apns:
+    carrier_settings = clear_apn_fields_in_default_value(carrier_settings)
+
+    cname = carrier_settings.canonical_name
+    if cname in tier1_carriers.values():
+      merge_carrier_settings(carrier_settings,
+                             '%s/setting/%s.textpb' % (FLAGS.data_dir, cname))
+    else:
+      other_carriers_patch.append(carrier_settings)
+      if cname not in other_carriers.values():
+        new_carriers.append(cname)
+
+  merge_multi_carrier_settings(other_carriers_patch,
+                               '%s/setting/others.textpb' % FLAGS.data_dir)
+
+  # Step 1b: populate carrier configs for new carriers.
+  add_carrierconfig_for_new_carriers(new_carriers, tier1_carriers,
+                                     other_carriers)
+
+  # Step 2: merge new carriers into non-tier1 carrier list.
+  add_new_carriers(new_carriers, OTHER_CARRIERS_TEXTPB)
+  # Update other_carriers map
+  other_carriers = get_knowncarriers([OTHER_CARRIERS_TEXTPB])
+
+  # Step 3: merge APNs into CarrierConfigs by mccmnc: for a carrier defined
+  # as mccmnc + gid/spn/imsi, if it doesn't have any APNs, it should use APNs
+  # from carrier defined as mccmnc only.
+  # Only handle non-tier1 carriers, as tier1 carriers are assumed to be better
+  # maintained and are already having APNs defined.
+  add_apns_for_other_carriers_by_mccmnc(apns, tier1_carriers, other_carriers)
+
+  # Step 4: clean up mcc-only carriers; they're used in step 3 but should not
+  # be in final carrier settings to avoid confusing CarrierSettings app.
+  cleanup_mcc_only_carriers()
+
+
+if __name__ == '__main__':
+  main()