Update NotoColorEmojiCompat to 2.034 (Emoji 14)

Bug: 208665210
Test: ./gradlew :emoji2:emoji2-bundled:cAT :emoji:emoji-bundled:cAT
Change-Id: I9e4687f5cf87224faaa55ab9955e509087eaf71e

4 files changed, 297 insertions, 10 deletions

diff --git a/emoji-compat/createfont.py b/emoji-compat/createfont.py
index 39204f8..f694cf2 100755
--- a/emoji-compat/createfont.py
+++ b/emoji-compat/createfont.py
@@ -49,19 +49,22 @@ import hashlib
 import itertools
 import json
 import os
+import re
 import shutil
 import subprocess
 import sys
 import tempfile
 from fontTools import ttLib
+from fontTools.ttLib.tables import otTables
+from nototools import font_data
 
 ########### UPDATE OR CHECK WHEN A NEW FONT IS BEING GENERATED ###########
 # Last Android SDK Version
-SDK_VERSION = 30
+SDK_VERSION = 31
 # metadata version that will be embedded into font. If there are updates to the font that would
 # cause data/emoji_metadata.txt to change, this integer number should be incremented. This number
 # defines in which EmojiCompat metadata version the emoji is added to the font.
-METADATA_VERSION = 7
+METADATA_VERSION = 8
 
 ####### main directories where output files are created #######
 SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
@@ -120,6 +123,12 @@ STD_VARIANTS_EMOJI_STYLE = 'EMOJI STYLE'
 DEFAULT_EMOJI_ID = 0xF0001
 EMOJI_STYLE_VS = 0xFE0F
 
+# The reference code point to be used for filling metrics of wartermark glyph
+WATERMARK_REF_CODE_POINT = 0x1F600
+# The code point and glyph name used for watermark.
+WATERMARK_NEW_CODE_POINT = 0x10FF00
+WATERMARK_NEW_GLYPH_ID = 'u10FF00'
+
 def to_hex_str(value):
     """Converts given int value to hex without the 0x prefix"""
     return format(value, 'X')
@@ -141,6 +150,12 @@ def prepend_header_to_file(file_path, header_path):
             original_file.seek(0)
             original_file.write(copyright_file.read() + "\n" + original_content)
 
+def is_ri(codepoint):
+  return 0x1F1E6 <= codepoint and codepoint <= 0x1F1FF
+
+def is_flag_seq(codepoints):
+  return all(is_ri(x) for x in codepoints)
+
 
 def update_flatbuffer_java_files(flatbuffer_java_dir, header_dir, target_dir):
     """Prepends headers to flatbuffer java files and copies to the final destination"""
@@ -338,7 +353,7 @@ def read_emoji_intervals(emoji_data_map, file_path, emoji_style_exceptions):
                 emoji_data_map[key] = emoji_data
 
 
-def read_emoji_sequences(emoji_data_map, file_path, optional=False):
+def read_emoji_sequences(emoji_data_map, file_path, optional=False, filter=None):
     """Reads the content of the file which contains emoji sequences. Creates EmojiData for each
     line and puts into emoji_data_map."""
     lines = read_emoji_lines(file_path, optional)
@@ -351,22 +366,29 @@ def read_emoji_sequences(emoji_data_map, file_path, optional=False):
             continue
         codepoints = [hex_str_to_int(x) for x in line.split(';')[0].strip().split(' ')]
         codepoints = [x for x in codepoints if x != EMOJI_STYLE_VS]
+        if filter:
+          if filter(codepoints):
+            continue
         key = codepoint_to_string(codepoints)
         if not key in emoji_data_map:
             emoji_data = _EmojiData(codepoints, False)
             emoji_data_map[key] = emoji_data
 
 
-def load_emoji_data_map(unicode_path):
+def load_emoji_data_map(unicode_path, without_flags):
     """Reads the emoji data files, constructs a map of space separated codepoints to EmojiData.
     :return: map of space separated codepoints to EmojiData
     """
+    if without_flags:
+      filter = lambda x: is_flag_seq(x)
+    else:
+      filter = None
     emoji_data_map = {}
     emoji_style_exceptions = get_emoji_style_exceptions(unicode_path)
     read_emoji_intervals(emoji_data_map, os.path.join(unicode_path, EMOJI_DATA_FILE),
                          emoji_style_exceptions)
     read_emoji_sequences(emoji_data_map, os.path.join(unicode_path, EMOJI_ZWJ_FILE))
-    read_emoji_sequences(emoji_data_map, os.path.join(unicode_path, EMOJI_SEQ_FILE))
+    read_emoji_sequences(emoji_data_map, os.path.join(unicode_path, EMOJI_SEQ_FILE), filter=filter)
 
     # Add the optional ANDROID_EMOJI_ZWJ_SEQ_FILE if it exists.
     read_emoji_sequences(emoji_data_map, os.path.join(unicode_path, ANDROID_EMOJI_ZWJ_SEQ_FILE),
@@ -472,11 +494,12 @@ def create_sha_from_source_files(font_paths):
 class EmojiFontCreator(object):
     """Creates the EmojiCompat font"""
 
-    def __init__(self, font_path, unicode_path):
+    def __init__(self, font_path, unicode_path, without_flags):
         validate_input_files(font_path, unicode_path, FLATBUFFER_MODULE_DIR)
 
         self.font_path = font_path
         self.unicode_path = unicode_path
+        self.without_flags = without_flags
         self.emoji_data_map = {}
         self.remapped_codepoints = {}
         self.glyph_to_image_metrics_map = {}
@@ -594,6 +617,8 @@ class EmojiFontCreator(object):
 
         total_emoji_count = 0
         for emoji_data in emoji_data_list:
+            if self.without_flags and is_flag_seq(emoji_data.codepoints):
+                continue  # Do not add flags emoji data if this is for subset font.
             element = emoji_data.create_json_element()
             output_json['list'].append(element)
             total_emoji_count = total_emoji_count + 1
@@ -614,6 +639,46 @@ class EmojiFontCreator(object):
             for emoji_data in emoji_data_list:
                 csvwriter.writerow(emoji_data.create_txt_row())
 
+    def add_watermark(self, ttf):
+        cmap = ttf.getBestCmap()
+        gsub = ttf['GSUB'].table
+
+        # Obtain Version string
+        m = re.search('^Version (\d*)\.(\d*)', font_data.font_version(ttf))
+        if not m:
+            raise ValueError('The font does not have proper version string.')
+        major = m.group(1)
+        minor = m.group(2)
+        # Replace the dot with space since NotoColorEmoji does not have glyph for dot.
+        glyphs = [cmap[ord(x)] for x in '%s %s' % (major, minor)]
+
+        # Update Glyph metrics
+        ttf.getGlyphOrder().append(WATERMARK_NEW_GLYPH_ID)
+        refGlyphId = cmap[WATERMARK_REF_CODE_POINT]
+        ttf['hmtx'].metrics[WATERMARK_NEW_GLYPH_ID] = ttf['hmtx'].metrics[refGlyphId]
+        ttf['vmtx'].metrics[WATERMARK_NEW_GLYPH_ID] = ttf['vmtx'].metrics[refGlyphId]
+
+        # Add new Glyph to cmap
+        font_data.add_to_cmap(ttf, { WATERMARK_NEW_CODE_POINT : WATERMARK_NEW_GLYPH_ID })
+
+        # Add lookup table for the version string.
+        lookups = gsub.LookupList.Lookup
+        new_lookup = otTables.Lookup()
+        new_lookup.LookupType = 2  # Multiple Substitution Subtable.
+        new_lookup.LookupFlag = 0
+        new_subtable = otTables.MultipleSubst()
+        new_subtable.mapping = { WATERMARK_NEW_GLYPH_ID : tuple(glyphs) }
+        new_lookup.SubTable = [ new_subtable ]
+        new_lookup_index = len(lookups)
+        lookups.append(new_lookup)
+
+        # Add feature
+        feature = next(x for x in gsub.FeatureList.FeatureRecord if x.FeatureTag == 'ccmp')
+        if not feature:
+            raise ValueError("Font doesn't contain ccmp feature.")
+
+        feature.Feature.LookupListIndex.append(new_lookup_index)
+
     def create_font(self):
         """Creates the EmojiCompat font.
         :param font_path: path to Android NotoColorEmoji font
@@ -623,7 +688,7 @@ class EmojiFontCreator(object):
         tmp_dir = tempfile.mkdtemp()
 
         # create emoji codepoints to EmojiData map
-        self.emoji_data_map = load_emoji_data_map(self.unicode_path)
+        self.emoji_data_map = load_emoji_data_map(self.unicode_path, self.without_flags)
 
         # read previous metadata file to update id, sdkAdded and compatAdded. emoji id that is
         # returned is either default or 1 greater than the largest id in previous data
@@ -671,6 +736,9 @@ class EmojiFontCreator(object):
             # inject metadata binary into font
             inject_meta_into_font(ttf, flatbuffer_bin_file)
 
+            # add wartermark glyph for manual verification.
+            self.add_watermark(ttf)
+
             # update CBDT and CBLC versions since older android versions cannot read > 2.0
             ttf['CBDT'].version = 2.0
             ttf['CBLC'].version = 2.0
@@ -698,14 +766,19 @@ def print_usage():
 
 def parse_args(argv):
     # parse manually to avoid any extra dependencies
+    if len(argv) == 4:
+      without_flags = argv[3] == '--without-flags'
+    else:
+      without_flags = False
+
     if len(argv) < 3:
         print_usage()
         sys.exit(1)
-    return (sys.argv[1], sys.argv[2])
+    return (sys.argv[1], sys.argv[2], without_flags)
 
 def main():
-    font_file, unicode_dir = parse_args(sys.argv)
-    EmojiFontCreator(font_file, unicode_dir).create_font()
+    font_file, unicode_dir, without_flags = parse_args(sys.argv)
+    EmojiFontCreator(font_file, unicode_dir, without_flags).create_font()
 
 
 if __name__ == '__main__':
diff --git a/emoji-compat/data/emoji_metadata.txt b/emoji-compat/data/emoji_metadata.txt
index 6bee6fa..d57d34e 100644
--- a/emoji-compat/data/emoji_metadata.txt
+++ b/emoji-compat/data/emoji_metadata.txt
@@ -3578,3 +3578,110 @@ F0E2C 30 7 1F9D4 1F3FF 200D 2640
 F0E2D 30 7 1F9D4 1F3FF 200D 2642

 F0E2E 30 7 1F9D4 200D 2640

 F0E2F 30 7 1F9D4 200D 2642

+F0E30 31 8 1F6DD

+F0E31 31 8 1F6DE

+F0E32 31 8 1F6DF

+F0E33 31 8 1F7F0

+F0E34 31 8 1F979

+F0E35 31 8 1F9CC

+F0E36 31 8 1FA7B

+F0E37 31 8 1FA7C

+F0E38 31 8 1FAA9

+F0E39 31 8 1FAAA

+F0E3A 31 8 1FAAB

+F0E3B 31 8 1FAAC

+F0E3C 31 8 1FAB7

+F0E3D 31 8 1FAB8

+F0E3E 31 8 1FAB9

+F0E3F 31 8 1FABA

+F0E40 31 8 1FAC3

+F0E41 31 8 1FAC4

+F0E42 31 8 1FAC5

+F0E43 31 8 1FAD7

+F0E44 31 8 1FAD8

+F0E45 31 8 1FAD9

+F0E46 31 8 1FAE0

+F0E47 31 8 1FAE1

+F0E48 31 8 1FAE2

+F0E49 31 8 1FAE3

+F0E4A 31 8 1FAE4

+F0E4B 31 8 1FAE5

+F0E4C 31 8 1FAE6

+F0E4D 31 8 1FAE7

+F0E4E 31 8 1FAF0

+F0E4F 31 8 1FAF1

+F0E50 31 8 1FAF2

+F0E51 31 8 1FAF3

+F0E52 31 8 1FAF4

+F0E53 31 8 1FAF5

+F0E54 31 8 1FAF6

+F0E55 31 8 1FAC3 1F3FB

+F0E56 31 8 1FAC3 1F3FC

+F0E57 31 8 1FAC3 1F3FD

+F0E58 31 8 1FAC3 1F3FE

+F0E59 31 8 1FAC3 1F3FF

+F0E5A 31 8 1FAC4 1F3FB

+F0E5B 31 8 1FAC4 1F3FC

+F0E5C 31 8 1FAC4 1F3FD

+F0E5D 31 8 1FAC4 1F3FE

+F0E5E 31 8 1FAC4 1F3FF

+F0E5F 31 8 1FAC5 1F3FB

+F0E60 31 8 1FAC5 1F3FC

+F0E61 31 8 1FAC5 1F3FD

+F0E62 31 8 1FAC5 1F3FE

+F0E63 31 8 1FAC5 1F3FF

+F0E64 31 8 1FAF0 1F3FB

+F0E65 31 8 1FAF0 1F3FC

+F0E66 31 8 1FAF0 1F3FD

+F0E67 31 8 1FAF0 1F3FE

+F0E68 31 8 1FAF0 1F3FF

+F0E69 31 8 1FAF1 1F3FB 200D 1FAF2 1F3FC

+F0E6A 31 8 1FAF1 1F3FB 200D 1FAF2 1F3FD

+F0E6B 31 8 1FAF1 1F3FB 200D 1FAF2 1F3FE

+F0E6C 31 8 1FAF1 1F3FB 200D 1FAF2 1F3FF

+F0E6D 31 8 1FAF1 1F3FC 200D 1FAF2 1F3FB

+F0E6E 31 8 1FAF1 1F3FC 200D 1FAF2 1F3FD

+F0E6F 31 8 1FAF1 1F3FC 200D 1FAF2 1F3FE

+F0E70 31 8 1FAF1 1F3FC 200D 1FAF2 1F3FF

+F0E71 31 8 1FAF1 1F3FD 200D 1FAF2 1F3FB

+F0E72 31 8 1FAF1 1F3FD 200D 1FAF2 1F3FC

+F0E73 31 8 1FAF1 1F3FD 200D 1FAF2 1F3FE

+F0E74 31 8 1FAF1 1F3FD 200D 1FAF2 1F3FF

+F0E75 31 8 1FAF1 1F3FE 200D 1FAF2 1F3FB

+F0E76 31 8 1FAF1 1F3FE 200D 1FAF2 1F3FC

+F0E77 31 8 1FAF1 1F3FE 200D 1FAF2 1F3FD

+F0E78 31 8 1FAF1 1F3FE 200D 1FAF2 1F3FF

+F0E79 31 8 1FAF1 1F3FF 200D 1FAF2 1F3FB

+F0E7A 31 8 1FAF1 1F3FF 200D 1FAF2 1F3FC

+F0E7B 31 8 1FAF1 1F3FF 200D 1FAF2 1F3FD

+F0E7C 31 8 1FAF1 1F3FF 200D 1FAF2 1F3FE

+F0E7D 31 8 1FAF1 1F3FB

+F0E7E 31 8 1FAF1 1F3FC

+F0E7F 31 8 1FAF1 1F3FD

+F0E80 31 8 1FAF1 1F3FE

+F0E81 31 8 1FAF1 1F3FF

+F0E82 31 8 1FAF2 1F3FB

+F0E83 31 8 1FAF2 1F3FC

+F0E84 31 8 1FAF2 1F3FD

+F0E85 31 8 1FAF2 1F3FE

+F0E86 31 8 1FAF2 1F3FF

+F0E87 31 8 1FAF3 1F3FB

+F0E88 31 8 1FAF3 1F3FC

+F0E89 31 8 1FAF3 1F3FD

+F0E8A 31 8 1FAF3 1F3FE

+F0E8B 31 8 1FAF3 1F3FF

+F0E8C 31 8 1FAF4 1F3FB

+F0E8D 31 8 1FAF4 1F3FC

+F0E8E 31 8 1FAF4 1F3FD

+F0E8F 31 8 1FAF4 1F3FE

+F0E90 31 8 1FAF4 1F3FF

+F0E91 31 8 1FAF5 1F3FB

+F0E92 31 8 1FAF5 1F3FC

+F0E93 31 8 1FAF5 1F3FD

+F0E94 31 8 1FAF5 1F3FE

+F0E95 31 8 1FAF5 1F3FF

+F0E96 31 8 1FAF6 1F3FB

+F0E97 31 8 1FAF6 1F3FC

+F0E98 31 8 1FAF6 1F3FD

+F0E99 31 8 1FAF6 1F3FE

+F0E9A 31 8 1FAF6 1F3FF

diff --git a/emoji-compat/font/NotoColorEmojiCompat.ttf b/emoji-compat/font/NotoColorEmojiCompat.ttf
index 9cc330f..7334ae8 100644
--- a/emoji-compat/font/NotoColorEmojiCompat.ttf
+++ b/emoji-compat/font/NotoColorEmojiCompat.ttf
diff --git a/emoji-compat/supported-emojis/emojis.txt b/emoji-compat/supported-emojis/emojis.txt
index a4818c0..a3ac299 100644
--- a/emoji-compat/supported-emojis/emojis.txt
+++ b/emoji-compat/supported-emojis/emojis.txt
@@ -2388,6 +2388,9 @@
 1F6D5
 1F6D6
 1F6D7
+1F6DD
+1F6DE
+1F6DF
 1F6E0 FE0F
 1F6E1 FE0F
 1F6E2 FE0F
@@ -2420,6 +2423,7 @@
 1F7E9
 1F7EA
 1F7EB
+1F7F0
 1F90C
 1F90C 1F3FB
 1F90C 1F3FC
@@ -2748,6 +2752,7 @@
 1F977 1F3FE
 1F977 1F3FF
 1F978
+1F979
 1F97A
 1F97B
 1F97C
@@ -2879,6 +2884,7 @@
 1F9C9
 1F9CA
 1F9CB
+1F9CC
 1F9CD
 1F9CD 1F3FB
 1F9CD 1F3FB 200D 2640 FE0F
@@ -3382,6 +3388,8 @@
 1FA78
 1FA79
 1FA7A
+1FA7B
+1FA7C
 1FA80
 1FA81
 1FA82
@@ -3414,6 +3422,10 @@
 1FAA6
 1FAA7
 1FAA8
+1FAA9
+1FAAA
+1FAAB
+1FAAC
 1FAB0
 1FAB1
 1FAB2
@@ -3421,9 +3433,31 @@
 1FAB4
 1FAB5
 1FAB6
+1FAB7
+1FAB8
+1FAB9
+1FABA
 1FAC0
 1FAC1
 1FAC2
+1FAC3
+1FAC3 1F3FB
+1FAC3 1F3FC
+1FAC3 1F3FD
+1FAC3 1F3FE
+1FAC3 1F3FF
+1FAC4
+1FAC4 1F3FB
+1FAC4 1F3FC
+1FAC4 1F3FD
+1FAC4 1F3FE
+1FAC4 1F3FF
+1FAC5
+1FAC5 1F3FB
+1FAC5 1F3FC
+1FAC5 1F3FD
+1FAC5 1F3FE
+1FAC5 1F3FF
 1FAD0
 1FAD1
 1FAD2
@@ -3431,6 +3465,79 @@
 1FAD4
 1FAD5
 1FAD6
+1FAD7
+1FAD8
+1FAD9
+1FAE0
+1FAE1
+1FAE2
+1FAE3
+1FAE4
+1FAE5
+1FAE6
+1FAE7
+1FAF0
+1FAF0 1F3FB
+1FAF0 1F3FC
+1FAF0 1F3FD
+1FAF0 1F3FE
+1FAF0 1F3FF
+1FAF1
+1FAF1 1F3FB
+1FAF1 1F3FB 200D 1FAF2 1F3FC
+1FAF1 1F3FB 200D 1FAF2 1F3FD
+1FAF1 1F3FB 200D 1FAF2 1F3FE
+1FAF1 1F3FB 200D 1FAF2 1F3FF
+1FAF1 1F3FC
+1FAF1 1F3FC 200D 1FAF2 1F3FB
+1FAF1 1F3FC 200D 1FAF2 1F3FD
+1FAF1 1F3FC 200D 1FAF2 1F3FE
+1FAF1 1F3FC 200D 1FAF2 1F3FF
+1FAF1 1F3FD
+1FAF1 1F3FD 200D 1FAF2 1F3FB
+1FAF1 1F3FD 200D 1FAF2 1F3FC
+1FAF1 1F3FD 200D 1FAF2 1F3FE
+1FAF1 1F3FD 200D 1FAF2 1F3FF
+1FAF1 1F3FE
+1FAF1 1F3FE 200D 1FAF2 1F3FB
+1FAF1 1F3FE 200D 1FAF2 1F3FC
+1FAF1 1F3FE 200D 1FAF2 1F3FD
+1FAF1 1F3FE 200D 1FAF2 1F3FF
+1FAF1 1F3FF
+1FAF1 1F3FF 200D 1FAF2 1F3FB
+1FAF1 1F3FF 200D 1FAF2 1F3FC
+1FAF1 1F3FF 200D 1FAF2 1F3FD
+1FAF1 1F3FF 200D 1FAF2 1F3FE
+1FAF2
+1FAF2 1F3FB
+1FAF2 1F3FC
+1FAF2 1F3FD
+1FAF2 1F3FE
+1FAF2 1F3FF
+1FAF3
+1FAF3 1F3FB
+1FAF3 1F3FC
+1FAF3 1F3FD
+1FAF3 1F3FE
+1FAF3 1F3FF
+1FAF4
+1FAF4 1F3FB
+1FAF4 1F3FC
+1FAF4 1F3FD
+1FAF4 1F3FE
+1FAF4 1F3FF
+1FAF5
+1FAF5 1F3FB
+1FAF5 1F3FC
+1FAF5 1F3FD
+1FAF5 1F3FE
+1FAF5 1F3FF
+1FAF6
+1FAF6 1F3FB
+1FAF6 1F3FC
+1FAF6 1F3FD
+1FAF6 1F3FE
+1FAF6 1F3FF
 203C FE0F
 2049 FE0F
 2122 FE0F