diff options
Diffstat (limited to 'scripts/unicode.py')
| -rwxr-xr-x | scripts/unicode.py | 748 |
1 files changed, 466 insertions, 282 deletions
diff --git a/scripts/unicode.py b/scripts/unicode.py index 7f5959d..2efb0b6 100755 --- a/scripts/unicode.py +++ b/scripts/unicode.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT +# Copyright 2011-2022 The Rust Project Developers. See the COPYRIGHT # file at the top-level directory of this distribution and at # http://rust-lang.org/COPYRIGHT. # @@ -16,11 +16,322 @@ # - UnicodeData.txt # # Since this should not require frequent updates, we just store this -# out-of-line and check the unicode.rs file into git. - -import fileinput, re, os, sys, operator +# out-of-line and check the generated module into git. + +import enum +import math +import os +import re +import sys + +NUM_CODEPOINTS = 0x110000 +"""An upper bound for which `range(0, NUM_CODEPOINTS)` contains Unicode's codespace.""" + +MAX_CODEPOINT_BITS = math.ceil(math.log2(NUM_CODEPOINTS - 1)) +"""The maximum number of bits required to represent a Unicode codepoint.""" + + +class OffsetType(enum.IntEnum): + """Represents the data type of a lookup table's offsets. Each variant's value represents the + number of bits required to represent that variant's type.""" + + U2 = 2 + """Offsets are 2-bit unsigned integers, packed four-per-byte.""" + U4 = 4 + """Offsets are 4-bit unsigned integers, packed two-per-byte.""" + U8 = 8 + """Each offset is a single byte (u8).""" + + +TABLE_CFGS = [ + (13, MAX_CODEPOINT_BITS, OffsetType.U8), + (6, 13, OffsetType.U8), + (0, 6, OffsetType.U2), +] +"""Represents the format of each level of the multi-level lookup table. +A level's entry is of the form `(low_bit, cap_bit, offset_type)`. +This means that every sub-table in that level is indexed by bits `low_bit..cap_bit` of the +codepoint and those tables offsets are stored according to `offset_type`. + +If this is edited, you must ensure that `emit_module` reflects your changes.""" + +MODULE_FILENAME = "tables.rs" +"""The filename of the emitted Rust module (will be created in the working directory)""" + +Codepoint = int +BitPos = int + + +def fetch_open(filename: str): + """Opens `filename` and return its corresponding file object. If `filename` isn't on disk, + fetches it from `http://www.unicode.org/Public/UNIDATA/`. Exits with code 1 on failure.""" + if not os.path.exists(os.path.basename(filename)): + os.system(f"curl -O http://www.unicode.org/Public/UNIDATA/{filename}") + try: + return open(filename, encoding="utf-8") + except OSError: + sys.stderr.write(f"cannot load {filename}") + sys.exit(1) + + +def load_unicode_version() -> "tuple[int, int, int]": + """Returns the current Unicode version by fetching and processing `ReadMe.txt`.""" + with fetch_open("ReadMe.txt") as readme: + pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode" + return tuple(map(int, re.search(pattern, readme.read()).groups())) + + +class EffectiveWidth(enum.IntEnum): + """Represents the width of a Unicode character. All East Asian Width classes resolve into + either `EffectiveWidth.NARROW`, `EffectiveWidth.WIDE`, or `EffectiveWidth.AMBIGUOUS`.""" + + ZERO = 0 + """ Zero columns wide. """ + NARROW = 1 + """ One column wide. """ + WIDE = 2 + """ Two columns wide. """ + AMBIGUOUS = 3 + """ Two columns wide in a CJK context. One column wide in all other contexts. """ + + +def load_east_asian_widths() -> "list[EffectiveWidth]": + """Return a list of effective widths, indexed by codepoint. + Widths are determined by fetching and parsing `EastAsianWidth.txt`. + + `Neutral`, `Narrow`, and `Halfwidth` characters are assigned `EffectiveWidth.NARROW`. + + `Wide` and `Fullwidth` characters are assigned `EffectiveWidth.WIDE`. + + `Ambiguous` chracters are assigned `EffectiveWidth.AMBIGUOUS`.""" + with fetch_open("EastAsianWidth.txt") as eaw: + # matches a width assignment for a single codepoint, i.e. "1F336;N # ..." + single = re.compile(r"^([0-9A-F]+);(\w+) +# (\w+)") + # matches a width assignment for a range of codepoints, i.e. "3001..3003;W # ..." + multiple = re.compile(r"^([0-9A-F]+)\.\.([0-9A-F]+);(\w+) +# (\w+)") + # map between width category code and condensed width + width_codes = { + **{c: EffectiveWidth.NARROW for c in ["N", "Na", "H"]}, + **{c: EffectiveWidth.WIDE for c in ["W", "F"]}, + "A": EffectiveWidth.AMBIGUOUS, + } -preamble = '''// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT + width_map = [] + current = 0 + for line in eaw.readlines(): + raw_data = None # (low, high, width) + if match := single.match(line): + raw_data = (match.group(1), match.group(1), match.group(2)) + elif match := multiple.match(line): + raw_data = (match.group(1), match.group(2), match.group(3)) + else: + continue + low = int(raw_data[0], 16) + high = int(raw_data[1], 16) + width = width_codes[raw_data[2]] + + assert current <= high + while current <= high: + # Some codepoints don't fall into any of the ranges in EastAsianWidth.txt. + # All such codepoints are implicitly given Neural width (resolves to narrow) + width_map.append(EffectiveWidth.NARROW if current < low else width) + current += 1 + + while len(width_map) < NUM_CODEPOINTS: + # Catch any leftover codepoints and assign them implicit Neutral/narrow width. + width_map.append(EffectiveWidth.NARROW) + + return width_map + + +def load_zero_widths() -> "list[bool]": + """Returns a list `l` where `l[c]` is true if codepoint `c` is considered a zero-width + character. `c` is considered a zero-width character if `c` is in general categories + `Cc`, `Cf`, `Mn`, or `Me` (determined by fetching and processing `UnicodeData.txt`).""" + with fetch_open("UnicodeData.txt") as categories: + zw_map = [] + current = 0 + for line in categories.readlines(): + if len(raw_data := line.split(";")) != 15: + continue + [codepoint, name, cat_code] = [ + int(raw_data[0], 16), + raw_data[1], + raw_data[2], + ] + zero_width = cat_code in ["Cc", "Cf", "Mn", "Me"] + + assert current <= codepoint + while current <= codepoint: + if name.endswith(", Last>") or current == codepoint: + # if name ends with Last, we backfill the width value to all codepoints since + # the previous codepoint (aka the start of the range) + zw_map.append(zero_width) + else: + # unassigned characters are implicitly given Neutral width, which is nonzero + zw_map.append(False) + current += 1 + + while len(zw_map) < NUM_CODEPOINTS: + # Catch any leftover codepoints. They must be unassigned (so nonzero width) + zw_map.append(False) + + return zw_map + + +class Bucket: + """A bucket contains a group of codepoints and an ordered width list. If one bucket's width + list overlaps with another's width list, those buckets can be merged via `try_extend`.""" + + def __init__(self): + """Creates an empty bucket.""" + self.entry_set = set() + self.widths = [] + + def append(self, codepoint: Codepoint, width: EffectiveWidth): + """Adds a codepoint/width pair to the bucket, and appends `width` to the width list.""" + self.entry_set.add((codepoint, width)) + self.widths.append(width) + + def try_extend(self, attempt: "Bucket") -> bool: + """If either `self` or `attempt`'s width list starts with the other bucket's width list, + set `self`'s width list to the longer of the two, add all of `attempt`'s codepoints + into `self`, and return `True`. Otherwise, return `False`.""" + (less, more) = (self.widths, attempt.widths) + if len(self.widths) > len(attempt.widths): + (less, more) = (attempt.widths, self.widths) + if less != more[: len(less)]: + return False + self.entry_set |= attempt.entry_set + self.widths = more + return True + + def entries(self) -> "list[tuple[Codepoint, EffectiveWidth]]": + """Return a list of the codepoint/width pairs in this bucket, sorted by codepoint.""" + result = list(self.entry_set) + result.sort() + return result + + def width(self) -> "EffectiveWidth": + """If all codepoints in this bucket have the same width, return that width; otherwise, + return `None`.""" + if len(self.widths) == 0: + return None + potential_width = self.widths[0] + for width in self.widths[1:]: + if potential_width != width: + return None + return potential_width + + +def make_buckets(entries, low_bit: BitPos, cap_bit: BitPos) -> "list[Bucket]": + """Partitions the `(Codepoint, EffectiveWidth)` tuples in `entries` into `Bucket`s. All + codepoints with identical bits from `low_bit` to `cap_bit` (exclusive) are placed in the + same bucket. Returns a list of the buckets in increasing order of those bits.""" + num_bits = cap_bit - low_bit + assert num_bits > 0 + buckets = [Bucket() for _ in range(0, 2 ** num_bits)] + mask = (1 << num_bits) - 1 + for (codepoint, width) in entries: + buckets[(codepoint >> low_bit) & mask].append(codepoint, width) + return buckets + + +class Table: + """Represents a lookup table. Each table contains a certain number of subtables; each + subtable is indexed by a contiguous bit range of the codepoint and contains a list + of `2**(number of bits in bit range)` entries. (The bit range is the same for all subtables.) + + Typically, tables contain a list of buckets of codepoints. Bucket `i`'s codepoints should + be indexed by sub-table `i` in the next-level lookup table. The entries of this table are + indexes into the bucket list (~= indexes into the sub-tables of the next-level table.) The + key to compression is that two different buckets in two different sub-tables may have the + same width list, which means that they can be merged into the same bucket. + + If no bucket contains two codepoints with different widths, calling `indices_to_widths` will + discard the buckets and convert the entries into `EffectiveWidth` values.""" + + def __init__( + self, entry_groups, low_bit: BitPos, cap_bit: BitPos, offset_type: OffsetType + ): + """Create a lookup table with a sub-table for each `(Codepoint, EffectiveWidth)` iterator + in `entry_groups`. Each sub-table is indexed by codepoint bits in `low_bit..cap_bit`, + and each table entry is represented in the format specified by `offset_type`. Asserts + that this table is actually representable with `offset_type`.""" + self.low_bit = low_bit + self.cap_bit = cap_bit + self.offset_type = offset_type + self.entries = [] + self.indexed = [] + + buckets = [] + for entries in entry_groups: + buckets.extend(make_buckets(entries, self.low_bit, self.cap_bit)) + + for bucket in buckets: + for (i, existing) in enumerate(self.indexed): + if existing.try_extend(bucket): + self.entries.append(i) + break + else: + self.entries.append(len(self.indexed)) + self.indexed.append(bucket) + + # Validate offset type + for index in self.entries: + assert index < (1 << int(self.offset_type)) + + def indices_to_widths(self): + """Destructively converts the indices in this table to the `EffectiveWidth` values of + their buckets. Assumes that no bucket contains codepoints with different widths.""" + self.entries = list(map(lambda i: int(self.indexed[i].width()), self.entries)) + del self.indexed + + def buckets(self): + """Returns an iterator over this table's buckets.""" + return self.indexed + + def to_bytes(self) -> "list[int]": + """Returns this table's entries as a list of bytes. The bytes are formatted according to + the `OffsetType` which the table was created with, converting any `EffectiveWidth` entries + to their enum variant's integer value. For example, with `OffsetType.U2`, each byte will + contain four packed 2-bit entries.""" + entries_per_byte = 8 // int(self.offset_type) + byte_array = [] + for i in range(0, len(self.entries), entries_per_byte): + byte = 0 + for j in range(0, entries_per_byte): + byte |= self.entries[i + j] << (j * int(self.offset_type)) + byte_array.append(byte) + return byte_array + + +def make_tables( + table_cfgs: "list[tuple[BitPos, BitPos, OffsetType]]", entries +) -> "list[Table]": + """Creates a table for each configuration in `table_cfgs`, with the first config corresponding + to the top-level lookup table, the second config corresponding to the second-level lookup + table, and so forth. `entries` is an iterator over the `(Codepoint, EffectiveWidth)` pairs + to include in the top-level table.""" + tables = [] + entry_groups = [entries] + for (low_bit, cap_bit, offset_type) in table_cfgs: + table = Table(entry_groups, low_bit, cap_bit, offset_type) + entry_groups = map(lambda bucket: bucket.entries(), table.buckets()) + tables.append(table) + return tables + + +def emit_module( + out_name: str, unicode_version: "tuple[int, int, int]", tables: "list[Table]" +): + """Outputs a Rust module to `out_name` using table data from `tables`. + If `TABLE_CFGS` is edited, you may need to edit the included code for `lookup_width`.""" + if os.path.exists(out_name): + os.remove(out_name) + with open(out_name, "w", newline="\n", encoding="utf-8") as module: + module.write( + """// Copyright 2012-2022 The Rust Project Developers. See the COPYRIGHT // file at the top-level directory of this distribution and at // http://rust-lang.org/COPYRIGHT. // @@ -31,291 +342,164 @@ preamble = '''// Copyright 2012-2015 The Rust Project Developers. See the COPYRI // except according to those terms. // NOTE: The following code was generated by "scripts/unicode.py", do not edit directly - -#![allow(missing_docs, non_upper_case_globals, non_snake_case)] -''' - -# Mapping taken from Table 12 from: -# http://www.unicode.org/reports/tr44/#General_Category_Values -expanded_categories = { - 'Lu': ['LC', 'L'], 'Ll': ['LC', 'L'], 'Lt': ['LC', 'L'], - 'Lm': ['L'], 'Lo': ['L'], - 'Mn': ['M'], 'Mc': ['M'], 'Me': ['M'], - 'Nd': ['N'], 'Nl': ['N'], 'No': ['No'], - 'Pc': ['P'], 'Pd': ['P'], 'Ps': ['P'], 'Pe': ['P'], - 'Pi': ['P'], 'Pf': ['P'], 'Po': ['P'], - 'Sm': ['S'], 'Sc': ['S'], 'Sk': ['S'], 'So': ['S'], - 'Zs': ['Z'], 'Zl': ['Z'], 'Zp': ['Z'], - 'Cc': ['C'], 'Cf': ['C'], 'Cs': ['C'], 'Co': ['C'], 'Cn': ['C'], -} - -# these are the surrogate codepoints, which are not valid rust characters -surrogate_codepoints = (0xd800, 0xdfff) - -def fetch(f): - if not os.path.exists(os.path.basename(f)): - os.system("curl -O http://www.unicode.org/Public/UNIDATA/%s" - % f) - - if not os.path.exists(os.path.basename(f)): - sys.stderr.write("cannot load %s" % f) - exit(1) - -def is_surrogate(n): - return surrogate_codepoints[0] <= n <= surrogate_codepoints[1] - -def load_unicode_data(f): - fetch(f) - gencats = {} - - udict = {} - range_start = -1 - for line in fileinput.input(f): - data = line.split(';') - if len(data) != 15: - continue - cp = int(data[0], 16) - if is_surrogate(cp): - continue - if range_start >= 0: - for i in range(range_start, cp): - udict[i] = data - range_start = -1 - if data[1].endswith(", First>"): - range_start = cp - continue - udict[cp] = data - - for code in udict: - [code_org, name, gencat, combine, bidi, - decomp, deci, digit, num, mirror, - old, iso, upcase, lowcase, titlecase ] = udict[code] - - # place letter in categories as appropriate - for cat in [gencat, "Assigned"] + expanded_categories.get(gencat, []): - if cat not in gencats: - gencats[cat] = [] - gencats[cat].append(code) - - gencats = group_cats(gencats) - - return gencats - -def group_cats(cats): - cats_out = {} - for cat in cats: - cats_out[cat] = group_cat(cats[cat]) - return cats_out - -def group_cat(cat): - cat_out = [] - letters = sorted(set(cat)) - cur_start = letters.pop(0) - cur_end = cur_start - for letter in letters: - assert letter > cur_end, \ - "cur_end: %s, letter: %s" % (hex(cur_end), hex(letter)) - if letter == cur_end + 1: - cur_end = letter - else: - cat_out.append((cur_start, cur_end)) - cur_start = cur_end = letter - cat_out.append((cur_start, cur_end)) - return cat_out - -def format_table_content(f, content, indent): - line = " "*indent - first = True - for chunk in content.split(","): - if len(line) + len(chunk) < 98: - if first: - line += chunk - else: - line += ", " + chunk - first = False - else: - f.write(line + ",\n") - line = " "*indent + chunk - f.write(line) - -# load all widths of want_widths, except those in except_cats -def load_east_asian_width(want_widths, except_cats): - f = "EastAsianWidth.txt" - fetch(f) - widths = {} - re1 = re.compile("^([0-9A-F]+);(\w+) +# (\w+)") - re2 = re.compile("^([0-9A-F]+)\.\.([0-9A-F]+);(\w+) +# (\w+)") - - for line in fileinput.input(f): - width = None - d_lo = 0 - d_hi = 0 - cat = None - m = re1.match(line) - if m: - d_lo = m.group(1) - d_hi = m.group(1) - width = m.group(2) - cat = m.group(3) - else: - m = re2.match(line) - if m: - d_lo = m.group(1) - d_hi = m.group(2) - width = m.group(3) - cat = m.group(4) - else: - continue - if cat in except_cats or width not in want_widths: - continue - d_lo = int(d_lo, 16) - d_hi = int(d_hi, 16) - if width not in widths: - widths[width] = [] - widths[width].append((d_lo, d_hi)) - return widths - -def escape_char(c): - return "'\\u{%x}'" % c - -def emit_table(f, name, t_data, t_type = "&'static [(char, char)]", is_pub=True, - pfun=lambda x: "(%s,%s)" % (escape_char(x[0]), escape_char(x[1])), is_const=True): - pub_string = "const" - if not is_const: - pub_string = "let" - if is_pub: - pub_string = "pub " + pub_string - f.write(" %s %s: %s = &[\n" % (pub_string, name, t_type)) - data = "" - first = True - for dat in t_data: - if not first: - data += "," - first = False - data += pfun(dat) - format_table_content(f, data, 8) - f.write("\n ];\n\n") - -def emit_charwidth_module(f, width_table): - f.write("pub mod charwidth {") - f.write(""" - use core::option::Option::{self, Some, None}; - use core::result::Result::{Ok, Err}; - +""" + ) + module.write( + f""" +/// The version of [Unicode](http://www.unicode.org/) +/// that this version of unicode-width is based on. +pub const UNICODE_VERSION: (u8, u8, u8) = {unicode_version}; +""" + ) + + module.write( + """ +pub mod charwidth { + use core::option::Option::{self, None, Some}; + + /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c` by + /// consulting a multi-level lookup table. + /// If `is_cjk == true`, ambiguous width characters are treated as double width; otherwise, + /// they're treated as single width. + /// + /// # Maintenance + /// The tables themselves are autogenerated but this function is hardcoded. You should have + /// nothing to worry about if you re-run `unicode.py` (for example, when updating Unicode.) + /// However, if you change the *actual structure* of the lookup tables (perhaps by editing the + /// `TABLE_CFGS` global in `unicode.py`) you must ensure that this code reflects those changes. #[inline] - fn bsearch_range_value_table(c: char, is_cjk: bool, r: &'static [(char, char, u8, u8)]) -> u8 { - use core::cmp::Ordering::{Equal, Less, Greater}; - match r.binary_search_by(|&(lo, hi, _, _)| { - if lo <= c && c <= hi { Equal } - else if hi < c { Less } - else { Greater } - }) { - Ok(idx) => { - let (_, _, r_ncjk, r_cjk) = r[idx]; - if is_cjk { r_cjk } else { r_ncjk } + fn lookup_width(c: char, is_cjk: bool) -> usize { + let cp = c as usize; + + let t1_offset = TABLES_0[cp >> 13 & 0xFF]; + + // Each sub-table in TABLES_1 is 7 bits, and each stored entry is a byte, + // so each sub-table is 128 bytes in size. + // (Sub-tables are selected using the computed offset from the previous table.) + let t2_offset = TABLES_1[128 * usize::from(t1_offset) + (cp >> 6 & 0x7F)]; + + // Each sub-table in TABLES_2 is 6 bits, but each stored entry is 2 bits. + // This is accomplished by packing four stored entries into one byte. + // So each sub-table is 2**(6-2) == 16 bytes in size. + // Since this is the last table, each entry represents an encoded width. + let packed_widths = TABLES_2[16 * usize::from(t2_offset) + (cp >> 2 & 0xF)]; + + // Extract the packed width + let width = packed_widths >> (2 * (cp & 0b11)) & 0b11; + + // A width of 3 signifies that the codepoint is ambiguous width. + if width == 3 { + if is_cjk { + 2 + } else { + 1 } - Err(_) => 1 + } else { + width.into() } } -""") - - f.write(""" +""" + ) + + module.write( + """ + /// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c`, or + /// `None` if `c` is a control character other than `'\\x00'`. + /// If `is_cjk == true`, ambiguous width characters are treated as double width; otherwise, + /// they're treated as single width. #[inline] pub fn width(c: char, is_cjk: bool) -> Option<usize> { - match c as usize { - _c @ 0 => Some(0), // null is zero width - cu if cu < 0x20 => None, // control sequences have no width - cu if cu < 0x7F => Some(1), // ASCII - cu if cu < 0xA0 => None, // more control sequences - _ => Some(bsearch_range_value_table(c, is_cjk, charwidth_table) as usize) + if c < '\\u{7F}' { + if c >= '\\u{20}' { + // U+0020 to U+007F (exclusive) are single-width ASCII codepoints + Some(1) + } else if c == '\\0' { + // U+0000 *is* a control code, but it's special-cased + Some(0) + } else { + // U+0001 to U+0020 (exclusive) are control codes + None + } + } else if c >= '\\u{A0}' { + // No characters >= U+00A0 are control codes, so we can consult the lookup tables + Some(lookup_width(c, is_cjk)) + } else { + // U+007F to U+00A0 (exclusive) are control codes + None } } +""" + ) + + subtable_count = 1 + for (i, table) in enumerate(tables): + new_subtable_count = len(table.buckets()) + if i == len(tables) - 1: + table.indices_to_widths() # for the last table, indices == widths + byte_array = table.to_bytes() + module.write( + f""" + /// Autogenerated. {subtable_count} sub-table(s). Consult [`lookup_width`] for layout info. + static TABLES_{i}: [u8; {len(byte_array)}] = [""" + ) + for (j, byte) in enumerate(byte_array): + # Add line breaks for every 15th entry (chosen to match what rustfmt does) + if j % 15 == 0: + module.write("\n ") + module.write(f" 0x{byte:02X},") + module.write("\n ];\n") + subtable_count = new_subtable_count + module.write("}\n") + + +def main(module_filename: str): + """Obtain character data from the latest version of Unicode, transform it into a multi-level + lookup table for character width, and write a Rust module utilizing that table to + `module_filename`. + + We obey the following rules in decreasing order of importance: + - The soft hyphen (`U+00AD`) is single-width. + - Hangul Jamo medial vowels & final consonants (`U+1160..=U+11FF`) are zero-width. + - All codepoints in general categories `Cc`, `Cf`, `Mn`, and `Me` are zero-width. + - All codepoints with an East Asian Width of `Ambigous` are ambiguous-width. + - All codepoints with an East Asian Width of `Wide` or `Fullwidth` are double-width. + - All other codepoints (including unassigned codepoints and codepoints with an East Asian Width + of `Neutral`, `Narrow`, or `Halfwidth`) are single-width. + + These rules are based off of Markus Kuhn's free `wcwidth()` implementation: + http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c""" + version = load_unicode_version() + print(f"Generating module for Unicode {version[0]}.{version[1]}.{version[2]}") + + eaw_map = load_east_asian_widths() + zw_map = load_zero_widths() + + # Characters marked as zero-width in zw_map should be zero-width in the final map + width_map = list( + map(lambda x: EffectiveWidth.ZERO if x[1] else x[0], zip(eaw_map, zw_map)) + ) + + # Override for soft hyphen + width_map[0x00AD] = EffectiveWidth.NARROW + + # Override for Hangul Jamo medial vowels & final consonants + for i in range(0x1160, 0x11FF + 1): + width_map[i] = EffectiveWidth.ZERO + + tables = make_tables(TABLE_CFGS, enumerate(width_map)) + + print("------------------------") + total_size = 0 + for (i, table) in enumerate(tables): + size_bytes = len(table.to_bytes()) + print(f"Table {i} Size: {size_bytes} bytes") + total_size += size_bytes + print("------------------------") + print(f" Total Size: {total_size} bytes") + + emit_module(module_filename, version, tables) + print(f'Wrote to "{module_filename}"') -""") - - f.write(" // character width table. Based on Markus Kuhn's free wcwidth() implementation,\n") - f.write(" // http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c\n") - emit_table(f, "charwidth_table", width_table, "&'static [(char, char, u8, u8)]", is_pub=False, - pfun=lambda x: "(%s,%s,%s,%s)" % (escape_char(x[0]), escape_char(x[1]), x[2], x[3])) - f.write("}\n\n") - -def remove_from_wtable(wtable, val): - wtable_out = [] - while wtable: - if wtable[0][1] < val: - wtable_out.append(wtable.pop(0)) - elif wtable[0][0] > val: - break - else: - (wt_lo, wt_hi, width, width_cjk) = wtable.pop(0) - if wt_lo == wt_hi == val: - continue - elif wt_lo == val: - wtable_out.append((wt_lo+1, wt_hi, width, width_cjk)) - elif wt_hi == val: - wtable_out.append((wt_lo, wt_hi-1, width, width_cjk)) - else: - wtable_out.append((wt_lo, val-1, width, width_cjk)) - wtable_out.append((val+1, wt_hi, width, width_cjk)) - if wtable: - wtable_out.extend(wtable) - return wtable_out - - - -def optimize_width_table(wtable): - wtable_out = [] - w_this = wtable.pop(0) - while wtable: - if w_this[1] == wtable[0][0] - 1 and w_this[2:3] == wtable[0][2:3]: - w_tmp = wtable.pop(0) - w_this = (w_this[0], w_tmp[1], w_tmp[2], w_tmp[3]) - else: - wtable_out.append(w_this) - w_this = wtable.pop(0) - wtable_out.append(w_this) - return wtable_out if __name__ == "__main__": - r = "tables.rs" - if os.path.exists(r): - os.remove(r) - with open(r, "w") as rf: - # write the file's preamble - rf.write(preamble) - - # download and parse all the data - fetch("ReadMe.txt") - with open("ReadMe.txt") as readme: - pattern = "for Version (\d+)\.(\d+)\.(\d+) of the Unicode" - unicode_version = re.search(pattern, readme.read()).groups() - rf.write(""" -/// The version of [Unicode](http://www.unicode.org/) -/// that this version of unicode-width is based on. -pub const UNICODE_VERSION: (u8, u8, u8) = (%s, %s, %s); - -""" % unicode_version) - gencats = load_unicode_data("UnicodeData.txt") - - ### character width module - width_table = [] - for zwcat in ["Me", "Mn", "Cf"]: - width_table.extend([(lo_hi[0], lo_hi[1], 0, 0) for lo_hi in gencats[zwcat]]) - width_table.append((4448, 4607, 0, 0)) - - # get widths, except those that are explicitly marked zero-width above - ea_widths = load_east_asian_width(["W", "F", "A"], ["Me", "Mn", "Cf"]) - # these are doublewidth - for dwcat in ["W", "F"]: - width_table.extend([(lo_hi1[0], lo_hi1[1], 2, 2) for lo_hi1 in ea_widths[dwcat]]) - width_table.extend([(lo_hi2[0], lo_hi2[1], 1, 2) for lo_hi2 in ea_widths["A"]]) - - width_table.sort(key=lambda w: w[0]) - - # soft hyphen is not zero width in preformatted text; it's used to indicate - # a hyphen inserted to facilitate a linebreak. - width_table = remove_from_wtable(width_table, 173) - - # optimize the width table by collapsing adjacent entities when possible - width_table = optimize_width_table(width_table) - emit_charwidth_module(rf, width_table) + main(MODULE_FILENAME) |