blob: 6b59fae1e20c3bfd3ce90b671c3a8d2158eedc46 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
|
#!/bin/sh
set -e
D="$(dirname "$0")"
# Convenience function for checking that a command exists.
requires() {
cmd="$1"
if ! command -v "$cmd" > /dev/null 2>&1; then
echo "DEPENDENCY MISSING: $cmd must be installed" >&2
exit 1
fi
}
# Test if an array ($2) contains a particular element ($1).
array_exists() {
needle="$1"
shift
for el in "$@"; do
if [ "$el" = "$needle" ]; then
return 0
fi
done
return 1
}
graphemes() {
regex="$(sh "$D/regex/grapheme.sh")"
echo "generating forward grapheme DFA"
ucd-generate dfa \
--name GRAPHEME_BREAK_FWD \
--sparse --minimize --anchored --state-size 2 \
src/unicode/fsm/ \
"$regex"
echo "generating reverse grapheme DFA"
ucd-generate dfa \
--name GRAPHEME_BREAK_REV \
--reverse --longest \
--sparse --minimize --anchored --state-size 2 \
src/unicode/fsm/ \
"$regex"
}
words() {
regex="$(sh "$D/regex/word.sh")"
echo "generating forward word DFA (this can take a while)"
ucd-generate dfa \
--name WORD_BREAK_FWD \
--sparse --minimize --anchored --state-size 4 \
src/unicode/fsm/ \
"$regex"
}
sentences() {
regex="$(sh "$D/regex/sentence.sh")"
echo "generating forward sentence DFA (this can take a while)"
ucd-generate dfa \
--name SENTENCE_BREAK_FWD \
--minimize \
--sparse --anchored --state-size 4 \
src/unicode/fsm/ \
"$regex"
}
regional_indicator() {
# For finding all occurrences of region indicators. This is used to handle
# regional indicators as a special case for the reverse grapheme iterator
# and the reverse word iterator.
echo "generating regional indicator DFA"
ucd-generate dfa \
--name REGIONAL_INDICATOR_REV \
--reverse \
--classes --minimize --anchored --premultiply --state-size 1 \
src/unicode/fsm/ \
"\p{gcb=Regional_Indicator}"
}
simple_word() {
echo "generating forward simple word DFA"
ucd-generate dfa \
--name SIMPLE_WORD_FWD \
--sparse --minimize --state-size 2 \
src/unicode/fsm/ \
"\w"
}
whitespace() {
echo "generating forward whitespace DFA"
ucd-generate dfa \
--name WHITESPACE_ANCHORED_FWD \
--anchored --classes --premultiply --minimize --state-size 1 \
src/unicode/fsm/ \
"\s+"
echo "generating reverse whitespace DFA"
ucd-generate dfa \
--name WHITESPACE_ANCHORED_REV \
--reverse \
--anchored --classes --premultiply --minimize --state-size 1 \
src/unicode/fsm/ \
"\s+"
}
main() {
if array_exists "-h" "$@" || array_exists "--help" "$@"; then
echo "Usage: $(basename "$0") [--list-commands] [<command>] ..." >&2
exit
fi
commands="
graphemes
sentences
words
regional-indicator
simple-word
whitespace
"
if array_exists "--list-commands" "$@"; then
for cmd in $commands; do
echo "$cmd"
done
exit
fi
# ucd-generate is used to compile regexes into DFAs.
requires ucd-generate
mkdir -p src/unicode/fsm/
cmds=$*
if [ $# -eq 0 ] || array_exists "all" "$@"; then
cmds=$commands
fi
for cmd in $cmds; do
if array_exists "$cmd" $commands; then
fun="$(echo "$cmd" | sed 's/-/_/g')"
eval "$fun"
else
echo "unrecognized command: $cmd" >&2
fi
done
}
main "$@"
|
