unicode-rs
diff --git a/‎scripts/unicode.py
Lines changed: 7 additions & 0 deletions b/‎scripts/unicode.py
Lines changed: 7 additions & 0 deletions
diff --git a/‎scripts/unicode_gen_breaktests.py
Lines changed: 15 additions & 0 deletions b/‎scripts/unicode_gen_breaktests.py
Lines changed: 15 additions & 0 deletions
@@ -351,3 +351,10 @@ def emit_break_module(f, break_table, break_cats, name):
             word_table.extend([(x, y, cat) for (x, y) in word_cats[cat]])
         word_table.sort(key=lambda w: w[0])
         emit_break_module(rf, word_table, word_cats.keys(), "word")
+
+        sentence_cats = load_properties("auxiliary/SentenceBreakProperty.txt", [])
+        sentence_table = []
+        for cat in sentence_cats:
+            sentence_table.extend([(x, y, cat) for (x, y) in sentence_cats[cat]])
+        sentence_table.sort(key=lambda w: w[0])
+        emit_break_module(rf, sentence_table, sentence_cats.keys(), "sentence")
@@ -190,8 +190,23 @@ def create_words_data(f):
     f.write("    // http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakTest.txt\n")
     unicode.emit_table(f, "TEST_WORD", test, wtype, True, showfun, True)
 
+def create_sentence_data(f):
+    d = load_test_data("auxiliary/SentenceBreakTest.txt")
+
+    test = []
+
+    for (c, i) in d:
+        allchars = [cn for s in c for cn in s]
+        test.append((allchars, c))
+
+    wtype = "&'static [(&'static str, &'static [&'static str])]"
+    f.write("    // official Unicode test data\n")
+    f.write("    // http://www.unicode.org/Public/UNIDATA/auxiliary/SentenceBreakTest.txt\n")
+    unicode.emit_table(f, "TEST_SENTENCE", test, wtype, True, showfun, True)
+
 if __name__ == "__main__":
     with open("testdata.rs", "w") as rf:
         rf.write(unicode.preamble)
         create_grapheme_data(rf)
         create_words_data(rf)
+        create_sentence_data(rf)