支持𩽾𩾌(ān kāng)之类的补充字符集 fix hankcs#1564

hankcs · hankcs · commit b9a899b1f54f · 2021-01-30T22:56:31.000-05:00
diff --git a/data/dictionary/pinyin/pinyin.txt b/data/dictionary/pinyin/pinyin.txt
@@ -30724,4 +30724,5 @@
 龥=yue4
 重启=chong2,qi3
 还款=huan2,kuan3
-侠传=xia2,zhuan4
+侠传=xia2,zhuan4
+𩽾𩾌=an1,kang1
diff --git a/src/main/java/com/hankcs/hanlp/dictionary/py/PinyinDictionary.java b/src/main/java/com/hankcs/hanlp/dictionary/py/PinyinDictionary.java
@@ -180,15 +180,17 @@ protected static List<Pinyin> segLongest(char[] charArray, AhoCorasickDoubleArra
     protected static List<Pinyin> segLongest(char[] charArray, AhoCorasickDoubleArrayTrie<Pinyin[]> trie, boolean remainNone)
     {
         final Pinyin[][] wordNet = new Pinyin[charArray.length][];
+        final int[] lengths = new int[charArray.length];
         trie.parseText(charArray, new AhoCorasickDoubleArrayTrie.IHit<Pinyin[]>()
         {
             @Override
             public void hit(int begin, int end, Pinyin[] value)
             {
                 int length = end - begin;
-                if (wordNet[begin] == null || length > wordNet[begin].length)
+                if (length > lengths[begin])
                 {
-                    wordNet[begin] = length == 1 ? new Pinyin[]{value[0]} : value;
+                    wordNet[begin] = value;
+                    lengths[begin] = length;
                 }
             }
         });
@@ -208,7 +210,7 @@ public void hit(int begin, int end, Pinyin[] value)
             {
                 pinyinList.add(pinyin);
             }
-            offset += wordNet[offset].length;
+            offset += lengths[offset];
         }
         return pinyinList;
     }
diff --git a/src/test/java/com/hankcs/hanlp/dictionary/py/PinyinDictionaryTest.java b/src/test/java/com/hankcs/hanlp/dictionary/py/PinyinDictionaryTest.java
@@ -0,0 +1,17 @@
+package com.hankcs.hanlp.dictionary.py;
+
+import com.hankcs.hanlp.HanLP;
+import junit.framework.TestCase;
+
+import java.util.Arrays;
+
+public class PinyinDictionaryTest extends TestCase
+{
+
+    public void testGet()
+    {
+        System.out.println(Arrays.toString(PinyinDictionary.get("鼖")));
+        System.out.println(PinyinDictionary.convertToPinyin("\uD867\uDF7E\uD867\uDF8C"));
+        System.out.println(HanLP.convertToPinyinList("\uD867\uDF7E\uD867\uDF8C"));
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -180,15 +180,17 @@ protected static List<Pinyin> segLongest(char[] charArray, AhoCorasickDoubleArra`
`180`	`180`	`protected static List<Pinyin> segLongest(char[] charArray, AhoCorasickDoubleArrayTrie<Pinyin[]> trie, boolean remainNone)`
`181`	`181`	`{`
`182`	`182`	`final Pinyin[][] wordNet = new Pinyin[charArray.length][];`
	`183`	`+ final int[] lengths = new int[charArray.length];`
`183`	`184`	`trie.parseText(charArray, new AhoCorasickDoubleArrayTrie.IHit<Pinyin[]>()`
`184`	`185`	`{`
`185`	`186`	`@Override`
`186`	`187`	`public void hit(int begin, int end, Pinyin[] value)`
`187`	`188`	`{`
`188`	`189`	`int length = end - begin;`
`189`		`- if (wordNet[begin] == null \|\| length > wordNet[begin].length)`
	`190`	`+ if (length > lengths[begin])`
`190`	`191`	`{`
`191`		`- wordNet[begin] = length == 1 ? new Pinyin[]{value[0]} : value;`
	`192`	`+ wordNet[begin] = value;`
	`193`	`+ lengths[begin] = length;`
`192`	`194`	`}`
`193`	`195`	`}`
`194`	`196`	`});`
`@@ -208,7 +210,7 @@ public void hit(int begin, int end, Pinyin[] value)`
`208`	`210`	`{`
`209`	`211`	`pinyinList.add(pinyin);`
`210`	`212`	`}`
`211`		`- offset += wordNet[offset].length;`
	`213`	`+ offset += lengths[offset];`
`212`	`214`	`}`
`213`	`215`	`return pinyinList;`
`214`	`216`	`}`