Optimize merging sub-tokens

hankcs · hankcs · commit 4467a3c88fcf · 2022-03-22T19:10:59.000-04:00
diff --git a/hanlp/components/tokenizers/transformer.py b/hanlp/components/tokenizers/transformer.py
@@ -181,8 +181,8 @@ def spans_to_tokens(self, spans, batch, rebuild_span=False):
                         combined = text[offsets[spans_per_sent[start - delta][0]][0]:
                                         offsets[spans_per_sent[end - delta - 1][1] - 1][1]]
                     else:
-                        combined = tokens[start:end]
-                    buffer.append(''.join(combined))
+                        combined = ''.join(tokens[start:end])
+                    buffer.append(combined)
                     offset = end
                     if rebuild_span:
                         start -= delta
diff --git a/hanlp/version.py b/hanlp/version.py
@@ -2,7 +2,7 @@
 # Author: hankcs
 # Date: 2019-12-28 19:26
 
-__version__ = '2.1.0-beta.18'
+__version__ = '2.1.0-beta.19'
 """HanLP version"""
 
 
