Give PadSequenceDataLoader the option to skip padding

hankcs · hankcs · commit 396568c35533 · 2022-04-19T13:59:12.000-04:00
diff --git a/hanlp/common/dataset.py b/hanlp/common/dataset.py
@@ -439,7 +439,7 @@ def tensorize(raw_batch: Dict[str, Any], vocabs: VocabDict, pad_dict: Dict[str,
             if vocab:
                 pad = vocab.safe_pad_token_idx
                 dtype = torch.long
-            elif pad_dict is not None and field in pad_dict:
+            elif pad_dict is not None and pad_dict.get(field, None) is not None:
                 pad = pad_dict[field]
                 dtype = dtype_of(pad)
             elif field.endswith('_offset') or field.endswith('_id') or field.endswith(
