File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,14 @@ def get_data(opts):
4242

4343
# Build a vocabulary object for these languages
4444
src_vocab = build_vocab_from_iterator(
45-
_yield_tokens(train_iterator, src_tokenizer, src_lang),
45+
_yield_tokens(train_iterator, src_tokenizer, True),
4646
min_freq=1,
4747
specials=list(special_symbols.keys()),
4848
special_first=True
4949
)
5050

5151
tgt_vocab = build_vocab_from_iterator(
52-
_yield_tokens(train_iterator, tgt_tokenizer, tgt_lang),
52+
_yield_tokens(train_iterator, tgt_tokenizer, False),
5353
min_freq=1,
5454
specials=list(special_symbols.keys()),
5555
special_first=True

0 commit comments

Comments
 (0)