File tree
Expand file treeCollapse file tree1 file changed
+2
-2
lines changed Expand file treeCollapse file tree1 file changed
+2
-2
lines changed Original file line number | Diff line number | Diff line change |
---|
@@ -42,14 +42,14 @@ def get_data(opts):
|
42 | 42 |
|
43 | 43 | # Build a vocabulary object for these languages
|
44 | 44 | src_vocab = build_vocab_from_iterator(
|
45 |
| -_yield_tokens(train_iterator, src_tokenizer, src_lang), |
| 45 | +_yield_tokens(train_iterator, src_tokenizer, True), |
46 | 46 | min_freq=1,
|
47 | 47 | specials=list(special_symbols.keys()),
|
48 | 48 | special_first=True
|
49 | 49 | )
|
50 | 50 |
|
51 | 51 | tgt_vocab = build_vocab_from_iterator(
|
52 |
| -_yield_tokens(train_iterator, tgt_tokenizer, tgt_lang), |
| 52 | +_yield_tokens(train_iterator, tgt_tokenizer, False), |
53 | 53 | min_freq=1,
|
54 | 54 | specials=list(special_symbols.keys()),
|
55 | 55 | special_first=True
|
|
You can’t perform that action at this time.
0 commit comments