Upgrade transformers==5.3.0 (#17784)
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com> Co-authored-by: Kangyan-Zhou <zky314343421@gmail.com> Co-authored-by: Alison Shao <alisonshao@mac.lan> Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
co-authored by
Kangyan-Zhou
Alison Shao
Mick
parent
e5750a572c
commit
d1e95af282
@@ -85,7 +85,7 @@ class TestScoreAPI(CustomTestCase):
|
||||
try:
|
||||
label_token_ids = []
|
||||
for token in tokens:
|
||||
encoding = tokenizer.encode_plus(token, add_special_tokens=False)
|
||||
encoding = tokenizer(token, add_special_tokens=False)
|
||||
token_ids = encoding["input_ids"]
|
||||
label_token_ids.append(token_ids[0])
|
||||
return label_token_ids
|
||||
|
||||
Reference in New Issue
Block a user