File tree Expand file tree Collapse file tree 1 file changed +3
-4
lines changed Expand file tree Collapse file tree 1 file changed +3
-4
lines changed Original file line number Diff line number Diff line change @@ -63,7 +63,7 @@ def all_special_ids(self) -> list[int]:
6363
6464 @property
6565 def bos_token_id (self ) -> int :
66- raise NotImplementedError
66+ return self . tokenizer . bos_token_id
6767
6868 @property
6969 def eos_token_id (self ) -> int :
@@ -135,13 +135,12 @@ def convert_tokens_to_string(self, tokens: list[str]) -> str:
135135 return self .tokenizer .convert_tokens_to_string (tokens )
136136
137137 def decode (self , ids : list [int ] | int , skip_special_tokens : bool = False ) -> str :
138+ skip_special_tokens = False
138139 return self .tokenizer .decode (ids , skip_special_tokens = skip_special_tokens )
139140
140141 def convert_ids_to_tokens (
141142 self ,
142143 ids : list [int ],
143144 skip_special_tokens : bool = False ,
144145 ) -> list [str ]:
145- return self .tokenizer .convert_ids_to_tokens (
146- ids , skip_special_tokens = skip_special_tokens
147- )
146+ return [self .decode (ids , skip_special_tokens = skip_special_tokens )]
You can’t perform that action at this time.
0 commit comments