Huggingface tokenizer
HuggingFaceTokenizer
Bases: BaseTokenizer
HuggingFaceTokenizer
Bases: BaseTokenizer
griptape/tokenizers/huggingface_tokenizer.py
max_input_tokens: int = field(default=Factory(lambda self: self.tokenizer.model_max_length, takes_self=True), kw_only=True)
class-attribute
instance-attribute
max_output_tokens: int = field(kw_only=True)
class-attribute
instance-attribute
model: str = field(init=False, kw_only=True)
class-attribute
instance-attribute
tokenizer: PreTrainedTokenizerBase = field(kw_only=True)
class-attribute
instance-attribute