Skip to content

Cohere tokenizer

CohereTokenizer

Bases: BaseTokenizer

Source code in griptape/griptape/tokenizers/cohere_tokenizer.py
@define(frozen=True)
class CohereTokenizer(BaseTokenizer):
    DEFAULT_MODEL = "command"
    DEFAULT_MAX_TOKENS = 2048

    model: str = field(kw_only=True)
    client: Client = field(kw_only=True)
    max_tokens: int = field(default=DEFAULT_MAX_TOKENS, kw_only=True)

    def count_tokens(self, text: str | list) -> int:
        if isinstance(text, str):
            return len(self.client.tokenize(text=text).tokens)
        else:
            raise ValueError("Text must be a string.")

DEFAULT_MAX_TOKENS = 2048 class-attribute instance-attribute

DEFAULT_MODEL = 'command' class-attribute instance-attribute

client: Client = field(kw_only=True) class-attribute instance-attribute

max_tokens: int = field(default=DEFAULT_MAX_TOKENS, kw_only=True) class-attribute instance-attribute

model: str = field(kw_only=True) class-attribute instance-attribute

count_tokens(text)

Source code in griptape/griptape/tokenizers/cohere_tokenizer.py
def count_tokens(self, text: str | list) -> int:
    if isinstance(text, str):
        return len(self.client.tokenize(text=text).tokens)
    else:
        raise ValueError("Text must be a string.")