Cohere tokenizer

CohereTokenizer

Bases: BaseTokenizer

Source code in griptape/tokenizers/cohere_tokenizer.py

@define()
class CohereTokenizer(BaseTokenizer):
    MODEL_PREFIXES_TO_MAX_INPUT_TOKENS = {"command": 4096}
    MODEL_PREFIXES_TO_MAX_OUTPUT_TOKENS = {"command": 4096}

    client: Client = field(kw_only=True)

    def count_tokens(self, text: str | list) -> int:
        if isinstance(text, str):
            return len(self.client.tokenize(text=text).tokens)
        else:
            raise ValueError("Text must be a string.")

MODEL_PREFIXES_TO_MAX_INPUT_TOKENS = {'command': 4096} class-attribute instance-attribute

MODEL_PREFIXES_TO_MAX_OUTPUT_TOKENS = {'command': 4096} class-attribute instance-attribute

client: Client = field(kw_only=True) class-attribute instance-attribute

count_tokens(text)

Source code in griptape/tokenizers/cohere_tokenizer.py

def count_tokens(self, text: str | list) -> int:
    if isinstance(text, str):
        return len(self.client.tokenize(text=text).tokens)
    else:
        raise ValueError("Text must be a string.")