Skip to content

Anthropic tokenizer

AnthropicTokenizer

Bases: BaseTokenizer

Source code in griptape/griptape/tokenizers/anthropic_tokenizer.py
@define(frozen=True)
class AnthropicTokenizer(BaseTokenizer):
    DEFAULT_MODEL = "claude-2.1"
    MODEL_PREFIXES_TO_MAX_TOKENS = {"claude-2.1": 200000, "claude": 100000}

    model: str = field(kw_only=True)
    max_tokens: int = field(kw_only=True, default=Factory(lambda self: self.default_max_tokens(), takes_self=True))
    client: Anthropic = field(
        default=Factory(lambda: import_optional_dependency("anthropic").Anthropic()), kw_only=True
    )

    def default_max_tokens(self) -> int:
        tokens = next(v for k, v in self.MODEL_PREFIXES_TO_MAX_TOKENS.items() if self.model.startswith(k))

        return tokens

    def count_tokens(self, text: str | list) -> int:
        if isinstance(text, str):
            return self.client.count_tokens(text)
        else:
            raise ValueError("Text must be a string.")

DEFAULT_MODEL = 'claude-2.1' class-attribute instance-attribute

MODEL_PREFIXES_TO_MAX_TOKENS = {'claude-2.1': 200000, 'claude': 100000} class-attribute instance-attribute

client: Anthropic = field(default=Factory(lambda : import_optional_dependency('anthropic').Anthropic()), kw_only=True) class-attribute instance-attribute

max_tokens: int = field(kw_only=True, default=Factory(lambda : self.default_max_tokens(), takes_self=True)) class-attribute instance-attribute

model: str = field(kw_only=True) class-attribute instance-attribute

count_tokens(text)

Source code in griptape/griptape/tokenizers/anthropic_tokenizer.py
def count_tokens(self, text: str | list) -> int:
    if isinstance(text, str):
        return self.client.count_tokens(text)
    else:
        raise ValueError("Text must be a string.")

default_max_tokens()

Source code in griptape/griptape/tokenizers/anthropic_tokenizer.py
def default_max_tokens(self) -> int:
    tokens = next(v for k, v in self.MODEL_PREFIXES_TO_MAX_TOKENS.items() if self.model.startswith(k))

    return tokens