Skip to content

Anthropic tokenizer

AnthropicTokenizer

Bases: BaseTokenizer

Source code in griptape/griptape/tokenizers/anthropic_tokenizer.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
@define(frozen=True)
class AnthropicTokenizer(BaseTokenizer):
    DEFAULT_MODEL = "claude-2"
    DEFAULT_MAX_TOKENS = 100000

    model: str = field(kw_only=True)

    @property
    def max_tokens(self) -> int:
        return self.DEFAULT_MAX_TOKENS

    def count_tokens(self, text: str) -> int:
        anthropic = import_optional_dependency("anthropic")

        return len(anthropic._client.sync_get_tokenizer().encode(text).ids)

DEFAULT_MAX_TOKENS = 100000 class-attribute instance-attribute

DEFAULT_MODEL = 'claude-2' class-attribute instance-attribute

max_tokens: int property

model: str = field(kw_only=True) class-attribute instance-attribute

count_tokens(text)

Source code in griptape/griptape/tokenizers/anthropic_tokenizer.py 
17
18
19
20
def count_tokens(self, text: str) -> int:
    anthropic = import_optional_dependency("anthropic")

    return len(anthropic._client.sync_get_tokenizer().encode(text).ids)