Skip to content

Bedrock cohere tokenizer

BedrockCohereTokenizer

Bases: SimpleTokenizer

Source code in griptape/tokenizers/bedrock_cohere_tokenizer.py 
 6
 7
 8
 9
10
11
12
13
14
@define()
class BedrockCohereTokenizer(SimpleTokenizer):
    # https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-embed.html
    DEFAULT_CHARACTERS_PER_TOKEN = 4
    MODEL_PREFIXES_TO_MAX_INPUT_TOKENS = {"cohere": 1024}
    MODEL_PREFIXES_TO_MAX_OUTPUT_TOKENS = {"cohere": 4096}

    model: str = field(kw_only=True)
    characters_per_token: int = field(default=DEFAULT_CHARACTERS_PER_TOKEN, kw_only=True)

DEFAULT_CHARACTERS_PER_TOKEN = 4 class-attribute instance-attribute

MODEL_PREFIXES_TO_MAX_INPUT_TOKENS = {'cohere': 1024} class-attribute instance-attribute

MODEL_PREFIXES_TO_MAX_OUTPUT_TOKENS = {'cohere': 4096} class-attribute instance-attribute

characters_per_token: int = field(default=DEFAULT_CHARACTERS_PER_TOKEN, kw_only=True) class-attribute instance-attribute

model: str = field(kw_only=True) class-attribute instance-attribute