Skip to content

Bedrock llama tokenizer

BedrockLlamaTokenizer

Bases: SimpleTokenizer

Source code in griptape/tokenizers/bedrock_llama_tokenizer.py 
 6
 7
 8
 9
10
11
12
13
14
@define()
class BedrockLlamaTokenizer(SimpleTokenizer):
    DEFAULT_CHARACTERS_PER_TOKEN = 6  # https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-prepare.html#model-customization-prepare-finetuning
    MODEL_PREFIXES_TO_MAX_INPUT_TOKENS = {"meta": 2048}
    MODEL_PREFIXES_TO_MAX_OUTPUT_TOKENS = {"meta": 2048}

    model: str = field(kw_only=True)
    characters_per_token: int = field(default=DEFAULT_CHARACTERS_PER_TOKEN, kw_only=True)
    stop_sequences: list[str] = field(factory=list, kw_only=True)

DEFAULT_CHARACTERS_PER_TOKEN = 6 class-attribute instance-attribute

MODEL_PREFIXES_TO_MAX_INPUT_TOKENS = {'meta': 2048} class-attribute instance-attribute

MODEL_PREFIXES_TO_MAX_OUTPUT_TOKENS = {'meta': 2048} class-attribute instance-attribute

characters_per_token: int = field(default=DEFAULT_CHARACTERS_PER_TOKEN, kw_only=True) class-attribute instance-attribute

model: str = field(kw_only=True) class-attribute instance-attribute

stop_sequences: list[str] = field(factory=list, kw_only=True) class-attribute instance-attribute