Base tokenizer

BaseTokenizer

Bases: ABC

Source code in griptape/griptape/tokenizers/base_tokenizer.py

@define(frozen=True)
class BaseTokenizer(ABC):
    stop_sequences: list[str] = field(default=Factory(lambda: [utils.constants.RESPONSE_STOP_SEQUENCE]), kw_only=True)

    @property
    @abstractmethod
    def max_tokens(self) -> int:
        ...

    def count_tokens_left(self, text: str | list) -> int:
        diff = self.max_tokens - self.count_tokens(text)

        if diff > 0:
            return diff
        else:
            return 0

    @abstractmethod
    def count_tokens(self, text: str | list) -> int:
        ...

max_tokens: int abstractmethod property

stop_sequences: list[str] = field(default=Factory(lambda : [utils.constants.RESPONSE_STOP_SEQUENCE]), kw_only=True) class-attribute instance-attribute

count_tokens(text) abstractmethod

Source code in griptape/griptape/tokenizers/base_tokenizer.py

@abstractmethod
def count_tokens(self, text: str | list) -> int:
    ...

count_tokens_left(text)

Source code in griptape/griptape/tokenizers/base_tokenizer.py

def count_tokens_left(self, text: str | list) -> int:
    diff = self.max_tokens - self.count_tokens(text)

    if diff > 0:
        return diff
    else:
        return 0