Skip to content

vllm.tokenizers.deepseekv32

DeepseekV32Tokenizer

Bases: HfTokenizer

Source code in vllm/tokenizers/deepseekv32.py 
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
@TokenizerRegistry.register("deepseek_v32")
class DeepseekV32Tokenizer(HfTokenizer):
    def __init__(self, tokenizer: TokenizerLike):
        self.tokenizer = tokenizer
        self.name_or_path = (
            tokenizer.name_or_path if hasattr(tokenizer, "name_or_path") else ""
        )

    @classmethod
    def from_pretrained(
        cls,
        path_or_repo_id: str | Path,
        *args,
        trust_remote_code: bool = False,
        revision: str | None = None,
        download_dir: str | None = None,
        **kwargs,
    ) -> "TokenizerLike":
        tokenizer = super().from_pretrained(
            path_or_repo_id,
            *args,
            trust_remote_code=trust_remote_code,
            revision=revision,
            download_dir=download_dir,
            **kwargs,
        )
        return DeepseekV32Tokenizer(tokenizer)

    def apply_chat_template(self, messages, tools=None, **kwargs):
        thinking = kwargs.get("thinking", False)
        thinking_mode = "thinking"
        if not thinking:
            thinking_mode = "chat"
        messages = messages.copy()
        drop_thinking = True
        if tools is not None and len(tools) > 0:
            messages.insert(0, {"role": "system"})
            messages[0]["tools"] = tools
            drop_thinking = False
        encode_config = dict(thinking_mode=thinking_mode, drop_thinking=drop_thinking)
        prompt_str = encode_messages(messages, **encode_config)  # type: ignore
        return prompt_str

    @property
    def all_special_tokens(self) -> list[str]:
        return self.tokenizer.all_special_tokens

    @property
    def all_special_ids(self) -> list[int]:
        return self.tokenizer.all_special_ids

    @property
    def bos_token_id(self) -> int:
        return self.tokenizer.bos_token_id

    @property
    def eos_token_id(self) -> int:
        return self.tokenizer.eos_token_id

    @property
    def pad_token_id(self) -> int:
        return self.tokenizer.pad_token_id

    @property
    def is_fast(self) -> bool:
        return self.tokenizer.is_fast

    @property
    def vocab_size(self) -> int:
        return self.tokenizer.vocab_size

    @property
    def max_token_id(self) -> int:
        return self.tokenizer.max_token_id

    @property
    def truncation_side(self) -> str:
        return self.tokenizer.truncation_side

    def __hash__(self) -> int:
        return hash(id(self))

    def __len__(self) -> int:
        # </think> is an added token in DeepseekV32 tokenizer
        return self.vocab_size + len(self.get_added_vocab())

    def __call__(
        self,
        text: str | list[str],
        text_pair: str | None = None,
        add_special_tokens: bool = True,
        truncation: bool = False,
        max_length: int | None = None,
    ) -> "BatchEncoding":
        return self.tokenizer(
            text,
            text_pair=text_pair,
            add_special_tokens=add_special_tokens,
            truncation=truncation,
            max_length=max_length,
        )

    def get_vocab(self) -> dict[str, int]:
        return self.tokenizer.get_vocab()

    def get_added_vocab(self) -> dict[str, int]:
        return self.tokenizer.get_added_vocab()

    def encode(
        self,
        text: str,
        truncation: bool | None = None,
        max_length: int | None = None,
        add_special_tokens: bool = True,
    ) -> list[int]:
        return self.tokenizer.encode(
            text,
            truncation=truncation,
            max_length=max_length,
            add_special_tokens=add_special_tokens,
        )

    def convert_tokens_to_string(self, tokens: list[str]) -> str:
        return self.tokenizer.convert_tokens_to_string(tokens)

    def decode(self, ids: list[int] | int, skip_special_tokens: bool = False) -> str:
        return self.tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)

    def convert_ids_to_tokens(
        self,
        ids: list[int],
        skip_special_tokens: bool = False,
    ) -> list[str]:
        return self.tokenizer.convert_ids_to_tokens(
            ids, skip_special_tokens=skip_special_tokens
        )

all_special_ids property 

all_special_ids: list[int]

all_special_tokens property 

all_special_tokens: list[str]

bos_token_id property 

bos_token_id: int

eos_token_id property 

eos_token_id: int

is_fast property 

is_fast: bool

max_token_id property 

max_token_id: int

name_or_path instance-attribute 

name_or_path = (
    name_or_path
    if hasattr(tokenizer, "name_or_path")
    else ""
)

pad_token_id property 

pad_token_id: int

tokenizer instance-attribute 

tokenizer = tokenizer

truncation_side property 

truncation_side: str

vocab_size property 

vocab_size: int

__call__ 

__call__(
    text: str | list[str],
    text_pair: str | None = None,
    add_special_tokens: bool = True,
    truncation: bool = False,
    max_length: int | None = None,
) -> BatchEncoding
Source code in vllm/tokenizers/deepseekv32.py 
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
def __call__(
    self,
    text: str | list[str],
    text_pair: str | None = None,
    add_special_tokens: bool = True,
    truncation: bool = False,
    max_length: int | None = None,
) -> "BatchEncoding":
    return self.tokenizer(
        text,
        text_pair=text_pair,
        add_special_tokens=add_special_tokens,
        truncation=truncation,
        max_length=max_length,
    )

__hash__ 

__hash__() -> int
Source code in vllm/tokenizers/deepseekv32.py 
92
93
def __hash__(self) -> int:
    return hash(id(self))

__init__ 

__init__(tokenizer: TokenizerLike)
Source code in vllm/tokenizers/deepseekv32.py 
15
16
17
18
19
def __init__(self, tokenizer: TokenizerLike):
    self.tokenizer = tokenizer
    self.name_or_path = (
        tokenizer.name_or_path if hasattr(tokenizer, "name_or_path") else ""
    )

__len__ 

__len__() -> int
Source code in vllm/tokenizers/deepseekv32.py 
95
96
97
def __len__(self) -> int:
    # </think> is an added token in DeepseekV32 tokenizer
    return self.vocab_size + len(self.get_added_vocab())

apply_chat_template 

apply_chat_template(messages, tools=None, **kwargs)
Source code in vllm/tokenizers/deepseekv32.py 
41
42
43
44
45
46
47
48
49
50
51
52
53
54
def apply_chat_template(self, messages, tools=None, **kwargs):
    thinking = kwargs.get("thinking", False)
    thinking_mode = "thinking"
    if not thinking:
        thinking_mode = "chat"
    messages = messages.copy()
    drop_thinking = True
    if tools is not None and len(tools) > 0:
        messages.insert(0, {"role": "system"})
        messages[0]["tools"] = tools
        drop_thinking = False
    encode_config = dict(thinking_mode=thinking_mode, drop_thinking=drop_thinking)
    prompt_str = encode_messages(messages, **encode_config)  # type: ignore
    return prompt_str

convert_ids_to_tokens 

convert_ids_to_tokens(
    ids: list[int], skip_special_tokens: bool = False
) -> list[str]
Source code in vllm/tokenizers/deepseekv32.py 
141
142
143
144
145
146
147
148
def convert_ids_to_tokens(
    self,
    ids: list[int],
    skip_special_tokens: bool = False,
) -> list[str]:
    return self.tokenizer.convert_ids_to_tokens(
        ids, skip_special_tokens=skip_special_tokens
    )

convert_tokens_to_string 

convert_tokens_to_string(tokens: list[str]) -> str
Source code in vllm/tokenizers/deepseekv32.py 
135
136
def convert_tokens_to_string(self, tokens: list[str]) -> str:
    return self.tokenizer.convert_tokens_to_string(tokens)

decode 

decode(
    ids: list[int] | int, skip_special_tokens: bool = False
) -> str
Source code in vllm/tokenizers/deepseekv32.py 
138
139
def decode(self, ids: list[int] | int, skip_special_tokens: bool = False) -> str:
    return self.tokenizer.decode(ids, skip_special_tokens=skip_special_tokens)

encode 

encode(
    text: str,
    truncation: bool | None = None,
    max_length: int | None = None,
    add_special_tokens: bool = True,
) -> list[int]
Source code in vllm/tokenizers/deepseekv32.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
def encode(
    self,
    text: str,
    truncation: bool | None = None,
    max_length: int | None = None,
    add_special_tokens: bool = True,
) -> list[int]:
    return self.tokenizer.encode(
        text,
        truncation=truncation,
        max_length=max_length,
        add_special_tokens=add_special_tokens,
    )

from_pretrained classmethod 

from_pretrained(
    path_or_repo_id: str | Path,
    *args,
    trust_remote_code: bool = False,
    revision: str | None = None,
    download_dir: str | None = None,
    **kwargs,
) -> TokenizerLike
Source code in vllm/tokenizers/deepseekv32.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
@classmethod
def from_pretrained(
    cls,
    path_or_repo_id: str | Path,
    *args,
    trust_remote_code: bool = False,
    revision: str | None = None,
    download_dir: str | None = None,
    **kwargs,
) -> "TokenizerLike":
    tokenizer = super().from_pretrained(
        path_or_repo_id,
        *args,
        trust_remote_code=trust_remote_code,
        revision=revision,
        download_dir=download_dir,
        **kwargs,
    )
    return DeepseekV32Tokenizer(tokenizer)

get_added_vocab 

get_added_vocab() -> dict[str, int]
Source code in vllm/tokenizers/deepseekv32.py 
118
119
def get_added_vocab(self) -> dict[str, int]:
    return self.tokenizer.get_added_vocab()

get_vocab 

get_vocab() -> dict[str, int]
Source code in vllm/tokenizers/deepseekv32.py 
115
116
def get_vocab(self) -> dict[str, int]:
    return self.tokenizer.get_vocab()