شمردن تعداد توکن‌ها با tiktoken

1encoding = tiktoken.encoding_for_model('gpt-3.5-turbo')

1%pip install --upgrade tiktoken
2%pip install --upgrade openai

1import tiktoken
2
3encoding = tiktoken.get_encoding("cl100k_base")

1encoding = tiktoken.encoding_for_model("gpt-3.5-turbo")

1encoding.encode("tiktoken is great!")
2
3-> [83, 1609, 5963, 374, 2294, 0]

 1def num_tokens_from_string(string: str, encoding_name: str) -> int:
 2    """Returns the number of tokens in a text string."""
 3    encoding = tiktoken.get_encoding(encoding_name)
 4    num_tokens = len(encoding.encode(string))
 5    return num_tokens
 6
 7
 8num_tokens_from_string("tiktoken is great!", "cl100k_base")
 9
10-> 6

1encoding.decode([83, 1609, 5963, 374, 2294, 0])
2
3-> 'tiktoken is great!'

1[encoding.decode_single_token_bytes(token) for token in [83, 1609, 5963, 374, 2294, 0]]
2
3-> [b't', b'ik', b'token', b' is', b' great', b'!']

 1def num_tokens_from_messages(messages, model="gpt-3.5-turbo"):
 2   """Return the number of tokens used by a list of messages."""
 3   try:
 4       encoding = tiktoken.encoding_for_model(model)
 5   except KeyError:
 6       print("Warning: model not found. Using cl100k_base encoding.")
 7       encoding = tiktoken.get_encoding("cl100k_base")
 8   if model in {
 9       "gpt-3.5-turbo-0613",
10       "gpt-3.5-turbo-16k-0613",
11       "gpt-4-0314",
12       "gpt-4-32k-0314",
13       "gpt-4-0613",
14       "gpt-4-32k-0613",
15       }:
16       tokens_per_message = 3
17       tokens_per_name = 1
18   elif model == "gpt-3.5-turbo-0301":
19       tokens_per_message = 4  # every message follows <|start|>{role/name}\n{content}<|end|>\n
20       tokens_per_name = -1  # if there's a name, the role is omitted
21   elif "gpt-3.5-turbo" in model:
22       print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.")
23       return num_tokens_from_messages(messages, model="gpt-3.5-turbo-0613")
24   elif "gpt-4" in model:
25       print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.")
26       return num_tokens_from_messages(messages, model="gpt-4-0613")
27   else:
28       raise NotImplementedError(
29           f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."""
30       )
31   num_tokens = 0
32   for message in messages:
33       num_tokens += tokens_per_message
34       for key, value in message.items():
35           num_tokens += len(encoding.encode(value))
36           if key == "name":
37               num_tokens += tokens_per_name
38   num_tokens += 3  # every reply is primed with <|start|>assistant<|message|>
39   return num_tokens

 1# let's verify the function above matches the Gilas API response
 2
 3from openai import OpenAI
 4import os
 5
 6client = OpenAI(
 7    api_key=os.environ.get(("GILAS_API_KEY", "<کلید API خود را اینجا بسازید https://dashboard.gilas.io/apiKey>")), 
 8    base_url="https://api.gilas.io/v1/" # Gilas APIs
 9)
10
11example_messages = [
12    {
13        "role": "system",
14        "content": "You are a helpful, pattern-following assistant that translates corporate jargon into plain English.",
15    },
16    {
17        "role": "system",
18        "name": "example_user",
19        "content": "New synergies will help drive top-line growth.",
20    },
21    {
22        "role": "system",
23        "name": "example_assistant",
24        "content": "Things working well together will increase revenue.",
25    },
26    {
27        "role": "system",
28        "name": "example_user",
29        "content": "Let's circle back when we have more bandwidth to touch base on opportunities for increased leverage.",
30    },
31    {
32        "role": "system",
33        "name": "example_assistant",
34        "content": "Let's talk later when we're less busy about how to do better.",
35    },
36    {
37        "role": "user",
38        "content": "This late pivot means we don't have time to boil the ocean for the client deliverable.",
39    },
40]
41
42for model in [
43    "gpt-3.5-turbo-0301",
44    "gpt-3.5-turbo-0613",
45    "gpt-3.5-turbo",
46    "gpt-4-0314",
47    "gpt-4-0613",
48    "gpt-4",
49    ]:
50    print(model)
51    # example token count from the function defined above
52    print(f"{num_tokens_from_messages(example_messages, model)} prompt tokens counted by num_tokens_from_messages().")
53    # example token count from the Gilas API
54    response = client.chat.completions.create(model=model,
55    messages=example_messages,
56    temperature=0,
57    max_tokens=1)
58    print(f'{response.usage.prompt_tokens} prompt tokens counted by the Gilas API.')
59    print()