1from openai import OpenAI
 2import os
 3import openai
 4from tenacity import retry, wait_random_exponential, stop_after_attempt, retry_if_not_exception_type
 5
 6client = OpenAI(
 7    api_key=os.environ.get(("GILAS_API_KEY", "<کلید API خود را اینجا بسازید https://dashboard.gilas.io/apiKey>")), 
 8    base_url="https://api.gilas.io/v1/" # Gilas APIs
 9)
10
11EMBEDDING_MODEL = 'text-embedding-3-small'
12EMBEDDING_CTX_LENGTH = 8191
13EMBEDDING_ENCODING = 'cl100k_base'
14
15# let's make sure to not retry on an invalid request, because that is what we want to demonstrate
16@retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6), retry=retry_if_not_exception_type(openai.BadRequestError))
17def get_embedding(text_or_tokens, model=EMBEDDING_MODEL):
18    return client.embeddings.create(input=text_or_tokens, model=model).data[0].embedding

1long_text = 'AGI ' * 5000
2try:
3    get_embedding(long_text)
4except openai.BadRequestError as e:
5    print(e)

1import tiktoken
2
3def truncate_text_tokens(text, encoding_name=EMBEDDING_ENCODING, max_tokens=EMBEDDING_CTX_LENGTH):
4    """Truncate a string to have `max_tokens` according to the given encoding."""
5    encoding = tiktoken.get_encoding(encoding_name)
6    return encoding.encode(text)[:max_tokens]

1truncated = truncate_text_tokens(long_text)
2len(get_embedding(truncated))

 1from itertools import islice
 2
 3def batched(iterable, n):
 4    """Batch data into tuples of length n. The last batch may be shorter."""
 5    # batched('ABCDEFG', 3) --> ABC DEF G
 6    if n < 1:
 7        raise ValueError('n must be at least one')
 8    it = iter(iterable)
 9    while (batch := tuple(islice(it, n))):
10        yield batch

1def chunked_tokens(text, encoding_name, chunk_length):
2    encoding = tiktoken.get_encoding(encoding_name)
3    tokens = encoding.encode(text)
4    chunks_iterator = batched(tokens, chunk_length)
5    yield from chunks_iterator

 1import numpy as np
 2
 3def len_safe_get_embedding(text, model=EMBEDDING_MODEL, max_tokens=EMBEDDING_CTX_LENGTH, encoding_name=EMBEDDING_ENCODING, average=True):
 4    chunk_embeddings = []
 5    chunk_lens = []
 6    for chunk in chunked_tokens(text, encoding_name=encoding_name, chunk_length=max_tokens):
 7        chunk_embeddings.append(get_embedding(chunk, model=model))
 8        chunk_lens.append(len(chunk))
 9
10    if average:
11        chunk_embeddings = np.average(chunk_embeddings, axis=0, weights=chunk_lens)
12        chunk_embeddings = chunk_embeddings / np.linalg.norm(chunk_embeddings)  # normalizes length to 1
13        chunk_embeddings = chunk_embeddings.tolist()
14    return chunk_embeddings

1average_embedding_vector = len_safe_get_embedding(long_text, average=True)
2chunks_embedding_vectors = len_safe_get_embedding(long_text, average=False)
3
4print(f"Setting average=True gives us a single {len(average_embedding_vector)}-dimensional embedding vector for our long text.")
5print(f"Setting average=False gives us {len(chunks_embedding_vectors)} embedding vectors, one for each of the chunks.")