1$ docker-compose up -d

1! pip install redis wget pandas openai

1from openai import OpenAI
2import os
3
4client = OpenAI(
5    api_key=os.environ.get(("GILAS_API_KEY", "<کلید API خود را اینجا بسازید https://dashboard.gilas.io/apiKey>")), 
6    base_url="https://api.gilas.io/v1/" # Gilas APIs
7)

 1import sys
 2import numpy as np
 3import pandas as pd
 4from typing import List
 5
 6# استفاده از تابع کمکی در nbutils.py برای دانلود و خواندن داده‌ها
 7# این باید بین 5-10 دقیقه طول بکشد
 8if os.getcwd() not in sys.path:
 9    sys.path.append(os.getcwd())
10import nbutils
11
12nbutils.download_wikipedia_data()
13data = nbutils.read_wikipedia_data()
14
15data.head()

 1import redis
 2from redis.commands.search.indexDefinition import (
 3    IndexDefinition,
 4    IndexType
 5)
 6from redis.commands.search.query import Query
 7from redis.commands.search.field import (
 8    TextField,
 9    VectorField
10)
11
12REDIS_HOST =  "localhost"
13REDIS_PORT = 6379
14REDIS_PASSWORD = ""
15
16# اتصال به Redis
17redis_client = redis.Redis(
18    host=REDIS_HOST,
19    port=REDIS_PORT,
20    password=REDIS_PASSWORD
21)
22redis_client.ping()

 1VECTOR_DIM = len(data['title_vector'][0]) # طول وکتورها
 2VECTOR_NUMBER = len(data)                 # تعداد اولیه وکتورها
 3INDEX_NAME = "embeddings-index"           # نام ایندکس جستجو
 4PREFIX = "doc"                            # پیشوند برای کلیدها
 5DISTANCE_METRIC = "COSINE"                # متریک فاصله برای وکتورها (مثلاً COSINE، IP، L2)
 6
 7# تعریف فیلدهای RediSearch برای هر یک از ستون‌های مجموعه داده
 8title = TextField(name="title")
 9url = TextField(name="url")
10text = TextField(name="text")
11title_embedding = VectorField("title_vector",
12    "FLAT", {
13        "TYPE": "FLOAT32",
14        "DIM": VECTOR_DIM,
15        "DISTANCE_METRIC": DISTANCE_METRIC,
16        "INITIAL_CAP": VECTOR_NUMBER,
17    }
18)
19text_embedding = VectorField("content_vector",
20    "FLAT", {
21        "TYPE": "FLOAT32",
22        "DIM": VECTOR_DIM,
23        "DISTANCE_METRIC": DISTANCE_METRIC,
24        "INITIAL_CAP": VECTOR_NUMBER,
25    }
26)
27fields = [title, url, text, title_embedding, text_embedding]
28
29# بررسی وجود ایندکس
30try:
31    redis_client.ft(INDEX_NAME).info()
32    print("ایندکس از قبل وجود دارد")
33except:
34    # ایجاد ایندکس RediSearch
35    redis_client.ft(INDEX_NAME).create_index(
36        fields = fields,
37        definition = IndexDefinition(prefix=[PREFIX], index_type=IndexType.HASH)
38)

 1def index_documents(client: redis.Redis, prefix: str, documents: pd.DataFrame):
 2    records = documents.to_dict("records")
 3    for doc in records:
 4        key = f"{prefix}:{str(doc['id'])}"
 5
 6        # ایجاد وکتورهای بایتی برای عنوان و محتوا
 7        title_embedding = np.array(doc["title_vector"], dtype=np.float32).tobytes()
 8        content_embedding = np.array(doc["content_vector"], dtype=np.float32).tobytes()
 9
10        # جایگزینی لیست اعداد اعشاری با وکتورهای بایتی
11        doc["title_vector"] = title_embedding
12        doc["content_vector"] = content_embedding
13
14        client.hset(key, mapping = doc)

1index_documents(redis_client, PREFIX, data)
2print(f"بارگذاری {redis_client.info()['db0']['keys']} سند در ایندکس جستجوی Redis با نام: {INDEX_NAME}")

 1def search_redis(
 2    redis_client: redis.Redis,
 3    user_query: str,
 4    index_name: str = "embeddings-index",
 5    vector_field: str = "title_vector",
 6    return_fields: list = ["title", "url", "text", "vector_score"],
 7    hybrid_fields = "*",
 8    k: int = 20,
 9    print_results: bool = True,
10) -> List[dict]:
11
12    # ایجاد وکتور جاسازی شده از پرس و جوی کاربر
13    embedded_query = client.embeddings.create(input=user_query,
14                                            model="text-embedding-3-small",
15                                            )["data"][0]['embedding']
16
17    # آماده‌سازی پرس و جو
18    base_query = f'{hybrid_fields}=>[KNN {k} @{vector_field} $vector AS vector_score]'
19    query = (
20        Query(base_query)
21         .return_fields(*return_fields)
22         .sort_by("vector_score")
23         .paging(0, k)
24         .dialect(2)
25    )
26    params_dict = {"vector": np.array(embedded_query).astype(dtype=np.float32).tobytes()}
27
28    # اجرای جستجوی وکتور
29    results = redis_client.ft(index_name).search(query, params_dict)
30    if print_results:
31        for i, article in enumerate(results.docs):
32            score = 1 - float(article.vector_score)
33            print(f"{i}. {article.title} (امتیاز: {round(score ,3) })")
34    return results.docs

1results = search_redis(redis_client, 'modern art in Europe', k=10)

1results = search_redis(redis_client, 'Famous battles in Scottish history', vector_field='content_vector', k=10)

 1def create_hybrid_field(field_name: str, value: str) -> str:
 2    return f'@{field_name}:"{value}"'
 3
 4# search the content vector for articles about famous battles in Scottish history and only include results with Scottish in the title
 5results = search_redis(redis_client,
 6                       "Famous battles in Scottish history",
 7                       vector_field="title_vector",
 8                       k=5,
 9                       hybrid_fields=create_hybrid_field("title", "Scottish")
10                       )

 1# run a hybrid query for articles about Art in the title vector and only include results with the phrase "Leonardo da Vinci" in the text
 2results = search_redis(redis_client,
 3                       "Art",
 4                       vector_field="title_vector",
 5                       k=5,
 6                       hybrid_fields=create_hybrid_field("text", "Leonardo da Vinci")
 7                       )
 8
 9# find specific mention of Leonardo da Vinci in the text that our full-text-search query returned
10mention = [sentence for sentence in results[0].text.split("\n") if "Leonardo da Vinci" in sentence][0]
11mention

 1# re-define RediSearch vector fields to use HNSW index
 2title_embedding = VectorField("title_vector",
 3    "HNSW", {
 4        "TYPE": "FLOAT32",
 5        "DIM": VECTOR_DIM,
 6        "DISTANCE_METRIC": DISTANCE_METRIC,
 7        "INITIAL_CAP": VECTOR_NUMBER
 8    }
 9)
10text_embedding = VectorField("content_vector",
11    "HNSW", {
12        "TYPE": "FLOAT32",
13        "DIM": VECTOR_DIM,
14        "DISTANCE_METRIC": DISTANCE_METRIC,
15        "INITIAL_CAP": VECTOR_NUMBER
16    }
17)
18fields = [title, url, text, title_embedding, text_embedding]

 1import time
 2# بررسی وجود ایندکس
 3HNSW_INDEX_NAME = INDEX_NAME + "_HNSW"
 4
 5try:
 6    redis_client.ft(HNSW_INDEX_NAME).info()
 7    print("ایندکس از قبل وجود دارد")
 8except:
 9    # ایجاد ایندکس RediSearch
10    redis_client.ft(HNSW_INDEX_NAME).create_index(
11        fields = fields,
12        definition = IndexDefinition(prefix=[PREFIX], index_type=IndexType.HASH)
13    )
14
15# since RediSearch creates the index in the background for existing documents, we will wait until
16# indexing is complete before running our queries. Although this is not necessary for the first query,
17# some queries may take longer to run if the index is not fully built. In general, Redis will perform
18# best when adding new documents to existing indices rather than new indices on existing documents.
19while redis_client.ft(HNSW_INDEX_NAME).info()["indexing"] == "1":
20    time.sleep(5)

1results = search_redis(redis_client, 'modern art in Europe', index_name=HNSW_INDEX_NAME, k=10)

 1# compare the results of the HNSW index to the FLAT index and time both queries
 2def time_queries(iterations: int = 10):
 3    print(" ----- ایندکس FLAT ----- ")
 4    t0 = time.time()
 5    for i in range(iterations):
 6        results_flat = search_redis(redis_client, 'modern art in Europe', k=10, print_results=False)
 7    t0 = (time.time() - t0) / iterations
 8    results_flat = search_redis(redis_client, 'modern art in Europe', k=10, print_results=True)
 9    print(f"زمان جستجوی ایندکس FLAT: {round(t0, 3)} ثانیه\n")
10    time.sleep(1)
11    print(" ----- ایندکس HNSW ------ ")
12    t1 = time.time()
13    for i in range(iterations):
14        results_hnsw = search_redis(redis_client, 'modern art in Europe', index_name=HNSW_INDEX_NAME, k=10, print_results=False)
15    t1 = (time.time() - t1) / iterations
16    results_hnsw = search_redis(redis_client, 'modern art in Europe', index_name=HNSW_INDEX_NAME, k=10, print_results=True)
17    print(f"زمان جستجوی ایندکس HNSW: {round(t1, 3)} ثانیه")
18    print(" ------------------------ ")
19time_queries()