خوشه‌بندی K-means در پایتون با استفاده از Gilas API

 1import pandas as pd
 2import tiktoken
 3
 4# load & inspect dataset
 5input_datapath = "data/fine_food_reviews_1k.csv"  # to save space, we provide a pre-filtered dataset
 6df = pd.read_csv(input_datapath, index_col=0)
 7df = df[["Time", "ProductId", "UserId", "Score", "Summary", "Text"]]
 8df = df.dropna()
 9df["combined"] = (
10    "Title: " + df.Summary.str.strip() + "; Content: " + df.Text.str.strip()
11)
12df.head(2)

 1
 2embedding_model = "text-embedding-3-small"
 3max_tokens = 8000  # the maximum for text-embedding-3-small is 8191
 4embedding_encoding = "cl100k_base"
 5
 6# subsample to 1k most recent reviews and remove samples that are too long
 7top_n = 1000
 8df = df.sort_values("Time").tail(top_n * 2)  # first cut to first 2k entries, assuming less than half will be filtered out
 9df.drop("Time", axis=1, inplace=True)
10
11encoding = tiktoken.get_encoding(embedding_encoding)
12
13# omit reviews that are too long to embed
14df["n_tokens"] = df.combined.apply(lambda x: len(encoding.encode(x)))
15df = df[df.n_tokens <= max_tokens].tail(top_n)

 1
 2from openai import OpenAI # for calling the OpenAI API
 3import os
 4
 5client = OpenAI(
 6    api_key=os.environ.get(("GILAS_API_KEY", "<کلید API خود را اینجا بسازید https://dashboard.gilas.io/apiKey>")), 
 7    base_url="https://api.gilas.io/v1/" # Gilas APIs
 8)
 9
10def get_embedding(query)
11    query_embedding_response = client.embeddings.create(
12        model=embedding_model,
13        input=query,
14    )
15    return query_embedding_response.data[0].embedding
16
17
18# This may take a few minutes
19df["embedding"] = df.combined.apply(lambda x: get_embedding(x))
20df.to_csv("data/fine_food_reviews_with_embeddings_1k.csv")

 1# imports
 2import numpy as np
 3import pandas as pd
 4from ast import literal_eval
 5
 6# load data
 7datafile_path = "./data/fine_food_reviews_with_embeddings_1k.csv"
 8
 9df = pd.read_csv(datafile_path)
10df["embedding"] = df.embedding.apply(literal_eval).apply(np.array)  # convert string to numpy array
11matrix = np.vstack(df.embedding.values)
12matrix.shape

 1from sklearn.cluster import KMeans
 2
 3n_clusters = 4
 4
 5kmeans = KMeans(n_clusters=n_clusters, init="k-means++", random_state=42)
 6kmeans.fit(matrix)
 7labels = kmeans.labels_
 8df["Cluster"] = labels
 9
10df.groupby("Cluster").Score.mean().sort_values()

 1from sklearn.manifold import TSNE
 2import matplotlib
 3import matplotlib.pyplot as plt
 4
 5tsne = TSNE(n_components=2, perplexity=15, random_state=42, init="random", learning_rate=200)
 6vis_dims2 = tsne.fit_transform(matrix)
 7
 8x = [x for x, y in vis_dims2]
 9y = [y for x, y in vis_dims2]
10
11for category, color in enumerate(["purple", "green", "red", "blue"]):
12    xs = np.array(x)[df.Cluster == category]
13    ys = np.array(y)[df.Cluster == category]
14    plt.scatter(xs, ys, color=color, alpha=0.3)
15
16    avg_x = xs.mean()
17    avg_y = ys.mean()
18
19    plt.scatter(avg_x, avg_y, marker="x", color=color, s=100)
20plt.title("Clusters identified visualized in language 2d using t-SNE")

 1
 2# Reading a review which belong to each group.
 3rev_per_cluster = 5
 4
 5for i in range(n_clusters):
 6    print(f"Cluster {i} Theme:", end=" ")
 7
 8    reviews = "\n".join(
 9        df[df.Cluster == i]
10        .combined.str.replace("Title: ", "")
11        .str.replace("\n\nContent: ", ":  ")
12        .sample(rev_per_cluster, random_state=42)
13        .values
14    )
15
16    messages = [
17        {"role": "user", "content": f'What do the following customer reviews have in common?\n\nCustomer reviews:\n"""\n{reviews}\n"""\n\nTheme:'}
18    ]
19
20    response = client.chat.completions.create(
21        model="gpt-4",
22        messages=messages,
23        temperature=0,
24        max_tokens=64,
25        top_p=1,
26        frequency_penalty=0,
27        presence_penalty=0)
28    print(response.choices[0].message.content.replace("\n", ""))
29
30    sample_cluster_rows = df[df.Cluster == i].sample(rev_per_cluster, random_state=42)
31    for j in range(rev_per_cluster):
32        print(sample_cluster_rows.Score.values[j], end=", ")
33        print(sample_cluster_rows.Summary.values[j], end=":   ")
34        print(sample_cluster_rows.Text.str[:70].values[j])
35
36    print("-" * 100)