تگ زدن تصاویر و تولید کپشن برای آنها

1# Install dependencies if needed
2%pip install openai
3%pip install scikit-learn

 1from IPython.display import Image, display
 2import pandas as pd
 3from sklearn.metrics.pairwise import cosine_similarity
 4import numpy as np
 5from openai import OpenAI
 6
 7# Initializing OpenAI client
 8client = OpenAI(
 9   api_key=os.environ.get(("GILAS_API_KEY", "<کلید API خود را اینجا بسازید https://dashboard.gilas.io/apiKey>")), 
10   base_url="https://api.gilas.io/v1/" # Gilas APIs
11)
12
13# Loading dataset
14dataset_path =  "data/amazon_furniture_dataset.csv"
15df = pd.read_csv(dataset_path)
16df.head()

 1system_prompt = '''
 2    You are an agent specialized in tagging images of furniture items, decorative items, or furnishings with relevant keywords that could be used to search for these items on a marketplace.
 3    
 4    You will be provided with an image and the title of the item that is depicted in the image, and your goal is to extract keywords for only the item specified. 
 5    
 6    Keywords should be concise and in lower case. 
 7    
 8    Keywords can describe things like:
 9    - Item type e.g. 'sofa bed', 'chair', 'desk', 'plant'
10    - Item material e.g. 'wood', 'metal', 'fabric'
11    - Item style e.g. 'scandinavian', 'vintage', 'industrial'
12    - Item color e.g. 'red', 'blue', 'white'
13    
14    Only deduce material, style or color keywords when it is obvious that they make the item depicted in the image stand out.
15
16    Return keywords in the format of an array of strings, like this:
17    ['desk', 'industrial', 'metal']
18    
19'''
20
21def analyze_image(img_url, title):
22    response = client.chat.completions.create(
23    model="gpt-4-turbo",
24    messages=[
25        {
26            "role": "system",
27            "content": system_prompt
28        },
29        {
30            "role": "user",
31            "content": [
32                {
33                    "type": "image_url",
34                    "image_url": img_url,
35                },
36            ],
37        },
38        {
39            "role": "user",
40            "content": title
41        }
42    ],
43        max_tokens=300,
44        top_p=0.1
45    )
46
47    return response.choices[0].message.content

1examples = df.iloc[:5]

1for index, ex in examples.iterrows():
2    url = ex['primary_image']
3    img = Image(url=url)
4    display(img)
5    result = analyze_image(url, ex['title'])
6    print(result)
7    print("\n\n")

1# Feel free to change the embedding model here
2def get_embedding(value, model="text-embedding-3-large"): 
3    embeddings = client.embeddings.create(
4      model=model,
5      input=value,
6      encoding_format="float"
7    )
8    return embeddings.data[0].embedding

1# Existing keywords
2keywords_list = ['industrial', 'metal', 'wood', 'vintage', 'bed']

1df_keywords = pd.DataFrame(keywords_list, columns=['keyword'])
2df_keywords['embedding'] = df_keywords['keyword'].apply(lambda x: get_embedding(x))
3df_keywords

 1def compare_keyword(keyword):
 2    embedded_value = get_embedding(keyword)
 3    df_keywords['similarity'] = df_keywords['embedding'].apply(lambda x: cosine_similarity(np.array(x).reshape(1,-1), np.array(embedded_value).reshape(1, -1)))
 4    most_similar = df_keywords.sort_values('similarity', ascending=False).iloc[0]
 5    return most_similar
 6
 7def replace_keyword(keyword, threshold = 0.6):
 8    most_similar = compare_keyword(keyword)
 9    if most_similar['similarity'] > threshold:
10        print(f"Replacing '{keyword}' with existing keyword: '{most_similar['keyword']}'")
11        return most_similar['keyword']
12    return keyword

1# Example keywords to compare to our list of existing keywords
2example_keywords = ['bed frame', 'wooden', 'vintage', 'old school', 'desk', 'table', 'old', 'metal', 'metallic', 'woody']
3final_keywords = []
4
5for k in example_keywords:
6    final_keywords.append(replace_keyword(k))
7    
8final_keywords = set(final_keywords)
9print(f"Final keywords: {final_keywords}")

1# Cleaning up dataset columns
2selected_columns = ['title', 'primary_image', 'style', 'material', 'color', 'url']
3df = df[selected_columns].copy()

 1describe_system_prompt = '''
 2    You are a system generating descriptions for furniture items, decorative items, or furnishings on an e-commerce website.
 3    Provided with an image and a title, you will describe the main item that you see in the image, giving details but staying concise.
 4    You can describe unambiguously what the item is and its material, color, and style if clearly identifiable.
 5    If there are multiple items depicted, refer to the title to understand which item you should describe.
 6    '''
 7
 8def describe_image(img_url, title):
 9    response = client.chat.completions.create(
10    temperature=0.2,
11    messages=[
12        {
13            "role": "system",
14            "content": describe_system_prompt
15        },
16        {
17            "role": "user",
18            "content": [
19                {
20                    "type": "image_url",
21                    "image_url": img_url,
22                },
23            ],
24        },
25        {
26            "role": "user",
27            "content": title
28        }
29    ],
30    max_tokens=300,
31    )
32
33    return response.choices[0].message.content

1for index, row in examples.iterrows():
2    print(f"{row['title'][:50]}{'...' if len(row['title']) > 50 else ''} - {row['url']} :\n")
3    img_description = describe_image(row['primary_image'], row['title'])
4    print(f"{img_description}\n--------------------------\n")

 1caption_system_prompt = '''
 2Your goal is to generate short, descriptive captions for images of furniture items, decorative items, or furnishings based on an image description.
 3You will be provided with a description of an item image and you will output a caption that captures the most important information about the item.
 4Your generated caption should be short (1 sentence), and include the most relevant information about the item.
 5The most important information could be: the type of the item, the style (if mentioned), the material if especially relevant and any distinctive features.
 6'''
 7
 8few_shot_examples = [
 9    {
10        "description": "This is a multi-layer metal shoe rack featuring a free-standing design. It has a clean, white finish that gives it a modern and versatile look, suitable for various home decors. The rack includes several horizontal shelves dedicated to organizing shoes, providing ample space for multiple pairs. Above the shoe storage area, there are 8 double hooks arranged in two rows, offering additional functionality for hanging items such as hats, scarves, or bags. The overall structure is sleek and space-saving, making it an ideal choice for placement in living rooms, bathrooms, hallways, or entryways where efficient use of space is essential.",
11        "caption": "White metal free-standing shoe rack"
12    },
13    {
14        "description": "The image shows a set of two dining chairs in black. These chairs are upholstered in a leather-like material, giving them a sleek and sophisticated appearance. The design features straight lines with a slight curve at the top of the high backrest, which adds a touch of elegance. The chairs have a simple, vertical stitching detail on the backrest, providing a subtle decorative element. The legs are also black, creating a uniform look that would complement a contemporary dining room setting. The chairs appear to be designed for comfort and style, suitable for both casual and formal dining environments.",
15        "caption": "Set of 2 modern black leather dining chairs"
16    },
17    {
18        "description": "This is a square plant repotting mat designed for indoor gardening tasks such as transplanting and changing soil for plants. It measures 26.8 inches by 26.8 inches and is made from a waterproof material, which appears to be a durable, easy-to-clean fabric in a vibrant green color. The edges of the mat are raised with integrated corner loops, likely to keep soil and water contained during gardening activities. The mat is foldable, enhancing its portability, and can be used as a protective surface for various gardening projects, including working with succulents. It's a practical accessory for garden enthusiasts and makes for a thoughtful gift for those who enjoy indoor plant care.",
19        "caption": "Waterproof square plant repotting mat"
20    }
21]
22
23formatted_examples = [[{
24    "role": "user",
25    "content": ex['description']
26},
27{
28    "role": "assistant", 
29    "content": ex['caption']
30}]
31    for ex in few_shot_examples
32]
33
34formatted_examples = [i for ex in formatted_examples for i in ex]

 1def caption_image(description, model="gpt-4-turbo"):
 2    messages = formatted_examples
 3    messages.insert(0, 
 4        {
 5            "role": "system",
 6            "content": caption_system_prompt
 7        })
 8    messages.append(
 9        {
10            "role": "user",
11            "content": description
12        })
13    response = client.chat.completions.create(
14    model=model,
15    temperature=0.2,
16    messages=messages
17    )
18
19    return response.choices[0].message.content

1examples = df.iloc[5:8]
2for index, row in examples.iterrows():
3    print(f"{row['title'][:50]}{'...' if len(row['title']) > 50 else ''} - {row['url']} :\n")
4    img_description = describe_image(row['primary_image'], row['title'])
5    print(f"{img_description}\n--------------------------\n")
6    img_caption = caption_image(img_description)
7    print(f"{img_caption}\n--------------------------\n")

1# Df we'll use to compare keywords
2df_keywords = pd.DataFrame(columns=['keyword', 'embedding'])
3df['keywords'] = ''
4df['img_description'] = ''
5df['caption'] = ''

 1# Function to replace a keyword with an existing keyword if it's too similar
 2def get_keyword(keyword, df_keywords, threshold = 0.6):
 3    embedded_value = get_embedding(keyword)
 4    df_keywords['similarity'] = df_keywords['embedding'].apply(lambda x: cosine_similarity(np.array(x).reshape(1,-1), np.array(embedded_value).reshape(1, -1)))
 5    sorted_keywords = df_keywords.copy().sort_values('similarity', ascending=False)
 6    if len(sorted_keywords) > 0 :
 7        most_similar = sorted_keywords.iloc[0]
 8        if most_similar['similarity'] > threshold:
 9            print(f"Replacing '{keyword}' with existing keyword: '{most_similar['keyword']}'")
10            return most_similar['keyword']
11    new_keyword = {
12        'keyword': keyword,
13        'embedding': embedded_value
14    }
15    df_keywords = pd.concat([df_keywords, pd.DataFrame([new_keyword])], ignore_index=True)
16    return keyword

 1import ast
 2
 3def tag_and_caption(row):
 4    keywords = analyze_image(row['primary_image'], row['title'])
 5    try:
 6        keywords = ast.literal_eval(keywords)
 7        mapped_keywords = [get_keyword(k, df_keywords) for k in keywords]
 8    except Exception as e:
 9        print(f"Error parsing keywords: {keywords}")
10        mapped_keywords = []
11    img_description = describe_image(row['primary_image'], row['title'])
12    caption = caption_image(img_description)
13    return {
14        'keywords': mapped_keywords,
15        'img_description': img_description,
16        'caption': caption
17    }

1df.shape
2
3(312, 9)

1# Saving locally for later
2data_path = "data/items_tagged_and_captioned.csv"
3df.to_csv(data_path, index=False)
4
5# Optional: load data from saved file
6df = pd.read_csv(data_path)

 1df_search = df.copy()
 2
 3def embed_tags_caption(x):
 4    if x['caption'] != '':
 5        keywords_string = ",".join(k for k in x['keywords']) + '\n'
 6        content = keywords_string + x['caption']
 7        embedding = get_embedding(content)
 8        return embedding
 9
10df_search['embedding'] = df_search.apply(lambda x: embed_tags_caption(x), axis=1)
11
12df_search.head()

1# Keep only the lines where we have embeddings
2df_search = df_search.dropna(subset=['embedding'])
3print(df_search.shape)
4
5(49, 10)

1# Saving locally for later
2data_embeddings_path = "data/items_tagged_and_captioned_embeddings.csv"
3df_search.to_csv(data_embeddings_path, index=False)
4
5# Optional: load data from saved file
6from ast import literal_eval
7df_search = pd.read_csv(data_embeddings_path)
8df_search["embedding"] = df_search.embedding.apply(literal_eval).apply(np.array)

 1# Searching for N most similar results
 2def search_from_input_text(query, n = 2):
 3    embedded_value = get_embedding(query)
 4    df_search['similarity'] = df_search['embedding'].apply(lambda x: cosine_similarity(np.array(x).reshape(1,-1), np.array(embedded_value).reshape(1, -1)))
 5    most_similar = df_search.sort_values('similarity', ascending=False).iloc[:n]
 6    return most_similar
 7
 8user_inputs = ['shoe storage', 'black metal side table', 'doormat', 'step bookshelf', 'ottoman']
 9
10for i in user_inputs:
11    print(f"Input: {i}\n")
12    res = search_from_input_text(i)
13    for index, row in res.iterrows():
14        similarity_score = row['similarity']
15        if isinstance(similarity_score, np.ndarray):
16            similarity_score = similarity_score[0][0]
17        print(f"{row['title'][:50]}{'...' if len(row['title']) > 50 else ''} ({row['url']}) - Similarity: {similarity_score:.2f}")
18        img = Image(url=row['primary_image'])
19        display(img)
20        print("\n\n")

 1# We'll take a mix of images: some we haven't seen and some that are already in the dataset
 2example_images = df.iloc[306:]['primary_image'].to_list() + df.iloc[5:10]['primary_image'].to_list()
 3
 4for i in example_images:
 5    img_description = describe_image(i, '')
 6    caption = caption_image(img_description)
 7    img = Image(url=i)
 8    print('Input: \n')
 9    display(img)
10    res = search_from_input_text(caption, 1).iloc[0]
11    similarity_score = res['similarity']
12    if isinstance(similarity_score, np.ndarray):
13        similarity_score = similarity_score[0][0]
14    print(f"{res['title'][:50]}{'...' if len(res['title']) > 50 else ''} ({res['url']}) - Similarity: {similarity_score:.2f}")
15    img_res = Image(url=res['primary_image'])
16    display(img_res)
17    print("\n\n")
18