1%pip install pdf2image
2%pip install pdfminer
3%pip install openai
4%pip install scikit-learn
5%pip install rich
6%pip install tqdm
7%pip install concurrent

 1# Imports
 2from pdf2image import convert_from_path
 3from pdf2image.exceptions import (
 4    PDFInfoNotInstalledError,
 5    PDFPageCountError,
 6    PDFSyntaxError
 7)
 8from pdfminer.high_level import extract_text
 9import base64
10from io import BytesIO
11import os
12import concurrent
13from tqdm import tqdm
14from openai import OpenAI
15import re
16import pandas as pd 
17from sklearn.metrics.pairwise import cosine_similarity
18import json
19import numpy as np
20from rich import print
21from ast import literal_eval

1def convert_doc_to_images(path):
2    images = convert_from_path(path)
3    return images
4
5def extract_text_from_doc(path):
6    text = extract_text(path)
7    page_text = []
8    return text

1file_path = "data/example_pdfs/fine-tuning-deck.pdf"
2
3images = convert_doc_to_images(file_path)

1text = extract_text_from_doc(file_path)

1for img in images:
2    display(img)

 1# imports
 2from openai import OpenAI # for calling the OpenAI API
 3import os
 4
 5client = OpenAI(
 6    api_key=os.environ.get(("GILAS_API_KEY", "<کلید API خود را اینجا بسازید https://dashboard.gilas.io/apiKey>")), 
 7    base_url="https://api.gilas.io/v1/" # Gilas APIs
 8)
 9
10
11# Converting images to base64 encoded images in a data URI format to use with the ChatCompletions API
12def get_img_uri(img):
13    buffer = BytesIO()
14    img.save(buffer, format="jpeg")
15    base64_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
16    data_uri = f"data:image/jpeg;base64,{base64_image}"
17    return data_uri

 1system_prompt = '''
 2You will be provided with an image of a pdf page or a slide. Your goal is to talk about the content that you see, in technical terms, as if you were delivering a presentation.
 3
 4If there are diagrams, describe the diagrams and explain their meaning.
 5For example: if there is a diagram describing a process flow, say something like "the process flow starts with X then we have Y and Z..."
 6
 7If there are tables, describe logically the content in the tables
 8For example: if there is a table listing items and prices, say something like "the prices are the following: A for X, B for Y..."
 9
10DO NOT include terms referring to the content format
11DO NOT mention the content type - DO focus on the content itself
12For example: if there is a diagram/chart and text on the image, talk about both without mentioning that one is a chart and the other is text.
13Simply describe what you see in the diagram and what you understand from the text.
14
15You should keep it concise, but keep in mind your audience cannot see the image so be exhaustive in describing the content.
16
17Exclude elements that are not relevant to the content:
18DO NOT mention page numbers or the position of the elements on the image.
19
20------
21
22If there is an identifiable title, identify the title to give the output in the following format:
23
24{TITLE}
25
26{Content description}
27
28If there is no clear title, simply return the content description.
29
30'''
31
32def analyze_image(img_url):
33    response = client.chat.completions.create(
34    model="gpt-4-vision-preview",
35    temperature=0,
36    messages=[
37        {
38            "role": "system",
39            "content": system_prompt
40        },
41        {
42            "role": "user",
43            "content": [
44                {
45                    "type": "image_url",
46                    "image_url": img_url,
47                },
48            ],
49        }
50    ],
51        max_tokens=300,
52        top_p=0.1
53    )
54
55    return response.choices[0].message.content

1img = images[2]
2data_uri = get_img_uri(img)
3
4
5res = analyze_image(data_uri)
6print(res)

1files_path = "data/example_pdfs"
2
3all_items = os.listdir(files_path)
4files = [item for item in all_items if os.path.isfile(os.path.join(files_path, item))]
5
6def analyze_doc_image(img):
7    img_uri = get_img_uri(img)
8    data = analyze_image(img_uri)
9    return data

 1docs = []
 2
 3for f in files:
 4    
 5    path = f"{files_path}/{f}"
 6    doc = {
 7        "filename": f
 8    }
 9    text = extract_text_from_doc(path)
10    doc['text'] = text
11    imgs = convert_doc_to_images(path)
12    pages_description = []
13    
14    print(f"Analyzing pages for doc {f}")
15    
16    # Concurrent execution
17    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
18        
19        # Removing 1st slide as it's usually just an intro
20        futures = [
21            executor.submit(analyze_doc_image, img)
22            for img in imgs[1:]
23        ]
24        
25        with tqdm(total=len(imgs)-1) as pbar:
26            for _ in concurrent.futures.as_completed(futures):
27                pbar.update(1)
28        
29        for f in futures:
30            res = f.result()
31            pages_description.append(res)
32        
33    doc['pages_description'] = pages_description
34    docs.append(doc)

1# Saving result to file for later
2json_path = "data/parsed_pdf_docs.json"
3
4with open(json_path, 'w') as f:
5    json.dump(docs, f)

1# Optional: load content from the saved file
2with open(json_path, 'r') as f:
3    docs = json.load(f)

 1# Chunking content by page and merging together slides text & description if applicable
 2content = []
 3for doc in docs:
 4    # Removing first slide as well
 5    text = doc['text'].split('\f')[1:]
 6    description = doc['pages_description']
 7    description_indexes = []
 8    for i in range(len(text)):
 9        slide_content = text[i] + '\n'
10        # Trying to find matching slide description
11        slide_title = text[i].split('\n')[0]
12        for j in range(len(description)):
13            description_title = description[j].split('\n')[0]
14            if slide_title.lower() == description_title.lower():
15                slide_content += description[j].replace(description_title, '')
16                # Keeping track of the descriptions added
17                description_indexes.append(j)
18        # Adding the slide content + matching slide description to the content pieces
19        content.append(slide_content) 
20    # Adding the slides descriptions that weren't used
21    for j in range(len(description)):
22        if j not in description_indexes:
23            content.append(description[j])

1for c in content:
2    print(c)
3    print("\n\n-------------------------------\n\n")

1# Cleaning up content
2# Removing trailing spaces, additional line breaks, page numbers and references to the content being a slide
3clean_content = []
4for c in content:
5    text = c.replace(' \n', '').replace('\n\n', '\n').replace('\n\n\n', '\n').strip()
6    text = re.sub(r"(?<=\n)\d{1,2}", "", text)
7    text = re.sub(r"\b(?:the|this)\s*slide\s*\w+\b", "", text, flags=re.IGNORECASE)
8    clean_content.append(text)

1for c in clean_content:
2    print(c)
3    print("\n\n-------------------------------\n\n")

1# Creating the embeddings
2# We'll save to a csv file here for testing purposes but this is where you should load content in your vectorDB.
3df = pd.DataFrame(clean_content, columns=['content'])
4print(df.shape)
5df.head()

1embeddings_model = "text-embedding-3-large"
2
3def get_embeddings(text):
4    embeddings = client.embeddings.create(
5      model="text-embedding-3-small",
6      input=text,
7      encoding_format="float"
8    )
9    return embeddings.data[0].embedding

1df['embeddings'] = df['content'].apply(lambda x: get_embeddings(x))
2df.head()

1# Saving locally for later
2data_path = "data/parsed_pdf_docs_with_embeddings.csv"
3df.to_csv(data_path, index=False)

1# Optional: load data from saved file
2df = pd.read_csv(data_path)
3df["embeddings"] = df.embeddings.apply(literal_eval).apply(np.array)

 1system_prompt = '''
 2    You will be provided with an input prompt and content as context that can be used to reply to the prompt.
 3    
 4    You will do 2 things:
 5    
 6    1. First, you will internally assess whether the content provided is relevant to reply to the input prompt. 
 7    
 8    2a. If that is the case, answer directly using this content. If the content is relevant, use elements found in the content to craft a reply to the input prompt.
 9
10    2b. If the content is not relevant, use your own knowledge to reply or say that you don't know how to respond if your knowledge is not sufficient to answer.
11    
12    Stay concise with your answer, replying specifically to the input prompt without mentioning additional information provided in the context content.
13'''
14
15model="gpt-4-turbo-preview"
16
17def search_content(df, input_text, top_k):
18    embedded_value = get_embeddings(input_text)
19    df["similarity"] = df.embeddings.apply(lambda x: cosine_similarity(np.array(x).reshape(1,-1), np.array(embedded_value).reshape(1, -1)))
20    res = df.sort_values('similarity', ascending=False).head(top_k)
21    return res
22
23def get_similarity(row):
24    similarity_score = row['similarity']
25    if isinstance(similarity_score, np.ndarray):
26        similarity_score = similarity_score[0][0]
27    return similarity_score
28
29def generate_output(input_prompt, similar_content, threshold = 0.5):
30    
31    content = similar_content.iloc[0]['content']
32    
33    # Adding more matching content if the similarity is above threshold
34    if len(similar_content) > 1:
35        for i, row in similar_content.iterrows():
36            similarity_score = get_similarity(row)
37            if similarity_score > threshold:
38                content += f"\n\n{row['content']}"
39            
40    prompt = f"INPUT PROMPT:\n{input_prompt}\n-------\nCONTENT:\n{content}"
41    
42    completion = client.chat.completions.create(
43        model=model,
44        temperature=0.5,
45        messages=[
46            {
47                "role": "system",
48                "content": system_prompt
49            },
50            {
51                "role": "user",
52                "content": prompt
53            }
54        ]
55    )
56
57    return completion.choices[0].message.content

 1# Example user queries related to the content
 2example_inputs = [
 3    'What are the main models you offer?',
 4    'Do you have a speech recognition model?',
 5    'Which embedding model should I use for non-English use cases?',
 6    'Can I introduce new knowledge in my LLM app using RAG?',
 7    'How many examples do I need to fine-tune a model?',
 8    'Which metric can I use to evaluate a summarization task?',
 9    'Give me a detailed example for an evaluation process where we are looking for a clear answer to compare to a ground truth.',
10]

 1# Running the RAG pipeline on each example
 2for ex in example_inputs:
 3    print(f"[deep_pink4][bold]QUERY:[/bold] {ex}[/deep_pink4]\n\n")
 4    matching_content = search_content(df, ex, 3)
 5    print(f"[grey37][b]Matching content:[/b][/grey37]\n")
 6    for i, match in matching_content.iterrows():
 7        print(f"[grey37][i]Similarity: {get_similarity(match):.2f}[/i][/grey37]")
 8        print(f"[grey37]{match['content'][:100]}{'...' if len(match['content']) > 100 else ''}[/[grey37]]\n\n")
 9    reply = generate_output(ex, matching_content)
10    print(f"[turquoise4][b]REPLY:[/b][/turquoise4]\n\n[spring_green4]{reply}[/spring_green4]\n\n--------------\n\n")