uploading all .py files

app.py

@@ -0,0 +1,92 @@
+import torch
+from transformers import LlavaForConditionalGeneration, BitsAndBytesConfig, AutoProcessor
+from transformers import MBartForConditionalGeneration, MBart50TokenizerFast
+import requests
+from PIL import Image
+import requests
+import gradio as gr
+
+
+# Load translation model and tokenizer
+translate_model_name = "facebook/mbart-large-50-many-to-many-mmt"
+translate_model = MBartForConditionalGeneration.from_pretrained(translate_model_name)
+tokenizer = MBart50TokenizerFast.from_pretrained(translate_model_name)
+
+# load the base model in 4 bit quantized
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+)
+
+# finetuned model adapter path (Hugging Face Hub)
+model_id = 'somnathsingh31/llava-1.5-7b-hf-ft-merged_model'
+
+# merge the models
+merged_model = LlavaForConditionalGeneration.from_pretrained(model_id,
+                                                           quantization_config=quantization_config,
+                                                           torch_dtype=torch.float16)
+
+# create processor from base model
+processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
+
+# function to translate
+def translate(text, source_lang, target_lang):
+    # Set source language
+    tokenizer.src_lang = source_lang
+
+    # Encode the text
+    encoded_text = tokenizer(text, return_tensors="pt")
+
+    # Force target language token
+    forced_bos_token_id = tokenizer.lang_code_to_id[target_lang]
+
+    # Generate the translation
+    generated_tokens = translate_model.generate(**encoded_text, forced_bos_token_id=forced_bos_token_id)
+
+    # Decode the translation
+    translation = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0]
+
+    return translation
+
+
+# function for making inference
+def ask_vlm(hindi_input_text, image):
+    # translate from Hindi to English
+    prompt_eng = translate(hindi_input_text, "hi_IN", "en_XX")
+    prompt = "USER: <image>\n" + prompt_eng + " ASSISTANT:"
+
+    # If image is uploaded, open the image from bytes, else open from URL
+    if hasattr(image, 'read'):
+        image = Image.open(image)
+    else:
+        image = Image.open(requests.get(image, stream=True).raw)
+    
+    inputs = processor(text=prompt, images=image, return_tensors="pt")
+    generate_ids = merged_model.generate(**inputs, max_new_tokens=250)
+    decoded_response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+    assistant_index = decoded_response.find("ASSISTANT:")
+
+    # Extract text after "ASSISTANT:"
+    if assistant_index != -1:
+        text_after_assistant = decoded_response[assistant_index + len("ASSISTANT:"):]
+        # Remove leading and trailing whitespace
+        text_after_assistant = text_after_assistant.strip()
+    else:
+        text_after_assistant = None
+
+    hindi_output_text = translate(text_after_assistant, "en_XX", "hi_IN")
+    return hindi_output_text
+
+# Define Gradio interface
+input_image = gr.inputs.Image(type="pil", label="Input Image (Upload or URL)")
+input_question = gr.inputs.Textbox(lines=2, label="Question (Hindi)")
+output_text = gr.outputs.Textbox(label="Response (Hindi)")
+
+# Create Gradio app
+gr.Interface(fn=ask_vlm, inputs=[input_question, input_image], outputs=output_text, title="Image and Text-based Dialogue System", description="Enter a question in Hindi and an image, either by uploading or providing URL, and get a response in Hindi.").launch()
+
+
+if __name__ ==  '__main__':
+    image_url = 'https://images.metmuseum.org/CRDImages/ad/original/138425.jpg'
+    user_query = 'यह किस प्रकार की कला है? विस्तार से बताइये'
+    output = ask_vlm(user_query, image_url)
+    print('Output:\n', output)

data_collection.py

@@ -0,0 +1,93 @@
+import pandas as pd
+import requests
+import json
+import cv2
+import numpy as np
+from bs4 import BeautifulSoup
+
+def download_metmuseum_data(ids):
+    data_final = []
+    for id in ids:
+        response = requests.get(f'https://collectionapi.metmuseum.org/public/collection/v1/objects/{id}')
+        if response.status_code == 200:
+            data_final.append(json.loads(response.text))
+    
+    with open('met_museum_collection.json', 'w') as json_file:
+        json.dump(data_final, json_file)
+
+def filter_and_save_data(keys_to_keep):
+    df = pd.read_json('/content/met_museum_collection.json')
+    final_data_df = df[keys_to_keep]
+    final_data_df.to_csv('final_data_df.csv', index=False)
+    final_data_df.to_json('final_data_json.json', orient='records', indent=4)
+    return final_data_df
+
+def download_and_convert_image(url):
+    response = requests.get(url)
+    if response.status_code == 200:
+        nparr = np.frombuffer(response.content, np.uint8)
+        img = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
+        return img
+    else:
+        return None
+
+def extract_description(url):
+    try:
+        response = requests.get(url)
+        if response.status_code == 200:
+            soup = BeautifulSoup(response.content, 'html.parser')
+            description_element = soup.find('div', class_='artwork__intro__desc')
+            if description_element:
+                description = description_element.get_text(strip=True)
+                return description
+            else:
+                return None
+        else:
+            return None
+    except Exception as e:
+        return str(e)
+
+def create_master_data(df):
+    df['description'] = df['objectURL'].apply(extract_description)
+    df.to_csv('master_data.csv', index=False)
+    df.to_json('master_data.json', orient='records', indent=4)
+
+
+def main():
+    # filtered ids based on Indian data from the overall data of metmuseum dataset
+    ids= [1986,7608,9748,38638,86116,86136,86167,86169,86171,86173,86176,86177,86178,86179,86186,86187,86189,86195,86216,86223,126982,127021,127511,308007,443097,443099,444411,444412,444539,444541,444546,444552,444557,444575,444577,444582,444588,444592,444609,444618,444623,444628,444634,444648,444650,444652,444657,444658,444661,444662,444667,444668,444670,444671,444672,444674,444675,444676,444678,444679,444681,444684,444689,444692,444694,444695,444720,444814,444816,444832,444836,444838,444854,444856,444857,444859,444860,444861,444863,444864,444865,444866,444867,444868,444963,444964,444984,444987,444988,444989,445003,445004,445005,445227,445231,445232,445233,445239,445245,445246,445261,445280,445362,445645,445646,445647,445648,445689,445690,445691,445693,445708,445869,445870,445871,445872,445873,445874,445875,445878,445879,445880,445881,445882,445889,445890,445894,445957,445958,445961,445963,445975,445976,445997,445998,445999,446000,446001,446002,446003,446004,446005,446007,446157,446158,446163,446164,446189,446266,446276,446278,446280,446290,446293,446299,446549,446556,446558,446560,446561,446562,446563,446564,446566,446567,446571,446572,446573,446574,446575,446576,446577,446578,446579,446580,446581,446582,446583,446584,446585,446586,446587,446588,446589,446591,446633,446634,446635,446638,446640,446643,446644,446646,446661,446662,446663,446704,446705,446706,446707,446708,446709,446710,446711,446712,446713,446714,446715,446716,446717,446718,446719,446720,446721,446722,446723,446724,446725,446726,446727,446728,446729,446730,446731,446732,446733,446734,446735,446736,446737,446738,446739,446740,446741,446742,446743,446744,446745,446746,446747,446748,446749,446750,446751,446752,446753,446754,446755,446756,446757,446759,446760,446761,446762,446763,446764,446765,446766,446767,446768,446769,446770,446771,446772,446773,446774,446775,446776,446777,446778,446779,446780,446781,446782,446783,446784,446785,446786,446787,446788,446789,446790,446791,446792,446793,446794,446795,446796,446797,446798,446799,446800,446801,446802,446804,446807,446808,446810,446816,446817,446855,446857,446859,446872,446882,446893,446967,446987,446988,446991,446992,446993,447021,447050,447051,447053,447054,447055,447062,447067,447082,447083,447084,447086,447087,447088,447089,447090,447091,447092,447093,447094,447095,447101,447102,447108,447109,447114,447118,447297,447298,447299,447300,447301,447302,447303,447304,447305,447306,447307,447308,447364,447365,447388,447389,447399,447401,447402,447403,447404,447405,447406,447407,447408,447409,447410,447411,447412,447413,447414,447415,447416,447417,447419,447421,447530,447531,447606,447627,447675,447703,447743,447744,447752,447755,447757,447758,447760,447773,447774,447775,447776,447780,447795,447796,447797,447799,447800,447801,447808,447816,447817,447821,447822,447848,447849,447867,447868,447918,447919,447939,447944,447946,447947,447948,447969,448015,448018,448147,448148,448149,448150,448151,448152,448153,448154,448155,448156,448157,448158,448159,448160,448161,448162,448163,448164,448165,448166,448167,448168,448178,448180,448188,448189,448190,448191,448206,448212,448213,448214,448215,448216,448241,448247,448257,448277,448278,448351,448352,448378,448379,448380,448450,448460,448464,448465,448466,448467,448468,448469,448470,448471,448472,448473,448474,448475,448476,448477,448478,448479,448480,448481,448484,448485,448486,448487,448488,448489,448491,448494,448495,448508,448517,448518,448519,448521,448522,448523,448524,448525,448526,448527,448528,448529,448530,448531,448532,448533,448534,448535,448536,448553,448583,448587,448589,448591,448657,448677,448923,448936,448937,449094,449105,450374,450395,450465,450469,450471,450496,450508,450529,450538,450572,450573,450584,450586,450592,450595,450601,450610,450613,450614,450618,450622,450625,450631,450639,450642,450665,450674,450677,450699,450708,450733,450750,450754,450964,450986,451009,451010,451018,451021,451118,451186,451195,451201,451257,451258,451259,451260,451261,451262,451263,451264,451265,451266,451267,451268,451269,451270,451271,451272,451273,451274,451275,451276,451277,451278,451279,451280,451281,451282,451283,451284,451285,451286,451287,451288,451289,451291,451292,451293,451294,451295,451296,451297,451313,451314,451315,451316,451317,451318,451334,451337,451393,451401,451409,451413,451479,451483,451485,451611,451683,451961,451982,452104,452192,452197,452205,452214,452294,452307,452310,452394,452395,452418,452419,452520,452521,452554,452680,452681,452734,452745,452746,452749,452776,452777,452780,452805,452811,452813,452814,452815,452818,452819,452836,452912,452913,452942,452943,452944,452963,453053,453054,453058,453059,453060,453061,453062,453063,453064,453065,453066,453067,453068,453069,453070,453071,453072,453073,453074,453075,453076,453077,453078,453079,453080,453081,453082,453083,453084,453085,453086,453087,453088,453089,453090,453091,453092,453093,453094,453095,453096,453097,453098,453099,453100,453101,453102,453103,453104,453105,453106,453107,453108,453109,453110,453111,453112,453113,453114,453115,453116,453117,453118,453119,453120,453121,453122,453123,453124,453125,453126,453127,453128,453129,453130,453131,453132,453133,453134,453135,453136,453137,453138,453139,453140,453141,453142,453143,453144,453145,453146,453147,453148,453149,453150,453151,453152,453153,453154,453159,453160,453161,453163,453169,453180,453183,453184,453188,453191,453192,453193,453194,453195,453206,453207,453210,453212,453213,453214,453215,453216,453217,453218,453219,453220,453221,453222,453223,453224,453225,453226,453227,453228,453234,453241,453242,453243,453253,453258,453259,453260,453261,453263,453265,453266,453267,453268,453271,453272,453273,453276,453280,453281,453287,453288,453309,453313,453328,453329,453330,453331,453332,453334,453335,453337,453338,453339,453340,453341,453343,453344,453348,453350,453354,453355,453356,453360,453366,453367,453384,453386,453387,453397,453412,453415,453417,453427,453428,453435,453436,453442,453479,453482,453498,453500,453564,453642,453673,453975,453983,454003,454011,454013,454027,454038,454049,454054,454064,454083,454084,454087,454088,454089,454090,454091,454093,454094,454095,454096,454097,454098,454099,454100,454113,454114,454115,454139,454343,454344,454357,454358,454359,454400,454406,454407,454408,454409,454415,454429,454445,454515,454530,454568,454601,454619,454620,454623,454624,454625,454713,454718,454738,454739,454768,455027,455028,455029,455030,455031,455032,455033,455034,455035,455036,455048,455060,455064,455285,455300,455544,455545,455546,455547,455548,456128,456312,456313,456314,456342,456875,456921,456922,456923,456926,456927,456928,456949,456951,456958,456959,456964,456966,456967,456968,456985,456986,457019,457020,457054,457058,457711,457728,457730,457743,457755,457758,457771,457781,457782,457783,457784,457788,457977,458260,458378,458437,458438,458439,458440,458444,458445,500709,500711,500712,500713,500714,500715,500716,500717,500718,500719,500720,500731,500732,500733,500734,500735,500736,500738,500739,500740,500741,500742,500743,500744,500745,500748,500749,500750,500751,500752,500754,500755,500756,500757,500758,500759,500760,500761,500763,500764,500767,500769,500771,500772,500773,500774,500775,500777,500778,500779,500781,500783,500785,500790,500791,500793,500794,500800,500801,500802,500806,500809,500835,500838,500844,500869,500909,500914,500994,501466,501811,501879,501899,501903,501981,502018,502072,502111,502149,502150,502236,502310,503431,503453,503554,503635,503672,503937,504217,505237,505818,506064,506151,632624,646829,659894,659901,659903,659904,659905,659906,662008,684250,688168,688233,688240,688242,698587,698590,698600,698602,698612,698613,698616,698646,698647,698648,698649,698651,698652,698655,698657,698660,698661,698662,698663,698664,698671,698672,698673,707408,717767,734073,752291,761737,772012,781487,781855,781858,781892,784283,784629,817378,820019,821905,822698,823157,823285,827351,835238,838357,843822,843823,853714,872977]
+
+    # download the data based on ids above
+    data_final = []
+
+    for id in ids:
+        response = requests.get(f'https://collectionapi.metmuseum.org/public/collection/v1/objects/{id}')
+        if response.status_code == 200:
+            data_final.append(json.loads(response.text))
+
+    # Save the data_final list as JSON
+    with open('met_museum_collection.json', 'w') as json_file:
+        json.dump(data_final, json_file)
+
+    # Read JSON file into a DataFrame
+    df = pd.read_json('met_museum_collection.json')
+
+    # List of keys to keep
+    keys_to_keep = [
+        'objectID', 'primaryImage', 'department', 'objectName', 'title', 'culture', 'period',
+        'artistDisplayName', 'objectDate', 'medium', 'dimensions', 'city',
+        'state', 'country', 'region', 'classification', 'repository', 'objectURL'
+    ]
+
+    # Filter data
+    final_data_df = df[keys_to_keep]
+
+    # create new column with description
+    final_data_df['description'] = final_data_df['objectURL'].apply(extract_description)
+
+    # Save as csv and json
+    final_data_df.to_csv('master_data.csv', index=False)
+    final_data_df.to_json('master_data.json', orient='records', indent=4)
+
+if __name__ == "__main__":
+    main()

dataset_preparation.py

@@ -0,0 +1,125 @@
+import pandas as pd
+import os
+import requests
+from PIL import Image
+from io import BytesIO
+from datasets import Dataset
+
+def modify_dataframe_and_extract_data(df):
+    data_list = []
+    for _, row in df.iterrows():
+        messages = []
+        for i in range(1, 5):
+            user_question = row[f'Question{i}']
+            user_answer = row[f'Answer{i}']
+            if user_question:
+                message_content = [{'index': None, 'text': user_question, 'type': 'text'}]
+                if i == 1:
+                    message_content.append({'index': 0, 'text': None, 'type': 'image'})
+                messages.append({'content': message_content, 'role': 'user'})
+                if user_answer:
+                    messages.append({'content': [{'index': None, 'text': user_answer, 'type': 'text'}], 'role': 'assistant'})
+        image = Image.open(row['imagePath'])
+        data_list.append({'messages': messages, 'images': [image]})
+    return {'messages': [data['messages'] for data in data_list], 'images': [data['images'] for data in data_list]}
+
+
+def download_and_resize_images(df, image_dir, target_size=(250, 250)):
+    image_paths = []
+    for index, row in df.iterrows():
+        image_url = row['primaryImageLink']
+        object_id = row['objectID']
+        if image_url:
+            # Extract filename from the URL
+            filename = os.path.join(image_dir, f"{object_id}.jpg")
+            # Download image from the URL
+            response = requests.get(image_url)
+            if response.status_code == 200:
+                # Open the image using PIL
+                image = Image.open(BytesIO(response.content))
+                # Resize the image
+                image = image.resize(target_size)
+                # Save the resized image
+                image.save(filename)
+                image_paths.append(filename)
+            else:
+                print(f"Failed to download image from {image_url}")
+                image_paths.append(None)
+        else:
+            image_paths.append(None)
+    return image_paths
+
+def split_data_dict(data_dict, train_ratio=0.7, test_ratio=0.2, val_ratio=0.1):
+    assert train_ratio + test_ratio + val_ratio == 1.0, "Ratios must sum up to 1.0"
+    
+    total_samples = len(data_dict['messages'])
+    train_size = int(total_samples * train_ratio)
+    test_size = int(total_samples * test_ratio)
+    val_size = int(total_samples * val_ratio)
+
+    train_data_dict = {
+        'messages': data_dict['messages'][:train_size],
+        'images': data_dict['images'][:train_size]
+    }
+    test_data_dict = {
+        'messages': data_dict['messages'][train_size:train_size + test_size],
+        'images': data_dict['images'][train_size:train_size + test_size]
+    }
+    val_data_dict = {
+        'messages': data_dict['messages'][-val_size:],
+        'images': data_dict['images'][-val_size:]
+    }
+
+    return train_data_dict, test_data_dict, val_data_dict
+
+
+def save_data_dict_as_arrow(data_dict, file_path):
+    # Convert the dictionary to a Dataset object
+    dataset = Dataset.from_dict(data_dict)
+    
+    # Save the dataset to an Arrow file
+    dataset.save_to_disk(file_path)
+
+if __name__ == "__main__":
+    # Example usage:
+
+    # df = pd.read_csv("/data/data_set_metmuseum.csv")
+    # df1 = df[['objectID', 'primaryImageLink', 'Question1', 'Answer1', 'Question2', 'Answer2', 'Question3', 'Answer3', 'Question4', 'Answer4']]
+    # df2 = df1.sample(frac=1)
+    # df3 = df2.head(250)
+
+    # df4 = df3.copy()
+
+    df4 = pd.read_csv("sampled_data250.csv")
+    paths = ['input_dataset', os.path.join('input_dataset', 'images'), 'output_dataset']
+    for path in paths:
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+    # Call the function to download and resize images
+    image_dir = 'input_dataset/images'
+    # image_paths = download_and_resize_images(df4, image_dir)
+
+    # Update the DataFrame with the resized image paths
+    # new_df = df4.copy()  # Create a copy of the original DataFrame
+    # df4['imagePath'] = image_paths  # Add a new column 'imagePath' containing the resized image paths
+    # df4 = df4.drop(['primaryImageLink'], axis=1)
+
+    # Call the function to modify the DataFrame and extract data
+    data_dict = modify_dataframe_and_extract_data(df4)
+    # split data_dict into train, test, valid
+    train_data_dict, test_data_dict, val_data_dict = split_data_dict(data_dict, train_ratio=0.6, test_ratio=0.2, val_ratio=0.2)
+
+    # save these as arrow dataset
+    save_data_dict_as_arrow(train_data_dict, os.path.join('output_dataset', 'train.arrow'))
+    save_data_dict_as_arrow(test_data_dict, os.path.join('output_dataset', 'test.arrow'))
+    save_data_dict_as_arrow(val_data_dict, os.path.join('output_dataset', 'val.arrow'))
+
+    # save to zip format
+    import shutil
+    shutil.make_archive("/content/input_dataset", "zip", "/content/input_dataset")
+    shutil.make_archive("/content/output_dataset", "zip", "/content/output_dataset")
+    
+    # read arrow from disk
+    test_data = Dataset.load_from_disk("output_dataset/test.arrow")
+    test_data

inference.py

@@ -0,0 +1,53 @@
+import torch
+from transformers import LlavaForConditionalGeneration, BitsAndBytesConfig, AutoProcessor
+from peft import PeftModel
+import requests
+from PIL import Image
+
+
+def load_base_model(model_id):
+    quantization_config = BitsAndBytesConfig(load_in_4bit=True)
+    base_model = LlavaForConditionalGeneration.from_pretrained(model_id,
+                                                               quantization_config=quantization_config,
+                                                               torch_dtype=torch.float16)
+    return base_model
+
+
+def load_peft_lora_adapter(base_model, peft_lora_adapter_path):
+    peft_lora_adapter = PeftModel.from_pretrained(base_model, peft_lora_adapter_path, adapter_name="lora_adapter")
+    return peft_lora_adapter
+
+
+def merge_adapters(base_model, peft_lora_adapter_path):
+    base_model.load_adapter(peft_lora_adapter_path, adapter_name="lora_adapter")
+    return base_model
+
+
+def main():
+    model_id = "llava-hf/llava-1.5-7b-hf"  # Actual base model id
+    peft_lora_adapter_path = 'somnathsingh31/llava-1.5-7b-hf-ft-museum'  # Actual adapter path
+
+    # Load the base model
+    base_model = load_base_model(model_id)
+
+    # Load the PEFT Lora model (adapter)
+    peft_lora_adapter = load_peft_lora_adapter(base_model, peft_lora_adapter_path)
+
+    # Merge the adapters into the base model
+    merged_model = merge_adapters(base_model, peft_lora_adapter_path)
+
+    prompt = "USER: <image>\nWhat is special in this chess set and pieces? \nASSISTANT:"
+    url = "https://images.metmuseum.org/CRDImages/ad/original/138425.jpg"
+    image = Image.open(requests.get(url, stream=True).raw)
+
+    processor = AutoProcessor.from_pretrained(model_id)
+    inputs = processor(text=prompt, images=image, return_tensors="pt")
+    
+    # ... process the image and create inputs ...
+    generate_ids = merged_model.generate(**inputs, max_new_tokens=150)
+    decoded_response = processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
+    print("Generated response:", decoded_response)
+
+
+if __name__ == "__main__":
+    main()

language_translation.py

@@ -0,0 +1,116 @@
+
+# !pip install googletrans==3.1.0a0
+# !pip install transformers sentencepiece
+
+from googletrans import Translator
+from transformers import MarianMTModel, MarianTokenizer  # transformer based pre-trained language translation model
+from transformers import MBartForConditionalGeneration, MBart50TokenizerFast
+
+
+
+def translate_hi2en_gtrans(sentence):
+    """
+    Function to translate from Hindi to English.
+
+    Args:
+    - sentence: string in Hindi
+
+    Returns:
+    - English translated text string
+
+    """
+    translator = Translator()
+    output = translator.translate(sentence, dest='en', src='hi')
+    return output.text
+
+def translate_en2hi_gtrans(sentence):
+    """
+    Function to translate from English to Hindi.
+
+    Args:
+    - sentence: string in English
+
+    Returns:
+    - Hindi translated text string
+
+    """
+    translator = Translator()
+    output = translator.translate(sentence, dest='hi', src='en')
+    return output.text
+
+# Translates text from source_lang to target_lang using the pre-trained model
+def translate_en_hi_transformer(text):
+    # Load the Pre-trained Model and Tokenizer for english to hindi
+    model_name_en_hi = "Helsinki-NLP/opus-mt-en-hi"  # English to Hindi translation model
+    tokenizer = MarianTokenizer.from_pretrained(model_name_en_hi)
+    model_en_hi = MarianMTModel.from_pretrained(model_name_en_hi)
+    encoded = tokenizer(text, return_tensors="pt")
+    translated = model_en_hi.generate(**encoded)
+    return tokenizer.batch_decode(translated, skip_special_tokens=True)[0]
+
+# Translates text from Hindi to english using the pre-trained model
+def translate_hi_en_transformer(text):
+    # Load the Pre-trained Model and Tokenizer for hindi to english
+    model_name_hi_en = "Helsinki-NLP/opus-mt-hi-en"  # Hindi to English translation model
+    tokenizer_hi = MarianTokenizer.from_pretrained(model_name_hi_en)
+    model_hi_en = MarianMTModel.from_pretrained(model_name_hi_en)
+    encoded = tokenizer_hi(text, return_tensors="pt")
+    translated = model_hi_en.generate(**encoded)
+    return tokenizer_hi.batch_decode(translated, skip_special_tokens=True)[0]
+
+def translate_mbart(text, source_lang, target_lang):
+    # Load model and tokenizer outside the function
+    model_name = "facebook/mbart-large-50-many-to-many-mmt"
+    model = MBartForConditionalGeneration.from_pretrained(model_name)
+    tokenizer = MBart50TokenizerFast.from_pretrained(model_name)
+
+    # Set source language
+    tokenizer.src_lang = source_lang
+    # Encode the text
+    encoded_text = tokenizer(text, return_tensors="pt")
+    # Force target language token
+    forced_bos_token_id = tokenizer.lang_code_to_id[target_lang]
+    # Generate the translation
+    generated_tokens = model.generate(**encoded_text, forced_bos_token_id=forced_bos_token_id)
+    # Decode the translation
+    translation = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0]
+    return translation
+
+
+if __name__ == "__main__":
+    print(translate_hi2en_gtrans("मैं खुश हूँ!!!"))
+    print(translate_en2hi_gtrans("I am happy!!!"))
+    import pandas as pd
+
+    # Read CSV file into a Pandas DataFrame
+    df_en = pd.read_csv('Data_with_QnA.csv', usecols=['Question1', 'Answer1', 'Question2', 'Answer2', 'Question3', 'Answer3', 'Question4', 'Answer4'])
+    df_en.head()
+
+    # convert all the question answers from English to Hindi
+    df_hi = df_en.applymap(translate_en2hi_gtrans)
+    df_hi.head()
+
+    # Save the modified DataFrame to a CSV file
+    df_hi.to_csv('Hindi_QnA.csv', index=False)
+
+    # English to Hindi example
+    english_text = " What is the material used to create the chess set?"
+    hindi_translation = translate_en_hi_transformer(english_text)
+    print(f"English: {english_text}")
+    print(f"Hindi: {hindi_translation}")
+
+    # Hindi to English example
+    hindi_text = "आपका दिन कैसा चल रहा है?"  # How is your day going?
+    english_translation = translate_hi_en_transformer(hindi_text)
+    print(f"Hindi: {hindi_text}")
+    print(f"English: {english_translation}")
+
+    # Example usage
+    hindi_text = "हिन्दी साहित्य पर अगर समुचित परिप्रेक्ष्य में विचार किया जाए तो स्पष्ट होता है कि हिन्दी साहित्य का इतिहास अत्यन्त विस्तृत व प्राचीन है। सुप्रसिद्ध भाषा वैज्ञानिक डॉ० हरदेव बाहरी के शब्दों में, हिन्दी साहित्य का इतिहास वस्तुतः वैदिक काल से आरम्भ होता है। यह कहना ही ठीक होगा कि वैदिक भाषा ही हिन्दी है। इस भाषा का दुर्भाग्य रहा है कि युग-य��ग में इसका नाम परिवर्तित होता रहा है। कभी 'वैदिक', कभी 'संस्कृत', कभी 'प्राकृत', कभी'अपभ्रंश' और अब - हिन्दी।[1] आलोचक कह सकते हैं कि 'वैदिक संस्कृत' और 'हिन्दी' में तो जमीन-आसमान का अन्तर है। पर ध्यान देने योग्य है कि हिब्रू, रूसी, चीनी, जर्मन और तमिल आदि जिन भाषाओं को 'बहुत पुरानी' बताया जाता है, उनके भी प्राचीन और वर्तमान रूपों में जमीन-आसमान का अन्तर है; पर लोगों ने उन भाषाओं के नाम नहीं बदले और उनके परिवर्तित स्वरूपों को 'प्राचीन', 'मध्यकालीन', 'आधुनिक' आदि कहा गया, जबकि 'हिन्दी' के सन्दर्भ में प्रत्येक युग की भाषा का नया नाम रखा जाता रहा।"
+    english_translation = translate_mbart(hindi_text, "hi_IN", "en_XX")
+    print(english_translation) 
+
+    english_text = "English literature, the body of written works produced in the English language by inhabitants of the British Isles (including Ireland) from the 7th century to the present day. The major literatures written in English outside the British Isles are treated separately under American literature, Australian literature, Canadian literature, and New Zealand literature. English literature has sometimes been stigmatized as insular. It can be argued that no single English novel attains the universality of the Russian writer Leo Tolstoy’s War and Peace or the French writer Gustave Flaubert’s Madame Bovary. Yet in the Middle Ages the Old English literature of the subjugated Saxons was leavened by the Latin and Anglo-Norman writings, eminently foreign in origin, in which the churchmen and the Norman conquerors expressed themselves. From this combination emerged a flexible and subtle linguistic instrument exploited by Geoffrey Chaucer and brought to supreme application by William Shakespeare. During the Renaissance the renewed interest in Classical learning and values had an important effect on English literature, as on all the arts; and ideas of Augustan literary propriety in the 18th century and reverence in the 19th century for a less specific, though still selectively viewed, Classical antiquity continued to shape the literature. All three of these impulses derived from a foreign source, namely the Mediterranean basin. The Decadents of the late 19th century and the Modernists of the early 20th looked to continental European individuals and movements for inspiration. Nor was attraction toward European intellectualism dead in the late 20th century, for by the mid-1980s the approach known as structuralism, a phenomenon predominantly French and German in origin, infused the very study of English literature itself in a host of published critical studies and university departments. Additional influence was exercised by deconstructionist analysis, based largely on the work of French philosopher Jacques Derrida."
+    hindi_translation = translate_mbart(english_text, "en_XX", "hi_IN")
+    print(hindi_translation) 
+

llava_finetune.py

@@ -0,0 +1,143 @@
+# install dependencies
+# !pip install -U "transformers>=4.39.0"
+# !pip install peft bitsandbytes
+# !pip install -U "trl>=0.8.3"
+
+import torch
+import io
+import PIL.Image as Image
+from transformers import AutoTokenizer, AutoProcessor, TrainingArguments, LlavaForConditionalGeneration, BitsAndBytesConfig
+from trl import SFTTrainer
+from peft import LoraConfig
+from datasets import Dataset
+from huggingface_hub import notebook_login
+
+
+def load_llava_model(model_id):
+    quantization_config = BitsAndBytesConfig(load_in_4bit=True)
+    model = LlavaForConditionalGeneration.from_pretrained(model_id, quantization_config=quantization_config,
+                                                          torch_dtype=torch.float16)
+    torch.cuda.empty_cache()
+    return model
+
+class LLavaDataCollator:
+    def __init__(self, processor):
+        self.processor = processor
+
+    def __call__(self, examples):
+        texts = []
+        images = []
+        for example in examples:
+            messages = example["messages"]
+            text = self.processor.tokenizer.apply_chat_template(
+                messages, tokenize=False, add_generation_prompt=False
+            )
+            texts.append(text)
+            img = Image.open(io.BytesIO(example['images'][0]['bytes']))
+            images.append(img)
+
+        batch = self.processor(texts, images, return_tensors="pt", padding=True)
+
+        labels = batch["input_ids"].clone()
+        if self.processor.tokenizer.pad_token_id is not None:
+            labels[labels == self.processor.tokenizer.pad_token_id] = -100
+        batch["labels"] = labels
+
+        return batch
+
+
+def load_datasets(train_path, test_path):
+    train_dataset = Dataset.load_from_disk(train_path)
+    eval_dataset = Dataset.load_from_disk(test_path)
+    return train_dataset, eval_dataset
+
+
+def configure_training_args(output_dir="llava-1.5-7b-hf-ft-museum"):
+    training_args = TrainingArguments(
+        output_dir=output_dir,
+        report_to="tensorboard",
+        learning_rate=1.4e-5,
+        per_device_train_batch_size=2,
+        gradient_accumulation_steps=1,
+        logging_steps=5,
+        num_train_epochs=5,
+        push_to_hub=True,
+        gradient_checkpointing=True,
+        remove_unused_columns=False,
+        fp16=True,
+        bf16=False
+    )
+    return training_args
+
+
+def configure_llora_config():
+    lora_config = LoraConfig(
+        r=32,
+        lora_alpha=16,
+        target_modules="all-linear"
+    )
+    return lora_config
+
+
+def configure_trainer(model, args, train_dataset, eval_dataset, lora_config, tokenizer, data_collator):
+    trainer = SFTTrainer(
+        model=model,
+        args=args,
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset,
+        peft_config=lora_config,
+        dataset_text_field="text",  # need a dummy field
+        tokenizer=tokenizer,
+        data_collator=data_collator,
+        dataset_kwargs={"skip_prepare_dataset": True},
+    )
+    return trainer
+
+
+def save_model(model, output_dir="llava_model_museum"):
+    training_args = TrainingArguments(output_dir=output_dir, push_to_hub=False)
+    model.save_pretrained(output_dir, push_to_hub=training_args.push_to_hub)
+
+
+def main():
+    # Load LLAVA model
+    model_id = "llava-hf/llava-1.5-7b-hf"
+    model = load_llava_model(model_id)
+
+    # Load datasets
+    train_path = '/kaggle/input/metmesuem-data/train.arrow'
+    test_path = '/kaggle/input/metmesuem-data/test.arrow'
+    train_dataset, eval_dataset = load_datasets(train_path, test_path)
+
+    # Login to Hugging-face Hub
+    notebook_login()
+
+    # Configure training arguments
+    args = configure_training_args()
+
+    # Configure LLORA config
+    lora_config = configure_llora_config()
+
+    # Configure trainer
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    processor = AutoProcessor.from_pretrained(model_id)
+    processor.tokenizer = tokenizer
+    data_collator = LLavaDataCollator(processor)
+    trainer = configure_trainer(model, args, train_dataset, eval_dataset, lora_config, tokenizer, data_collator)
+
+    # Load and set Tensorboard for logging
+    # %load_ext tensorboard
+    # %tensorboard --logdir llava-1.5-7b-hf-ft-museum
+
+    # Train model
+    trainer.train()
+
+    # Save the Lora adapters to Hugging Face Hub
+    trainer.push_to_hub()
+
+    # Save model
+    save_model(model)
+
+
+if __name__ == "__main__":
+    main()

translation_model_evaluation.py

@@ -0,0 +1,84 @@
+import nltk
+from nltk.translate.meteor_score import meteor_score
+from nltk.translate.bleu_score import corpus_bleu
+from indicnlp.tokenize import indic_tokenize
+import pandas as pd
+import numpy as np
+
+def configure_nltk():
+    nltk.download('punkt')
+    nltk.download('wordnet')
+
+def tokenize_hindi(text):
+    return indic_tokenize.trivial_tokenize(text, lang='hi')
+
+def calculate_meteor_score(ref, output):
+    ref_tokens = tokenize_hindi(ref)
+    output_tokens = tokenize_hindi(output)
+    return meteor_score([ref_tokens], output_tokens)
+
+def calculate_bleu_score(ref, output):
+    ref_tokens = [tokenize_hindi(ref)]
+    output_tokens = tokenize_hindi(output)
+    return corpus_bleu(ref_tokens, [output_tokens])
+
+def read_hindi_data(filepath):
+    hindi_df = pd.read_csv(filepath)
+    hindi_df = hindi_df.replace('नेन', np.nan)
+    hindi_df = hindi_df.dropna()
+    return hindi_df
+
+def create_reference_df(hindi_df):
+    reference_df = pd.DataFrame()
+    for i in range(1, 3):
+        reference_df[f'Question{i}'] = hindi_df[f'Question{i}']
+        reference_df[f'Answer{i}'] = hindi_df[f'Answer{i}']
+    return reference_df
+
+def select_first_n_rows(df, n=10):
+    return df.iloc[:n]
+
+def calculate_scores(reference_df, n=10):
+    meteor_scores = []
+    bleu_scores = []
+
+    for i in range(1, 3):  
+        ref_question_col = f'Question{i}'
+        ref_answer_col = f'Answer{i}'
+        output_question_col = f'Question{i}_hindi'
+        output_answer_col = f'Answer{i}_hindi'
+
+        for ref_question, output_question in zip(reference_df[ref_question_col], reference_df[output_question_col]):
+            meteor_scores.append(calculate_meteor_score(ref_question, output_question))
+            bleu_scores.append(calculate_bleu_score(ref_question, output_question))
+
+        for ref_answer, output_answer in zip(reference_df[ref_answer_col], reference_df[output_answer_col]):
+            meteor_scores.append(calculate_meteor_score(ref_answer, output_answer))
+            bleu_scores.append(calculate_bleu_score(ref_answer, output_answer))
+
+    average_meteor_score = sum(meteor_scores) / len(meteor_scores)
+    average_bleu_score = sum(bleu_scores) / len(bleu_scores)
+    
+    return average_meteor_score, average_bleu_score
+
+def main():
+    # Configure NLTK
+    configure_nltk()
+
+    # Read Hindi data
+    hindi_df = read_hindi_data('/content/Hindi_QnA.csv')
+
+    # Create reference dataframe
+    reference_df = create_reference_df(hindi_df)
+
+    # Select first 10 rows
+    reference_df_first10 = select_first_n_rows(reference_df)
+
+    # Calculate scores
+    average_meteor_score, average_bleu_score = calculate_scores(reference_df_first10)
+
+    print("Average METEOR Score:", average_meteor_score)
+    print("Average BLEU Score:", average_bleu_score)
+
+if __name__ == "__main__":
+    main()

vqa_collection.py

@@ -0,0 +1,89 @@
+
+# !pip install -q -U google-generativeai
+
+import time
+import json
+import pandas as pd
+from google.colab import userdata
+import google.generativeai as genai
+
+def configure_genai():
+    # Used to securely store your API key
+    GOOGLE_API_KEY = userdata.get('GOOGLE_API_KEY')
+    genai.configure(api_key=GOOGLE_API_KEY)
+
+def generate_art_questions(data_json, model):
+    art_texts = []
+    i = 0
+
+    for data_entry in data_json:
+        # Prompt for generating questions based on art description and metadata
+        prompt = f"Following is the description and meta data of an art. Based on this generate 4 questions and corresponding answer. Denote each question as **Question and answer as **Answer. {data_entry}"
+        # Generate content using the model
+        response = model.generate_content(prompt)
+        art_texts.append(response.text)
+        i += 1
+        if i == 60:
+            print("60 completed")
+            i = 0
+            time.sleep(10)
+    
+    return art_texts
+
+def extract_qa_pairs(text):
+    qa_pairs = {}
+    current_q = 1
+    current_a = 1
+
+    for line in text.splitlines():
+        if line.startswith("**Question"):
+            qa_pairs[f"Question{current_q}"] = line.strip().split(":", 1)[1].strip("**")  # Extract question
+            current_q += 1
+        elif line.startswith(("**Answer", "Answer")):
+            qa_pairs[f"Answer{current_a}"] = line.strip().split(":", 1)[1].strip("**")  # Extract answer
+            current_a += 1
+
+    return qa_pairs
+
+def create_qa_dataframe(art_texts):
+    df = pd.DataFrame(columns=["Question1", "Answer1", "Question2", "Answer2", "Question3", "Answer3", "Question4", "Answer4"])
+
+    for art_text in art_texts:
+        qa_dict = extract_qa_pairs(art_text)
+
+        if len(list(qa_dict.values())) == 8:
+            # Fill the DataFrame with extracted data
+            df.loc[len(df)] = list(qa_dict.values())  # Efficiently fill with all values
+        else:
+            df.loc[len(df)] = [None]*8
+    
+    return df
+
+def main():
+    # Configure GenAI
+    configure_genai()
+
+    # Initialize Generative Model
+    model = genai.GenerativeModel('gemini-pro')
+
+    # Read the data from CSV
+    data = pd.read_csv('/content/Data_for_questioning.csv')
+
+    # Convert DataFrame to JSON
+    data.to_json('final_data_json.json', orient='records', indent=4)
+
+    # Open the JSON file and load its contents
+    with open('/content/final_data_json.json', 'r') as f:
+        data_json = json.load(f)
+
+    # Generate questions for art descriptions
+    art_texts = generate_art_questions(data_json, model)
+
+    # Create DataFrame with questions and answers
+    df = create_qa_dataframe(art_texts)
+
+    # Save DataFrame to CSV
+    df.to_csv('QnA.csv', index=False)
+
+if __name__ == "__main__":
+    main()