import os.path import openai from openai import OpenAI import os from groq import Groq import requests import time from html.parser import HTMLParser from bs4 import BeautifulSoup import json from datetime import datetime import pandas as pd from serpapi import GoogleSearch import gradio as gr GROQ_API_KEY=getenv("GROQ_API_KEY") client_groq = Groq(api_key=GROQ_API_KEY,) openai_key=getenv("OPENAI_API_KEY") os.environ["OPENAI_API_KEY"] = openai_key client = OpenAI() SERPAPI_KEY=getenv("SERPAPI_KEY") api_key = os.getenv("API_KEY") def scrape_website(url): headers = {'User-Agent': 'Mozilla/5.0'} try: response = requests.get(url, headers=headers, timeout=20) response.encoding = response.apparent_encoding # Set encoding to match the content if response.status_code == 200: page_content = response.content soup = BeautifulSoup(page_content, 'html.parser') paragraphs = soup.find_all('p') scraped_data = [p.get_text() for p in paragraphs] formatted_data = u"\n".join(scraped_data) return formatted_data # Return only content else: return "Failed to retrieve the webpage (Status Code: {})".format(response.status_code) except requests.exceptions.ReadTimeout: # Handle the timeout exception return "Request timed out after 20 seconds." except requests.exceptions.SSLError as e: return "Request Error: {}".format(e) except requests.exceptions.RequestException as e: # Handle other requests-related exceptions return "An error occurred: {}".format(e) def update_dataframe_with_results(organic_results): # Prepare data for DataFrame max_chars = 100000 # Maximum characters allowed in a cell data = [] for result in organic_results: # Scrape the website content scraped_content = scrape_website(result.get('link')) # Truncate the content if it exceeds the limit if len(scraped_content) > max_chars: scraped_content = scraped_content[:max_chars] data.append({ "Title": result.get('title'), "Link": result.get('link'), "Snippet": result.get('snippet'), "Displayed Link": result.get('displayed_link'), "Date": result.get('date'), # Might not always be present "Rich Snippet": result.get('rich_snippet'), # Might not always be present "Scraped Content": scraped_content # Add scraped content }) df = pd.DataFrame(data) return df def opencall(text,user_query): print("Calling opencall function with", len(text), "characters") #completion = client_groq.chat.completions.create( completion = client.chat.completions.create( model="gpt-4-0125-preview", #model="mixtral-8x7b-32768", temperature=0.1, messages=[ {"role": "system", "content": "You are a helpful assistant, specialised in preparing contents for preparing a presentation."}, {"role": "system", "content": "Your task is to prepare a base report on the topics, themes and trends addressed in the latest conferences, seminars and symposiums." }, {"role": "system", "content": "For this matter I will be providing you in the Information Pool a compilation of several scraped google search results from the latest conferences, seminars and symposiums on the topic: "+user_query}, {"role": "system", "content": "Each piece of Scraped Content start with the tag '### Title:' indicating the title, followed by the URL reference '### Link:' , followed by the contents '### Content:'"}, {"role": "system", "content": "Process all the information in the Information Pool to provide:"}, {"role": "system", "content": "1) Perspective of Relevant Information: Assess and extract the most relevant information from the point of view of this aspect: "+user_query+"."}, {"role": "system", "content": "2) Perspective of Key Topics: Highlight the key topics and themes.Cite the URLs that source those topics and themes"}, {"role": "system", "content": "3) Perspective of Emergent Trends: Highlight the emergent trends.Cite the URLs that source those trends."}, {"role": "system", "content": "In the response, use the indicated structure of 1)Perspective of Relevant Information 2)Perspective of Key Topics 3)Perspective of Emergent Trends"}, {"role": "user", "content":"Information Pool:"+text} ] ) response = completion.choices[0].message.content response = response + "\n" + "XXXXX" + "\n" return response def split_large_content(content, max_length=30000): # Extract the title and source URL, assuming they end with the second newline title_and_source_end = content.find('\n\n') + 2 title_and_source = content[:title_and_source_end] title_and_source_length = len(title_and_source) # Ensure each segment has space for the title and source by reducing max_length max_segment_length = max_length - title_and_source_length segments = [] content_body = content[title_and_source_end:] # Start splitting the content_body into segments while len(content_body) > 0: # Take a slice of content up to max_segment_length segment = content_body[:max_segment_length] # If we're not at the end of content_body, back-track to the last complete word if len(content_body) > max_segment_length: last_space = segment.rfind(' ') segment = segment[:last_space] # Add the title and source URL to the start of this segment full_segment = title_and_source + segment segments.append(full_segment) # Move forward in content_body by the length of the segment minus the title/source content_body = content_body[len(segment):] return segments def main(df,google_search_query): # Initialize a string to accumulate the information information_pool = "" archivo1="" # Open or create a plain text file in append mode with open('respuestas.txt', mode='a', encoding='utf-8') as file: # Iterate over the rows of the DataFrame for index, row in df.iterrows(): # Combine title, link, and content into a single string document_name = row['Title'] # Using title as document_name raw_content = str(row['Scraped Content']) # Convert to string to ensure compatibility link = row['Link'] # Retrieve link for additional usage or logging # Assuming process_document_content is a function you've defined to process the content processed_content = "### Title: " + row['Title'] + "\n" + "### Link: " + row['Link'] + "\n" + "### Content: " + str(row['Scraped Content']) + "\n" + "\n" print(document_name, ":", len(processed_content)) #print("Contenido:", processed_content) print("acumulado:", len(information_pool + processed_content)) # Handle long content by splitting and processing in segments if len(processed_content) > 30000: content_segments = split_large_content(processed_content) for segment in content_segments: print("EN C, Nuevo valor de Text:", len(segment)) #print("segmen:",segment) response = opencall(segment,google_search_query) # Replace 'opencall' with your actual function call archivo1=archivo1+response+'\n' file.write(response + '\n') else: # Check if adding processed content exceeds the size limit if len(information_pool + processed_content) <= 30000: information_pool += processed_content print("EN A, Nuevo valor de Text:", len(information_pool)) else: # Process current accumulated content and start new accumulation print("EN B1, llamando con valor de Text:", len(information_pool)) #print("Information pool", information_pool) response = opencall(information_pool,google_search_query) file.write(response + '\n') archivo1=archivo1+response+'\n' information_pool = processed_content print("EN B2, nuevo valor de Text:", len(information_pool), " Con documento:", document_name) # Handle any remaining content after loop if information_pool: print("Final call") response = opencall(information_pool,google_search_query) file.write(response + '\n') archivo1=archivo1+response+'\n' return archivo1 def rearrange_text(text): # Split the text into batches using 'XXXXX' batches = text.split('XXXXX') # Initialize variables to store concatenated texts all_texta = "" all_textb = "" all_textc = "" # Define markers for different sections markers = { 'texta_marker': "Perspective of Relevant Information", 'textb_marker': "Perspective of Key Emerging Aspects", 'textc_marker': "Perspective of Key Entities" } # Process each batch for batch in batches: # Initialize indices for each section texta_start = batch.find(markers['texta_marker']) textb_start = batch.find(markers['textb_marker']) textc_start = batch.find(markers['textc_marker']) # Extract TEXTA, TEXTB, and TEXTC using the found indices # Check if the markers are found; if not, skip to the next marker texta = batch[texta_start:textb_start] if textb_start != -1 else batch[texta_start:] textb = batch[textb_start:textc_start] if textc_start != -1 else batch[textb_start:] textc = batch[textc_start:] # Remove the markers from the beginning of each text texta = texta.replace(markers['texta_marker'], '').strip() textb = textb.replace(markers['textb_marker'], '').strip() textc = textc.replace(markers['textc_marker'], '').strip() # Concatenate texts from all batches all_texta += "\n" + texta if all_texta else texta all_textb += "\n" + textb if all_textb else textb all_textc += "\n" + textc if all_textc else textc # You can now use all_texta, all_textb, and all_textc for further summarization or processing return all_texta, all_textb, all_textc def resumen(text): texta, textb, textc = rearrange_text(text) completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[ {"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."}, {"role": "system", "content": "Your task is to provide an integrated comprehensive 2000 words narrative of the different points indicated in the Information Pool text for a internal report on recent news." }, {"role": "system", "content": "Instructions. Elaborate the text following these rules:" }, {"role": "system", "content": "Be exhaustive, comprehensive and detailed in addressing the relation of different points indicated in the Information Pool text." }, {"role": "system", "content": "Arrange paragraphs and information around each entity or related entities and concepts, integrating them with a fluent narrative." }, {"role": "system", "content": "Start directly with the narrative, do not introduce the text, as it is part of a broader report." }, {"role": "system", "content": "Use a formal writing style, yet plain and easy to read. Avoid pomposity and making up artificial descriptions. The audience is well acquainted with technical and defence/military vocabulary, information and entities. " }, {"role": "user", "content":"Information Pool:"+texta} ] ) response1 = completion.choices[0].message.content if completion.choices[0].message else "" response_1="1) Perspective of Relevant Information:"+"\n"+response1+"\n" completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[ {"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."}, {"role": "system", "content": "Your task is to provide a comprehensive and integrated relation of about 2000 words in length of the different emerging aspects indicated in the Information Pool text for a internal report on recent news." }, {"role": "system", "content": "Instructions. Elaborate the text following these rules:" }, {"role": "system", "content": "Be exhaustive, comprehensive and detailed in the relation." }, {"role": "system", "content": "Arrange paragraphs and information around each entity or related entities and concepts." }, {"role": "system", "content": "Start directly with the relation, do not introduce the text, as it is part of a broader report." }, {"role": "system", "content": "Use a formal writing style, yet plain and easy to read. The audience is well acquainted with technical and defence/military vocabulary, information and entities. " }, {"role": "user", "content":"Information Pool:"+textb} ] ) response2 = completion.choices[0].message.content if completion.choices[0].message else "" response_2=" 2)Perspective of Key emerging aspects:"+"\n"+response2+"\n" completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[ {"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."}, {"role": "system", "content": "Your task is to consolidate and sore the relation of the different entities indicated in the Information Pool text for a internal report on recent news." }, {"role": "system", "content": "Instructions. Elaborate the text following these rules:" }, {"role": "system", "content": "Be exhaustive in the sorting. Sort around similar entry types: Organization, Program, Technology, Entity, ... You can merge similar entry types (i.e. Technologies and Technology Terms and Concepts, People and Officials,...)" }, {"role": "system", "content": "Arrange and integrate entries around similar or related concepts. Discard duplicated concepts or elements." }, {"role": "system", "content": "Start directly with the relation, do not introduce the text, as it is part of a broader report." }, {"role": "system", "content": "The audience is well acquainted with technical and defence/military vocabulary, information and entities. " }, {"role": "user", "content":"Information Pool:"+textc} ] ) response3 = completion.choices[0].message.content if completion.choices[0].message else "" response_3=" 3)Perspective of of Key Entities"+"\n"+response3+"\n" compilacion=response_1+"\n"+response_2+"\n"+response_3 print(compilacion) print("\n\n") print("\n\n") return compilacion # Define the function to get news results def get_organic_results(query, periodo_tbs, num_results): params = { "q": query, "num": str(num_results), "tbs": periodo_tbs, # quiero los resultados del último año "api_key": SERPAPI_KEY } search = GoogleSearch(params) results = search.get_dict() organic_results = results.get("organic_results", []) # Change from "news_results" to "organic_results" for result in organic_results: title = result.get('title') print("Title:", title) print() # Print a newline for better readability between results return organic_results def process_inputs(task_type, topic, integration_period, num_results): # Construct the query based on user input google_search_query = f'"{topic}" Conferences OR seminars OR SYMPOSIUMS' periodo_tbs = integration_period num_resultados = int(num_results) # Fetch results based on the user's query results = get_organic_results(google_search_query, periodo_tbs, num_resultados) df = update_dataframe_with_results(results) archivo1 = main(df, google_search_query) resumen_text = resumen(archivo1) return archivo1,resumen_text # Create the Gradio blocks interface with gr.Blocks() as app: with gr.Row(): with gr.Column(): task_type = gr.Dropdown(choices=["Conferencias", "Seminarios", "Simposios"], label="Selecciona el tipo de tarea:") topic = gr.Textbox(label="Aspecto o Tema sobre el que trabajar", placeholder="Ingrese el tema aquí...") integration_period = gr.Dropdown(choices=["1M", "3M", "6M", "1Y"], label="Periodo de integración de información") num_results = gr.Number(label="Número de resultados sobre los que trabajar", value=10) submit_button = gr.Button("Submit") with gr.Column(): output_text_intermedio = gr.Textbox(label="Resultados Intermedios", interactive=True, lines=10) output_text_final = gr.Textbox(label="Resultados Compilados", interactive=True, lines=10) # Define what happens when you click the Submit button submit_button.click( fn=process_inputs, inputs=[task_type, topic, integration_period, num_results], outputs=[output_text_intermedio,output_text_final] ) if __name__ == "__main__": app.launch()