Update app.py

app.py

@@ -1,227 +1,191 @@
-import datetime
 import os
-import csv
 import time
 import hashlib
 import logging
-import gradio as gr
+import datetime
+import csv
+from urllib.parse import urlparse
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.chrome.options import Options
-from webdriver_manager.chrome import ChromeDriverManager
-from huggingface_hub import InferenceClient
-import random
-import yaml
-import requests
-from bs4 import BeautifulSoup
-
-urls = [
-    "https://twitter.com/wlcscrdp",
-    "https://www.facebook.com/aurorareddevils/",
-    "https://www.facebook.com/brightpanthers/",
-    "https://www.facebook.com/carrollcountychamberin/",
-    "https://www.facebook.com/Culver.Cavs.MHS",
-    "https://www.facebook.com/culver.elementary.school",
-    "https://www.facebook.com/CulverCommunitySchools",
-    "https://www.facebook.com/DillsboroBulldogs/",
-    "https://www.facebook.com/ECMSTROJANS",
-    "https://www.facebook.com/enjoywhitecountyIN/",
-    "https://www.facebook.com/farmersvilleelementary",
-    "https://www.facebook.com/groups/SDMSparents",
-    "https://www.facebook.com/jghsart/",
-    "https://www.facebook.com/jgmusicdept",
-    "https://www.facebook.com/John-Glenn-Education-Foundation-208326199636364/",
-    "https://www.facebook.com/John-Glenn-High-School-1102148953201006/",
-    "https://www.facebook.com/John-Glenn-Theatre-Company-383638295064502/",
-    "https://www.facebook.com/JohnGlennFalconsAthletics",
-    "https://www.facebook.com/KIRPC-Head-Start-1485812354989001",
-    "https://www.facebook.com/KIRPC1",
-    "https://www.facebook.com/LHNEeagles",
-    "https://www.facebook.com/LuceElementarySchool/",
-    "https://www.facebook.com/marrselementary",
-    "https://www.facebook.com/messhiners/",
-    "https://www.facebook.com/monticellocitypool",
-    "https://www.facebook.com/monticelloinwastewater/",
-    "https://www.facebook.com/MooresHillBobcats/",
-    "https://www.facebook.com/msdmv",
-    "https://www.facebook.com/msdnorthposey",
-    "https://www.facebook.com/MUTPL/",
-    "https://www.facebook.com/MVJHS/",
-    "https://www.facebook.com/mvshs",
-    "https://www.facebook.com/njspjrsrhighschool?mibextid=b06tZ0",
-    "https://www.facebook.com/NorthElementaryStars/",
-    "https://www.facebook.com/NorthLibertyElementary/",
-    "https://www.facebook.com/northposey/",
-    "https://www.facebook.com/northposeyhs/",
-    "https://www.facebook.com/NPJuniorHigh",
-    "https://www.facebook.com/Prairie-Heights-Elementary-659322230934707/",
-    "https://www.facebook.com/Prairie-Heights-High-School-2027713067459043/",
-    "https://www.facebook.com/PrairieHeightsPanthers/",
-    "https://www.facebook.com/profile.php?id=100057030237096",
-    "https://www.facebook.com/profile.php?id=100057451179651",
-    "https://www.facebook.com/profile.php?id=100063463513451",
-    "https://www.facebook.com/profile.php?id=100063612319256",
-    "https://www.facebook.com/profile.php?id=100064532596422",
-    "https://www.facebook.com/profile.php?id=100067180226810",
-    "https://www.facebook.com/profile.php?id=61563484312348",
-    "https://www.facebook.com/PTOSWES/",
-    "https://www.facebook.com/RandolphSouthern/",
-    "https://www.facebook.com/RochesterMiddleSchool",
-    "https://www.facebook.com/RochesterZebraNewTechHigh",
-    "https://www.facebook.com/rockportelementarysouthspencer/",
-    "https://www.facebook.com/satellitesathletics/",
-    "https://www.facebook.com/seymourcommunityschools/",
-    "https://www.facebook.com/SeymourHighSchool/",
-    "https://www.facebook.com/SouthDearbornHighSchool/",
-    "https://www.facebook.com/southarbornschools/",
-    "https://www.facebook.com/SouthDearbornSquires/",
-    "https://www.facebook.com/southspencerhighschool",
-    "https://www.facebook.com/southspencermiddleschool/",
-    "https://www.facebook.com/SouthSpencerSchools",
-    "https://www.facebook.com/SouthTerracePanthers/",
-    "https://www.facebook.com/sunmantigers/",
-    "https://www.facebook.com/SWShelbySpartan/",
-    "https://www.facebook.com/TallTimbersMarina",
-    "https://www.facebook.com/WabashValleyESC/",
-    "https://www.facebook.com/Walkerton-Elementary-School-283088605088622/",
-    "https://www.facebook.com/westcentralcte/",
-    "https://www.facebook.com/westelementary",
-    "https://www.facebook.com/wlcscrdp",
-    "https://www.instagram.com/mutpl/",
-    "https://www.instagram.com/northposeyhsathletics",
-    "https://www.instagram.com/rchsprincipalcook/",
-    "https://www.instagram.com/southdearbornhighschool/",
-    "https://www.instagram.com/southdearbornschools/",
-    "https://www.instagram.com/westcentralcte/",
-    "https://www.tiktok.com/@mutplteen"
-    ]
-
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.exceptions import NoSuchElementException
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+from transformers import pipeline
+import feedparser
 
 # Configure logging
 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
 
 # Define constants
-DATE_TIME_STR = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+DEFAULT_FILE_PATH = "scraped_data"
 PURPOSE = f"You go to Culvers sites, you continuously seek changes on them since your last observation. Anything new that gets logged and dumped into csv, stored in your log folder at user/app/scraped_data."
 HISTORY = []
 CURRENT_TASK = None
-DEFAULT_FILE_PATH = "user/app/scraped_data/culver/culvers_changes.csv"
 
-# Ensure the directory exists
-os.makedirs(os.path.dirname(DEFAULT_FILE_PATH), exist_ok=True)
+# Define the list of URLs to monitor (you can add more URLs here)
+URLS_TO_MONITOR = ["https://twitter.com/wlcscrdp", "https://www.facebook.com/aurorareddevils/", "https://www.facebook.com/brightpanthers/", "https://www.facebook.com/carrollcountychamberin/", "https://www.facebook.com/Culver.Cavs.MHS", "https://www.facebook.com/culver.elementary.school", "https://www.facebook.com/CulverCommunitySchools", "https://www.facebook.com/DillsboroBulldogs/", "https://www.facebook.com/ECMSTROJANS", "https://www.facebook.com/enjoywhitecountyIN/", "https://www.facebook.com/farmersvilleelementary", "https://www.facebook.com/groups/SDMSparents", "https://www.facebook.com/jghsart/", "https://www.facebook.com/jgmusicdept", "https://www.facebook.com/John-Glenn-Education-Foundation-208326199636364/", "https://www.facebook.com/John-Glenn-High-School-1102148953201006/", "https://www.facebook.com/John-Glenn-Theatre-Company-383638295064502/", "https://www.facebook.com/JohnGlennFalconsAthletics", "https://www.facebook.com/KIRPC-Head-Start-1485812354989001", "https://www.facebook.com/KIRPC1", "https://www.facebook.com/LHNEeagles", "https://www.facebook.com/LuceElementarySchool/", "https://www.facebook.com/marrselementary", "https://www.facebook.com/messhiners/", "https://www.facebook.com/monticellocitypool", "https://www.facebook.com/monticelloinwastewater/", "https://www.facebook.com/MooresHillBobcats/", "https://www.facebook.com/msdmv", "https://www.facebook.com/msdnorthposey", "https://www.facebook.com/MUTPL/", "https://www.facebook.com/MVJHS/", "https://www.facebook.com/mvshs", "https://www.facebook.com/njspjrsrhighschool?mibextid=b06tZ0", "https://www.facebook.com/NorthElementaryStars/", "https://www.facebook.com/NorthLibertyElementary/", "https://www.facebook.com/northposey/", "https://www.facebook.com/northposeyhs/", "https://www.facebook.com/NPJuniorHigh", "https://www.facebook.com/Prairie-Heights-Elementary-659322230934707/", "https://www.facebook.com/Prairie-Heights-High-School-2027713067459043/", "https://www.facebook.com/PrairieHeightsPanthers/", "https://www.facebook.com/profile.php?id=100057030237096", "https://www.facebook.com/profile.php?id=100057451179651", "https://www.facebook.com/profile.php?id=100063463513451", "https://www.facebook.com/profile.php?id=100063612319256", "https://www.facebook.com/profile.php?id=100064532596422", "https://www.facebook.com/profile.php?id=100067180226810", "https://www.facebook.com/profile.php?id=61563484312348", "https://www.facebook.com/PTOSWES/", "https://www.facebook.com/RandolphSouthern/", "https://www.facebook.com/RochesterMiddleSchool", "https://www.facebook.com/RochesterZebraNewTechHigh", "https://www.facebook.com/rockportelementarysouthspencer/", "https://www.facebook.com/satellitesathletics/", "https://www.facebook.com/seymourcommunityschools/", "https://www.facebook.com/SeymourHighSchool/", "https://www.facebook.com/SouthDearbornHighSchool/", "https://www.facebook.com/southarbornschools/", "https://www.facebook.com/SouthDearbornSquires/", "https://www.facebook.com/southspencerhighschool", "https://www.facebook.com/southspencermiddleschool/", "https://www.facebook.com/SouthSpencerSchools", "https://www.facebook.com/SouthTerracePanthers/", "https://www.facebook.com/sunmantigers/", "https://www.facebook.com/SWShelbySpartan/", "https://www.facebook.com/TallTimbersMarina", "https://www.facebook.com/WabashValleyESC/", "https://www.facebook.com/Walkerton-Elementary-School-283088605088622/", "https://www.facebook.com/westcentralcte/", "https://www.facebook.com/westelementary", "https://www.facebook.com/wlcscrdp", "https://www.instagram.com/mutpl/", "https://www.instagram.com/northposeyhsathletics", "https://www.instagram.com/rchsprincipalcook/", "https://www.instagram.com/southdearbornhighschool/", "https://www.instagram.com/southdearbornschools/", "https://www.instagram.com/westcentralcte/", "https://www.tiktok.com/@mutplteen"]
 
 # Function to monitor URLs for changes
 def monitor_urls(storage_location, urls, scrape_interval, content_type):
     global HISTORY
-    previous_hashes = [""] * len(urls)
+    previous_hashes = {url: "" for url in urls}  # Use a dictionary for better organization
 
     try:
-        with webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=Options()) as driver:
+        with webdriver.Chrome(service=Service(webdriver.ChromeDriverManager().install()), options=Options()) as driver:
             while True:
-                for i, url in enumerate(urls):
+                for url in urls:
                     try:
                         driver.get(url)
                         time.sleep(2)  # Wait for the page to load
                         if content_type == "text":
                             current_content = driver.page_source
                         elif content_type == "media":
-                            current_content = driver.find_elements_by_tag_name("img")
+                            current_content = driver.find_elements(By.TAG_NAME, "img")
                         else:
                             current_content = driver.page_source
                         current_hash = hashlib.md5(str(current_content).encode('utf-8')).hexdigest()
-                        if current_hash != previous_hashes[i]:
-                            previous_hashes[i] = current_hash
+                        if current_hash != previous_hashes[url]:
+                            previous_hashes[url] = current_hash
                             date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                             HISTORY.append(f"Change detected at {url} on {date_time_str}")
-                            with open(storage_location, "a", newline="") as csvfile:
+                            with open(os.path.join(storage_location, f"{urlparse(url).hostname}_changes.csv"), "a", newline="") as csvfile:
                                 csv_writer = csv.DictWriter(csvfile, fieldnames=["date", "time", "url", "change"])
                                 csv_writer.writerow({"date": date_time_str.split()[0], "time": date_time_str.split()[1], "url": url, "change": "Content changed"})
                                 logging.info(f"Change detected at {url} on {date_time_str}")
-                    except Exception as e:
+                    except (NoSuchElementException, Exception) as e:
                         logging.error(f"Error accessing {url}: {e}")
                 time.sleep(scrape_interval * 60)  # Check every scrape_interval minutes
     except Exception as e:
         logging.error(f"Error starting ChromeDriver: {e}")
 
-# Define main function to handle user input
-def handle_input(storage_location, urls, scrape_interval, content_type):
+# Function to start scraping
+def start_scraping(storage_location, urls, scrape_interval, content_type):
     global CURRENT_TASK, HISTORY
 
     CURRENT_TASK = f"Monitoring URLs: {', '.join(urls)}"
     HISTORY.append(f"Task started: {CURRENT_TASK}")
-    monitor_urls(storage_location, urls, scrape_interval, content_type)
-    return TASK_PROMPT.format(task=CURRENT_TASK, history="\n".join(map(str, HISTORY)))
-
-# Load custom prompts
-try:
-    with open("custom_prompts.yaml", "r") as fp:
-        custom_prompts = yaml.safe_load(fp)
-except FileNotFoundError:
-    custom_prompts = {"WEB_DEV": "", "AI_SYSTEM_PROMPT": "", "PYTHON_CODE_DEV": "", "CODE_GENERATION": "", "CODE_INTERPRETATION": "", "CODE_TRANSLATION": "", "CODE_IMPLEMENTATION": ""}
 
-# Define agents
-AGENTS = ["WEB_DEV", "AI_SYSTEM_PROMPT", "PYTHON_CODE_DEV", "CODE_GENERATION", "CODE_INTERPRETATION", "CODE_TRANSLATION", "CODE_IMPLEMENTATION"]
-
-# Define the Mistral inference client
-client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
-
-# Define the chat response function
-def respond(message, history, system_message, max_tokens, temperature, top_p):
-    return generate(message, history, system_message, max_tokens, temperature, top_p)
+    for url in urls:
+        # Create a folder for the URL
+        hostname = urlparse(url).hostname
+        folder_path = os.path.join(storage_location, hostname)
+        os.makedirs(folder_path, exist_ok=True)
+
+        # Log the initial observation
+        try:
+            with webdriver.Chrome(service=Service(webdriver.ChromeDriverManager().install()), options=Options()) as driver:
+                driver.get(url)
+                time.sleep(2)  # Wait for the page to load
+                if content_type == "text":
+                    initial_content = driver.page_source
+                elif content_type == "media":
+                    initial_content = driver.find_elements(By.TAG_NAME, "img")
+                else:
+                    initial_content = driver.page_source
+                initial_hash = hashlib.md5(str(initial_content).encode('utf-8')).hexdigest()
+                HISTORY.append(f"Initial observation at {url}: {initial_hash}")
+                with open(os.path.join(folder_path, f"{hostname}_initial_observation.txt"), "w") as file:
+                    file.write(f"Initial observation at {url}: {initial_hash}")
+        except (NoSuchElementException, Exception) as e:
+            HISTORY.append(f"Error accessing {url}: {e}")
+
+    # Monitor the URLs
+    monitor_urls(storage_location, urls, scrape_interval, content_type)
 
-# Function to start scraping
-def start_scraping(storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type):
-    urls = [url for url in [url1, url2, url3, url4, url5, url6, url7, url8, url9, url10] if url]
-    handle_input(storage_location, urls, scrape_interval, content_type)
     return f"Started scraping {', '.join(urls)} every {scrape_interval} minutes."
 
 # Function to display CSV content
-def display_csv(storage_location):
-    if os.path.exists(storage_location):
-        with open(storage_location, "r") as file:
+def display_csv(url):
+    hostname = urlparse(url).hostname
+    folder_path = os.path.join(DEFAULT_FILE_PATH, hostname)
+    csv_path = os.path.join(folder_path, f"{hostname}_changes.csv")
+    if os.path.exists(csv_path):
+        with open(csv_path, "r") as file:
             return file.read()
     else:
         return "No data available."
 
+# Define the chat response function using the Mistral model
+def respond(message, history, system_message, max_tokens, temperature, top_p):
+    model = AutoModelForSeq2SeqLM.from_pretrained_model("mistralai/Mixtral-8x7B-Instruct-v0.1")
+    tokenizer = AutoTokenizer.from_pretrained_model("mistralai/Mixtral-8x7B-Instruct-v0.1")
+    pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
+    response = pipe(f"User: {message}\nHistory: {history}\nSystem: {system_message}", max_length=max_tokens, temperature=temperature, top_p=top_p)[0]
+    return response
+
+# Function to generate RSS feed for a given URL
+def generate_rss_feed(url):
+    hostname = urlparse(url).hostname
+    folder_path = os.path.join(DEFAULT_FILE_PATH, hostname)
+    csv_path = os.path.join(folder_path, f"{hostname}_changes.csv")
+    if os.path.exists(csv_path):
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            feed = feedparser.parse(f"rss.xml")  # Create a new feed object
+            feed.feed.title = f"Changes for {hostname}"
+            feed.feed.link = url
+            feed.feed.description = "Recent changes detected on the website."
+            feed.entries = []
+            for row in reader:
+                feed.entries.append({
+                    "title": f"Change detected at {row['url']}",
+                    "link": row['url'],
+                    "description": f"Content changed on {row['date']} at {row['time']}",
+                    "published": datetime.datetime.strptime(f"{row['date']} {row['time']}", "%Y-%m-%d %H:%M:%S").isoformat(),
+                })
+            return feed.entries
+    else:
+        return "No data available."
+
+# Function to handle user input and generate response
+def chat_interface(message, history, system_message, max_tokens, temperature, top_p, storage_location, urls, scrape_interval, content_type):
+    response = respond(message, history, system_message, max_tokens, temperature, top_p)
+    history.append((message, response))
+    return history, response
+
+app.py
+
+import gradio as gr
+from agent import start_scraping, display_csv, generate_rss_feed, chat_interface
+
 # Create Gradio interface
-def chat_interface(message, system_message, max_tokens, temperature, top_p, storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type):
-    global HISTORY
-    response = respond(message, HISTORY, system_message, max_tokens, temperature, top_p)
-    HISTORY.append((message, response))
-    return HISTORY, ""
-
-demo = gr.Blocks()
-
-with demo:
-    with gr.Row():
-        with gr.Column():
-            message = gr.Textbox(label="Message")
-            system_message = gr.Textbox(value="You are a friendly Chatbot.", label="System message")
-            max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
-            temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
-            top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
-            storage_location = gr.Textbox(value=DEFAULT_FILE_PATH, label="Storage Location")
-            url1 = gr.Textbox(value="https://www.culver.k12.in/", label="URL 1")
-            url2 = gr.Textbox(value="https://www.facebook.com/CulverCommunitySchools", label="URL 2")
-            url3 = gr.Textbox(label="URL 3")
-            url4 = gr.Textbox(label="URL 4")
-            url5 = gr.Textbox(label="URL 5")
-            url6 = gr.Textbox(label="URL 6")
-            url7 = gr.Textbox(label="URL 7")
-            url8 = gr.Textbox(label="URL 8")
-            url9 = gr.Textbox(label="URL 9")
-            url10 = gr.Textbox(label="URL 10")
-            scrape_interval = gr.Slider(minimum=1, maximum=60, value=5, step=1, label="Scrape Interval (minutes)")
-            content_type = gr.Radio(choices=["text", "media", "both"], value="text", label="Content Type")
-            start_button = gr.Button("Start Scraping")
-            csv_output = gr.Textbox(label="CSV Output", interactive=False)
-        
-        with gr.Column():
-            chat_history = gr.Chatbot(label="Chat History")
-            response_box = gr.Textbox(label="Response")
-    
-    start_button.click(start_scraping, inputs=[storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type], outputs=csv_output)
-    message.submit(chat_interface, inputs=[message, system_message, max_tokens, temperature, top_p, storage_location, url1, url2, url3, url4, url5, url6, url7, url8, url9, url10, scrape_interval, content_type], outputs=[chat_history, response_box])
+def create_interface():
+    with gr.Blocks() as demo:
+        with gr.Row():
+            with gr.Column():
+                message = gr.Textbox(label="Message")
+                system_message = gr.Textbox(value="You are a helpful assistant.", label="System message")
+                max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
+                temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
+                top_p = gr.Slider(minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p (nucleus sampling)")
+                storage_location = gr.Textbox(value="scraped_data", label="Storage Location")
+                urls = gr.Textbox(label="URLs (comma separated)")
+                scrape_interval = gr.Slider(minimum=1, maximum=60, value=5, step=1, label="Scrape Interval (minutes)")
+                content_type = gr.Radio(choices=["text", "media", "both"], value="text", label="Content Type")
+                start_button = gr.Button("Start Scraping")
+                csv_output = gr.Textbox(label="CSV Output", interactive=False)
+
+            with gr.Column():
+                chat_history = gr.Chatbot(label="Chat History")
+                response_box = gr.Textbox(label="Response")
+
+        start_button.click(start_scraping, inputs=[storage_location, urls, scrape_interval, content_type], outputs=csv_output)
+        message.submit(chat_interface, inputs=[message, chat_history, system_message, max_tokens, temperature, top_p, storage_location, urls, scrape_interval, content_type], outputs=[chat_history, response_box])
+
+        # Add a button to display the RSS feed for a selected URL
+        with gr.Row():
+            selected_url = gr.Textbox(label="Select URL for RSS Feed")
+            rss_button = gr.Button("Generate RSS Feed")
+            rss_output = gr.Textbox(label="RSS Feed Output", interactive=False)
+
+        rss_button.click(generate_rss_feed, inputs=[selected_url], outputs=rss_output)
+
+    return demo
 
 if __name__ == "__main__":
+    demo = create_interface()
     demo.launch()