Update app.py

app.py

@@ -5,23 +5,29 @@ import time
 import hashlib
 import threading
 from pathlib import Path
+import logging
 
 import gradio as gr
 from selenium import webdriver
 from selenium.webdriver.chrome.service import Service
 from selenium.webdriver.chrome.options import Options
 from selenium.webdriver.common.by import By
+from selenium.common.exceptions import WebDriverException, NoSuchElementException
 from webdriver_manager.chrome import ChromeDriverManager
 from huggingface_hub import InferenceClient
 import mysql.connector
 import feedparser  # For parsing RSS feeds
 import sqlite3  # For simple local storage if needed
 
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
 # Configuration (replace with your actual values or environment variables)
 DB_HOST = os.environ.get("DB_HOST", "your_host")
 DB_USER = os.environ.get("DB_USER", "your_user")
 DB_PASSWORD = os.environ.get("DB_PASSWORD", "your_password")
 DB_NAME = os.environ.get("DB_NAME", "your_database")
+HUGGINGFACE_API_KEY = os.environ.get("HUGGINGFACE_API_KEY")  # Add API key
 
 # Global variables
 monitoring_thread = None
@@ -29,6 +35,7 @@ stop_event = threading.Event()
 db_connection = None
 current_task = None
 history = []
+url_monitoring_intervals = {}  # Store monitoring intervals for each URL
 
 # Function to establish a database connection
 def get_db_connection():
@@ -43,7 +50,7 @@ def get_db_connection():
             )
             return db_connection
         except Exception as e:
-            print(f"Error connecting to database: {e}")
+            logging.error(f"Error connecting to database: {e}")
             return None
     else:
         return db_connection
@@ -71,7 +78,7 @@ create_articles_table()
 
 # Function to monitor URLs for changes
 def monitor_urls(target_urls, storage_location, feed_rss, stop_event):
-    global history
+    global history, url_monitoring_intervals
     previous_hashes = {url: "" for url in target_urls}
     options = Options()
     options.headless = True
@@ -85,16 +92,23 @@ def monitor_urls(target_urls, storage_location, feed_rss, stop_event):
         while not stop_event.is_set():
             for url in target_urls:
                 try:
+                    # Dynamic monitoring interval
+                    interval = url_monitoring_intervals.get(url, 300)  # Default 5 minutes
+
                     driver.get(url)
-                    time.sleep(2)
+                    time.sleep(2)  # Allow page to load
                     current_content = driver.page_source
                     current_hash = hashlib.md5(current_content.encode('utf-8')).hexdigest()
 
                     if current_hash != previous_hashes[url]:
                         previous_hashes[url] = current_hash
                         timestamp = datetime.datetime.now()
-                        title_element = driver.find_element(By.TAG_NAME, "title")
-                        title = title_element.text if title_element else "No Title"
+
+                        try:
+                            title_element = driver.find_element(By.TAG_NAME, "title")
+                            title = title_element.text
+                        except NoSuchElementException:
+                            title = "No Title"
 
                         history.append(f"Change detected at {url} on {timestamp.strftime('%Y-%m-%d %H:%M:%S')}")
 
@@ -104,17 +118,27 @@ def monitor_urls(target_urls, storage_location, feed_rss, stop_event):
                         if feed_rss:
                             save_to_database(url, title, current_content, current_hash)
 
-                except Exception as e:
-                    print(f"Error accessing {url}: {e}")
+                        # Adjust monitoring interval based on change frequency (example)
+                        url_monitoring_intervals[url] = 60  # Check more frequently after a change
+
+                    else:
+                        # Increase interval if no changes detected (example)
+                        url_monitoring_intervals[url] = min(url_monitoring_intervals[url] + 60, 600) # Max 10 min
+
+                except WebDriverException as e:
+                    logging.error(f"Error accessing {url}: {e}")
+
+                if stop_event.is_set():
+                    break  # Exit inner loop if stop event is set
 
             if not stop_event.is_set():
-                time.sleep(300)  # Wait for 5 minutes
+                time.sleep(interval) 
 
     except Exception as e:
-        print(f"Unexpected error in monitoring thread: {e}")
+        logging.error(f"Unexpected error in monitoring thread: {e}")
     finally:
         driver.quit()
-        print("Monitoring thread has been stopped.")
+        logging.info("Monitoring thread has been stopped.")
 
 # Function to save data to local storage (CSV)
 def save_to_storage(storage_location, url, title, content, timestamp):
@@ -123,7 +147,7 @@ def save_to_storage(storage_location, url, title, content, timestamp):
             csv_writer = csv.writer(csvfile)
             csv_writer.writerow([timestamp.strftime("%Y-%m-%d %H:%M:%S"), url, title, content])
     except Exception as e:
-        print(f"Error saving to storage: {e}")
+        logging.error(f"Error saving to storage: {e}")
 
 # Function to save data to the database
 def save_to_database(url, title, content, hash):
@@ -136,7 +160,7 @@ def save_to_database(url, title, content, hash):
             cursor.execute(sql, val)
             conn.commit()
         except Exception as e:
-            print(f"Error saving to database: {e}")
+            logging.error(f"Error saving to database: {e}")
         finally:
             cursor.close()
 
@@ -165,7 +189,7 @@ def generate_rss_feed():
 
             return feedparser.FeedGenerator().feed_from_dictionary(feed).writeString('utf-8')
         except Exception as e:
-            print(f"Error generating RSS feed: {e}")
+            logging.error(f"Error generating RSS feed: {e}")
         finally:
             cursor.close()
     return None
@@ -201,12 +225,15 @@ def stop_monitoring():
 
 # Function to handle chatbot responses
 def chatbot_response(message, history):
-    # Replace this with your actual chatbot logic using 'mistralai/Mixtral-8x7B-Instruct-v0.1'
-    # You'll need to load and use the model from Hugging Face's InferenceClient
-    client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1")
-    response = client.inference(message)
-    history.append((message, response))
-    return history, history
+    try:
+        client = InferenceClient("mistralai/Mixtral-8x7B-Instruct-v0.1", token=HUGGINGFACE_API_KEY)
+        response = client.inference(message)
+        history.append((message, response))
+        return history, history
+    except Exception as e:
+        logging.error(f"Error getting chatbot response: {e}")
+        history.append((message, "Error: Could not get a response from the chatbot."))
+        return history, history
 
 # --- Gradio Interface ---
 with gr.Blocks() as demo:
@@ -223,7 +250,7 @@ with gr.Blocks() as demo:
             storage_location = gr.Textbox(
                 label="Storage Location (CSV file path)",
                 placeholder="/path/to/your/file.csv",
-                visible=False
+                visible=False  # You can enable this if you want CSV storage
             )
         with gr.Row():
             feed_rss_checkbox = gr.Checkbox(label="Enable RSS Feed")
@@ -253,11 +280,15 @@ with gr.Blocks() as demo:
 
     # Start monitoring button click
     def on_start_click(target_urls_str, storage_loc, feed_enabled):
-        global history
+        global history, url_monitoring_intervals
         try:
             target_urls = [url.strip() for url.strip() in target_urls_str.split(",")]
             if not all(target_urls):
                 return "Please enter valid URLs.", history
+            
+            # Reset monitoring intervals when starting
+            url_monitoring_intervals = {url: 300 for url in target_urls} 
+
             status, history = start_monitoring(target_urls, storage_loc if storage_loc else None, feed_enabled)
             return status, history
         except Exception as e: