Delete app.py

app.py

@@ -1,1569 +0,0 @@
-limport datetimeimport osimport csvimport timeimport hashlibimport loggingfrom collections import defaultdictimport mysql.connectorimport threadingfrom urllib.parse import urlparseimport gradio as grfrom selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as   
-
-
-
- ECfrom selenium.common.exceptions import NoSuchElementException, StaleElementReferenceException, TimeoutExceptionfrom selenium.webdriver.chrome.service   
-
-
-
- import Servicefrom selenium.webdriver.chrome.options import Optionsfrom webdriver_manager.chrome import ChromeDriverManager   
-
-
-
-from huggingface_hub import InferenceClient, loginfrom transformers import AutoTokenizer, AutoModelForCausalLM, pipelineimport randomimport yamlimport torchimport pandas as pdimport xml.etree.ElementTree as ETimport reimport spacyimport unittestfrom dotenv import load_dotenvimport nltk# Initialize NLTK resources (you may need to download these)
-
-nltk.download('punkt')nltk.download('averaged_perceptron_tagger')
-
-nltk.download('maxent_ne_chunker')
-
-nltk.download('words')   
-
-
-
-# Load spaCy model
-
-nlp = spacy.load("en_core_web_sm")   
-
-
-
-# Dictionary to store model loading functions
-
-model_loaders = {
-
-    "Falcon": lambda: load_model("tiiuae/falcon-7b"),
-
-    "Flan-T5": lambda: load_model("google/flan-t5-xl"),
-
-    "Flan-T5-Small": lambda: load_model("google/flan-t5-small")  # Add a smaller model
-
-}# Load environment variables from .env file
-
-load_dotenv()
-
-
-
-HUGGINGFACE_TOKEN = os.getenv("HUGGINGFACE_TOKEN")if not HUGGINGFACE_TOKEN:
-
-    raise ValueError("HUGGINGFACE_TOKEN is not set in the environment variables.")
-
-
-
-login(token=HUGGINGFACE_TOKEN, add_to_git_credential=True)# Configure logging
-
-logging.basicConfig(
-
-    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
-
-)# Define constants
-
-DEFAULT_FILE_PATH = "scraped_data"
-
-PURPOSE = (
-
-    "You monitor urls. You log what you observe. You seek any changes on them since your last observation. "
-
-    "Anything new gets logged and dumped into csv, stored in your log folder at user/app/scraped_data."
-
-)# Global variables for task management
-
-HISTORY = []
-
-CURRENT_TASK = None
-
-STOP_THREADS = False  # Flag to stop scraping threads# Database Pooling Configuration
-
-DB_POOL_NAME = "mypool"
-
-DB_POOL_SIZE = 5  # Adjust based on expected loadtry:
-
-    dbconfig = {
-
-        "host": os.getenv("DB_HOST"),
-
-        "user": os.getenv("DB_USER"),
-
-        "password": os.getenv("DB_PASSWORD"),
-
-        "database": os.getenv("DB_NAME"),
-
-    }
-
-    connection_pool = mysql.connector.pooling.MySQLConnectionPool(
-
-        pool_name=DB_POOL_NAME,
-
-        pool_size=DB_POOL_SIZE,
-
-        pool_reset_session=True,
-
-        **dbconfig
-
-    )
-
-    logging.info("Database connection pool created successfully.")except mysql.connector.Error as err:
-
-    logging.warning(f"Database connection pool creation failed: {err}")
-
-    connection_pool = None  # Will use CSV as fallback# Function to get a database connection from the pooldef get_db_connection():
-
-    """
-
-    Retrieves a connection from the pool. Returns None if pool is not available.
-
-    """
-
-    if connection_pool:
-
-        try:
-
-            connection = connection_pool.get_connection()
-
-            if connection.is_connected():
-
-                return connection
-
-        except mysql.connector.Error as err:
-
-            logging.error(f"Error getting connection from pool: {err}")
-
-    return None# Initialize Database: Create tables and indexesdef initialize_database():
-
-    """
-
-    Initializes the database by creating necessary tables and indexes if they do not exist.
-
-    """
-
-    connection = get_db_connection()
-
-    if connection is None:
-
-        logging.info("Database initialization skipped. Using CSV storage.")
-
-        return
-
-
-
-    cursor = connection.cursor()
-
-    try:
-
-        # Create table for scraped data
-
-        create_scraped_data_table = """
-
-        CREATE TABLE IF NOT EXISTS scraped_data (
-
-            id INT AUTO_INCREMENT PRIMARY KEY,
-
-            url VARCHAR(255) NOT NULL,
-
-            content_hash VARCHAR(64) NOT NULL,
-
-            change_detected DATETIME NOT NULL
-
-        )
-
-        """
-
-        cursor.execute(create_scraped_data_table)
-
-        logging.info("Table 'scraped_data' is ready.")
-
-
-
-        # Create indexes for performance
-
-        create_index_url = "CREATE INDEX IF NOT EXISTS idx_url ON scraped_data(url)"
-
-        create_index_change = "CREATE INDEX IF NOT EXISTS idx_change_detected ON scraped_data(change_detected)"
-
-        cursor.execute(create_index_url)
-
-        cursor.execute(create_index_change)
-
-        logging.info("Indexes on 'url' and 'change_detected' columns created.")
-
-
-
-        # Create table for action logs
-
-        create_action_logs_table = """
-
-        CREATE TABLE IF NOT EXISTS action_logs (
-
-            id INT AUTO_INCREMENT PRIMARY KEY,
-
-            action VARCHAR(255) NOT NULL,
-
-            timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
-
-        )
-
-        """
-
-        cursor.execute(create_action_logs_table)
-
-        logging.info("Table 'action_logs' is ready.")
-
-
-
-    except mysql.connector.Error as err:
-
-        logging.error(f"Error initializing database: {err}")
-
-    finally:
-
-        cursor.close()
-
-        connection.close()
-
-        logging.info("Database initialization complete.")# Function to create WebDriverdef create_driver(options: Options) -> webdriver.Chrome:
-
-    """
-
-    Initializes and returns a Selenium Chrome WebDriver instance.
-
-    """
-
-    try:
-
-        driver = webdriver.Chrome(
-
-            service=Service(ChromeDriverManager().install()), options=options
-
-        )
-
-        logging.info("ChromeDriver initialized successfully.")
-
-        return driver
-
-    except Exception as exception:
-
-        logging.error(f"Error initializing ChromeDriver: {exception}")
-
-        return None# Function to log changes to CSVdef log_to_csv(storage_location: str, url: str, content_hash: str, change_detected: str):
-
-    """
-
-    Logs the change to a CSV file in the storage_location.
-
-    """
-
-    try:
-
-        os.makedirs(storage_location, exist_ok=True)
-
-        csv_file_path = os.path.join(storage_location, f"{urlparse(url).hostname}_changes.csv")
-
-        file_exists = os.path.isfile(csv_file_path)
-
-
-
-        with open(csv_file_path, "a", newline="", encoding="utf-8") as csvfile:
-
-            fieldnames = ["date", "time", "url", "content_hash", "change"]
-
-            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
-
-            if not file_exists:
-
-                writer.writeheader()
-
-            writer.writerow(
-
-                {
-
-                    "date": change_detected.split()[0],
-
-                    "time": change_detected.split()[1],
-
-                    "url": url,
-
-                    "content_hash": content_hash,
-
-                    "change": "Content changed",
-
-                }
-
-            )
-
-        logging.info(f"Change detected at {url} on {change_detected} and logged to CSV.")
-
-    except Exception as e:
-
-        logging.error(f"Error logging data to CSV: {e}")# Function to get initial observationdef get_initial_observation(
-
-    driver: webdriver.Chrome, url: str, content_type: str, selector: str = None) -> str:
-
-    """
-
-    Retrieves the initial content from the URL and returns its MD5 hash.
-
-    """
-
-    try:
-
-        driver.get(url)
-
-        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
-
-        time.sleep(2)  # Additional wait for dynamic content
-
-
-
-        if content_type == "text":
-
-            initial_content = driver.page_source
-
-        elif content_type == "media":
-
-            if selector:
-
-                try:
-
-                    elements = WebDriverWait(driver, 5).until(
-
-                        EC.presence_of_all_elements_located((By.CSS_SELECTOR, selector))
-
-                    )
-
-                    initial_content = [element.get_attribute("src") for element in elements]
-
-                except TimeoutException:                     logging.warning(f"Timeout waiting for media elements with selector '{selector}' on {url}")
-
-                    initial_content = []
-
-            else:
-
-                elements = driver.find_elements(By.TAG_NAME, "img")
-
-                initial_content = [element.get_attribute("src") for element in elements]
-
-        else:
-
-            initial_content = driver.page_source
-
-
-
-        initial_hash = hashlib.md5(str(initial_content).encode("utf-8")).hexdigest()
-
-        logging.info(f"Initial hash for {url}: {initial_hash}")
-
-        return initial_hash
-
-    except Exception as exception:
-
-        logging.error(f"Error accessing {url}: {exception}")
-
-        return None# Function to monitor URLs for changesdef monitor_urls(
-
-    storage_location: str,
-
-    urls: list,
-
-    scrape_interval: int,
-
-    content_type: str,
-
-    selector: str = None,
-
-    progress: gr.Progress = None):
-
-    """
-
-    Monitors the specified URLs for changes and logs any detected changes to the database or CSV.
-
-    """
-
-    global HISTORY, STOP_THREADS
-
-    previous_hashes = {url: "" for url in urls}
-
-
-
-    options = Options()
-
-    options.add_argument("--headless")
-
-    options.add_argument("--no-sandbox")
-
-    options.add_argument("--disable-dev-shm-usage")   
-
-
-
-
-
-
-
-    driver = create_driver(options)
-
-    if driver is None:
-
-        logging.error("WebDriver could not be initialized. Exiting monitor.")
-
-        return
-
-
-
-    try:
-
-        while not STOP_THREADS:
-
-            for url in urls:
-
-                if STOP_THREADS:
-
-                    break
-
-                try:
-
-                    driver.get(url)
-
-                    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
-
-                    time.sleep(2)  # Additional wait for dynamic content
-
-
-
-                    if content_type == "text":
-
-                        current_content = driver.page_source
-
-                    elif content_type == "media":
-
-                        if selector:
-
-                            try:
-
-                                elements = WebDriverWait(driver, 5).until(
-
-                                    EC.presence_of_all_elements_located((By.CSS_SELECTOR, selector))
-
-                                )
-
-                                current_content = [element.get_attribute("src") for element in elements]
-
-                            except TimeoutException:
-
-                                logging.warning(f"Timeout waiting for media elements with selector '{selector}' on {url}")
-
-                                current_content = []
-
-                        else:
-
-                            elements = driver.find_elements(By.TAG_NAME, "img")
-
-                            current_content = [element.get_attribute("src") for element in elements]
-
-                    else:
-
-                        current_content = driver.page_source
-
-
-
-                    current_hash = hashlib.md5(str(current_content).encode("utf-8")).hexdigest()
-
-                    if current_hash != previous_hashes[url]:
-
-                        previous_hashes[url] = current_hash
-
-                        date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-
-                        HISTORY.append(f"Change detected at {url} on {date_time_str}")
-
-
-
-                        # Attempt to log to database
-
-                        connection = get_db_connection()
-
-                        if connection:
-
-                            try:
-
-                                cursor = connection.cursor()
-
-                                insert_query = """
-
-                                INSERT INTO scraped_data (url, content_hash, change_detected)
-
-                                VALUES (%s, %s, %s)
-
-                                """
-
-                                cursor.execute(insert_query, (url, current_hash, date_time_str))
-
-                                connection.commit()
-
-                                logging.info(f"Change detected at {url} on {date_time_str} and logged to database.")
-
-                            except mysql.connector.Error as err:
-
-                                logging.error(f"Error inserting data into database: {err}")
-
-                                # Fallback to CSV
-
-                                log_to_csv(storage_location, url, current_hash, date_time_str)
-
-                            finally:
-
-                                cursor.close()
-
-                                connection.close()
-
-                        else:
-
-                            # Fallback to CSV
-
-                            log_to_csv(storage_location, url, current_hash, date_time_str)
-
-
-
-                        # Update progress
-
-                        if progress:
-
-                            progress(1)
-
-                except (
-
-                    NoSuchElementException,
-
-                    StaleElementReferenceException,
-
-                    TimeoutException,
-
-                    Exception,
-
-                ) as e:
-
-                    logging.error(f"Error accessing {url}: {e}")
-
-                    if progress:
-
-                        progress(1)
-
-            time.sleep(scrape_interval * 60)  # Wait for the next scrape interval
-
-    finally:
-
-        driver.quit()
-
-        logging.info("ChromeDriver session ended.")# Function to start scrapingdef start_scraping(
-
-    storage_location: str,
-
-    urls: str,
-
-    scrape_interval: int,
-
-    content_type: str,
-
-    selector: str = None,
-
-    progress: gr.Progress = None) -> str:
-
-    """
-
-    Starts the scraping process in a separate thread with progress indication.
-
-    """
-
-    global CURRENT_TASK, HISTORY, STOP_THREADS
-
-
-
-    if STOP_THREADS:
-
-        STOP_THREADS = False  # Reset the flag if previously stopped
-
-
-
-    url_list = [url.strip() for url in urls.split(",") if url.strip()]
-
-    CURRENT_TASK = f"Monitoring URLs: {', '.join(url_list)}"
-
-    HISTORY.append(f"Task started: {CURRENT_TASK}")
-
-    logging.info(f"Task started: {CURRENT_TASK}")
-
-
-
-    # Initialize database tables
-
-    initialize_database()
-
-
-
-    # Log initial observations
-
-    def log_initial_observations():
-
-        options = Options()
-
-        options.add_argument("--headless")
-
-        options.add_argument("--no-sandbox")
-
-        options.add_argument("--disable-dev-shm-usage")   
-
-
-
-
-
-
-
-        driver = create_driver(options)
-
-        if driver is None:
-
-            return
-
-
-
-        for url in url_list:
-
-            if STOP_THREADS:
-
-                break
-
-            try:
-
-                initial_hash = get_initial_observation(driver, url, content_type, selector)
-
-                if initial_hash:
-
-                    date_time_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-
-                    HISTORY.append(f"Initial observation at {url}: {initial_hash}")
-
-
-
-                    # Attempt to log to database
-
-                    connection = get_db_connection()
-
-                    if connection:
-
-                        try:
-
-                            cursor = connection.cursor()
-
-                            insert_query = """
-
-                            INSERT INTO scraped_data (url, content_hash, change_detected)
-
-                            VALUES (%s, %s, %s)
-
-                            """
-
-                            cursor.execute(insert_query, (url, initial_hash, date_time_str))
-
-                            connection.commit()
-
-                            logging.info(f"Initial observation logged for {url} in database.")
-
-                        except mysql.connector.Error as err:
-
-                            logging.error(f"Error inserting initial observation into database: {err}")
-
-                            # Fallback to CSV
-
-                            log_to_csv(storage_location, url, initial_hash, date_time_str)
-
-                        finally:
-
-                            cursor.close()
-
-                            connection.close()
-
-                    else:
-
-                        # Fallback to CSV
-
-                        log_to_csv(storage_location, url, initial_hash, date_time_str)
-
-            except Exception as e:
-
-                HISTORY.append(f"Error accessing {url}: {e}")
-
-                logging.error(f"Error accessing {url}: {e}")
-
-        driver.quit()
-
-
-
-    # Start logging initial observations
-
-    initial_thread = threading.Thread(target=log_initial_observations, daemon=True)
-
-    initial_thread.start()
-
-
-
-    # Start the monitoring thread with progress
-
-    monitor_thread = threading.Thread(
-
-        target=monitor_urls,
-
-        args=(storage_location, url_list, scrape_interval, content_type, selector, progress),
-
-        daemon=True,
-
-    )
-
-    monitor_thread.start()
-
-    logging.info("Started scraping thread.")
-
-    return f"Started scraping {', '.join(url_list)} every {scrape_interval} minutes."
-
-# Function to stop scraping
-def stop_scraping() -> str:
-    """
-    Stops all ongoing scraping threads.
-    """
-    global STOP_THREADS
-    STOP_THREADS = True
-    HISTORY.append("Scraping stopped by user.")
-    logging.info("Scraping stop signal sent.")
-    return "Scraping has been stopped."
-
-# Function to display CSV content from MySQL or CSV
-def display_csv(storage_location: str, url: str) -> str:
-    """
-    Fetches and returns the scraped data for a given URL from the MySQL database or CSV.
-    """
-    try:
-        connection = get_db_connection()
-        if connection:
-            try:
-                cursor = connection.cursor(dictionary=True)
-                query = "SELECT * FROM scraped_data WHERE url = %s ORDER BY change_detected DESC"
-                cursor.execute(query, (url,))
-                results = cursor.fetchall()
-
-                if not results:
-                    return "No data available for the selected URL."
-
-                df = pd.DataFrame(results)
-                cursor.close()
-                connection.close()
-                return df.to_string(index=False)
-            except mysql.connector.Error as err:
-                logging.error(f"Error fetching data from database: {err}")
-                # Fallback to CSV
-        else:
-            logging.info("No database connection. Fetching data from CSV.")
-
-        # Fallback to CSV
-        hostname = urlparse(url).hostname
-        csv_path = os.path.join(storage_location, f"{hostname}_changes.csv")
-        if os.path.exists(csv_path):
-            df = pd.read_csv(csv_path)
-            return df.to_string(index=False)
-        else:
-            return "No data available."
-
-    except Exception as e:
-        logging.error(f"Error fetching data for {url}: {e}")
-        return f"Error fetching data for {url}: {e}"
-
-# Function to generate RSS feed from MySQL or CSV data
-def generate_rss_feed(storage_location: str, url: str) -> str:
-    """
-    Generates an RSS feed for the latest changes detected on a given URL from the MySQL database or CSV.
-    """
-    try:
-        connection = get_db_connection()
-        rss_feed = ""
-
-        if connection:
-            try:
-                cursor = connection.cursor(dictionary=True)
-                query = "SELECT * FROM scraped_data WHERE url = %s ORDER BY change_detected DESC LIMIT 10"
-                cursor.execute(query, (url,))
-                results = cursor.fetchall()
-
-                if not results:
-                    return "No changes detected to include in RSS feed."
-
-                # Create the root RSS element
-                rss = ET.Element("rss", version="2.0")
-                channel = ET.SubElement(rss, "channel")
-
-                # Add channel elements
-                title = ET.SubElement(channel, "title")
-                title.text = f"RSS Feed for {urlparse(url).hostname}"
-
-                link = ET.SubElement(channel, "link")
-                link.text = url
-
-                description = ET.SubElement(channel, "description")
-                description.text = "Recent changes detected on the website."
-
-                # Add items to the feed
-                for row in results:
-                    item = ET.SubElement(channel, "item")
-
-                    item_title = ET.SubElement(item, "title")
-                    item_title.text = f"Change detected at {row['url']}"
-
-                    item_link = ET.SubElement(item, "link")
-                    item_link.text = row["url"]
-
-                    item_description = ET.SubElement(item, "description")
-                    item_description.text = f"Content changed on {row['change_detected']}"
-
-                    pub_date = ET.SubElement(item, "pubDate")
-                    pub_date.text = datetime.datetime.strptime(
-                        str(row['change_detected']), "%Y-%m-%d %H:%M:%S"
-                    ).strftime("%a, %d %b %Y %H:%M:%S +0000")
-
-                # Generate the XML string
-                rss_feed = ET.tostring(rss, encoding="utf-8", method="xml").decode("utf-8")
-                cursor.close()
-                connection.close()
-                return rss_feed
-            except mysql.connector.Error as err:
-                logging.error(f"Error fetching data from database: {err}")
-                # Fallback to CSV
-        else:
-            logging.info("No database connection. Generating RSS feed from CSV.")
-
-        # Fallback to CSV
-        hostname = urlparse(url).hostname
-        csv_path = os.path.join(storage_location, f"{hostname}_changes.csv")
-        if os.path.exists(csv_path):
-            df = pd.read_csv(csv_path).tail(10)
-            if df.empty:
-                return "No changes detected to include in RSS feed."
-
-            # Create the root RSS element
-            rss = ET.Element("rss", version="2.0")
-            channel = ET.SubElement(rss, "channel")
-
-            # Add channel elements
-            title = ET.SubElement(channel, "title")
-            title.text = f"RSS Feed for {hostname}"
-
-            link = ET.SubElement(channel, "link")
-            link.text = url
-
-            description = ET.SubElement(channel, "description")
-            description.text = "Recent changes detected on the website."
-
-            # Add items to the feed
-            for _, row in df.iterrows():
-                item = ET.SubElement(channel, "item")
-
-                item_title = ET.SubElement(item, "title")
-                item_title.text = f"Change detected at {row['url']}"
-
-                item_link = ET.SubElement(item, "link")
-                item_link.text = row["url"]
-
-                item_description = ET.SubElement(item, "description")
-                item_description.text = f"Content changed on {row['date']} at {row['time']}"
-
-                pub_date = ET.SubElement(item, "pubDate")
-                pub_date.text = datetime.datetime.strptime(
-                    f"{row['date']} {row['time']}", "%Y-%m-%d %H:%M:%S"
-                ).strftime("%a, %d %b %Y %H:%M:%S +0000")
-
-            # Generate the XML string
-            rss_feed = ET.tostring(rss, encoding="utf-8", method="xml").decode("utf-8")
-            return rss_feed
-        else:
-            return "No data available."
-
-    except Exception as e:
-        logging.error(f"Error generating RSS feed for {url}: {e}")
-        return f"Error generating RSS feed for {url}: {e}"
-
-# Function to parse user commands using spaCy
-def parse_command(message: str) -> tuple:
-    """
-    Parses the user message using spaCy to identify if it contains a command.
-    Returns the command and its parameters if found, else (None, None).
-    """
-    doc = nlp(message.lower())
-    command = None
-    params = {}
-
-    # Define command patterns
-    if "filter" in message.lower():
-        # Example: "Filter apples, oranges in column Description"
-        match = re.search(r"filter\s+([\w\s,]+)\s+in\s+column\s+(\w+)", message, re.IGNORECASE)
-        if match:
-            words = [word.strip() for word in match.group(1).split(",")]
-            column = match.group(2)
-            command = "filter"
-            params = {"words": words, "column": column}
-
-    elif "sort" in message.lower():
-        # Example: "Sort Price ascending"
-        match = re.search(r"sort\s+(\w+)\s+(ascending|descending)", message, re.IGNORECASE)
-        if match:
-            column = match.group(1)
-            order = match.group(2)
-            command = "sort"
-            params = {"column": column, "order": order}
-
-    elif "export to csv as" in message.lower():
-        # Example: "Export to CSV as filtered_data.csv"
-        match = re.search(r"export\s+to\s+csv\s+as\s+([\w\-]+\.csv)", message, re.IGNORECASE)
-        if match:
-            filename = match.group(1)
-            command = "export"
-            params = {"filename": filename}
-
-    elif "log action" in message.lower():
-        # Example: "Log action Filtered data for specific fruits"
-        match = re.search(r"log\s+action\s+(.+)", message, re.IGNORECASE)
-        if match:
-            action = match.group(1)
-            command = "log"
-            params = {"action": action}
-
-    return command, params
-
-# Function to execute parsed commands
-def execute_command(command: str, params: dict) -> str:
-    """
-    Executes the corresponding function based on the command and parameters.
-    """
-    if command == "filter":
-        words = params["words"]
-        column = params["column"]
-        return filter_data(column, words)
-    elif command == "sort":
-        column = params["column"]
-        order = params["order"]
-        return sort_data(column, order)
-    elif command == "export":
-        filename = params["filename"]
-        return export_csv(filename)
-    elif command == "log":
-        action = params["action"]
-        return log_action(action)
-    else:
-        return "Unknown command."
-
-# Data Manipulation Functions
-def filter_data(column: str, words: list) -> str:
-    """
-    Filters the scraped data to include only rows where the specified column contains the given words.
-    Saves the filtered data to a new CSV file.
-    """
-    try:
-        storage_location = DEFAULT_FILE_PATH
-
-        connection = get_db_connection()
-        if connection:
-            try:
-                cursor = connection.cursor(dictionary=True)
-                # Fetch all data
-                query = "SELECT * FROM scraped_data"
-                cursor.execute(query)
-                results = cursor.fetchall()
-
-                if not results:
-                    return "No data available to filter."
-
-                df = pd.DataFrame(results)
-                # Create a regex pattern to match any of the words
-                pattern = '|'.join(words)
-                if column not in df.columns:
-                    return f"Column '{column}' does not exist in the data."
-
-                filtered_df = df[df[column].astype(str).str.contains(pattern, case=False, na=False)]
-
-                if filtered_df.empty:
-                    return f"No records found with words {words} in column '{column}'."
-
-                # Save the filtered data to a new CSV
-                timestamp = int(time.time())
-                filtered_csv = os.path.join(storage_location, f"filtered_data_{timestamp}.csv")
-                filtered_df.to_csv(filtered_csv, index=False)
-                logging.info(f"Data filtered on column '{column}' for words {words}.")
-                return f"Data filtered and saved to {filtered_csv}."
-            except mysql.connector.Error as err:
-                logging.error(f"Error fetching data from database: {err}")
-                # Fallback to CSV
-        else:
-            logging.info("No database connection. Filtering data from CSV.")
-
-        # Fallback to CSV
-        csv_files = [f for f in os.listdir(storage_location) if f.endswith("_changes.csv") or f.endswith("_filtered.csv") or f.endswith("_sorted_asc.csv") or f.endswith("_sorted_desc.csv")]
-        if not csv_files:
-            return "No CSV files found to filter."
-
-        # Assume the latest CSV is the target
-        latest_csv = max([os.path.join(storage_location, f) for f in csv_files], key=os.path.getmtime)
-        df = pd.read_csv(latest_csv)
-
-        if column not in df.columns:
-            return f"Column '{column}' does not exist in the data."
-
-        filtered_df = df[df[column].astype(str).str.contains('|'.join(words), case=False, na=False)]
-
-        if filtered_df.empty:
-            return f"No records found with words {words} in column '{column}'."
-
-        # Save the filtered data to a new CSV
-        timestamp = int(time.time())
-        filtered_csv = latest_csv.replace(".csv", f"_filtered_{timestamp}.csv")
-        filtered_df.to_csv(filtered_csv, index=False)
-        logging.info(f"Data filtered on column '{column}' for words {words}.")
-        return f"Data filtered and saved to {filtered_csv}."
-    except Exception as e:
-        logging.error(f"Error filtering data: {e}")
-        return f"Error filtering data: {e}"
-
-def sort_data(column: str, order: str) -> str:
-    """
-    Sorts the scraped data based on the specified column and order.
-    Saves the sorted data to a new CSV file.
-    """
-    try:
-        storage_location = DEFAULT_FILE_PATH
-
-        connection = get_db_connection()
-        if connection:
-            try:
-                cursor = connection.cursor(dictionary=True)
-                # Fetch all data
-                query = "SELECT * FROM scraped_data"
-                cursor.execute(query)
-                results = cursor.fetchall()
-
-                if not results:
-                    return "No data available to sort."
-
-                df = pd.DataFrame(results)
-                if column not in df.columns:
-                    return f"Column '{column}' does not exist in the data."
-
-                ascending = True if order.lower() == "ascending" else False
-                sorted_df = df.sort_values(by=column, ascending=ascending)
-
-                # Save the sorted data to a new CSV
-                timestamp = int(time.time())
-                sorted_csv = os.path.join(storage_location, f"sorted_data_{column}_{order.lower()}_{timestamp}.csv")
-                sorted_df.to_csv(sorted_csv, index=False)
-                logging.info(f"Data sorted on column '{column}' in {order} order.")
-                return f"Data sorted and saved to {sorted_csv}."
-            except mysql.connector.Error as err:
-                logging.error(f"Error fetching data from database: {err}")
-                # Fallback to CSV
-        else:
-            logging.info("No database connection. Sorting data from CSV.")
-
-        # Fallback to CSV
-        csv_files = [f for f in os.listdir(storage_location) if f.endswith("_changes.csv") or f.endswith("_filtered.csv") or f.endswith("_sorted_asc.csv") or f.endswith("_sorted_desc.csv")]
-        if not csv_files:
-            return "No CSV files found to sort."
-
-        # Assume the latest CSV is the target
-        latest_csv = max([os.path.join(storage_location, f) for f in csv_files], key=os.path.getmtime)
-        df = pd.read_csv(latest_csv)
-
-        if column not in df.columns:
-            return f"Column '{column}' does not exist in the data."
-
-        ascending = True if order.lower() == "ascending" else False
-        sorted_df = df.sort_values(by=column, ascending=ascending)
-
-        # Save the sorted data to a new CSV
-        timestamp = int(time.time())
-        sorted_csv = latest_csv.replace(".csv", f"_sorted_{order.lower()}_{timestamp}.csv")
-        sorted_df.to_csv(sorted_csv, index=False)
-        logging.info(f"Data sorted on column '{column}' in {order} order.")
-        return f"Data sorted and saved to {sorted_csv}."
-    except Exception as e:
-        logging.error(f"Error sorting data: {e}")
-        return f"Error sorting data: {e}"
-
-def export_csv(filename: str) -> str:
-    """
-    Exports the latest scraped data to a specified CSV filename.
-    """
-    try:
-        storage_location = DEFAULT_FILE_PATH
-
-        connection = get_db_connection()
-        if connection:
-            try:
-                cursor = connection.cursor(dictionary=True)
-                # Fetch all data
-                query = "SELECT * FROM scraped_data"
-                cursor.execute(query)
-                results = cursor.fetchall()
-
-                if not results:
-                    return "No data available to export."
-
-                df = pd.DataFrame(results)
-                export_path = os.path.join(storage_location, filename)
-                df.to_csv(export_path, index=False)
-                logging.info(f"Data exported to {export_path}.")
-                return f"Data exported to {export_path}."
-            except mysql.connector.Error as err:
-                logging.error(f"Error exporting data from database: {err}")
-                # Fallback to CSV
-        else:
-            logging.info("No database connection. Exporting data from CSV.")
-
-        # Fallback to CSV
-        csv_files = [f for f in os.listdir(storage_location) if f.endswith("_changes.csv") or f.endswith("_filtered.csv") or f.endswith("_sorted_asc.csv") or f.endswith("_sorted_desc.csv")]
-        if not csv_files:
-            return "No CSV files found to export."
-
-        # Assume the latest CSV is the target
-        latest_csv = max([os.path.join(storage_location, f) for f in csv_files], key=os.path.getmtime)
-        df = pd.read_csv(latest_csv)
-        export_path = os.path.join(storage_location, filename)
-        df.to_csv(export_path, index=False)
-        logging.info(f"Data exported to {export_path}.")
-        return f"Data exported to {export_path}."
-    except Exception as e:
-        logging.error(f"Error exporting CSV: {e}")
-        return f"Error exporting CSV: {e}"
-
-def log_action(action: str) -> str:
-    """
-    Logs a custom action message to the MySQL database or CSV.
-    """
-    try:
-        connection = get_db_connection()
-        if connection:
-            try:
-                cursor = connection.cursor()
-                insert_query = """
-                INSERT INTO action_logs (action)
-                VALUES (%s)
-                """
-                cursor.execute(insert_query, (action,))
-                connection.commit()
-                logging.info(f"Action logged in database: {action}")
-                cursor.close()
-                connection.close()
-                return f"Action logged: {action}"
-            except mysql.connector.Error as err:
-                logging.error(f"Error logging action to database: {err}")
-                # Fallback to CSV
-        else:
-            logging.info("No database connection. Logging action to CSV.")
-
-        # Fallback to CSV
-        storage_location = DEFAULT_FILE_PATH
-        try:
-            os.makedirs(storage_location, exist_ok=True)
-            csv_file_path = os.path.join(storage_location, "action_logs.csv")
-            file_exists = os.path.isfile(csv_file_path)
-
-            with open(csv_file_path, "a", newline="", encoding="utf-8") as csvfile:
-                fieldnames = ["timestamp", "action"]
-                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
-                if not file_exists:
-                    writer.writeheader()
-                writer.writerow(
-                    {
-                        "timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
-                        "action": action,
-                    }
-                )
-            logging.info(f"Action logged to CSV: {action}")
-            return f"Action logged: {action}"
-        except Exception as e:
-            logging.error(f"Error logging action to CSV: {e}")
-            return f"Error logging action: {e}"
-    except Exception as e:
-        logging.error(f"Error logging action: {e}")
-        return f"Error logging action: {e}"
-
-# Function to get the latest CSV file based on modification time
-def get_latest_csv() -> str:
-    """
-    Retrieves the latest CSV file from the storage directory based on modification time.
-    """
-    try:
-        storage_location = "/home/users/app/scraped_data"
-        csv_files = [f for f in os.listdir(storage_location) if f.endswith(".csv")]
-        if not csv_files:
-            return None
-
-        latest_csv = max([os.path.join(storage_location, f) for f in csv_files], key=os.path.getmtime)
-        return latest_csv
-    except Exception as e:
-        logging.error(f"Error retrieving latest CSV: {e}")
-        return None
-
-def respond(
-    message: str,
-    history: list,
-    system_message: str,
-    max_tokens: int,
-    temperature: float,
-    top_p: float,
-) -> str:
-    """
-    Generates a response using OpenLlamaForCausalLM.
-    """
-    try:
-        # Check if the message contains a command
-        command, params = parse_command(message)
-        if command:
-            # Execute the corresponding function
-            response = execute_command(command, params)
-        else:
-            # Generate a regular response using OpenLlama
-            prompt = (
-                f"System: {system_message}\n"
-                f"History: {history}\n"
-                f"User: {message}\n"
-                f"Assistant:"
-            )
-            response = openllama_pipeline(
-                prompt,
-                max_length=max_tokens,
-                temperature=temperature,
-                top_p=top_p,
-            )[0]["generated_text"]
-
-
-            # Extract the assistant's reply
-            response = response.split("Assistant:")[-1].strip()
-        return response
-    except Exception as e:
-        logging.error(f"Error generating response: {e}")
-        return "Error generating response."
-
-# Define the Gradio interface
-def create_interface() -> gr.Blocks():
-    """
-    Defines and returns the Gradio interface for the application.
-    """
-    with gr.Blocks() as demo:
-        gr.Markdown("# All-in-One Scraper, Database, and RSS Feeder")
-
-        with gr.Row():
-            with gr.Column():
-                # Scraping Controls
-                storage_location = gr.Textbox(
-                    value=DEFAULT_FILE_PATH, label="Storage Location"
-                )
-                urls = gr.Textbox(
-                    label="URLs (comma separated)",
-                    placeholder="https://example.com, https://anotherexample.com",
-                )
-                scrape_interval = gr.Slider(
-                    minimum=1,
-                    maximum=60,
-                    value=5,
-                    step=1,
-                    label="Scrape Interval (minutes)",
-                )
-                content_type = gr.Radio(
-                    choices=["text", "media", "both"],
-                    value="text",
-                    label="Content Type",
-                )
-                selector = gr.Textbox(
-                    label="CSS Selector for Media (Optional)",
-                    placeholder="e.g., img.main-image",
-                )
-                start_button = gr.Button("Start Scraping")
-                stop_button = gr.Button("Stop Scraping")
-                status_output = gr.Textbox(
-                    label="Status Output", interactive=False, lines=2
-                )
-
-            with gr.Column():
-                # Chat Interface
-                chat_history = gr.Chatbot(label="Chat History", type='messages')
-                with gr.Row():
-                    message = gr.Textbox(label="Message", placeholder="Type your message here...")
-                system_message = gr.Textbox(
-                    value="You are a helpful assistant.", label="System message"
-                )
-                max_tokens = gr.Slider(
-                    minimum=1,
-                    maximum=2048,
-                    value=512,
-                    step=1,
-                    label="Max new tokens",
-                )
-                temperature = gr.Slider(
-                    minimum=0.1,
-                    maximum=4.0,
-                    value=0.7,
-                    step=0.1,
-                    label="Temperature",
-                )
-                top_p = gr.Slider(
-                    minimum=0.1,
-                    maximum=1.0,
-                    value=0.95,
-                    step=0.05,
-                    label="Top-p (nucleus sampling)",
-                )
-                response_box = gr.Textbox(label="Response", interactive=False, lines=2)
-
-        with gr.Row():
-            with gr.Column():
-                # CSV Display Controls
-                selected_url_csv = gr.Textbox(
-                    label="Select URL for CSV Content",
-                    placeholder="https://example.com",
-                )
-                csv_button = gr.Button("Display CSV Content")
-                csv_content_output = gr.Textbox(
-                    label="CSV Content Output", interactive=False, lines=10
-                )
-
-            with gr.Column():
-                # RSS Feed Generation Controls
-                selected_url_rss = gr.Textbox(
-                    label="Select URL for RSS Feed",
-                    placeholder="https://example.com",
-                )
-                rss_button = gr.Button("Generate RSS Feed")
-                rss_output = gr.Textbox(
-                    label="RSS Feed Output", interactive=False, lines=20
-                )
-
-        # Historical Data View
-        with gr.Row():
-            with gr.Column():
-                historical_view_url = gr.Textbox(
-                    label="Select URL for Historical Data",
-                    placeholder="https://example.com",
-                )
-                historical_button = gr.Button("View Historical Data")
-                historical_output = gr.Dataframe(
-                    headers=["ID", "URL", "Content Hash", "Change Detected"],
-                    label="Historical Data",
-                    interactive=False
-                )
-
-        # Connect buttons to their respective functions
-        start_button.click(
-            fn=start_scraping,
-            inputs=[
-                storage_location,
-                urls,
-                scrape_interval,
-                content_type,
-                selector,
-            ],
-            outputs=status_output,
-        )
-
-        stop_button.click(fn=stop_scraping, outputs=status_output)
-
-        csv_button.click(
-            fn=display_csv,
-            inputs=[storage_location, selected_url_csv],
-            outputs=csv_content_output,
-        )
-
-        rss_button.click(
-            fn=generate_rss_feed,
-            inputs=[storage_location, selected_url_rss],
-            outputs=rss_output,
-        )
-
-        historical_button.click(
-            fn=display_historical_data,
-            inputs=[storage_location, historical_view_url],
-            outputs=historical_output,
-        )
-
-        # Connect message submission to the chat interface
-        def update_chat(message_input, history, system_msg, max_toks, temp, top_p_val):
-            if not message_input.strip():
-                return history, "Please enter a message."
-
-            response = respond(
-                message_input,
-                history,
-                system_msg,
-                max_toks,
-                temp,
-                top_p_val,
-            )
-            history.append((message_input, response))
-            return history, response
-
-        message.submit(
-            update_chat,
-            inputs=[
-                message,
-                chat_history,
-                system_message,
-                max_tokens,
-                temperature,
-                top_p,
-            ],
-            outputs=[chat_history, response_box],
-        )
-
-    return demo
-# Function to display historical data
-def display_historical_data(storage_location: str, url: str):
-    """
-    Retrieves and displays historical scraping data for a given URL.
-    """
-    try:
-        connection = get_db_connection()
-        if connection:
-            try:
-                cursor = connection.cursor(dictionary=True)
-                query = "SELECT * FROM scraped_data WHERE url = %s ORDER BY change_detected DESC"
-                cursor.execute(query, (url,))
-                results = cursor.fetchall()
-
-                if not results:
-                    return pd.DataFrame()
-
-                df = pd.DataFrame(results)
-                cursor.close()
-                connection.close()
-                return df
-            except mysql.connector.Error as err:
-                logging.error(f"Error fetching historical data from database: {err}")
-                # Fallback to CSV
-        else:
-            logging.info("No database connection. Fetching historical data from CSV.")
-
-        # Fallback to CSV
-        hostname = urlparse(url).hostname
-        csv_path = os.path.join(storage_location, f"{hostname}_changes.csv")
-        if os.path.exists(csv_path):
-            df = pd.read_csv(csv_path)
-            return df
-        else:
-            return pd.DataFrame()
-    except Exception as e:
-        logging.error(f"Error fetching historical data for {url}: {e}")
-        return pd.DataFrame()
-
-    def load_model():
-        """
-        Loads the openLlama model and tokenizer once and returns the pipeline.
-        """
-    try:
-        model_name = "openlm-research/open_llama_3b_v2"
-        tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, legacy=False)
-        model = AutoModelForCausalLM.from_pretrained(model_name)
-
-        # This should be inside the try block
-        max_supported_length = 2048
-
-        openllama_pipeline = pipeline(
-            "text-generation",
-            model=model,
-            tokenizer=tokenizer,
-            truncation=True,
-            max_length=max_supported_length,
-            temperature=0.7,
-            top_p=0.95,
-            device=0 if torch.cuda.is_available() else -1,
-        )
-        logging.info("Model loaded successfully.")
-        return openllama_pipeline  # Return the pipeline
-    except Exception as e:
-        logging.error(f"Error loading google/flan-t5-xl model: {e}")
-        return None
-
-def load_model(model_name: str):
-    """
-    Loads the specified model and tokenizer.
-    """
-    try:
-        tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False, legacy=False)
-        model = AutoModelForCausalLM.from_pretrained(model_name)
-        # This should be inside the try block
-        max_supported_length = 2048  # Get this from the model config
-        openllama_pipeline = pipeline(
-            "text-generation",
-            model=model,
-            tokenizer=tokenizer,
-            truncation=True,
-            max_length=max_supported_length,
-            temperature=0.7,
-            top_p=0.95,
-            device=0 if torch.cuda.is_available() else -1,
-        )
-        logging.info(f"{model_name} loaded successfully.")
-        return openllama_pipeline
-    except Exception as e:
-        logging.error(f"Error loading {model_name} model: {e}")
-        return None
-
-# Automated Testing using unittest
-class TestApp(unittest.TestCase):
-    def test_parse_command_filter(self):
-        command = "Filter apples, oranges in column Description"
-        parsed_command = parse_command(command)
-        self.assertEqual(parsed_command[0], "filter")
-        self.assertListEqual(parsed_command[1]["words"], ["apples", "oranges"])
-        self.assertEqual(parsed_command[1]["column"], "Description")
-
-    def test_parse_command_sort(self):
-        command = "Sort Price ascending"
-        parsed_command = parse_command(command)
-        self.assertEqual(parsed_command[0], "sort")
-        self.assertEqual(parsed_command[1]["column"], "Price")
-        self.assertEqual(parsed_command[1]["order"], "ascending")
-
-    def test_parse_command_export(self):
-        command = "Export to CSV as filtered_data.csv"
-        parsed_command = parse_command(command)
-        self.assertEqual(parsed_command[0], "export")
-        self.assertEqual(parsed_command[1]["filename"], "filtered_data.csv")
-
-    def test_parse_command_log(self):
-        command = "Log action Filtered data for specific fruits"
-        parsed_command = parse_command(command)
-        self.assertEqual(parsed_command[0], "log")
-        self.assertEqual(parsed_command[1]["action"], "Filtered data for specific fruits")
-
-    def test_database_connection(self):
-        connection = get_db_connection()
-        # Connection may be None if not configured; adjust the test accordingly
-        if connection:
-            self.assertTrue(connection.is_connected())
-            connection.close()
-        else:
-            self.assertIsNone(connection)
-
-def main():
-    # Initialize and run the application
-    logging.info("Starting the application...")
-    model = load_model()
-    if model:
-        logging.info("Application started successfully.")
-        print("Main function executed")
-        print("Creating interface...")
-        demo = create_interface()
-        print("Launching interface...")
-        demo.launch(server_name="0.0.0.0", server_port=7860)
-    else:
-        logging.error("Failed to start the application.")
-
-# Main execution
-if __name__ == "__main__":
-    # Initialize database
-    initialize_database()
-
-    # Create and launch Gradio interface
-    demo = create_interface()
-    demo.launch()
-
-    # Run automated tests
-    unittest.main(argv=[''], exit=False)