Scraper+

app.py

@@ -2,50 +2,97 @@ import streamlit as st
 import requests
 from bs4 import BeautifulSoup
 import re
+from requests.sessions import Session
+from langdetect import detect
+from googletrans import Translator
 
-def scrape_visible_text_from_url(url):
+def scrape_visible_text_from_url(url, query_selector=None, email=None, password=None, login_url=None):
     try:
-        response = requests.get(url)
-        response.raise_for_status()
+        session = Session()
+
+        # Handle authentication if credentials are provided
+        if email and password and login_url:
+            login_data = {
+                'email': email,
+                'password': password
+                # Include other necessary fields as required by the website
+            }
+            response = session.post(login_url, data=login_data)
+            response.raise_for_status()
+        else:
+            response = session.get(url)
+            response.raise_for_status()
+
         soup = BeautifulSoup(response.content, 'html.parser')
 
-        
+        # Remove unwanted tags
         for tag in soup(["script", "style", "meta", "link", "noscript", "header", "footer", "aside", "nav", "img"]):
             tag.extract()
 
-       
-        header_content = soup.find("header")
-        header_text = header_content.get_text() if header_content else ""
+        # Use query selector if provided
+        if query_selector:
+            elements = soup.select(query_selector)
+            text_content = " ".join([element.get_text() for element in elements])
+        else:
+            # Extract header content
+            header_content = soup.find("header")
+            header_text = header_content.get_text() if header_content else ""
+
+            # Extract paragraph content
+            paragraph_content = soup.find_all("p")
+            paragraph_text = " ".join([p.get_text() for p in paragraph_content])
+
+            text_content = f"{header_text}\n\n{paragraph_text}"
 
- 
-        paragraph_content = soup.find_all("p")
-        paragraph_text = " ".join([p.get_text() for p in paragraph_content])
+        # Clean up whitespace
+        visible_text = re.sub(r'\s+', ' ', text_content).strip()
 
-        
-        visible_text = f"{header_text}\n\n{paragraph_text}"
+        # Translate non-English text
+        translator = Translator()
+        sentences = re.split(r'(?<=[.!?]) +', visible_text)
+        translated_sentences = []
+        for sentence in sentences:
+            try:
+                lang = detect(sentence)
+                if lang != 'en':
+                    translation = translator.translate(sentence, dest='en').text
+                    translated_sentences.append(translation)
+                else:
+                    translated_sentences.append(sentence)
+            except Exception:
+                translated_sentences.append(sentence)
+        translated_text = ' '.join(translated_sentences)
 
-        
-        visible_text = re.sub(r'\s+', ' ', visible_text)
-        return visible_text.strip()
+        return translated_text
     except Exception as e:
         st.error(f"Error occurred while scraping the data: {e}")
         return None
 
-#ST
-
 def main():
     st.title("Web Data Scraper")
 
-    
     url_input = st.text_input("Enter the URL 👉✏️:", "")
 
-    if st.button("Load Datum 🧈"):
+    query_selector = st.text_input("Enter a query selector (optional):", "")
+
+    email = st.text_input("Email (if authentication required):", "")
+
+    password = st.text_input("Password (if authentication required):", "", type="password")
+
+    login_url = st.text_input("Enter the login URL (if authentication required):", "")
+
+    if st.button("Load Data 🧈"):
         if url_input:
-           
-            data = scrape_visible_text_from_url(url_input)
+            data = scrape_visible_text_from_url(
+                url=url_input,
+                query_selector=query_selector if query_selector else None,
+                email=email if email else None,
+                password=password if password else None,
+                login_url=login_url if login_url else None
+            )
             if data:
                 st.success("Data text successfully scraped!")
-                st.subheader("Scraped Text :")
+                st.subheader("Scraped Text:")
                 st.write(data)
             else:
                 st.warning("Failed to load data from the URL.")
@@ -53,4 +100,4 @@ def main():
             st.warning("Please enter a valid URL.")
 
 if __name__ == "__main__":
-    main()
+    main()