app

app.py

@@ -2,55 +2,163 @@ import streamlit as st
 import requests
 from bs4 import BeautifulSoup
 import re
+import os
+from urllib.parse import urljoin
 
-# Function to scrape only visible text from the given URL
-def scrape_visible_text_from_url(url):
+def download_images(url, max_images=10):
+    """
+    Download images from the given URL
+    
+    Args:
+        url (str): URL of the webpage
+        max_images (int): Maximum number of images to download
+    
+    Returns:
+        tuple: List of downloaded image paths and total image count
+    """
     try:
+        # Create a directory to save images if it doesn't exist
+        os.makedirs('downloaded_images', exist_ok=True)
+        
+        # Send a request to the URL
         response = requests.get(url)
         response.raise_for_status()
+        
+        # Parse the HTML content
         soup = BeautifulSoup(response.content, 'html.parser')
+        
+        # Find all image tags
+        img_tags = soup.find_all('img', src=True)
+        
+        # List to store downloaded image paths
+        downloaded_images = []
+        
+        # Download images
+        for i, img in enumerate(img_tags[:max_images], 1):
+            # Get the image source URL
+            img_url = img['src']
+            
+            # Handle relative URLs
+            if not img_url.startswith(('http://', 'https://')):
+                img_url = urljoin(url, img_url)
+            
+            try:
+                # Download the image
+                img_response = requests.get(img_url)
+                img_response.raise_for_status()
+                
+                # Generate a unique filename
+                filename = f'downloaded_images/image_{i}.{img_url.split(".")[-1].split("?")[0]}'
+                
+                # Save the image
+                with open(filename, 'wb') as f:
+                    f.write(img_response.content)
+                
+                downloaded_images.append(filename)
+            
+            except Exception as img_error:
+                st.warning(f"Could not download image {i}: {img_error}")
+        
+        return downloaded_images, len(img_tags)
+    
+    except Exception as e:
+        st.error(f"Error occurred while downloading images: {e}")
+        return [], 0
 
+def scrape_visible_text_from_url(url):
+    """
+    Scrape visible text from the given URL
+    
+    Args:
+        url (str): URL of the webpage
+    
+    Returns:
+        str: Extracted visible text
+    """
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.content, 'html.parser')
+        
         # Remove script, style, and other non-visible tags
-        for tag in soup(["script", "style", "meta", "link", "noscript", "header", "footer", "aside", "nav", "img"]):
+        for tag in soup(["script", "style", "meta", "link", "noscript", "header", "footer", "aside", "nav"]):
             tag.extract()
-
+        
         # Get the header content
         header_content = soup.find("header")
         header_text = header_content.get_text() if header_content else ""
-
+        
         # Get the paragraph content
         paragraph_content = soup.find_all("p")
         paragraph_text = " ".join([p.get_text() for p in paragraph_content])
-
+        
         # Combine header and paragraph text
         visible_text = f"{header_text}\n\n{paragraph_text}"
-
+        
         # Remove multiple whitespaces and newlines
         visible_text = re.sub(r'\s+', ' ', visible_text)
         return visible_text.strip()
+    
     except Exception as e:
         st.error(f"Error occurred while scraping the data: {e}")
         return None
 
-# Streamlit UI
 def main():
-    st.title("Web Data Scraper")
-
+    """
+    Main Streamlit application
+    """
+    st.title("Web Data Scraper with Image Downloader")
+    
     # Get the URL from the user
     url_input = st.text_input("Enter the URL of the web page:", "")
-
-    if st.button("Scrape Visible Text"):
+    
+    # Maximum images to download slider
+    max_images = st.slider("Maximum number of images to download", 1, 20, 10)
+    
+    if st.button("Scrape Content"):
         if url_input:
             # Extract visible text from the URL
-            data = scrape_visible_text_from_url(url_input)
-            if data:
-                st.success("Visible text successfully scraped!")
+            text_data = scrape_visible_text_from_url(url_input)
+            
+            # Download images
+            downloaded_images, total_images = download_images(url_input, max_images)
+            
+            if text_data:
+                st.success("Content successfully scraped!")
+                
+                # Display text
                 st.subheader("Scraped Text:")
-                st.write(data)
+                st.write(text_data)
+                
+                # Display and manage images
+                st.subheader(f"Images (Downloaded {len(downloaded_images)} out of {total_images} total)")
+                
+                # Create columns for images
+                if downloaded_images:
+                    cols = st.columns(min(len(downloaded_images), 3))
+                    for i, img_path in enumerate(downloaded_images):
+                        with cols[i % 3]:
+                            st.image(img_path, use_column_width=True)
+                    
+                    # Provide download option for all images
+                    with open('downloaded_images.zip', 'wb') as zipf:
+                        import zipfile
+                        with zipfile.ZipFile(zipf, 'w') as zip_file:
+                            for img_path in downloaded_images:
+                                zip_file.write(img_path, os.path.basename(img_path))
+                    
+                    st.download_button(
+                        label="Download All Images",
+                        data=open('downloaded_images.zip', 'rb').read(),
+                        file_name='downloaded_images.zip',
+                        mime='application/zip'
+                    )
+                else:
+                    st.warning("No images found on the page.")
             else:
-                st.warning("Failed to scrape visible text from the URL.")
+                st.warning("Failed to scrape content from the URL.")
         else:
             st.warning("Please enter a valid URL.")
 
 if __name__ == "__main__":
-    main()
+    main()