app

app.py

@@ -5,21 +5,17 @@ import re
 import os
 from urllib.parse import urljoin
 
-def download_images(url, max_images=10):
+def scrape_web_content(url):
     """
-    Download images from the given URL
+    Scrape the web content while preserving its original formatting
     
     Args:
         url (str): URL of the webpage
-        max_images (int): Maximum number of images to download
     
     Returns:
-        tuple: List of downloaded image paths and total image count
+        dict: Extracted content with text, HTML, and images
     """
     try:
-        # Create a directory to save images if it doesn't exist
-        os.makedirs('downloaded_images', exist_ok=True)
-        
         # Send a request to the URL
         response = requests.get(url)
         response.raise_for_status()
@@ -27,22 +23,21 @@ def download_images(url, max_images=10):
         # Parse the HTML content
         soup = BeautifulSoup(response.content, 'html.parser')
         
-        # Find all image tags
-        img_tags = soup.find_all('img', src=True)
-        
-        # List to store downloaded image paths
-        downloaded_images = []
+        # Create a directory to save images if it doesn't exist
+        os.makedirs('downloaded_images', exist_ok=True)
         
         # Download images
-        for i, img in enumerate(img_tags[:max_images], 1):
-            # Get the image source URL
-            img_url = img['src']
-            
-            # Handle relative URLs
-            if not img_url.startswith(('http://', 'https://')):
-                img_url = urljoin(url, img_url)
-            
+        downloaded_images = []
+        img_tags = soup.find_all('img', src=True)
+        for i, img in enumerate(img_tags[:10], 1):
             try:
+                # Get the image source URL
+                img_url = img['src']
+                
+                # Handle relative URLs
+                if not img_url.startswith(('http://', 'https://')):
+                    img_url = urljoin(url, img_url)
+                
                 # Download the image
                 img_response = requests.get(img_url)
                 img_response.raise_for_status()
@@ -54,97 +49,90 @@ def download_images(url, max_images=10):
                 with open(filename, 'wb') as f:
                     f.write(img_response.content)
                 
+                # Update the image tag in the soup to point to local file
+                img['src'] = filename
                 downloaded_images.append(filename)
             
             except Exception as img_error:
                 st.warning(f"Could not download image {i}: {img_error}")
         
-        return downloaded_images, len(img_tags)
-    
-    except Exception as e:
-        st.error(f"Error occurred while downloading images: {e}")
-        return [], 0
-
-def scrape_visible_text_from_url(url):
-    """
-    Scrape visible text from the given URL
-    
-    Args:
-        url (str): URL of the webpage
-    
-    Returns:
-        str: Extracted visible text
-    """
-    try:
-        response = requests.get(url)
-        response.raise_for_status()
-        soup = BeautifulSoup(response.content, 'html.parser')
-        
-        # Remove script, style, and other non-visible tags
-        for tag in soup(["script", "style", "meta", "link", "noscript", "header", "footer", "aside", "nav"]):
-            tag.extract()
-        
-        # Get the header content
-        header_content = soup.find("header")
-        header_text = header_content.get_text() if header_content else ""
+        # Remove unwanted tags
+        for tag in soup(["script", "style", "meta", "link", "noscript"]):
+            tag.decompose()
         
-        # Get the paragraph content
-        paragraph_content = soup.find_all("p")
-        paragraph_text = " ".join([p.get_text() for p in paragraph_content])
+        # Convert remaining soup to HTML string
+        formatted_html = str(soup)
         
-        # Combine header and paragraph text
-        visible_text = f"{header_text}\n\n{paragraph_text}"
+        # Extract plain text for preview
+        plain_text = soup.get_text(separator='\n', strip=True)
         
-        # Remove multiple whitespaces and newlines
-        visible_text = re.sub(r'\s+', ' ', visible_text)
-        return visible_text.strip()
+        return {
+            'html': formatted_html,
+            'plain_text': plain_text,
+            'images': downloaded_images
+        }
     
     except Exception as e:
-        st.error(f"Error occurred while scraping the data: {e}")
+        st.error(f"Error occurred while scraping the content: {e}")
         return None
 
 def main():
     """
     Main Streamlit application
     """
-    st.title("Web Data Scraper with Image Downloader")
+    st.title("Web Content Scraper with Preserved Formatting")
     
     # Get the URL from the user
     url_input = st.text_input("Enter the URL of the web page:", "")
     
-    # Maximum images to download slider
-    max_images = st.slider("Maximum number of images to download", 1, 20, 10)
+    # Option to choose display mode
+    display_mode = st.radio("Display Mode:", 
+                             ["Full HTML", "Plain Text", "Side-by-Side"])
     
     if st.button("Scrape Content"):
         if url_input:
-            # Extract visible text from the URL
-            text_data = scrape_visible_text_from_url(url_input)
+            # Scrape the content
+            scraped_content = scrape_web_content(url_input)
             
-            # Download images
-            downloaded_images, total_images = download_images(url_input, max_images)
-            
-            if text_data:
+            if scraped_content:
                 st.success("Content successfully scraped!")
                 
-                # Display text
-                st.subheader("Scraped Text:")
-                st.write(text_data)
+                # Display content based on selected mode
+                if display_mode == "Full HTML":
+                    # Display full HTML with preserved formatting
+                    st.markdown("### Formatted Web Content")
+                    st.components.v1.html(scraped_content['html'], height=600, scrolling=True)
+                
+                elif display_mode == "Plain Text":
+                    # Display plain text
+                    st.markdown("### Plain Text Content")
+                    st.text_area("Scraped Text:", scraped_content['plain_text'], height=400)
                 
-                # Display and manage images
-                st.subheader(f"Images (Downloaded {len(downloaded_images)} out of {total_images} total)")
+                else:  # Side-by-Side
+                    # Split the screen to show HTML and plain text
+                    col1, col2 = st.columns(2)
+                    
+                    with col1:
+                        st.markdown("### Formatted HTML")
+                        st.components.v1.html(scraped_content['html'], height=600, scrolling=True)
+                    
+                    with col2:
+                        st.markdown("### Plain Text")
+                        st.text_area("Scraped Text:", scraped_content['plain_text'], height=600)
                 
-                # Create columns for images
-                if downloaded_images:
-                    cols = st.columns(min(len(downloaded_images), 3))
-                    for i, img_path in enumerate(downloaded_images):
+                # Display images
+                if scraped_content['images']:
+                    st.subheader("Downloaded Images")
+                    cols = st.columns(min(len(scraped_content['images']), 3))
+                    for i, img_path in enumerate(scraped_content['images']):
                         with cols[i % 3]:
                             st.image(img_path, use_column_width=True)
                     
-                    # Provide download option for all images
+                    # Zip and download option for images
                     with open('downloaded_images.zip', 'wb') as zipf:
                         import zipfile
                         with zipfile.ZipFile(zipf, 'w') as zip_file:
-                            for img_path in downloaded_images:
+                            for img_path in scraped_content['images']:
                                 zip_file.write(img_path, os.path.basename(img_path))
                     
                     st.download_button(
@@ -153,8 +141,6 @@ def main():
                         file_name='downloaded_images.zip',
                         mime='application/zip'
                     )
-                else:
-                    st.warning("No images found on the page.")
             else:
                 st.warning("Failed to scrape content from the URL.")
         else: