visit_webpage_tool.py

from transformers import Tool
import requests
from markdownify import markdownify as md
from requests.exceptions import RequestException
import re

class VisitWebpageTool(Tool):
    name = "visit_webpage"
    description = "Visits a webpage at the given URL and returns its content as a markdown string."
    inputs = {
        "url": {
            "type": "text",
            "description": "The URL of the webpage to visit.",
        }
    }
    output_type = "text"

    def forward(self, url: str) -> str:
        try:
            # Send a GET request to the URL
            response = requests.get(url)
            response.raise_for_status()  # Raise an exception for bad status codes

            # Convert the HTML content to Markdown
            markdown_content = md(response.text).strip()

            # Remove multiple line breaks
            markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)

            return markdown_content

        except RequestException as e:
            return f"Error fetching the webpage: {str(e)}"
        except Exception as e:
            return f"An unexpected error occurred: {str(e)}"