Spaces:
Running
Running
File size: 2,173 Bytes
1f9c201 4fafc06 1f9c201 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 |
import gradio as gr
from bs4 import BeautifulSoup
from markdownify import MarkdownConverter
from playwright.sync_api import sync_playwright
def md(soup, **options):
return MarkdownConverter(**options).convert_soup(soup)
def main_fn(url: str, check: list[int], request: gr.Request):
user_agent = request.headers["user-agent"]
with sync_playwright() as p:
browser = p.chromium.launch(
args=[
"--single-process",
"--no-zygote",
"--no-sandbox",
"--disable-gpu",
"--disable-dev-shm-usage",
"--headless=new",
]
)
context = browser.new_context(user_agent=user_agent)
page = context.new_page()
response = page.goto(url=url)
status = response.status
content = page.content()
title = page.title()
browser.close()
soup = BeautifulSoup(content, features="html.parser")
for tag in ["script", "style"]:
target = soup.find_all(tag)
for t in target:
t.clear
body = soup.find("body")
main = soup.find("main")
body = md(body)
if main:
body = md(main, strip=check)
return f"{title}\n======\n\n{body}"
demo = gr.Interface(
main_fn,
title="URL to Markdown V2",
description="""<div style="width: fit-content; margin: 0 auto;">It gets the HTML given by the URL and converts it to Markdown. It uses Playwright, so it also supports dynamically generated HTML such as React.</div>
<div style="width: fit-content; margin: 0 auto;">URLで与えたHTMLを取得してMarkdownに変換します。Playwright を使用しているのでReactなどの動的に生成されるHTMLにも対応しています</div>""",
inputs=[
gr.Text(label="URL", placeholder="https://*****"),
gr.CheckboxGroup(
label="Ignore tags（無視するタグ）",
choices=["a", "img", "noscript"],
value=["a", "img"],
),
],
outputs=[gr.TextArea(label="Markdown", show_copy_button=True)],
allow_flagging="never",
)
demo.launch(server_name="0.0.0.0")
|