s > 9 correctly.
+ parent_list = None
+ for list in self.list:
+ self.o(
+ " " if parent_list == "ol" and list.name == "ul" else " "
+ )
+ parent_list = list.name
+
+ if li.name == "ul":
+ self.o(self.ul_item_mark + " ")
+ elif li.name == "ol":
+ li.num += 1
+ self.o(str(li.num) + ". ")
+ self.start = True
+
+ if tag in ["table", "tr", "td", "th"]:
+ if self.ignore_tables:
+ if tag == "tr":
+ if start:
+ pass
+ else:
+ self.soft_br()
+ else:
+ pass
+
+ elif self.bypass_tables:
+ if start:
+ self.soft_br()
+ if tag in ["td", "th"]:
+ if start:
+ self.o("<{}>\n\n".format(tag))
+ else:
+ self.o("\n".format(tag))
+ else:
+ if start:
+ self.o("<{}>".format(tag))
+ else:
+ self.o("".format(tag))
+
+ else:
+ if tag == "table":
+ if start:
+ self.table_start = True
+ if self.pad_tables:
+ self.o("<" + config.TABLE_MARKER_FOR_PAD + ">")
+ self.o(" \n")
+ else:
+ if self.pad_tables:
+ # add break in case the table is empty or its 1 row table
+ self.soft_br()
+ self.o("")
+ self.o(" \n")
+ if tag in ["td", "th"] and start:
+ if self.split_next_td:
+ self.o("| ")
+ self.split_next_td = True
+
+ if tag == "tr" and start:
+ self.td_count = 0
+ if tag == "tr" and not start:
+ self.split_next_td = False
+ self.soft_br()
+ if tag == "tr" and not start and self.table_start:
+ # Underline table header
+ self.o("|".join(["---"] * self.td_count))
+ self.soft_br()
+ self.table_start = False
+ if tag in ["td", "th"] and start:
+ self.td_count += 1
+
+ if tag == "pre":
+ if start:
+ self.startpre = True
+ self.pre = True
+ else:
+ self.pre = False
+ if self.mark_code:
+ self.out("\n[/code]")
+ self.p()
+
+ if tag in ["sup", "sub"] and self.include_sup_sub:
+ if start:
+ self.o("<{}>".format(tag))
+ else:
+ self.o("".format(tag))
+
+ # TODO: Add docstring for these one letter functions
+ def pbr(self) -> None:
+ "Pretty print has a line break"
+ if self.p_p == 0:
+ self.p_p = 1
+
+ def p(self) -> None:
+ "Set pretty print to 1 or 2 lines"
+ self.p_p = 1 if self.single_line_break else 2
+
+ def soft_br(self) -> None:
+ "Soft breaks"
+ self.pbr()
+ self.br_toggle = " "
+
+ def o(
+ self, data: str, puredata: bool = False, force: Union[bool, str] = False
+ ) -> None:
+ """
+ Deal with indentation and whitespace
+ """
+ if self.abbr_data is not None:
+ self.abbr_data += data
+
+ if not self.quiet:
+ if self.google_doc:
+ # prevent white space immediately after 'begin emphasis'
+ # marks ('**' and '_')
+ lstripped_data = data.lstrip()
+ if self.drop_white_space and not (self.pre or self.code):
+ data = lstripped_data
+ if lstripped_data != "":
+ self.drop_white_space = 0
+
+ if puredata and not self.pre:
+ # This is a very dangerous call ... it could mess up
+ # all handling of when not handled properly
+ # (see entityref)
+ data = re.sub(r"\s+", r" ", data)
+ if data and data[0] == " ":
+ self.space = True
+ data = data[1:]
+ if not data and not force:
+ return
+
+ if self.startpre:
+ # self.out(" :") #TODO: not output when already one there
+ if not data.startswith("\n") and not data.startswith("\r\n"):
+ # stuff...
+ data = "\n" + data
+ if self.mark_code:
+ self.out("\n[code]")
+ self.p_p = 0
+
+ bq = ">" * self.blockquote
+ if not (force and data and data[0] == ">") and self.blockquote:
+ bq += " "
+
+ if self.pre:
+ if not self.list:
+ bq += " "
+ # else: list content is already partially indented
+ bq += " " * len(self.list)
+ data = data.replace("\n", "\n" + bq)
+
+ if self.startpre:
+ self.startpre = False
+ if self.list:
+ # use existing initial indentation
+ data = data.lstrip("\n")
+
+ if self.start:
+ self.space = False
+ self.p_p = 0
+ self.start = False
+
+ if force == "end":
+ # It's the end.
+ self.p_p = 0
+ self.out("\n")
+ self.space = False
+
+ if self.p_p:
+ self.out((self.br_toggle + "\n" + bq) * self.p_p)
+ self.space = False
+ self.br_toggle = ""
+
+ if self.space:
+ if not self.lastWasNL:
+ self.out(" ")
+ self.space = False
+
+ if self.a and (
+ (self.p_p == 2 and self.links_each_paragraph) or force == "end"
+ ):
+ if force == "end":
+ self.out("\n")
+
+ newa = []
+ for link in self.a:
+ if self.outcount > link.outcount:
+ self.out(
+ " ["
+ + str(link.count)
+ + "]: "
+ + urlparse.urljoin(self.baseurl, link.attrs["href"])
+ )
+ if "title" in link.attrs and link.attrs["title"] is not None:
+ self.out(" (" + link.attrs["title"] + ")")
+ self.out("\n")
+ else:
+ newa.append(link)
+
+ # Don't need an extra line when nothing was done.
+ if self.a != newa:
+ self.out("\n")
+
+ self.a = newa
+
+ if self.abbr_list and force == "end":
+ for abbr, definition in self.abbr_list.items():
+ self.out(" *[" + abbr + "]: " + definition + "\n")
+
+ self.p_p = 0
+ self.out(data)
+ self.outcount += 1
+
+ def handle_data(self, data: str, entity_char: bool = False) -> None:
+ if not data:
+ # Data may be empty for some HTML entities. For example,
+ # LEFT-TO-RIGHT MARK.
+ return
+
+ if self.stressed:
+ data = data.strip()
+ self.stressed = False
+ self.preceding_stressed = True
+ elif self.preceding_stressed:
+ if (
+ re.match(r"[^][(){}\s.!?]", data[0])
+ and not hn(self.current_tag)
+ and self.current_tag not in ["a", "code", "pre"]
+ ):
+ # should match a letter or common punctuation
+ data = " " + data
+ self.preceding_stressed = False
+
+ if self.style:
+ self.style_def.update(dumb_css_parser(data))
+
+ if self.maybe_automatic_link is not None:
+ href = self.maybe_automatic_link
+ if (
+ href == data
+ and self.absolute_url_matcher.match(href)
+ and self.use_automatic_links
+ ):
+ self.o("<" + data + ">")
+ self.empty_link = False
+ return
+ else:
+ self.o("[")
+ self.maybe_automatic_link = None
+ self.empty_link = False
+
+ if not self.code and not self.pre and not entity_char:
+ data = escape_md_section(data, snob=self.escape_snob, escape_dot=self.escape_dot, escape_plus=self.escape_plus, escape_dash=self.escape_dash)
+ self.preceding_data = data
+ self.o(data, puredata=True)
+
+ def charref(self, name: str) -> str:
+ if name[0] in ["x", "X"]:
+ c = int(name[1:], 16)
+ else:
+ c = int(name)
+
+ if not self.unicode_snob and c in unifiable_n:
+ return unifiable_n[c]
+ else:
+ try:
+ return chr(c)
+ except ValueError: # invalid unicode
+ return ""
+
+ def entityref(self, c: str) -> str:
+ if not self.unicode_snob and c in config.UNIFIABLE:
+ return config.UNIFIABLE[c]
+ try:
+ ch = html.entities.html5[c + ";"]
+ except KeyError:
+ return "&" + c + ";"
+ return config.UNIFIABLE[c] if c == "nbsp" else ch
+
+ def google_nest_count(self, style: Dict[str, str]) -> int:
+ """
+ Calculate the nesting count of google doc lists
+
+ :type style: dict
+
+ :rtype: int
+ """
+ nest_count = 0
+ if "margin-left" in style:
+ nest_count = int(style["margin-left"][:-2]) // self.google_list_indent
+
+ return nest_count
+
+ def optwrap(self, text: str) -> str:
+ """
+ Wrap all paragraphs in the provided text.
+
+ :type text: str
+
+ :rtype: str
+ """
+ if not self.body_width:
+ return text
+
+ result = ""
+ newlines = 0
+ # I cannot think of a better solution for now.
+ # To avoid the non-wrap behaviour for entire paras
+ # because of the presence of a link in it
+ if not self.wrap_links:
+ self.inline_links = False
+ for para in text.split("\n"):
+ if len(para) > 0:
+ if not skipwrap(
+ para, self.wrap_links, self.wrap_list_items, self.wrap_tables
+ ):
+ indent = ""
+ if para.startswith(" " + self.ul_item_mark):
+ # list item continuation: add a double indent to the
+ # new lines
+ indent = " "
+ elif para.startswith("> "):
+ # blockquote continuation: add the greater than symbol
+ # to the new lines
+ indent = "> "
+ wrapped = wrap(
+ para,
+ self.body_width,
+ break_long_words=False,
+ subsequent_indent=indent,
+ )
+ result += "\n".join(wrapped)
+ if para.endswith(" "):
+ result += " \n"
+ newlines = 1
+ elif indent:
+ result += "\n"
+ newlines = 1
+ else:
+ result += "\n\n"
+ newlines = 2
+ else:
+ # Warning for the tempted!!!
+ # Be aware that obvious replacement of this with
+ # line.isspace()
+ # DOES NOT work! Explanations are welcome.
+ if not config.RE_SPACE.match(para):
+ result += para + "\n"
+ newlines = 1
+ else:
+ if newlines < 2:
+ result += "\n"
+ newlines += 1
+ return result
+
+def html2text(html: str, baseurl: str = "", bodywidth: Optional[int] = None) -> str:
+ if bodywidth is None:
+ bodywidth = config.BODY_WIDTH
+ h = HTML2Text(baseurl=baseurl, bodywidth=bodywidth)
+
+ return h.handle(html)
+
+class CustomHTML2Text(HTML2Text):
+ def __init__(self, *args, handle_code_in_pre=False, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.inside_pre = False
+ self.inside_code = False
+ self.preserve_tags = set() # Set of tags to preserve
+ self.current_preserved_tag = None
+ self.preserved_content = []
+ self.preserve_depth = 0
+ self.handle_code_in_pre = handle_code_in_pre
+
+ # Configuration options
+ self.skip_internal_links = False
+ self.single_line_break = False
+ self.mark_code = False
+ self.include_sup_sub = False
+ self.body_width = 0
+ self.ignore_mailto_links = True
+ self.ignore_links = False
+ self.escape_backslash = False
+ self.escape_dot = False
+ self.escape_plus = False
+ self.escape_dash = False
+ self.escape_snob = False
+
+ def update_params(self, **kwargs):
+ """Update parameters and set preserved tags."""
+ for key, value in kwargs.items():
+ if key == 'preserve_tags':
+ self.preserve_tags = set(value)
+ elif key == 'handle_code_in_pre':
+ self.handle_code_in_pre = value
+ else:
+ setattr(self, key, value)
+
+ def handle_tag(self, tag, attrs, start):
+ # Handle preserved tags
+ if tag in self.preserve_tags:
+ if start:
+ if self.preserve_depth == 0:
+ self.current_preserved_tag = tag
+ self.preserved_content = []
+ # Format opening tag with attributes
+ attr_str = ''.join(f' {k}="{v}"' for k, v in attrs.items() if v is not None)
+ self.preserved_content.append(f'<{tag}{attr_str}>')
+ self.preserve_depth += 1
+ return
+ else:
+ self.preserve_depth -= 1
+ if self.preserve_depth == 0:
+ self.preserved_content.append(f'')
+ # Output the preserved HTML block with proper spacing
+ preserved_html = ''.join(self.preserved_content)
+ self.o('\n' + preserved_html + '\n')
+ self.current_preserved_tag = None
+ return
+
+ # If we're inside a preserved tag, collect all content
+ if self.preserve_depth > 0:
+ if start:
+ # Format nested tags with attributes
+ attr_str = ''.join(f' {k}="{v}"' for k, v in attrs.items() if v is not None)
+ self.preserved_content.append(f'<{tag}{attr_str}>')
+ else:
+ self.preserved_content.append(f'')
+ return
+
+ # Handle pre tags
+ if tag == 'pre':
+ if start:
+ self.o('```\n') # Markdown code block start
+ self.inside_pre = True
+ else:
+ self.o('\n```\n') # Markdown code block end
+ self.inside_pre = False
+ elif tag == 'code':
+ if self.inside_pre and not self.handle_code_in_pre:
+ # Ignore code tags inside pre blocks if handle_code_in_pre is False
+ return
+ if start:
+ self.o('`') # Markdown inline code start
+ self.inside_code = True
+ else:
+ self.o('`') # Markdown inline code end
+ self.inside_code = False
+ else:
+ super().handle_tag(tag, attrs, start)
+
+ def handle_data(self, data, entity_char=False):
+ """Override handle_data to capture content within preserved tags."""
+ if self.preserve_depth > 0:
+ self.preserved_content.append(data)
+ return
+
+ if self.inside_pre:
+ # Output the raw content for pre blocks, including content inside code tags
+ self.o(data) # Directly output the data as-is (preserve newlines)
+ return
+ if self.inside_code:
+ # Inline code: no newlines allowed
+ self.o(data.replace('\n', ' '))
+ return
+
+ # Default behavior for other tags
+ super().handle_data(data, entity_char)
+
+
+ # # Handle pre tags
+ # if tag == 'pre':
+ # if start:
+ # self.o('```\n')
+ # self.inside_pre = True
+ # else:
+ # self.o('\n```')
+ # self.inside_pre = False
+ # # elif tag in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+ # # pass
+ # else:
+ # super().handle_tag(tag, attrs, start)
+
+ # def handle_data(self, data, entity_char=False):
+ # """Override handle_data to capture content within preserved tags."""
+ # if self.preserve_depth > 0:
+ # self.preserved_content.append(data)
+ # return
+ # super().handle_data(data, entity_char)
diff --git a/crawl4ai/html2text/__main__.py b/crawl4ai/html2text/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4e28416e104515e90fca4b69cc60d0c61fd15d61
--- /dev/null
+++ b/crawl4ai/html2text/__main__.py
@@ -0,0 +1,3 @@
+from .cli import main
+
+main()
diff --git a/crawl4ai/html2text/_typing.py b/crawl4ai/html2text/_typing.py
new file mode 100644
index 0000000000000000000000000000000000000000..eed83251cd381e68c0c5062ac3a50b97fbc3a483
--- /dev/null
+++ b/crawl4ai/html2text/_typing.py
@@ -0,0 +1,2 @@
+class OutCallback:
+ def __call__(self, s: str) -> None: ...
diff --git a/crawl4ai/html2text/cli.py b/crawl4ai/html2text/cli.py
new file mode 100644
index 0000000000000000000000000000000000000000..015322743d7bebb535b105d493cd6d23da64f303
--- /dev/null
+++ b/crawl4ai/html2text/cli.py
@@ -0,0 +1,330 @@
+import argparse
+import sys
+
+from . import HTML2Text, __version__, config
+
+
+def main() -> None:
+ baseurl = ""
+
+ class bcolors:
+ HEADER = "\033[95m"
+ OKBLUE = "\033[94m"
+ OKGREEN = "\033[92m"
+ WARNING = "\033[93m"
+ FAIL = "\033[91m"
+ ENDC = "\033[0m"
+ BOLD = "\033[1m"
+ UNDERLINE = "\033[4m"
+
+ p = argparse.ArgumentParser()
+ p.add_argument(
+ "--default-image-alt",
+ dest="default_image_alt",
+ default=config.DEFAULT_IMAGE_ALT,
+ help="The default alt string for images with missing ones",
+ )
+ p.add_argument(
+ "--pad-tables",
+ dest="pad_tables",
+ action="store_true",
+ default=config.PAD_TABLES,
+ help="pad the cells to equal column width in tables",
+ )
+ p.add_argument(
+ "--no-wrap-links",
+ dest="wrap_links",
+ action="store_false",
+ default=config.WRAP_LINKS,
+ help="don't wrap links during conversion",
+ )
+ p.add_argument(
+ "--wrap-list-items",
+ dest="wrap_list_items",
+ action="store_true",
+ default=config.WRAP_LIST_ITEMS,
+ help="wrap list items during conversion",
+ )
+ p.add_argument(
+ "--wrap-tables",
+ dest="wrap_tables",
+ action="store_true",
+ default=config.WRAP_TABLES,
+ help="wrap tables",
+ )
+ p.add_argument(
+ "--ignore-emphasis",
+ dest="ignore_emphasis",
+ action="store_true",
+ default=config.IGNORE_EMPHASIS,
+ help="don't include any formatting for emphasis",
+ )
+ p.add_argument(
+ "--reference-links",
+ dest="inline_links",
+ action="store_false",
+ default=config.INLINE_LINKS,
+ help="use reference style links instead of inline links",
+ )
+ p.add_argument(
+ "--ignore-links",
+ dest="ignore_links",
+ action="store_true",
+ default=config.IGNORE_ANCHORS,
+ help="don't include any formatting for links",
+ )
+ p.add_argument(
+ "--ignore-mailto-links",
+ action="store_true",
+ dest="ignore_mailto_links",
+ default=config.IGNORE_MAILTO_LINKS,
+ help="don't include mailto: links",
+ )
+ p.add_argument(
+ "--protect-links",
+ dest="protect_links",
+ action="store_true",
+ default=config.PROTECT_LINKS,
+ help="protect links from line breaks surrounding them with angle brackets",
+ )
+ p.add_argument(
+ "--ignore-images",
+ dest="ignore_images",
+ action="store_true",
+ default=config.IGNORE_IMAGES,
+ help="don't include any formatting for images",
+ )
+ p.add_argument(
+ "--images-as-html",
+ dest="images_as_html",
+ action="store_true",
+ default=config.IMAGES_AS_HTML,
+ help=(
+ "Always write image tags as raw html; preserves `height`, `width` and "
+ "`alt` if possible."
+ ),
+ )
+ p.add_argument(
+ "--images-to-alt",
+ dest="images_to_alt",
+ action="store_true",
+ default=config.IMAGES_TO_ALT,
+ help="Discard image data, only keep alt text",
+ )
+ p.add_argument(
+ "--images-with-size",
+ dest="images_with_size",
+ action="store_true",
+ default=config.IMAGES_WITH_SIZE,
+ help=(
+ "Write image tags with height and width attrs as raw html to retain "
+ "dimensions"
+ ),
+ )
+ p.add_argument(
+ "-g",
+ "--google-doc",
+ action="store_true",
+ dest="google_doc",
+ default=False,
+ help="convert an html-exported Google Document",
+ )
+ p.add_argument(
+ "-d",
+ "--dash-unordered-list",
+ action="store_true",
+ dest="ul_style_dash",
+ default=False,
+ help="use a dash rather than a star for unordered list items",
+ )
+ p.add_argument(
+ "-e",
+ "--asterisk-emphasis",
+ action="store_true",
+ dest="em_style_asterisk",
+ default=False,
+ help="use an asterisk rather than an underscore for emphasized text",
+ )
+ p.add_argument(
+ "-b",
+ "--body-width",
+ dest="body_width",
+ type=int,
+ default=config.BODY_WIDTH,
+ help="number of characters per output line, 0 for no wrap",
+ )
+ p.add_argument(
+ "-i",
+ "--google-list-indent",
+ dest="list_indent",
+ type=int,
+ default=config.GOOGLE_LIST_INDENT,
+ help="number of pixels Google indents nested lists",
+ )
+ p.add_argument(
+ "-s",
+ "--hide-strikethrough",
+ action="store_true",
+ dest="hide_strikethrough",
+ default=False,
+ help="hide strike-through text. only relevant when -g is " "specified as well",
+ )
+ p.add_argument(
+ "--escape-all",
+ action="store_true",
+ dest="escape_snob",
+ default=False,
+ help=(
+ "Escape all special characters. Output is less readable, but avoids "
+ "corner case formatting issues."
+ ),
+ )
+ p.add_argument(
+ "--bypass-tables",
+ action="store_true",
+ dest="bypass_tables",
+ default=config.BYPASS_TABLES,
+ help="Format tables in HTML rather than Markdown syntax.",
+ )
+ p.add_argument(
+ "--ignore-tables",
+ action="store_true",
+ dest="ignore_tables",
+ default=config.IGNORE_TABLES,
+ help="Ignore table-related tags (table, th, td, tr) " "while keeping rows.",
+ )
+ p.add_argument(
+ "--single-line-break",
+ action="store_true",
+ dest="single_line_break",
+ default=config.SINGLE_LINE_BREAK,
+ help=(
+ "Use a single line break after a block element rather than two line "
+ "breaks. NOTE: Requires --body-width=0"
+ ),
+ )
+ p.add_argument(
+ "--unicode-snob",
+ action="store_true",
+ dest="unicode_snob",
+ default=config.UNICODE_SNOB,
+ help="Use unicode throughout document",
+ )
+ p.add_argument(
+ "--no-automatic-links",
+ action="store_false",
+ dest="use_automatic_links",
+ default=config.USE_AUTOMATIC_LINKS,
+ help="Do not use automatic links wherever applicable",
+ )
+ p.add_argument(
+ "--no-skip-internal-links",
+ action="store_false",
+ dest="skip_internal_links",
+ default=config.SKIP_INTERNAL_LINKS,
+ help="Do not skip internal links",
+ )
+ p.add_argument(
+ "--links-after-para",
+ action="store_true",
+ dest="links_each_paragraph",
+ default=config.LINKS_EACH_PARAGRAPH,
+ help="Put links after each paragraph instead of document",
+ )
+ p.add_argument(
+ "--mark-code",
+ action="store_true",
+ dest="mark_code",
+ default=config.MARK_CODE,
+ help="Mark program code blocks with [code]...[/code]",
+ )
+ p.add_argument(
+ "--decode-errors",
+ dest="decode_errors",
+ default=config.DECODE_ERRORS,
+ help=(
+ "What to do in case of decode errors.'ignore', 'strict' and 'replace' are "
+ "acceptable values"
+ ),
+ )
+ p.add_argument(
+ "--open-quote",
+ dest="open_quote",
+ default=config.OPEN_QUOTE,
+ help="The character used to open quotes",
+ )
+ p.add_argument(
+ "--close-quote",
+ dest="close_quote",
+ default=config.CLOSE_QUOTE,
+ help="The character used to close quotes",
+ )
+ p.add_argument(
+ "--version", action="version", version=".".join(map(str, __version__))
+ )
+ p.add_argument("filename", nargs="?")
+ p.add_argument("encoding", nargs="?", default="utf-8")
+ p.add_argument(
+ "--include-sup-sub",
+ dest="include_sup_sub",
+ action="store_true",
+ default=config.INCLUDE_SUP_SUB,
+ help="Include the sup and sub tags",
+ )
+ args = p.parse_args()
+
+ if args.filename and args.filename != "-":
+ with open(args.filename, "rb") as fp:
+ data = fp.read()
+ else:
+ data = sys.stdin.buffer.read()
+
+ try:
+ html = data.decode(args.encoding, args.decode_errors)
+ except UnicodeDecodeError as err:
+ warning = bcolors.WARNING + "Warning:" + bcolors.ENDC
+ warning += " Use the " + bcolors.OKGREEN
+ warning += "--decode-errors=ignore" + bcolors.ENDC + " flag."
+ print(warning)
+ raise err
+
+ h = HTML2Text(baseurl=baseurl)
+ # handle options
+ if args.ul_style_dash:
+ h.ul_item_mark = "-"
+ if args.em_style_asterisk:
+ h.emphasis_mark = "*"
+ h.strong_mark = "__"
+
+ h.body_width = args.body_width
+ h.google_list_indent = args.list_indent
+ h.ignore_emphasis = args.ignore_emphasis
+ h.ignore_links = args.ignore_links
+ h.ignore_mailto_links = args.ignore_mailto_links
+ h.protect_links = args.protect_links
+ h.ignore_images = args.ignore_images
+ h.images_as_html = args.images_as_html
+ h.images_to_alt = args.images_to_alt
+ h.images_with_size = args.images_with_size
+ h.google_doc = args.google_doc
+ h.hide_strikethrough = args.hide_strikethrough
+ h.escape_snob = args.escape_snob
+ h.bypass_tables = args.bypass_tables
+ h.ignore_tables = args.ignore_tables
+ h.single_line_break = args.single_line_break
+ h.inline_links = args.inline_links
+ h.unicode_snob = args.unicode_snob
+ h.use_automatic_links = args.use_automatic_links
+ h.skip_internal_links = args.skip_internal_links
+ h.links_each_paragraph = args.links_each_paragraph
+ h.mark_code = args.mark_code
+ h.wrap_links = args.wrap_links
+ h.wrap_list_items = args.wrap_list_items
+ h.wrap_tables = args.wrap_tables
+ h.pad_tables = args.pad_tables
+ h.default_image_alt = args.default_image_alt
+ h.open_quote = args.open_quote
+ h.close_quote = args.close_quote
+ h.include_sup_sub = args.include_sup_sub
+
+ sys.stdout.write(h.handle(html))
diff --git a/crawl4ai/html2text/config.py b/crawl4ai/html2text/config.py
new file mode 100644
index 0000000000000000000000000000000000000000..d14ed64f90772ea9a3e92cc850b659f6f31756f0
--- /dev/null
+++ b/crawl4ai/html2text/config.py
@@ -0,0 +1,172 @@
+import re
+
+# Use Unicode characters instead of their ascii pseudo-replacements
+UNICODE_SNOB = False
+
+# Marker to use for marking tables for padding post processing
+TABLE_MARKER_FOR_PAD = "special_marker_for_table_padding"
+# Escape all special characters. Output is less readable, but avoids
+# corner case formatting issues.
+ESCAPE_SNOB = False
+ESCAPE_BACKSLASH = False
+ESCAPE_DOT = False
+ESCAPE_PLUS = False
+ESCAPE_DASH = False
+
+# Put the links after each paragraph instead of at the end.
+LINKS_EACH_PARAGRAPH = False
+
+# Wrap long lines at position. 0 for no wrapping.
+BODY_WIDTH = 78
+
+# Don't show internal links (href="#local-anchor") -- corresponding link
+# targets won't be visible in the plain text file anyway.
+SKIP_INTERNAL_LINKS = True
+
+# Use inline, rather than reference, formatting for images and links
+INLINE_LINKS = True
+
+# Protect links from line breaks surrounding them with angle brackets (in
+# addition to their square brackets)
+PROTECT_LINKS = False
+# WRAP_LINKS = True
+WRAP_LINKS = True
+
+# Wrap list items.
+WRAP_LIST_ITEMS = False
+
+# Wrap tables
+WRAP_TABLES = False
+
+# Number of pixels Google indents nested lists
+GOOGLE_LIST_INDENT = 36
+
+# Values Google and others may use to indicate bold text
+BOLD_TEXT_STYLE_VALUES = ("bold", "700", "800", "900")
+
+IGNORE_ANCHORS = False
+IGNORE_MAILTO_LINKS = False
+IGNORE_IMAGES = False
+IMAGES_AS_HTML = False
+IMAGES_TO_ALT = False
+IMAGES_WITH_SIZE = False
+IGNORE_EMPHASIS = False
+MARK_CODE = False
+DECODE_ERRORS = "strict"
+DEFAULT_IMAGE_ALT = ""
+PAD_TABLES = False
+
+# Convert links with same href and text to format
+# if they are absolute links
+USE_AUTOMATIC_LINKS = True
+
+# For checking space-only lines on line 771
+RE_SPACE = re.compile(r"\s\+")
+
+RE_ORDERED_LIST_MATCHER = re.compile(r"\d+\.\s")
+RE_UNORDERED_LIST_MATCHER = re.compile(r"[-\*\+]\s")
+RE_MD_CHARS_MATCHER = re.compile(r"([\\\[\]\(\)])")
+RE_MD_CHARS_MATCHER_ALL = re.compile(r"([`\*_{}\[\]\(\)#!])")
+
+# to find links in the text
+RE_LINK = re.compile(r"(\[.*?\] ?\(.*?\))|(\[.*?\]:.*?)")
+
+# to find table separators
+RE_TABLE = re.compile(r" \| ")
+
+RE_MD_DOT_MATCHER = re.compile(
+ r"""
+ ^ # start of line
+ (\s*\d+) # optional whitespace and a number
+ (\.) # dot
+ (?=\s) # lookahead assert whitespace
+ """,
+ re.MULTILINE | re.VERBOSE,
+)
+RE_MD_PLUS_MATCHER = re.compile(
+ r"""
+ ^
+ (\s*)
+ (\+)
+ (?=\s)
+ """,
+ flags=re.MULTILINE | re.VERBOSE,
+)
+RE_MD_DASH_MATCHER = re.compile(
+ r"""
+ ^
+ (\s*)
+ (-)
+ (?=\s|\-) # followed by whitespace (bullet list, or spaced out hr)
+ # or another dash (header or hr)
+ """,
+ flags=re.MULTILINE | re.VERBOSE,
+)
+RE_SLASH_CHARS = r"\`*_{}[]()#+-.!"
+RE_MD_BACKSLASH_MATCHER = re.compile(
+ r"""
+ (\\) # match one slash
+ (?=[%s]) # followed by a char that requires escaping
+ """
+ % re.escape(RE_SLASH_CHARS),
+ flags=re.VERBOSE,
+)
+
+UNIFIABLE = {
+ "rsquo": "'",
+ "lsquo": "'",
+ "rdquo": '"',
+ "ldquo": '"',
+ "copy": "(C)",
+ "mdash": "--",
+ "nbsp": " ",
+ "rarr": "->",
+ "larr": "<-",
+ "middot": "*",
+ "ndash": "-",
+ "oelig": "oe",
+ "aelig": "ae",
+ "agrave": "a",
+ "aacute": "a",
+ "acirc": "a",
+ "atilde": "a",
+ "auml": "a",
+ "aring": "a",
+ "egrave": "e",
+ "eacute": "e",
+ "ecirc": "e",
+ "euml": "e",
+ "igrave": "i",
+ "iacute": "i",
+ "icirc": "i",
+ "iuml": "i",
+ "ograve": "o",
+ "oacute": "o",
+ "ocirc": "o",
+ "otilde": "o",
+ "ouml": "o",
+ "ugrave": "u",
+ "uacute": "u",
+ "ucirc": "u",
+ "uuml": "u",
+ "lrm": "",
+ "rlm": "",
+}
+
+# Format tables in HTML rather than Markdown syntax
+BYPASS_TABLES = False
+# Ignore table-related tags (table, th, td, tr) while keeping rows
+IGNORE_TABLES = False
+
+
+# Use a single line break after a block element rather than two line breaks.
+# NOTE: Requires body width setting to be 0.
+SINGLE_LINE_BREAK = False
+
+
+# Use double quotation marks when converting the tag.
+OPEN_QUOTE = '"'
+CLOSE_QUOTE = '"'
+
+# Include the and tags
+INCLUDE_SUP_SUB = False
diff --git a/crawl4ai/html2text/elements.py b/crawl4ai/html2text/elements.py
new file mode 100644
index 0000000000000000000000000000000000000000..2533ec084e664f6c4cd19adb175325de0c844d55
--- /dev/null
+++ b/crawl4ai/html2text/elements.py
@@ -0,0 +1,18 @@
+from typing import Dict, Optional
+
+
+class AnchorElement:
+ __slots__ = ["attrs", "count", "outcount"]
+
+ def __init__(self, attrs: Dict[str, Optional[str]], count: int, outcount: int):
+ self.attrs = attrs
+ self.count = count
+ self.outcount = outcount
+
+
+class ListElement:
+ __slots__ = ["name", "num"]
+
+ def __init__(self, name: str, num: int):
+ self.name = name
+ self.num = num
diff --git a/crawl4ai/html2text/utils.py b/crawl4ai/html2text/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..1909d2cf754b57c8d1fba112a1a1eb3af81a8d3b
--- /dev/null
+++ b/crawl4ai/html2text/utils.py
@@ -0,0 +1,303 @@
+import html.entities
+from typing import Dict, List, Optional
+
+from . import config
+
+unifiable_n = {
+ html.entities.name2codepoint[k]: v
+ for k, v in config.UNIFIABLE.items()
+ if k != "nbsp"
+}
+
+
+def hn(tag: str) -> int:
+ if tag[0] == "h" and len(tag) == 2:
+ n = tag[1]
+ if "0" < n <= "9":
+ return int(n)
+ return 0
+
+
+def dumb_property_dict(style: str) -> Dict[str, str]:
+ """
+ :returns: A hash of css attributes
+ """
+ return {
+ x.strip().lower(): y.strip().lower()
+ for x, y in [z.split(":", 1) for z in style.split(";") if ":" in z]
+ }
+
+
+def dumb_css_parser(data: str) -> Dict[str, Dict[str, str]]:
+ """
+ :type data: str
+
+ :returns: A hash of css selectors, each of which contains a hash of
+ css attributes.
+ :rtype: dict
+ """
+ # remove @import sentences
+ data += ";"
+ importIndex = data.find("@import")
+ while importIndex != -1:
+ data = data[0:importIndex] + data[data.find(";", importIndex) + 1 :]
+ importIndex = data.find("@import")
+
+ # parse the css. reverted from dictionary comprehension in order to
+ # support older pythons
+ pairs = [x.split("{") for x in data.split("}") if "{" in x.strip()]
+ try:
+ elements = {a.strip(): dumb_property_dict(b) for a, b in pairs}
+ except ValueError:
+ elements = {} # not that important
+
+ return elements
+
+
+def element_style(
+ attrs: Dict[str, Optional[str]],
+ style_def: Dict[str, Dict[str, str]],
+ parent_style: Dict[str, str],
+) -> Dict[str, str]:
+ """
+ :type attrs: dict
+ :type style_def: dict
+ :type style_def: dict
+
+ :returns: A hash of the 'final' style attributes of the element
+ :rtype: dict
+ """
+ style = parent_style.copy()
+ if "class" in attrs:
+ assert attrs["class"] is not None
+ for css_class in attrs["class"].split():
+ css_style = style_def.get("." + css_class, {})
+ style.update(css_style)
+ if "style" in attrs:
+ assert attrs["style"] is not None
+ immediate_style = dumb_property_dict(attrs["style"])
+ style.update(immediate_style)
+
+ return style
+
+
+def google_list_style(style: Dict[str, str]) -> str:
+ """
+ Finds out whether this is an ordered or unordered list
+
+ :type style: dict
+
+ :rtype: str
+ """
+ if "list-style-type" in style:
+ list_style = style["list-style-type"]
+ if list_style in ["disc", "circle", "square", "none"]:
+ return "ul"
+
+ return "ol"
+
+
+def google_has_height(style: Dict[str, str]) -> bool:
+ """
+ Check if the style of the element has the 'height' attribute
+ explicitly defined
+
+ :type style: dict
+
+ :rtype: bool
+ """
+ return "height" in style
+
+
+def google_text_emphasis(style: Dict[str, str]) -> List[str]:
+ """
+ :type style: dict
+
+ :returns: A list of all emphasis modifiers of the element
+ :rtype: list
+ """
+ emphasis = []
+ if "text-decoration" in style:
+ emphasis.append(style["text-decoration"])
+ if "font-style" in style:
+ emphasis.append(style["font-style"])
+ if "font-weight" in style:
+ emphasis.append(style["font-weight"])
+
+ return emphasis
+
+
+def google_fixed_width_font(style: Dict[str, str]) -> bool:
+ """
+ Check if the css of the current element defines a fixed width font
+
+ :type style: dict
+
+ :rtype: bool
+ """
+ font_family = ""
+ if "font-family" in style:
+ font_family = style["font-family"]
+ return "courier new" == font_family or "consolas" == font_family
+
+
+def list_numbering_start(attrs: Dict[str, Optional[str]]) -> int:
+ """
+ Extract numbering from list element attributes
+
+ :type attrs: dict
+
+ :rtype: int or None
+ """
+ if "start" in attrs:
+ assert attrs["start"] is not None
+ try:
+ return int(attrs["start"]) - 1
+ except ValueError:
+ pass
+
+ return 0
+
+
+def skipwrap(
+ para: str, wrap_links: bool, wrap_list_items: bool, wrap_tables: bool
+) -> bool:
+ # If it appears to contain a link
+ # don't wrap
+ if not wrap_links and config.RE_LINK.search(para):
+ return True
+ # If the text begins with four spaces or one tab, it's a code block;
+ # don't wrap
+ if para[0:4] == " " or para[0] == "\t":
+ return True
+
+ # If the text begins with only two "--", possibly preceded by
+ # whitespace, that's an emdash; so wrap.
+ stripped = para.lstrip()
+ if stripped[0:2] == "--" and len(stripped) > 2 and stripped[2] != "-":
+ return False
+
+ # I'm not sure what this is for; I thought it was to detect lists,
+ # but there's a case in one of the tests that
+ # also depends upon it.
+ if stripped[0:1] in ("-", "*") and not stripped[0:2] == "**":
+ return not wrap_list_items
+
+ # If text contains a pipe character it is likely a table
+ if not wrap_tables and config.RE_TABLE.search(para):
+ return True
+
+ # If the text begins with a single -, *, or +, followed by a space,
+ # or an integer, followed by a ., followed by a space (in either
+ # case optionally proceeded by whitespace), it's a list; don't wrap.
+ return bool(
+ config.RE_ORDERED_LIST_MATCHER.match(stripped)
+ or config.RE_UNORDERED_LIST_MATCHER.match(stripped)
+ )
+
+
+def escape_md(text: str) -> str:
+ """
+ Escapes markdown-sensitive characters within other markdown
+ constructs.
+ """
+ return config.RE_MD_CHARS_MATCHER.sub(r"\\\1", text)
+
+
+def escape_md_section(
+ text: str,
+ escape_backslash: bool = True,
+ snob: bool = False,
+ escape_dot: bool = True,
+ escape_plus: bool = True,
+ escape_dash: bool = True
+) -> str:
+ """
+ Escapes markdown-sensitive characters across whole document sections.
+ Each escaping operation can be controlled individually.
+ """
+ if escape_backslash:
+ text = config.RE_MD_BACKSLASH_MATCHER.sub(r"\\\1", text)
+
+ if snob:
+ text = config.RE_MD_CHARS_MATCHER_ALL.sub(r"\\\1", text)
+
+ if escape_dot:
+ text = config.RE_MD_DOT_MATCHER.sub(r"\1\\\2", text)
+
+ if escape_plus:
+ text = config.RE_MD_PLUS_MATCHER.sub(r"\1\\\2", text)
+
+ if escape_dash:
+ text = config.RE_MD_DASH_MATCHER.sub(r"\1\\\2", text)
+
+ return text
+
+def reformat_table(lines: List[str], right_margin: int) -> List[str]:
+ """
+ Given the lines of a table
+ padds the cells and returns the new lines
+ """
+ # find the maximum width of the columns
+ max_width = [len(x.rstrip()) + right_margin for x in lines[0].split("|")]
+ max_cols = len(max_width)
+ for line in lines:
+ cols = [x.rstrip() for x in line.split("|")]
+ num_cols = len(cols)
+
+ # don't drop any data if colspan attributes result in unequal lengths
+ if num_cols < max_cols:
+ cols += [""] * (max_cols - num_cols)
+ elif max_cols < num_cols:
+ max_width += [len(x) + right_margin for x in cols[-(num_cols - max_cols) :]]
+ max_cols = num_cols
+
+ max_width = [
+ max(len(x) + right_margin, old_len) for x, old_len in zip(cols, max_width)
+ ]
+
+ # reformat
+ new_lines = []
+ for line in lines:
+ cols = [x.rstrip() for x in line.split("|")]
+ if set(line.strip()) == set("-|"):
+ filler = "-"
+ new_cols = [
+ x.rstrip() + (filler * (M - len(x.rstrip())))
+ for x, M in zip(cols, max_width)
+ ]
+ new_lines.append("|-" + "|".join(new_cols) + "|")
+ else:
+ filler = " "
+ new_cols = [
+ x.rstrip() + (filler * (M - len(x.rstrip())))
+ for x, M in zip(cols, max_width)
+ ]
+ new_lines.append("| " + "|".join(new_cols) + "|")
+ return new_lines
+
+
+def pad_tables_in_text(text: str, right_margin: int = 1) -> str:
+ """
+ Provide padding for tables in the text
+ """
+ lines = text.split("\n")
+ table_buffer = [] # type: List[str]
+ table_started = False
+ new_lines = []
+ for line in lines:
+ # Toggle table started
+ if config.TABLE_MARKER_FOR_PAD in line:
+ table_started = not table_started
+ if not table_started:
+ table = reformat_table(table_buffer, right_margin)
+ new_lines.extend(table)
+ table_buffer = []
+ new_lines.append("")
+ continue
+ # Process lines
+ if table_started:
+ table_buffer.append(line)
+ else:
+ new_lines.append(line)
+ return "\n".join(new_lines)
diff --git a/crawl4ai/install.py b/crawl4ai/install.py
new file mode 100644
index 0000000000000000000000000000000000000000..7efb6800b1d7eb9a9edf5fea639f92c982fbe7b8
--- /dev/null
+++ b/crawl4ai/install.py
@@ -0,0 +1,83 @@
+import subprocess
+import sys
+import asyncio
+from .async_logger import AsyncLogger, LogLevel
+
+# Initialize logger
+logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
+
+def post_install():
+ """Run all post-installation tasks"""
+ logger.info("Running post-installation setup...", tag="INIT")
+ install_playwright()
+ run_migration()
+ logger.success("Post-installation setup completed!", tag="COMPLETE")
+
+def install_playwright():
+ logger.info("Installing Playwright browsers...", tag="INIT")
+ try:
+ # subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chrome"])
+ subprocess.check_call([sys.executable, "-m", "playwright", "install", "--with-deps", "--force", "chromium"])
+ logger.success("Playwright installation completed successfully.", tag="COMPLETE")
+ except subprocess.CalledProcessError as e:
+ # logger.error(f"Error during Playwright installation: {e}", tag="ERROR")
+ logger.warning(f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation.")
+ except Exception as e:
+ # logger.error(f"Unexpected error during Playwright installation: {e}", tag="ERROR")
+ logger.warning(f"Please run '{sys.executable} -m playwright install --with-deps' manually after the installation.")
+
+def run_migration():
+ """Initialize database during installation"""
+ try:
+ logger.info("Starting database initialization...", tag="INIT")
+ from crawl4ai.async_database import async_db_manager
+
+ asyncio.run(async_db_manager.initialize())
+ logger.success("Database initialization completed successfully.", tag="COMPLETE")
+ except ImportError:
+ logger.warning("Database module not found. Will initialize on first use.")
+ except Exception as e:
+ logger.warning(f"Database initialization failed: {e}")
+ logger.warning("Database will be initialized on first use")
+
+async def run_doctor():
+ """Test if Crawl4AI is working properly"""
+ logger.info("Running Crawl4AI health check...", tag="INIT")
+ try:
+ from .async_webcrawler import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+
+ browser_config = BrowserConfig(
+ headless=True,
+ browser_type="chromium",
+ ignore_https_errors=True,
+ light_mode=True,
+ viewport_width=1280,
+ viewport_height=720
+ )
+
+ run_config = CrawlerRunConfig(
+ cache_mode=CacheMode.BYPASS,
+ screenshot=True,
+ )
+
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ logger.info("Testing crawling capabilities...", tag="TEST")
+ result = await crawler.arun(
+ url="https://crawl4ai.com",
+ config=run_config
+ )
+
+ if result and result.markdown:
+ logger.success("✅ Crawling test passed!", tag="COMPLETE")
+ return True
+ else:
+ raise Exception("Failed to get content")
+
+ except Exception as e:
+ logger.error(f"❌ Test failed: {e}", tag="ERROR")
+ return False
+
+def doctor():
+ """Entry point for the doctor command"""
+ import asyncio
+ return asyncio.run(run_doctor())
diff --git a/crawl4ai/js_snippet/__init__.py b/crawl4ai/js_snippet/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..73b0c2dd343d45483a5783ecc8d05fa459af9cd6
--- /dev/null
+++ b/crawl4ai/js_snippet/__init__.py
@@ -0,0 +1,15 @@
+import os, sys
+
+# Create a function get name of a js script, then load from the CURRENT folder of this script and return its content as string, make sure its error free
+def load_js_script(script_name):
+ # Get the path of the current script
+ current_script_path = os.path.dirname(os.path.realpath(__file__))
+ # Get the path of the script to load
+ script_path = os.path.join(current_script_path, script_name + '.js')
+ # Check if the script exists
+ if not os.path.exists(script_path):
+ raise ValueError(f"Script {script_name} not found in the folder {current_script_path}")
+ # Load the content of the script
+ with open(script_path, 'r') as f:
+ script_content = f.read()
+ return script_content
diff --git a/crawl4ai/js_snippet/navigator_overrider.js b/crawl4ai/js_snippet/navigator_overrider.js
new file mode 100644
index 0000000000000000000000000000000000000000..f341ceeb743bfaea669a7bdf378844586f52c5f2
--- /dev/null
+++ b/crawl4ai/js_snippet/navigator_overrider.js
@@ -0,0 +1,25 @@
+// Pass the Permissions Test.
+const originalQuery = window.navigator.permissions.query;
+window.navigator.permissions.query = (parameters) =>
+ parameters.name === "notifications"
+ ? Promise.resolve({ state: Notification.permission })
+ : originalQuery(parameters);
+Object.defineProperty(navigator, "webdriver", {
+ get: () => undefined,
+});
+window.navigator.chrome = {
+ runtime: {},
+ // Add other properties if necessary
+};
+Object.defineProperty(navigator, "plugins", {
+ get: () => [1, 2, 3, 4, 5],
+});
+Object.defineProperty(navigator, "languages", {
+ get: () => ["en-US", "en"],
+});
+Object.defineProperty(document, "hidden", {
+ get: () => false,
+});
+Object.defineProperty(document, "visibilityState", {
+ get: () => "visible",
+});
diff --git a/crawl4ai/js_snippet/remove_overlay_elements.js b/crawl4ai/js_snippet/remove_overlay_elements.js
new file mode 100644
index 0000000000000000000000000000000000000000..0400d89c40a9c0206ddf6d8110d0c2939a29af8e
--- /dev/null
+++ b/crawl4ai/js_snippet/remove_overlay_elements.js
@@ -0,0 +1,119 @@
+async () => {
+ // Function to check if element is visible
+ const isVisible = (elem) => {
+ const style = window.getComputedStyle(elem);
+ return style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0";
+ };
+
+ // Common selectors for popups and overlays
+ const commonSelectors = [
+ // Close buttons first
+ 'button[class*="close" i]',
+ 'button[class*="dismiss" i]',
+ 'button[aria-label*="close" i]',
+ 'button[title*="close" i]',
+ 'a[class*="close" i]',
+ 'span[class*="close" i]',
+
+ // Cookie notices
+ '[class*="cookie-banner" i]',
+ '[id*="cookie-banner" i]',
+ '[class*="cookie-consent" i]',
+ '[id*="cookie-consent" i]',
+
+ // Newsletter/subscription dialogs
+ '[class*="newsletter" i]',
+ '[class*="subscribe" i]',
+
+ // Generic popups/modals
+ '[class*="popup" i]',
+ '[class*="modal" i]',
+ '[class*="overlay" i]',
+ '[class*="dialog" i]',
+ '[role="dialog"]',
+ '[role="alertdialog"]',
+ ];
+
+ // Try to click close buttons first
+ for (const selector of commonSelectors.slice(0, 6)) {
+ const closeButtons = document.querySelectorAll(selector);
+ for (const button of closeButtons) {
+ if (isVisible(button)) {
+ try {
+ button.click();
+ await new Promise((resolve) => setTimeout(resolve, 100));
+ } catch (e) {
+ console.log("Error clicking button:", e);
+ }
+ }
+ }
+ }
+
+ // Remove remaining overlay elements
+ const removeOverlays = () => {
+ // Find elements with high z-index
+ const allElements = document.querySelectorAll("*");
+ for (const elem of allElements) {
+ const style = window.getComputedStyle(elem);
+ const zIndex = parseInt(style.zIndex);
+ const position = style.position;
+
+ if (
+ isVisible(elem) &&
+ (zIndex > 999 || position === "fixed" || position === "absolute") &&
+ (elem.offsetWidth > window.innerWidth * 0.5 ||
+ elem.offsetHeight > window.innerHeight * 0.5 ||
+ style.backgroundColor.includes("rgba") ||
+ parseFloat(style.opacity) < 1)
+ ) {
+ elem.remove();
+ }
+ }
+
+ // Remove elements matching common selectors
+ for (const selector of commonSelectors) {
+ const elements = document.querySelectorAll(selector);
+ elements.forEach((elem) => {
+ if (isVisible(elem)) {
+ elem.remove();
+ }
+ });
+ }
+ };
+
+ // Remove overlay elements
+ removeOverlays();
+
+ // Remove any fixed/sticky position elements at the top/bottom
+ const removeFixedElements = () => {
+ const elements = document.querySelectorAll("*");
+ elements.forEach((elem) => {
+ const style = window.getComputedStyle(elem);
+ if ((style.position === "fixed" || style.position === "sticky") && isVisible(elem)) {
+ elem.remove();
+ }
+ });
+ };
+
+ removeFixedElements();
+
+ // Remove empty block elements as: div, p, span, etc.
+ const removeEmptyBlockElements = () => {
+ const blockElements = document.querySelectorAll(
+ "div, p, span, section, article, header, footer, aside, nav, main, ul, ol, li, dl, dt, dd, h1, h2, h3, h4, h5, h6"
+ );
+ blockElements.forEach((elem) => {
+ if (elem.innerText.trim() === "") {
+ elem.remove();
+ }
+ });
+ };
+
+ // Remove margin-right and padding-right from body (often added by modal scripts)
+ document.body.style.marginRight = "0px";
+ document.body.style.paddingRight = "0px";
+ document.body.style.overflow = "auto";
+
+ // Wait a bit for any animations to complete
+ await new Promise((resolve) => setTimeout(resolve, 100));
+};
diff --git a/crawl4ai/js_snippet/update_image_dimensions.js b/crawl4ai/js_snippet/update_image_dimensions.js
new file mode 100644
index 0000000000000000000000000000000000000000..709a35d5143227718ef2a5c29385f1346af4de40
--- /dev/null
+++ b/crawl4ai/js_snippet/update_image_dimensions.js
@@ -0,0 +1,54 @@
+() => {
+ return new Promise((resolve) => {
+ const filterImage = (img) => {
+ // Filter out images that are too small
+ if (img.width < 100 && img.height < 100) return false;
+
+ // Filter out images that are not visible
+ const rect = img.getBoundingClientRect();
+ if (rect.width === 0 || rect.height === 0) return false;
+
+ // Filter out images with certain class names (e.g., icons, thumbnails)
+ if (img.classList.contains("icon") || img.classList.contains("thumbnail")) return false;
+
+ // Filter out images with certain patterns in their src (e.g., placeholder images)
+ if (img.src.includes("placeholder") || img.src.includes("icon")) return false;
+
+ return true;
+ };
+
+ const images = Array.from(document.querySelectorAll("img")).filter(filterImage);
+ let imagesLeft = images.length;
+
+ if (imagesLeft === 0) {
+ resolve();
+ return;
+ }
+
+ const checkImage = (img) => {
+ if (img.complete && img.naturalWidth !== 0) {
+ img.setAttribute("width", img.naturalWidth);
+ img.setAttribute("height", img.naturalHeight);
+ imagesLeft--;
+ if (imagesLeft === 0) resolve();
+ }
+ };
+
+ images.forEach((img) => {
+ checkImage(img);
+ if (!img.complete) {
+ img.onload = () => {
+ checkImage(img);
+ };
+ img.onerror = () => {
+ imagesLeft--;
+ if (imagesLeft === 0) resolve();
+ };
+ }
+ });
+
+ // Fallback timeout of 5 seconds
+ // setTimeout(() => resolve(), 5000);
+ resolve();
+ });
+};
diff --git a/crawl4ai/llmtxt.py b/crawl4ai/llmtxt.py
new file mode 100644
index 0000000000000000000000000000000000000000..94efe0767995af580e2f75c9b6a13f4be0f8d811
--- /dev/null
+++ b/crawl4ai/llmtxt.py
@@ -0,0 +1,498 @@
+import os
+from pathlib import Path
+import re
+from typing import Dict, List, Tuple, Optional, Any
+import json
+from tqdm import tqdm
+import time
+import psutil
+import numpy as np
+from rank_bm25 import BM25Okapi
+from nltk.tokenize import word_tokenize
+from nltk.corpus import stopwords
+from nltk.stem import WordNetLemmatizer
+from litellm import completion, batch_completion
+from .async_logger import AsyncLogger
+import litellm
+import pickle
+import hashlib # <--- ADDED for file-hash
+from fnmatch import fnmatch
+import glob
+
+litellm.set_verbose = False
+
+def _compute_file_hash(file_path: Path) -> str:
+ """Compute MD5 hash for the file's entire content."""
+ hash_md5 = hashlib.md5()
+ with file_path.open("rb") as f:
+ for chunk in iter(lambda: f.read(4096), b""):
+ hash_md5.update(chunk)
+ return hash_md5.hexdigest()
+
+class AsyncLLMTextManager:
+ def __init__(
+ self,
+ docs_dir: Path,
+ logger: Optional[AsyncLogger] = None,
+ max_concurrent_calls: int = 5,
+ batch_size: int = 3
+ ) -> None:
+ self.docs_dir = docs_dir
+ self.logger = logger
+ self.max_concurrent_calls = max_concurrent_calls
+ self.batch_size = batch_size
+ self.bm25_index = None
+ self.document_map: Dict[str, Any] = {}
+ self.tokenized_facts: List[str] = []
+ self.bm25_index_file = self.docs_dir / "bm25_index.pkl"
+
+ async def _process_document_batch(self, doc_batch: List[Path]) -> None:
+ """Process a batch of documents in parallel"""
+ contents = []
+ for file_path in doc_batch:
+ try:
+ with open(file_path, 'r', encoding='utf-8') as f:
+ contents.append(f.read())
+ except Exception as e:
+ self.logger.error(f"Error reading {file_path}: {str(e)}")
+ contents.append("") # Add empty content to maintain batch alignment
+
+ prompt = """Given a documentation file, generate a list of atomic facts where each fact:
+1. Represents a single piece of knowledge
+2. Contains variations in terminology for the same concept
+3. References relevant code patterns if they exist
+4. Is written in a way that would match natural language queries
+
+Each fact should follow this format:
+: | |
+
+Example Facts:
+browser_config: Configure headless mode and browser type for AsyncWebCrawler | headless, browser_type, chromium, firefox | BrowserConfig(browser_type="chromium", headless=True)
+redis_connection: Redis client connection requires host and port configuration | redis setup, redis client, connection params | Redis(host='localhost', port=6379, db=0)
+pandas_filtering: Filter DataFrame rows using boolean conditions | dataframe filter, query, boolean indexing | df[df['column'] > 5]
+
+Wrap your response in ... tags.
+"""
+
+ # Prepare messages for batch processing
+ messages_list = [
+ [
+ {"role": "user", "content": f"{prompt}\n\nGenerate index for this documentation:\n\n{content}"}
+ ]
+ for content in contents if content
+ ]
+
+ try:
+ responses = batch_completion(
+ model="anthropic/claude-3-5-sonnet-latest",
+ messages=messages_list,
+ logger_fn=None
+ )
+
+ # Process responses and save index files
+ for response, file_path in zip(responses, doc_batch):
+ try:
+ index_content_match = re.search(
+ r'(.*?) ',
+ response.choices[0].message.content,
+ re.DOTALL
+ )
+ if not index_content_match:
+ self.logger.warning(f"No ... content found for {file_path}")
+ continue
+
+ index_content = re.sub(
+ r"\n\s*\n", "\n", index_content_match.group(1)
+ ).strip()
+ if index_content:
+ index_file = file_path.with_suffix('.q.md')
+ with open(index_file, 'w', encoding='utf-8') as f:
+ f.write(index_content)
+ self.logger.info(f"Created index file: {index_file}")
+ else:
+ self.logger.warning(f"No index content found in response for {file_path}")
+
+ except Exception as e:
+ self.logger.error(f"Error processing response for {file_path}: {str(e)}")
+
+ except Exception as e:
+ self.logger.error(f"Error in batch completion: {str(e)}")
+
+ def _validate_fact_line(self, line: str) -> Tuple[bool, Optional[str]]:
+ if "|" not in line:
+ return False, "Missing separator '|'"
+
+ parts = [p.strip() for p in line.split("|")]
+ if len(parts) != 3:
+ return False, f"Expected 3 parts, got {len(parts)}"
+
+ concept_part = parts[0]
+ if ":" not in concept_part:
+ return False, "Missing ':' in concept definition"
+
+ return True, None
+
+ def _load_or_create_token_cache(self, fact_file: Path) -> Dict:
+ """
+ Load token cache from .q.tokens if present and matching file hash.
+ Otherwise return a new structure with updated file-hash.
+ """
+ cache_file = fact_file.with_suffix(".q.tokens")
+ current_hash = _compute_file_hash(fact_file)
+
+ if cache_file.exists():
+ try:
+ with open(cache_file, "r") as f:
+ cache = json.load(f)
+ # If the hash matches, return it directly
+ if cache.get("content_hash") == current_hash:
+ return cache
+ # Otherwise, we signal that it's changed
+ self.logger.info(f"Hash changed for {fact_file}, reindex needed.")
+ except json.JSONDecodeError:
+ self.logger.warning(f"Corrupt token cache for {fact_file}, rebuilding.")
+ except Exception as e:
+ self.logger.warning(f"Error reading cache for {fact_file}: {str(e)}")
+
+ # Return a fresh cache
+ return {"facts": {}, "content_hash": current_hash}
+
+ def _save_token_cache(self, fact_file: Path, cache: Dict) -> None:
+ cache_file = fact_file.with_suffix(".q.tokens")
+ # Always ensure we're saving the correct file-hash
+ cache["content_hash"] = _compute_file_hash(fact_file)
+ with open(cache_file, "w") as f:
+ json.dump(cache, f)
+
+ def preprocess_text(self, text: str) -> List[str]:
+ parts = [x.strip() for x in text.split("|")] if "|" in text else [text]
+ # Remove : after the first word of parts[0]
+ parts[0] = re.sub(r"^(.*?):", r"\1", parts[0])
+
+ lemmatizer = WordNetLemmatizer()
+ stop_words = set(stopwords.words("english")) - {
+ "how", "what", "when", "where", "why", "which",
+ }
+
+ tokens = []
+ for part in parts:
+ if "(" in part and ")" in part:
+ code_tokens = re.findall(
+ r'[\w_]+(?=\()|[\w_]+(?==[\'"]{1}[\w_]+[\'"]{1})', part
+ )
+ tokens.extend(code_tokens)
+
+ words = word_tokenize(part.lower())
+ tokens.extend(
+ [
+ lemmatizer.lemmatize(token)
+ for token in words
+ if token not in stop_words
+ ]
+ )
+
+ return tokens
+
+ def maybe_load_bm25_index(self, clear_cache=False) -> bool:
+ """
+ Load existing BM25 index from disk, if present and clear_cache=False.
+ """
+ if not clear_cache and os.path.exists(self.bm25_index_file):
+ self.logger.info("Loading existing BM25 index from disk.")
+ with open(self.bm25_index_file, "rb") as f:
+ data = pickle.load(f)
+ self.tokenized_facts = data["tokenized_facts"]
+ self.bm25_index = data["bm25_index"]
+ return True
+ return False
+
+ def build_search_index(self, clear_cache=False) -> None:
+ """
+ Checks for new or modified .q.md files by comparing file-hash.
+ If none need reindexing and clear_cache is False, loads existing index if available.
+ Otherwise, reindexes only changed/new files and merges or creates a new index.
+ """
+ # If clear_cache is True, we skip partial logic: rebuild everything from scratch
+ if clear_cache:
+ self.logger.info("Clearing cache and rebuilding full search index.")
+ if self.bm25_index_file.exists():
+ self.bm25_index_file.unlink()
+
+ process = psutil.Process()
+ self.logger.info("Checking which .q.md files need (re)indexing...")
+
+ # Gather all .q.md files
+ q_files = [self.docs_dir / f for f in os.listdir(self.docs_dir) if f.endswith(".q.md")]
+
+ # We'll store known (unchanged) facts in these lists
+ existing_facts: List[str] = []
+ existing_tokens: List[List[str]] = []
+
+ # Keep track of invalid lines for logging
+ invalid_lines = []
+ needSet = [] # files that must be (re)indexed
+
+ for qf in q_files:
+ token_cache_file = qf.with_suffix(".q.tokens")
+
+ # If no .q.tokens or clear_cache is True → definitely reindex
+ if clear_cache or not token_cache_file.exists():
+ needSet.append(qf)
+ continue
+
+ # Otherwise, load the existing cache and compare hash
+ cache = self._load_or_create_token_cache(qf)
+ # If the .q.tokens was out of date (i.e. changed hash), we reindex
+ if len(cache["facts"]) == 0 or cache.get("content_hash") != _compute_file_hash(qf):
+ needSet.append(qf)
+ else:
+ # File is unchanged → retrieve cached token data
+ for line, cache_data in cache["facts"].items():
+ existing_facts.append(line)
+ existing_tokens.append(cache_data["tokens"])
+ self.document_map[line] = qf # track the doc for that fact
+
+ if not needSet and not clear_cache:
+ # If no file needs reindexing, try loading existing index
+ if self.maybe_load_bm25_index(clear_cache=False):
+ self.logger.info("No new/changed .q.md files found. Using existing BM25 index.")
+ return
+ else:
+ # If there's no existing index, we must build a fresh index from the old caches
+ self.logger.info("No existing BM25 index found. Building from cached facts.")
+ if existing_facts:
+ self.logger.info(f"Building BM25 index with {len(existing_facts)} cached facts.")
+ self.bm25_index = BM25Okapi(existing_tokens)
+ self.tokenized_facts = existing_facts
+ with open(self.bm25_index_file, "wb") as f:
+ pickle.dump({
+ "bm25_index": self.bm25_index,
+ "tokenized_facts": self.tokenized_facts
+ }, f)
+ else:
+ self.logger.warning("No facts found at all. Index remains empty.")
+ return
+
+ # ----------------------------------------------------- /Users/unclecode/.crawl4ai/docs/14_proxy_security.q.q.tokens '/Users/unclecode/.crawl4ai/docs/14_proxy_security.q.md'
+ # If we reach here, we have new or changed .q.md files
+ # We'll parse them, reindex them, and then combine with existing_facts
+ # -----------------------------------------------------
+
+ self.logger.info(f"{len(needSet)} file(s) need reindexing. Parsing now...")
+
+ # 1) Parse the new or changed .q.md files
+ new_facts = []
+ new_tokens = []
+ with tqdm(total=len(needSet), desc="Indexing changed files") as file_pbar:
+ for file in needSet:
+ # We'll build up a fresh cache
+ fresh_cache = {"facts": {}, "content_hash": _compute_file_hash(file)}
+ try:
+ with open(file, "r", encoding="utf-8") as f_obj:
+ content = f_obj.read().strip()
+ lines = [l.strip() for l in content.split("\n") if l.strip()]
+
+ for line in lines:
+ is_valid, error = self._validate_fact_line(line)
+ if not is_valid:
+ invalid_lines.append((file, line, error))
+ continue
+
+ tokens = self.preprocess_text(line)
+ fresh_cache["facts"][line] = {
+ "tokens": tokens,
+ "added": time.time(),
+ }
+ new_facts.append(line)
+ new_tokens.append(tokens)
+ self.document_map[line] = file
+
+ # Save the new .q.tokens with updated hash
+ self._save_token_cache(file, fresh_cache)
+
+ mem_usage = process.memory_info().rss / 1024 / 1024
+ self.logger.debug(f"Memory usage after {file.name}: {mem_usage:.2f}MB")
+
+ except Exception as e:
+ self.logger.error(f"Error processing {file}: {str(e)}")
+
+ file_pbar.update(1)
+
+ if invalid_lines:
+ self.logger.warning(f"Found {len(invalid_lines)} invalid fact lines:")
+ for file, line, error in invalid_lines:
+ self.logger.warning(f"{file}: {error} in line: {line[:50]}...")
+
+ # 2) Merge newly tokenized facts with the existing ones
+ all_facts = existing_facts + new_facts
+ all_tokens = existing_tokens + new_tokens
+
+ # 3) Build BM25 index from combined facts
+ self.logger.info(f"Building BM25 index with {len(all_facts)} total facts (old + new).")
+ self.bm25_index = BM25Okapi(all_tokens)
+ self.tokenized_facts = all_facts
+
+ # 4) Save the updated BM25 index to disk
+ with open(self.bm25_index_file, "wb") as f:
+ pickle.dump({
+ "bm25_index": self.bm25_index,
+ "tokenized_facts": self.tokenized_facts
+ }, f)
+
+ final_mem = process.memory_info().rss / 1024 / 1024
+ self.logger.info(f"Search index updated. Final memory usage: {final_mem:.2f}MB")
+
+ async def generate_index_files(self, force_generate_facts: bool = False, clear_bm25_cache: bool = False) -> None:
+ """
+ Generate index files for all documents in parallel batches
+
+ Args:
+ force_generate_facts (bool): If True, regenerate indexes even if they exist
+ clear_bm25_cache (bool): If True, clear existing BM25 index cache
+ """
+ self.logger.info("Starting index generation for documentation files.")
+
+ md_files = [
+ self.docs_dir / f for f in os.listdir(self.docs_dir)
+ if f.endswith('.md') and not any(f.endswith(x) for x in ['.q.md', '.xs.md'])
+ ]
+
+ # Filter out files that already have .q files unless force=True
+ if not force_generate_facts:
+ md_files = [
+ f for f in md_files
+ if not (self.docs_dir / f.name.replace('.md', '.q.md')).exists()
+ ]
+
+ if not md_files:
+ self.logger.info("All index files exist. Use force=True to regenerate.")
+ else:
+ # Process documents in batches
+ for i in range(0, len(md_files), self.batch_size):
+ batch = md_files[i:i + self.batch_size]
+ self.logger.info(f"Processing batch {i//self.batch_size + 1}/{(len(md_files)//self.batch_size) + 1}")
+ await self._process_document_batch(batch)
+
+ self.logger.info("Index generation complete, building/updating search index.")
+ self.build_search_index(clear_cache=clear_bm25_cache)
+
+ def generate(self, sections: List[str], mode: str = "extended") -> str:
+ # Get all markdown files
+ all_files = glob.glob(str(self.docs_dir / "[0-9]*.md")) + \
+ glob.glob(str(self.docs_dir / "[0-9]*.xs.md"))
+
+ # Extract base names without extensions
+ base_docs = {Path(f).name.split('.')[0] for f in all_files
+ if not Path(f).name.endswith('.q.md')}
+
+ # Filter by sections if provided
+ if sections:
+ base_docs = {doc for doc in base_docs
+ if any(section.lower() in doc.lower() for section in sections)}
+
+ # Get file paths based on mode
+ files = []
+ for doc in sorted(base_docs, key=lambda x: int(x.split('_')[0]) if x.split('_')[0].isdigit() else 999999):
+ if mode == "condensed":
+ xs_file = self.docs_dir / f"{doc}.xs.md"
+ regular_file = self.docs_dir / f"{doc}.md"
+ files.append(str(xs_file if xs_file.exists() else regular_file))
+ else:
+ files.append(str(self.docs_dir / f"{doc}.md"))
+
+ # Read and format content
+ content = []
+ for file in files:
+ try:
+ with open(file, 'r', encoding='utf-8') as f:
+ fname = Path(file).name
+ content.append(f"{'#'*20}\n# {fname}\n{'#'*20}\n\n{f.read()}")
+ except Exception as e:
+ self.logger.error(f"Error reading {file}: {str(e)}")
+
+ return "\n\n---\n\n".join(content) if content else ""
+
+ def search(self, query: str, top_k: int = 5) -> str:
+ if not self.bm25_index:
+ return "No search index available. Call build_search_index() first."
+
+ query_tokens = self.preprocess_text(query)
+ doc_scores = self.bm25_index.get_scores(query_tokens)
+
+ mean_score = np.mean(doc_scores)
+ std_score = np.std(doc_scores)
+ score_threshold = mean_score + (0.25 * std_score)
+
+ file_data = self._aggregate_search_scores(
+ doc_scores=doc_scores,
+ score_threshold=score_threshold,
+ query_tokens=query_tokens,
+ )
+
+ ranked_files = sorted(
+ file_data.items(),
+ key=lambda x: (
+ x[1]["code_match_score"] * 2.0
+ + x[1]["match_count"] * 1.5
+ + x[1]["total_score"]
+ ),
+ reverse=True,
+ )[:top_k]
+
+ results = []
+ for file, _ in ranked_files:
+ main_doc = str(file).replace(".q.md", ".md")
+ if os.path.exists(self.docs_dir / main_doc):
+ with open(self.docs_dir / main_doc, "r", encoding='utf-8') as f:
+ only_file_name = main_doc.split("/")[-1]
+ content = [
+ "#" * 20,
+ f"# {only_file_name}",
+ "#" * 20,
+ "",
+ f.read()
+ ]
+ results.append("\n".join(content))
+
+ return "\n\n---\n\n".join(results)
+
+ def _aggregate_search_scores(
+ self, doc_scores: List[float], score_threshold: float, query_tokens: List[str]
+ ) -> Dict:
+ file_data = {}
+
+ for idx, score in enumerate(doc_scores):
+ if score <= score_threshold:
+ continue
+
+ fact = self.tokenized_facts[idx]
+ file_path = self.document_map[fact]
+
+ if file_path not in file_data:
+ file_data[file_path] = {
+ "total_score": 0,
+ "match_count": 0,
+ "code_match_score": 0,
+ "matched_facts": [],
+ }
+
+ components = fact.split("|") if "|" in fact else [fact]
+
+ code_match_score = 0
+ if len(components) == 3:
+ code_ref = components[2].strip()
+ code_tokens = self.preprocess_text(code_ref)
+ code_match_score = len(set(query_tokens) & set(code_tokens)) / len(query_tokens)
+
+ file_data[file_path]["total_score"] += score
+ file_data[file_path]["match_count"] += 1
+ file_data[file_path]["code_match_score"] = max(
+ file_data[file_path]["code_match_score"], code_match_score
+ )
+ file_data[file_path]["matched_facts"].append(fact)
+
+ return file_data
+
+ def refresh_index(self) -> None:
+ """Convenience method for a full rebuild."""
+ self.build_search_index(clear_cache=True)
diff --git a/crawl4ai/markdown_generation_strategy.py b/crawl4ai/markdown_generation_strategy.py
new file mode 100644
index 0000000000000000000000000000000000000000..89e5e34e624c7212cffe1e19cb4e00bbcf2bfa5f
--- /dev/null
+++ b/crawl4ai/markdown_generation_strategy.py
@@ -0,0 +1,225 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any, Tuple
+from .models import MarkdownGenerationResult
+from .html2text import CustomHTML2Text
+from .content_filter_strategy import RelevantContentFilter, BM25ContentFilter
+import re
+from urllib.parse import urljoin
+
+# Pre-compile the regex pattern
+LINK_PATTERN = re.compile(r'!?\[([^\]]+)\]\(([^)]+?)(?:\s+"([^"]*)")?\)')
+
+def fast_urljoin(base: str, url: str) -> str:
+ """Fast URL joining for common cases."""
+ if url.startswith(('http://', 'https://', 'mailto:', '//')):
+ return url
+ if url.startswith('/'):
+ # Handle absolute paths
+ if base.endswith('/'):
+ return base[:-1] + url
+ return base + url
+ return urljoin(base, url)
+
+class MarkdownGenerationStrategy(ABC):
+ """Abstract base class for markdown generation strategies."""
+ def __init__(self, content_filter: Optional[RelevantContentFilter] = None, options: Optional[Dict[str, Any]] = None):
+ self.content_filter = content_filter
+ self.options = options or {}
+
+ @abstractmethod
+ def generate_markdown(self,
+ cleaned_html: str,
+ base_url: str = "",
+ html2text_options: Optional[Dict[str, Any]] = None,
+ content_filter: Optional[RelevantContentFilter] = None,
+ citations: bool = True,
+ **kwargs) -> MarkdownGenerationResult:
+ """Generate markdown from cleaned HTML."""
+ pass
+
+class DefaultMarkdownGenerator(MarkdownGenerationStrategy):
+ """
+ Default implementation of markdown generation strategy.
+
+ How it works:
+ 1. Generate raw markdown from cleaned HTML.
+ 2. Convert links to citations.
+ 3. Generate fit markdown if content filter is provided.
+ 4. Return MarkdownGenerationResult.
+
+ Args:
+ content_filter (Optional[RelevantContentFilter]): Content filter for generating fit markdown.
+ options (Optional[Dict[str, Any]]): Additional options for markdown generation. Defaults to None.
+
+ Returns:
+ MarkdownGenerationResult: Result containing raw markdown, fit markdown, fit HTML, and references markdown.
+ """
+ def __init__(self, content_filter: Optional[RelevantContentFilter] = None, options: Optional[Dict[str, Any]] = None):
+ super().__init__(content_filter, options)
+
+ def convert_links_to_citations(self, markdown: str, base_url: str = "") -> Tuple[str, str]:
+ """
+ Convert links in markdown to citations.
+
+ How it works:
+ 1. Find all links in the markdown.
+ 2. Convert links to citations.
+ 3. Return converted markdown and references markdown.
+
+ Note:
+ This function uses a regex pattern to find links in markdown.
+
+ Args:
+ markdown (str): Markdown text.
+ base_url (str): Base URL for URL joins.
+
+ Returns:
+ Tuple[str, str]: Converted markdown and references markdown.
+ """
+ link_map = {}
+ url_cache = {} # Cache for URL joins
+ parts = []
+ last_end = 0
+ counter = 1
+
+ for match in LINK_PATTERN.finditer(markdown):
+ parts.append(markdown[last_end:match.start()])
+ text, url, title = match.groups()
+
+ # Use cached URL if available, otherwise compute and cache
+ if base_url and not url.startswith(('http://', 'https://', 'mailto:')):
+ if url not in url_cache:
+ url_cache[url] = fast_urljoin(base_url, url)
+ url = url_cache[url]
+
+ if url not in link_map:
+ desc = []
+ if title: desc.append(title)
+ if text and text != title: desc.append(text)
+ link_map[url] = (counter, ": " + " - ".join(desc) if desc else "")
+ counter += 1
+
+ num = link_map[url][0]
+ parts.append(f"{text}⟨{num}⟩" if not match.group(0).startswith('!') else f"![{text}⟨{num}⟩]")
+ last_end = match.end()
+
+ parts.append(markdown[last_end:])
+ converted_text = ''.join(parts)
+
+ # Pre-build reference strings
+ references = ["\n\n## References\n\n"]
+ references.extend(
+ f"⟨{num}⟩ {url}{desc}\n"
+ for url, (num, desc) in sorted(link_map.items(), key=lambda x: x[1][0])
+ )
+
+ return converted_text, ''.join(references)
+
+ def generate_markdown(self,
+ cleaned_html: str,
+ base_url: str = "",
+ html2text_options: Optional[Dict[str, Any]] = None,
+ options: Optional[Dict[str, Any]] = None,
+ content_filter: Optional[RelevantContentFilter] = None,
+ citations: bool = True,
+ **kwargs) -> MarkdownGenerationResult:
+ """
+ Generate markdown with citations from cleaned HTML.
+
+ How it works:
+ 1. Generate raw markdown from cleaned HTML.
+ 2. Convert links to citations.
+ 3. Generate fit markdown if content filter is provided.
+ 4. Return MarkdownGenerationResult.
+
+ Args:
+ cleaned_html (str): Cleaned HTML content.
+ base_url (str): Base URL for URL joins.
+ html2text_options (Optional[Dict[str, Any]]): HTML2Text options.
+ options (Optional[Dict[str, Any]]): Additional options for markdown generation.
+ content_filter (Optional[RelevantContentFilter]): Content filter for generating fit markdown.
+ citations (bool): Whether to generate citations.
+
+ Returns:
+ MarkdownGenerationResult: Result containing raw markdown, fit markdown, fit HTML, and references markdown.
+ """
+ try:
+ # Initialize HTML2Text with default options for better conversion
+ h = CustomHTML2Text(baseurl=base_url)
+ default_options = {
+ 'body_width': 0, # Disable text wrapping
+ 'ignore_emphasis': False,
+ 'ignore_links': False,
+ 'ignore_images': False,
+ 'protect_links': True,
+ 'single_line_break': True,
+ 'mark_code': True,
+ 'escape_snob': False
+ }
+
+ # Update with custom options if provided
+ if html2text_options:
+ default_options.update(html2text_options)
+ elif options:
+ default_options.update(options)
+ elif self.options:
+ default_options.update(self.options)
+
+ h.update_params(**default_options)
+
+ # Ensure we have valid input
+ if not cleaned_html:
+ cleaned_html = ""
+ elif not isinstance(cleaned_html, str):
+ cleaned_html = str(cleaned_html)
+
+ # Generate raw markdown
+ try:
+ raw_markdown = h.handle(cleaned_html)
+ except Exception as e:
+ raw_markdown = f"Error converting HTML to markdown: {str(e)}"
+
+ raw_markdown = raw_markdown.replace(' ```', '```')
+
+ # Convert links to citations
+ markdown_with_citations: str = raw_markdown
+ references_markdown: str = ""
+ if citations:
+ try:
+ markdown_with_citations, references_markdown = self.convert_links_to_citations(
+ raw_markdown, base_url
+ )
+ except Exception as e:
+ markdown_with_citations = raw_markdown
+ references_markdown = f"Error generating citations: {str(e)}"
+
+ # Generate fit markdown if content filter is provided
+ fit_markdown: Optional[str] = ""
+ filtered_html: Optional[str] = ""
+ if content_filter or self.content_filter:
+ try:
+ content_filter = content_filter or self.content_filter
+ filtered_html = content_filter.filter_content(cleaned_html)
+ filtered_html = '\n'.join('{}
'.format(s) for s in filtered_html)
+ fit_markdown = h.handle(filtered_html)
+ except Exception as e:
+ fit_markdown = f"Error generating fit markdown: {str(e)}"
+ filtered_html = ""
+
+ return MarkdownGenerationResult(
+ raw_markdown=raw_markdown or "",
+ markdown_with_citations=markdown_with_citations or "",
+ references_markdown=references_markdown or "",
+ fit_markdown=fit_markdown or "",
+ fit_html=filtered_html or "",
+ )
+ except Exception as e:
+ # If anything fails, return empty strings with error message
+ error_msg = f"Error in markdown generation: {str(e)}"
+ return MarkdownGenerationResult(
+ raw_markdown=error_msg,
+ markdown_with_citations=error_msg,
+ references_markdown="",
+ fit_markdown="",
+ fit_html="",
+ )
diff --git a/crawl4ai/migrations.py b/crawl4ai/migrations.py
new file mode 100644
index 0000000000000000000000000000000000000000..3386b0fb433b4ba116476ee225606eab2cb3a956
--- /dev/null
+++ b/crawl4ai/migrations.py
@@ -0,0 +1,168 @@
+import os
+import asyncio
+import logging
+from pathlib import Path
+import aiosqlite
+from typing import Optional
+import xxhash
+import aiofiles
+import shutil
+import time
+from datetime import datetime
+from .async_logger import AsyncLogger, LogLevel
+
+# Initialize logger
+logger = AsyncLogger(log_level=LogLevel.DEBUG, verbose=True)
+
+# logging.basicConfig(level=logging.INFO)
+# logger = logging.getLogger(__name__)
+
+class DatabaseMigration:
+ def __init__(self, db_path: str):
+ self.db_path = db_path
+ self.content_paths = self._ensure_content_dirs(os.path.dirname(db_path))
+
+ def _ensure_content_dirs(self, base_path: str) -> dict:
+ dirs = {
+ 'html': 'html_content',
+ 'cleaned': 'cleaned_html',
+ 'markdown': 'markdown_content',
+ 'extracted': 'extracted_content',
+ 'screenshots': 'screenshots'
+ }
+ content_paths = {}
+ for key, dirname in dirs.items():
+ path = os.path.join(base_path, dirname)
+ os.makedirs(path, exist_ok=True)
+ content_paths[key] = path
+ return content_paths
+
+ def _generate_content_hash(self, content: str) -> str:
+ x = xxhash.xxh64()
+ x.update(content.encode())
+ content_hash = x.hexdigest()
+ return content_hash
+ # return hashlib.sha256(content.encode()).hexdigest()
+
+ async def _store_content(self, content: str, content_type: str) -> str:
+ if not content:
+ return ""
+
+ content_hash = self._generate_content_hash(content)
+ file_path = os.path.join(self.content_paths[content_type], content_hash)
+
+ if not os.path.exists(file_path):
+ async with aiofiles.open(file_path, 'w', encoding='utf-8') as f:
+ await f.write(content)
+
+ return content_hash
+
+ async def migrate_database(self):
+ """Migrate existing database to file-based storage"""
+ # logger.info("Starting database migration...")
+ logger.info("Starting database migration...", tag="INIT")
+
+ try:
+ async with aiosqlite.connect(self.db_path) as db:
+ # Get all rows
+ async with db.execute(
+ '''SELECT url, html, cleaned_html, markdown,
+ extracted_content, screenshot FROM crawled_data'''
+ ) as cursor:
+ rows = await cursor.fetchall()
+
+ migrated_count = 0
+ for row in rows:
+ url, html, cleaned_html, markdown, extracted_content, screenshot = row
+
+ # Store content in files and get hashes
+ html_hash = await self._store_content(html, 'html')
+ cleaned_hash = await self._store_content(cleaned_html, 'cleaned')
+ markdown_hash = await self._store_content(markdown, 'markdown')
+ extracted_hash = await self._store_content(extracted_content, 'extracted')
+ screenshot_hash = await self._store_content(screenshot, 'screenshots')
+
+ # Update database with hashes
+ await db.execute('''
+ UPDATE crawled_data
+ SET html = ?,
+ cleaned_html = ?,
+ markdown = ?,
+ extracted_content = ?,
+ screenshot = ?
+ WHERE url = ?
+ ''', (html_hash, cleaned_hash, markdown_hash,
+ extracted_hash, screenshot_hash, url))
+
+ migrated_count += 1
+ if migrated_count % 100 == 0:
+ logger.info(f"Migrated {migrated_count} records...", tag="INIT")
+
+
+ await db.commit()
+ logger.success(f"Migration completed. {migrated_count} records processed.", tag="COMPLETE")
+
+ except Exception as e:
+ # logger.error(f"Migration failed: {e}")
+ logger.error(
+ message="Migration failed: {error}",
+ tag="ERROR",
+ params={"error": str(e)}
+ )
+ raise e
+
+async def backup_database(db_path: str) -> str:
+ """Create backup of existing database"""
+ if not os.path.exists(db_path):
+ logger.info("No existing database found. Skipping backup.", tag="INIT")
+ return None
+
+ # Create backup with timestamp
+ timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
+ backup_path = f"{db_path}.backup_{timestamp}"
+
+ try:
+ # Wait for any potential write operations to finish
+ await asyncio.sleep(1)
+
+ # Create backup
+ shutil.copy2(db_path, backup_path)
+ logger.info(f"Database backup created at: {backup_path}", tag="COMPLETE")
+ return backup_path
+ except Exception as e:
+ # logger.error(f"Backup failed: {e}")
+ logger.error(
+ message="Migration failed: {error}",
+ tag="ERROR",
+ params={"error": str(e)}
+ )
+ raise e
+
+async def run_migration(db_path: Optional[str] = None):
+ """Run database migration"""
+ if db_path is None:
+ db_path = os.path.join(Path.home(), ".crawl4ai", "crawl4ai.db")
+
+ if not os.path.exists(db_path):
+ logger.info("No existing database found. Skipping migration.", tag="INIT")
+ return
+
+ # Create backup first
+ backup_path = await backup_database(db_path)
+ if not backup_path:
+ return
+
+ migration = DatabaseMigration(db_path)
+ await migration.migrate_database()
+
+def main():
+ """CLI entry point for migration"""
+ import argparse
+ parser = argparse.ArgumentParser(description='Migrate Crawl4AI database to file-based storage')
+ parser.add_argument('--db-path', help='Custom database path')
+ args = parser.parse_args()
+
+ asyncio.run(run_migration(args.db_path))
+
+if __name__ == "__main__":
+ main()
\ No newline at end of file
diff --git a/crawl4ai/model_loader.py b/crawl4ai/model_loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..d1872d7e48decbb1860caf25684a6d27004f19e0
--- /dev/null
+++ b/crawl4ai/model_loader.py
@@ -0,0 +1,256 @@
+from functools import lru_cache
+from pathlib import Path
+import subprocess, os
+import shutil
+import tarfile
+from .model_loader import *
+import argparse
+import urllib.request
+from crawl4ai.config import MODEL_REPO_BRANCH
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+@lru_cache()
+def get_available_memory(device):
+ import torch
+ if device.type == 'cuda':
+ return torch.cuda.get_device_properties(device).total_memory
+ elif device.type == 'mps':
+ return 48 * 1024 ** 3 # Assuming 8GB for MPS, as a conservative estimate
+ else:
+ return 0
+
+@lru_cache()
+def calculate_batch_size(device):
+ available_memory = get_available_memory(device)
+
+ if device.type == 'cpu':
+ return 16
+ elif device.type in ['cuda', 'mps']:
+ # Adjust these thresholds based on your model size and available memory
+ if available_memory >= 31 * 1024 ** 3: # > 32GB
+ return 256
+ elif available_memory >= 15 * 1024 ** 3: # > 16GB to 32GB
+ return 128
+ elif available_memory >= 8 * 1024 ** 3: # 8GB to 16GB
+ return 64
+ else:
+ return 32
+ else:
+ return 16 # Default batch size
+
+@lru_cache()
+def get_device():
+ import torch
+ if torch.cuda.is_available():
+ device = torch.device('cuda')
+ elif torch.backends.mps.is_available():
+ device = torch.device('mps')
+ else:
+ device = torch.device('cpu')
+ return device
+
+def set_model_device(model):
+ device = get_device()
+ model.to(device)
+ return model, device
+
+@lru_cache()
+def get_home_folder():
+ home_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+ os.makedirs(home_folder, exist_ok=True)
+ os.makedirs(f"{home_folder}/cache", exist_ok=True)
+ os.makedirs(f"{home_folder}/models", exist_ok=True)
+ return home_folder
+
+@lru_cache()
+def load_bert_base_uncased():
+ from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
+ tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', resume_download=None)
+ model = BertModel.from_pretrained('bert-base-uncased', resume_download=None)
+ model.eval()
+ model, device = set_model_device(model)
+ return tokenizer, model
+
+@lru_cache()
+def load_HF_embedding_model(model_name="BAAI/bge-small-en-v1.5") -> tuple:
+ """Load the Hugging Face model for embedding.
+
+ Args:
+ model_name (str, optional): The model name to load. Defaults to "BAAI/bge-small-en-v1.5".
+
+ Returns:
+ tuple: The tokenizer and model.
+ """
+ from transformers import BertTokenizer, BertModel, AutoTokenizer, AutoModel
+ tokenizer = AutoTokenizer.from_pretrained(model_name, resume_download=None)
+ model = AutoModel.from_pretrained(model_name, resume_download=None)
+ model.eval()
+ model, device = set_model_device(model)
+ return tokenizer, model
+
+@lru_cache()
+def load_text_classifier():
+ from transformers import AutoTokenizer, AutoModelForSequenceClassification
+ from transformers import pipeline
+ import torch
+
+ tokenizer = AutoTokenizer.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
+ model = AutoModelForSequenceClassification.from_pretrained("dstefa/roberta-base_topic_classification_nyt_news")
+ model.eval()
+ model, device = set_model_device(model)
+ pipe = pipeline("text-classification", model=model, tokenizer=tokenizer)
+ return pipe
+
+@lru_cache()
+def load_text_multilabel_classifier():
+ from transformers import AutoModelForSequenceClassification, AutoTokenizer
+ import numpy as np
+ from scipy.special import expit
+ import torch
+
+ # # Check for available device: CUDA, MPS (for Apple Silicon), or CPU
+ # if torch.cuda.is_available():
+ # device = torch.device("cuda")
+ # elif torch.backends.mps.is_available():
+ # device = torch.device("mps")
+ # else:
+ # device = torch.device("cpu")
+ # # return load_spacy_model(), torch.device("cpu")
+
+
+ MODEL = "cardiffnlp/tweet-topic-21-multi"
+ tokenizer = AutoTokenizer.from_pretrained(MODEL, resume_download=None)
+ model = AutoModelForSequenceClassification.from_pretrained(MODEL, resume_download=None)
+ model.eval()
+ model, device = set_model_device(model)
+ class_mapping = model.config.id2label
+
+ def _classifier(texts, threshold=0.5, max_length=64):
+ tokens = tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=max_length)
+ tokens = {key: val.to(device) for key, val in tokens.items()} # Move tokens to the selected device
+
+ with torch.no_grad():
+ output = model(**tokens)
+
+ scores = output.logits.detach().cpu().numpy()
+ scores = expit(scores)
+ predictions = (scores >= threshold) * 1
+
+ batch_labels = []
+ for prediction in predictions:
+ labels = [class_mapping[i] for i, value in enumerate(prediction) if value == 1]
+ batch_labels.append(labels)
+
+ return batch_labels
+
+ return _classifier, device
+
+@lru_cache()
+def load_nltk_punkt():
+ import nltk
+ try:
+ nltk.data.find('tokenizers/punkt')
+ except LookupError:
+ nltk.download('punkt')
+ return nltk.data.find('tokenizers/punkt')
+
+@lru_cache()
+def load_spacy_model():
+ import spacy
+ name = "models/reuters"
+ home_folder = get_home_folder()
+ model_folder = Path(home_folder) / name
+
+ # Check if the model directory already exists
+ if not (model_folder.exists() and any(model_folder.iterdir())):
+ repo_url = "https://github.com/unclecode/crawl4ai.git"
+ branch = MODEL_REPO_BRANCH
+ repo_folder = Path(home_folder) / "crawl4ai"
+
+ print("[LOG] ⏬ Downloading Spacy model for the first time...")
+
+ # Remove existing repo folder if it exists
+ if repo_folder.exists():
+ try:
+ shutil.rmtree(repo_folder)
+ if model_folder.exists():
+ shutil.rmtree(model_folder)
+ except PermissionError:
+ print("[WARNING] Unable to remove existing folders. Please manually delete the following folders and try again:")
+ print(f"- {repo_folder}")
+ print(f"- {model_folder}")
+ return None
+
+ try:
+ # Clone the repository
+ subprocess.run(
+ ["git", "clone", "-b", branch, repo_url, str(repo_folder)],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ check=True
+ )
+
+ # Create the models directory if it doesn't exist
+ models_folder = Path(home_folder) / "models"
+ models_folder.mkdir(parents=True, exist_ok=True)
+
+ # Copy the reuters model folder to the models directory
+ source_folder = repo_folder / "models" / "reuters"
+ shutil.copytree(source_folder, model_folder)
+
+ # Remove the cloned repository
+ shutil.rmtree(repo_folder)
+
+ print("[LOG] ✅ Spacy Model downloaded successfully")
+ except subprocess.CalledProcessError as e:
+ print(f"An error occurred while cloning the repository: {e}")
+ return None
+ except Exception as e:
+ print(f"An error occurred: {e}")
+ return None
+
+ try:
+ return spacy.load(str(model_folder))
+ except Exception as e:
+ print(f"Error loading spacy model: {e}")
+ return None
+
+def download_all_models(remove_existing=False):
+ """Download all models required for Crawl4AI."""
+ if remove_existing:
+ print("[LOG] Removing existing models...")
+ home_folder = get_home_folder()
+ model_folders = [
+ os.path.join(home_folder, "models/reuters"),
+ os.path.join(home_folder, "models"),
+ ]
+ for folder in model_folders:
+ if Path(folder).exists():
+ shutil.rmtree(folder)
+ print("[LOG] Existing models removed.")
+
+ # Load each model to trigger download
+ # print("[LOG] Downloading BERT Base Uncased...")
+ # load_bert_base_uncased()
+ # print("[LOG] Downloading BGE Small EN v1.5...")
+ # load_bge_small_en_v1_5()
+ # print("[LOG] Downloading ONNX model...")
+ # load_onnx_all_MiniLM_l6_v2()
+ print("[LOG] Downloading text classifier...")
+ _, device = load_text_multilabel_classifier()
+ print(f"[LOG] Text classifier loaded on {device}")
+ print("[LOG] Downloading custom NLTK Punkt model...")
+ load_nltk_punkt()
+ print("[LOG] ✅ All models downloaded successfully.")
+
+def main():
+ print("[LOG] Welcome to the Crawl4AI Model Downloader!")
+ print("[LOG] This script will download all the models required for Crawl4AI.")
+ parser = argparse.ArgumentParser(description="Crawl4AI Model Downloader")
+ parser.add_argument('--remove-existing', action='store_true', help="Remove existing models before downloading")
+ args = parser.parse_args()
+
+ download_all_models(remove_existing=args.remove_existing)
+
+if __name__ == "__main__":
+ main()
diff --git a/crawl4ai/models.py b/crawl4ai/models.py
new file mode 100644
index 0000000000000000000000000000000000000000..6fb362a349fae94fdf96f8f5b1518371930b6e7b
--- /dev/null
+++ b/crawl4ai/models.py
@@ -0,0 +1,61 @@
+from pydantic import BaseModel, HttpUrl
+from typing import List, Dict, Optional, Callable, Awaitable, Union, Any
+from dataclasses import dataclass
+from .ssl_certificate import SSLCertificate
+
+@dataclass
+class TokenUsage:
+ completion_tokens: int = 0
+ prompt_tokens: int = 0
+ total_tokens: int = 0
+ completion_tokens_details: Optional[dict] = None
+ prompt_tokens_details: Optional[dict] = None
+
+
+class UrlModel(BaseModel):
+ url: HttpUrl
+ forced: bool = False
+
+class MarkdownGenerationResult(BaseModel):
+ raw_markdown: str
+ markdown_with_citations: str
+ references_markdown: str
+ fit_markdown: Optional[str] = None
+ fit_html: Optional[str] = None
+
+class CrawlResult(BaseModel):
+ url: str
+ html: str
+ success: bool
+ cleaned_html: Optional[str] = None
+ media: Dict[str, List[Dict]] = {}
+ links: Dict[str, List[Dict]] = {}
+ downloaded_files: Optional[List[str]] = None
+ screenshot: Optional[str] = None
+ pdf : Optional[bytes] = None
+ markdown: Optional[Union[str, MarkdownGenerationResult]] = None
+ markdown_v2: Optional[MarkdownGenerationResult] = None
+ fit_markdown: Optional[str] = None
+ fit_html: Optional[str] = None
+ extracted_content: Optional[str] = None
+ metadata: Optional[dict] = None
+ error_message: Optional[str] = None
+ session_id: Optional[str] = None
+ response_headers: Optional[dict] = None
+ status_code: Optional[int] = None
+ ssl_certificate: Optional[SSLCertificate] = None
+ class Config:
+ arbitrary_types_allowed = True
+
+class AsyncCrawlResponse(BaseModel):
+ html: str
+ response_headers: Dict[str, str]
+ status_code: int
+ screenshot: Optional[str] = None
+ pdf_data: Optional[bytes] = None
+ get_delayed_content: Optional[Callable[[Optional[float]], Awaitable[str]]] = None
+ downloaded_files: Optional[List[str]] = None
+ ssl_certificate: Optional[SSLCertificate] = None
+
+ class Config:
+ arbitrary_types_allowed = True
diff --git a/crawl4ai/prompts.py b/crawl4ai/prompts.py
new file mode 100644
index 0000000000000000000000000000000000000000..7a963e6d9215da02def72b686cd87abb8ebfed5b
--- /dev/null
+++ b/crawl4ai/prompts.py
@@ -0,0 +1,204 @@
+PROMPT_EXTRACT_BLOCKS = """Here is the URL of the webpage:
+{URL}
+
+And here is the cleaned HTML content of that webpage:
+
+{HTML}
+
+
+Your task is to break down this HTML content into semantically relevant blocks, and for each block, generate a JSON object with the following keys:
+
+- index: an integer representing the index of the block in the content
+- tags: a list of semantic tags that are relevant to the content of the block
+- content: a list of strings containing the text content of the block
+- questions: a list of 3 questions that a user may ask about the content in this block
+
+To generate the JSON objects:
+
+1. Carefully read through the HTML content and identify logical breaks or shifts in the content that would warrant splitting it into separate blocks.
+
+2. For each block:
+ a. Assign it an index based on its order in the content.
+ b. Analyze the content and generate a list of relevant semantic tags that describe what the block is about.
+ c. Extract the text content, clean it up if needed, and store it as a list of strings in the "content" field.
+ d. Come up with 3 questions that a user might ask about this specific block of content, based on the tags and content. The questions should be relevant and answerable by the content in the block.
+
+3. Ensure that the order of the JSON objects matches the order of the blocks as they appear in the original HTML content.
+
+4. Double-check that each JSON object includes all required keys (index, tags, content, questions) and that the values are in the expected format (integer, list of strings, etc.).
+
+5. Make sure the generated JSON is complete and parsable, with no errors or omissions.
+
+6. Make sure to escape any special characters in the HTML content, and also single or double quote to avoid JSON parsing issues.
+
+Please provide your output within tags, like this:
+
+
+[{
+ "index": 0,
+ "tags": ["introduction", "overview"],
+ "content": ["This is the first paragraph of the article, which provides an introduction and overview of the main topic."],
+ "questions": [
+ "What is the main topic of this article?",
+ "What can I expect to learn from reading this article?",
+ "Is this article suitable for beginners or experts in the field?"
+ ]
+},
+{
+ "index": 1,
+ "tags": ["history", "background"],
+ "content": ["This is the second paragraph, which delves into the history and background of the topic.",
+ "It provides context and sets the stage for the rest of the article."],
+ "questions": [
+ "What historical events led to the development of this topic?",
+ "How has the understanding of this topic evolved over time?",
+ "What are some key milestones in the history of this topic?"
+ ]
+}]
+
+
+Remember, the output should be a complete, parsable JSON wrapped in tags, with no omissions or errors. The JSON objects should semantically break down the content into relevant blocks, maintaining the original order."""
+
+PROMPT_EXTRACT_BLOCKS = """Here is the URL of the webpage:
+{URL}
+
+And here is the cleaned HTML content of that webpage:
+
+{HTML}
+
+
+Your task is to break down this HTML content into semantically relevant blocks, and for each block, generate a JSON object with the following keys:
+
+- index: an integer representing the index of the block in the content
+- content: a list of strings containing the text content of the block
+
+To generate the JSON objects:
+
+1. Carefully read through the HTML content and identify logical breaks or shifts in the content that would warrant splitting it into separate blocks.
+
+2. For each block:
+ a. Assign it an index based on its order in the content.
+ b. Analyze the content and generate ONE semantic tag that describe what the block is about.
+ c. Extract the text content, EXACTLY SAME AS THE GIVE DATA, clean it up if needed, and store it as a list of strings in the "content" field.
+
+3. Ensure that the order of the JSON objects matches the order of the blocks as they appear in the original HTML content.
+
+4. Double-check that each JSON object includes all required keys (index, tag, content) and that the values are in the expected format (integer, list of strings, etc.).
+
+5. Make sure the generated JSON is complete and parsable, with no errors or omissions.
+
+6. Make sure to escape any special characters in the HTML content, and also single or double quote to avoid JSON parsing issues.
+
+7. Never alter the extracted content, just copy and paste it as it is.
+
+Please provide your output within tags, like this:
+
+
+[{
+ "index": 0,
+ "tags": ["introduction"],
+ "content": ["This is the first paragraph of the article, which provides an introduction and overview of the main topic."]
+},
+{
+ "index": 1,
+ "tags": ["background"],
+ "content": ["This is the second paragraph, which delves into the history and background of the topic.",
+ "It provides context and sets the stage for the rest of the article."]
+}]
+
+
+Remember, the output should be a complete, parsable JSON wrapped in tags, with no omissions or errors. The JSON objects should semantically break down the content into relevant blocks, maintaining the original order."""
+
+PROMPT_EXTRACT_BLOCKS_WITH_INSTRUCTION = """Here is the URL of the webpage:
+{URL}
+
+And here is the cleaned HTML content of that webpage:
+
+{HTML}
+
+
+Your task is to break down this HTML content into semantically relevant blocks, following the provided user's REQUEST, and for each block, generate a JSON object with the following keys:
+
+- index: an integer representing the index of the block in the content
+- content: a list of strings containing the text content of the block
+
+This is the user's REQUEST, pay attention to it:
+
+{REQUEST}
+
+
+To generate the JSON objects:
+
+1. Carefully read through the HTML content and identify logical breaks or shifts in the content that would warrant splitting it into separate blocks.
+
+2. For each block:
+ a. Assign it an index based on its order in the content.
+ b. Analyze the content and generate ONE semantic tag that describe what the block is about.
+ c. Extract the text content, EXACTLY SAME AS GIVE DATA, clean it up if needed, and store it as a list of strings in the "content" field.
+
+3. Ensure that the order of the JSON objects matches the order of the blocks as they appear in the original HTML content.
+
+4. Double-check that each JSON object includes all required keys (index, tag, content) and that the values are in the expected format (integer, list of strings, etc.).
+
+5. Make sure the generated JSON is complete and parsable, with no errors or omissions.
+
+6. Make sure to escape any special characters in the HTML content, and also single or double quote to avoid JSON parsing issues.
+
+7. Never alter the extracted content, just copy and paste it as it is.
+
+Please provide your output within tags, like this:
+
+
+[{
+ "index": 0,
+ "tags": ["introduction"],
+ "content": ["This is the first paragraph of the article, which provides an introduction and overview of the main topic."]
+},
+{
+ "index": 1,
+ "tags": ["background"],
+ "content": ["This is the second paragraph, which delves into the history and background of the topic.",
+ "It provides context and sets the stage for the rest of the article."]
+}]
+
+
+**Make sure to follow the user instruction to extract blocks aligin with the instruction.**
+
+Remember, the output should be a complete, parsable JSON wrapped in tags, with no omissions or errors. The JSON objects should semantically break down the content into relevant blocks, maintaining the original order."""
+
+PROMPT_EXTRACT_SCHEMA_WITH_INSTRUCTION = """Here is the content from the URL:
+{URL}
+
+
+{HTML}
+
+
+The user has made the following request for what information to extract from the above content:
+
+
+{REQUEST}
+
+
+
+{SCHEMA}
+
+
+Please carefully read the URL content and the user's request. If the user provided a desired JSON schema in the above, extract the requested information from the URL content according to that schema. If no schema was provided, infer an appropriate JSON schema based on the user's request that will best capture the key information they are looking for.
+
+Extraction instructions:
+Return the extracted information as a list of JSON objects, with each object in the list corresponding to a block of content from the URL, in the same order as it appears on the page. Wrap the entire JSON list in ... XML tags.
+
+Quality Reflection:
+Before outputting your final answer, double check that the JSON you are returning is complete, containing all the information requested by the user, and is valid JSON that could be parsed by json.loads() with no errors or omissions. The outputted JSON objects should fully match the schema, either provided or inferred.
+
+Quality Score:
+After reflecting, score the quality and completeness of the JSON data you are about to return on a scale of 1 to 5. Write the score inside tags.
+
+Avoid Common Mistakes:
+- Do NOT add any comments using "//" or "#" in the JSON output. It causes parsing errors.
+- Make sure the JSON is properly formatted with curly braces, square brackets, and commas in the right places.
+- Do not miss closing tag at the end of the JSON output.
+- Do not generate the Python coee show me how to do the task, this is your task to extract the information and return it in JSON format.
+
+Result
+Output the final list of JSON objects, wrapped in ... XML tags. Make sure to close the tag properly."""
diff --git a/crawl4ai/ssl_certificate.py b/crawl4ai/ssl_certificate.py
new file mode 100644
index 0000000000000000000000000000000000000000..97529e3e1d4aa358003b4cca3b7c1ee8929bd852
--- /dev/null
+++ b/crawl4ai/ssl_certificate.py
@@ -0,0 +1,181 @@
+"""SSL Certificate class for handling certificate operations."""
+
+import ssl
+import socket
+import base64
+import json
+from typing import Dict, Any, Optional
+from urllib.parse import urlparse
+import OpenSSL.crypto
+from pathlib import Path
+
+
+class SSLCertificate:
+ """
+ A class representing an SSL certificate with methods to export in various formats.
+
+ Attributes:
+ cert_info (Dict[str, Any]): The certificate information.
+
+ Methods:
+ from_url(url: str, timeout: int = 10) -> Optional['SSLCertificate']: Create SSLCertificate instance from a URL.
+ from_file(file_path: str) -> Optional['SSLCertificate']: Create SSLCertificate instance from a file.
+ from_binary(binary_data: bytes) -> Optional['SSLCertificate']: Create SSLCertificate instance from binary data.
+ export_as_pem() -> str: Export the certificate as PEM format.
+ export_as_der() -> bytes: Export the certificate as DER format.
+ export_as_json() -> Dict[str, Any]: Export the certificate as JSON format.
+ export_as_text() -> str: Export the certificate as text format.
+ """
+ def __init__(self, cert_info: Dict[str, Any]):
+ self._cert_info = self._decode_cert_data(cert_info)
+
+ @staticmethod
+ def from_url(url: str, timeout: int = 10) -> Optional['SSLCertificate']:
+ """
+ Create SSLCertificate instance from a URL.
+
+ Args:
+ url (str): URL of the website.
+ timeout (int): Timeout for the connection (default: 10).
+
+ Returns:
+ Optional[SSLCertificate]: SSLCertificate instance if successful, None otherwise.
+ """
+ try:
+ hostname = urlparse(url).netloc
+ if ':' in hostname:
+ hostname = hostname.split(':')[0]
+
+ context = ssl.create_default_context()
+ with socket.create_connection((hostname, 443), timeout=timeout) as sock:
+ with context.wrap_socket(sock, server_hostname=hostname) as ssock:
+ cert_binary = ssock.getpeercert(binary_form=True)
+ x509 = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_ASN1, cert_binary)
+
+ cert_info = {
+ "subject": dict(x509.get_subject().get_components()),
+ "issuer": dict(x509.get_issuer().get_components()),
+ "version": x509.get_version(),
+ "serial_number": hex(x509.get_serial_number()),
+ "not_before": x509.get_notBefore(),
+ "not_after": x509.get_notAfter(),
+ "fingerprint": x509.digest("sha256").hex(),
+ "signature_algorithm": x509.get_signature_algorithm(),
+ "raw_cert": base64.b64encode(cert_binary)
+ }
+
+ # Add extensions
+ extensions = []
+ for i in range(x509.get_extension_count()):
+ ext = x509.get_extension(i)
+ extensions.append({
+ "name": ext.get_short_name(),
+ "value": str(ext)
+ })
+ cert_info["extensions"] = extensions
+
+ return SSLCertificate(cert_info)
+
+ except Exception as e:
+ return None
+
+ @staticmethod
+ def _decode_cert_data(data: Any) -> Any:
+ """Helper method to decode bytes in certificate data."""
+ if isinstance(data, bytes):
+ return data.decode('utf-8')
+ elif isinstance(data, dict):
+ return {
+ (k.decode('utf-8') if isinstance(k, bytes) else k): SSLCertificate._decode_cert_data(v)
+ for k, v in data.items()
+ }
+ elif isinstance(data, list):
+ return [SSLCertificate._decode_cert_data(item) for item in data]
+ return data
+
+ def to_json(self, filepath: Optional[str] = None) -> Optional[str]:
+ """
+ Export certificate as JSON.
+
+ Args:
+ filepath (Optional[str]): Path to save the JSON file (default: None).
+
+ Returns:
+ Optional[str]: JSON string if successful, None otherwise.
+ """
+ json_str = json.dumps(self._cert_info, indent=2, ensure_ascii=False)
+ if filepath:
+ Path(filepath).write_text(json_str, encoding='utf-8')
+ return None
+ return json_str
+
+ def to_pem(self, filepath: Optional[str] = None) -> Optional[str]:
+ """
+ Export certificate as PEM.
+
+ Args:
+ filepath (Optional[str]): Path to save the PEM file (default: None).
+
+ Returns:
+ Optional[str]: PEM string if successful, None otherwise.
+ """
+ try:
+ x509 = OpenSSL.crypto.load_certificate(
+ OpenSSL.crypto.FILETYPE_ASN1,
+ base64.b64decode(self._cert_info['raw_cert'])
+ )
+ pem_data = OpenSSL.crypto.dump_certificate(
+ OpenSSL.crypto.FILETYPE_PEM,
+ x509
+ ).decode('utf-8')
+
+ if filepath:
+ Path(filepath).write_text(pem_data, encoding='utf-8')
+ return None
+ return pem_data
+ except Exception as e:
+ return None
+
+ def to_der(self, filepath: Optional[str] = None) -> Optional[bytes]:
+ """
+ Export certificate as DER.
+
+ Args:
+ filepath (Optional[str]): Path to save the DER file (default: None).
+
+ Returns:
+ Optional[bytes]: DER bytes if successful, None otherwise.
+ """
+ try:
+ der_data = base64.b64decode(self._cert_info['raw_cert'])
+ if filepath:
+ Path(filepath).write_bytes(der_data)
+ return None
+ return der_data
+ except Exception:
+ return None
+
+ @property
+ def issuer(self) -> Dict[str, str]:
+ """Get certificate issuer information."""
+ return self._cert_info.get('issuer', {})
+
+ @property
+ def subject(self) -> Dict[str, str]:
+ """Get certificate subject information."""
+ return self._cert_info.get('subject', {})
+
+ @property
+ def valid_from(self) -> str:
+ """Get certificate validity start date."""
+ return self._cert_info.get('not_before', '')
+
+ @property
+ def valid_until(self) -> str:
+ """Get certificate validity end date."""
+ return self._cert_info.get('not_after', '')
+
+ @property
+ def fingerprint(self) -> str:
+ """Get certificate fingerprint."""
+ return self._cert_info.get('fingerprint', '')
diff --git a/crawl4ai/user_agent_generator.py b/crawl4ai/user_agent_generator.py
new file mode 100644
index 0000000000000000000000000000000000000000..6679bb1b25a2f020b403eb230a8a640194062f19
--- /dev/null
+++ b/crawl4ai/user_agent_generator.py
@@ -0,0 +1,305 @@
+import random
+from typing import Optional, Literal, List, Dict, Tuple
+import re
+
+
+class UserAgentGenerator:
+ """
+ Generate random user agents with specified constraints.
+
+ Attributes:
+ desktop_platforms (dict): A dictionary of possible desktop platforms and their corresponding user agent strings.
+ mobile_platforms (dict): A dictionary of possible mobile platforms and their corresponding user agent strings.
+ browser_combinations (dict): A dictionary of possible browser combinations and their corresponding user agent strings.
+ rendering_engines (dict): A dictionary of possible rendering engines and their corresponding user agent strings.
+ chrome_versions (list): A list of possible Chrome browser versions.
+ firefox_versions (list): A list of possible Firefox browser versions.
+ edge_versions (list): A list of possible Edge browser versions.
+ safari_versions (list): A list of possible Safari browser versions.
+ ios_versions (list): A list of possible iOS browser versions.
+ android_versions (list): A list of possible Android browser versions.
+
+ Methods:
+ generate_user_agent(
+ platform: Literal["desktop", "mobile"] = "desktop",
+ browser: str = "chrome",
+ rendering_engine: str = "chrome_webkit",
+ chrome_version: Optional[str] = None,
+ firefox_version: Optional[str] = None,
+ edge_version: Optional[str] = None,
+ safari_version: Optional[str] = None,
+ ios_version: Optional[str] = None,
+ android_version: Optional[str] = None
+ ): Generates a random user agent string based on the specified parameters.
+ """
+ def __init__(self):
+ # Previous platform definitions remain the same...
+ self.desktop_platforms = {
+ "windows": {
+ "10_64": "(Windows NT 10.0; Win64; x64)",
+ "10_32": "(Windows NT 10.0; WOW64)",
+ },
+ "macos": {
+ "intel": "(Macintosh; Intel Mac OS X 10_15_7)",
+ "newer": "(Macintosh; Intel Mac OS X 10.15; rv:109.0)",
+ },
+ "linux": {
+ "generic": "(X11; Linux x86_64)",
+ "ubuntu": "(X11; Ubuntu; Linux x86_64)",
+ "chrome_os": "(X11; CrOS x86_64 14541.0.0)",
+ }
+ }
+
+ self.mobile_platforms = {
+ "android": {
+ "samsung": "(Linux; Android 13; SM-S901B)",
+ "pixel": "(Linux; Android 12; Pixel 6)",
+ "oneplus": "(Linux; Android 13; OnePlus 9 Pro)",
+ "xiaomi": "(Linux; Android 12; M2102J20SG)",
+ },
+ "ios": {
+ "iphone": "(iPhone; CPU iPhone OS 16_5 like Mac OS X)",
+ "ipad": "(iPad; CPU OS 16_5 like Mac OS X)",
+ }
+ }
+
+ # Browser Combinations
+ self.browser_combinations = {
+ 1: [
+ ["chrome"],
+ ["firefox"],
+ ["safari"],
+ ["edge"]
+ ],
+ 2: [
+ ["gecko", "firefox"],
+ ["chrome", "safari"],
+ ["webkit", "safari"]
+ ],
+ 3: [
+ ["chrome", "safari", "edge"],
+ ["webkit", "chrome", "safari"]
+ ]
+ }
+
+ # Rendering Engines with versions
+ self.rendering_engines = {
+ "chrome_webkit": "AppleWebKit/537.36",
+ "safari_webkit": "AppleWebKit/605.1.15",
+ "gecko": [ # Added Gecko versions
+ "Gecko/20100101",
+ "Gecko/20100101", # Firefox usually uses this constant version
+ "Gecko/2010010",
+ ]
+ }
+
+ # Browser Versions
+ self.chrome_versions = [
+ "Chrome/119.0.6045.199",
+ "Chrome/118.0.5993.117",
+ "Chrome/117.0.5938.149",
+ "Chrome/116.0.5845.187",
+ "Chrome/115.0.5790.171",
+ ]
+
+ self.edge_versions = [
+ "Edg/119.0.2151.97",
+ "Edg/118.0.2088.76",
+ "Edg/117.0.2045.47",
+ "Edg/116.0.1938.81",
+ "Edg/115.0.1901.203",
+ ]
+
+ self.safari_versions = [
+ "Safari/537.36", # For Chrome-based
+ "Safari/605.1.15",
+ "Safari/604.1",
+ "Safari/602.1",
+ "Safari/601.5.17",
+ ]
+
+ # Added Firefox versions
+ self.firefox_versions = [
+ "Firefox/119.0",
+ "Firefox/118.0.2",
+ "Firefox/117.0.1",
+ "Firefox/116.0",
+ "Firefox/115.0.3",
+ "Firefox/114.0.2",
+ "Firefox/113.0.1",
+ "Firefox/112.0",
+ "Firefox/111.0.1",
+ "Firefox/110.0",
+ ]
+
+ def get_browser_stack(self, num_browsers: int = 1) -> List[str]:
+ """
+ Get a valid combination of browser versions.
+
+ How it works:
+ 1. Check if the number of browsers is supported.
+ 2. Randomly choose a combination of browsers.
+ 3. Iterate through the combination and add browser versions.
+ 4. Return the browser stack.
+
+ Args:
+ num_browsers: Number of browser specifications (1-3)
+
+ Returns:
+ List[str]: A list of browser versions.
+ """
+ if num_browsers not in self.browser_combinations:
+ raise ValueError(f"Unsupported number of browsers: {num_browsers}")
+
+ combination = random.choice(self.browser_combinations[num_browsers])
+ browser_stack = []
+
+ for browser in combination:
+ if browser == "chrome":
+ browser_stack.append(random.choice(self.chrome_versions))
+ elif browser == "firefox":
+ browser_stack.append(random.choice(self.firefox_versions))
+ elif browser == "safari":
+ browser_stack.append(random.choice(self.safari_versions))
+ elif browser == "edge":
+ browser_stack.append(random.choice(self.edge_versions))
+ elif browser == "gecko":
+ browser_stack.append(random.choice(self.rendering_engines["gecko"]))
+ elif browser == "webkit":
+ browser_stack.append(self.rendering_engines["chrome_webkit"])
+
+ return browser_stack
+
+ def generate(self,
+ device_type: Optional[Literal['desktop', 'mobile']] = None,
+ os_type: Optional[str] = None,
+ device_brand: Optional[str] = None,
+ browser_type: Optional[Literal['chrome', 'edge', 'safari', 'firefox']] = None,
+ num_browsers: int = 3) -> str:
+ """
+ Generate a random user agent with specified constraints.
+
+ Args:
+ device_type: 'desktop' or 'mobile'
+ os_type: 'windows', 'macos', 'linux', 'android', 'ios'
+ device_brand: Specific device brand
+ browser_type: 'chrome', 'edge', 'safari', or 'firefox'
+ num_browsers: Number of browser specifications (1-3)
+ """
+ # Get platform string
+ platform = self.get_random_platform(device_type, os_type, device_brand)
+
+ # Start with Mozilla
+ components = ["Mozilla/5.0", platform]
+
+ # Add browser stack
+ browser_stack = self.get_browser_stack(num_browsers)
+
+ # Add appropriate legacy token based on browser stack
+ if "Firefox" in str(browser_stack):
+ components.append(random.choice(self.rendering_engines["gecko"]))
+ elif "Chrome" in str(browser_stack) or "Safari" in str(browser_stack):
+ components.append(self.rendering_engines["chrome_webkit"])
+ components.append("(KHTML, like Gecko)")
+
+ # Add browser versions
+ components.extend(browser_stack)
+
+ return " ".join(components)
+
+ def generate_with_client_hints(self, **kwargs) -> Tuple[str, str]:
+ """Generate both user agent and matching client hints"""
+ user_agent = self.generate(**kwargs)
+ client_hints = self.generate_client_hints(user_agent)
+ return user_agent, client_hints
+
+ def get_random_platform(self, device_type, os_type, device_brand):
+ """Helper method to get random platform based on constraints"""
+ platforms = self.desktop_platforms if device_type == 'desktop' else \
+ self.mobile_platforms if device_type == 'mobile' else \
+ {**self.desktop_platforms, **self.mobile_platforms}
+
+ if os_type:
+ for platform_group in [self.desktop_platforms, self.mobile_platforms]:
+ if os_type in platform_group:
+ platforms = {os_type: platform_group[os_type]}
+ break
+
+ os_key = random.choice(list(platforms.keys()))
+ if device_brand and device_brand in platforms[os_key]:
+ return platforms[os_key][device_brand]
+ return random.choice(list(platforms[os_key].values()))
+
+ def parse_user_agent(self, user_agent: str) -> Dict[str, str]:
+ """Parse a user agent string to extract browser and version information"""
+ browsers = {
+ 'chrome': r'Chrome/(\d+)',
+ 'edge': r'Edg/(\d+)',
+ 'safari': r'Version/(\d+)',
+ 'firefox': r'Firefox/(\d+)'
+ }
+
+ result = {}
+ for browser, pattern in browsers.items():
+ match = re.search(pattern, user_agent)
+ if match:
+ result[browser] = match.group(1)
+
+ return result
+
+ def generate_client_hints(self, user_agent: str) -> str:
+ """Generate Sec-CH-UA header value based on user agent string"""
+ browsers = self.parse_user_agent(user_agent)
+
+ # Client hints components
+ hints = []
+
+ # Handle different browser combinations
+ if 'chrome' in browsers:
+ hints.append(f'"Chromium";v="{browsers["chrome"]}"')
+ hints.append('"Not_A Brand";v="8"')
+
+ if 'edge' in browsers:
+ hints.append(f'"Microsoft Edge";v="{browsers["edge"]}"')
+ else:
+ hints.append(f'"Google Chrome";v="{browsers["chrome"]}"')
+
+ elif 'firefox' in browsers:
+ # Firefox doesn't typically send Sec-CH-UA
+ return '""'
+
+ elif 'safari' in browsers:
+ # Safari's format for client hints
+ hints.append(f'"Safari";v="{browsers["safari"]}"')
+ hints.append('"Not_A Brand";v="8"')
+
+ return ', '.join(hints)
+
+# Example usage:
+if __name__ == "__main__":
+ generator = UserAgentGenerator()
+ print(generator.generate())
+
+ print("\nSingle browser (Chrome):")
+ print(generator.generate(num_browsers=1, browser_type='chrome'))
+
+ print("\nTwo browsers (Gecko/Firefox):")
+ print(generator.generate(num_browsers=2))
+
+ print("\nThree browsers (Chrome/Safari/Edge):")
+ print(generator.generate(num_browsers=3))
+
+ print("\nFirefox on Linux:")
+ print(generator.generate(
+ device_type='desktop',
+ os_type='linux',
+ browser_type='firefox',
+ num_browsers=2
+ ))
+
+ print("\nChrome/Safari/Edge on Windows:")
+ print(generator.generate(
+ device_type='desktop',
+ os_type='windows',
+ num_browsers=3
+ ))
\ No newline at end of file
diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..6fd7429f19df0ac1700dbac7b760cdc125697e14
--- /dev/null
+++ b/crawl4ai/utils.py
@@ -0,0 +1,1660 @@
+import time
+from urllib.parse import urlparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from bs4 import BeautifulSoup, Comment, element, Tag, NavigableString
+import json
+import html
+import re
+import os
+import platform
+from .prompts import PROMPT_EXTRACT_BLOCKS
+from .config import *
+from pathlib import Path
+from typing import Dict, Any
+from urllib.parse import urljoin
+import requests
+from requests.exceptions import InvalidSchema
+from typing import Optional, Tuple, Dict, Any
+import xxhash
+from colorama import Fore, Style, init
+import textwrap
+import cProfile
+import pstats
+from functools import wraps
+import asyncio
+
+
+class InvalidCSSSelectorError(Exception):
+ pass
+
+def create_box_message(message: str, type: str = "info", width: int = 120, add_newlines: bool = True, double_line: bool = False) -> str:
+ """
+ Create a styled message box with colored borders and formatted text.
+
+ How it works:
+ 1. Determines box style and colors based on the message type (e.g., info, warning).
+ 2. Wraps text to fit within the specified width.
+ 3. Constructs a box using characters (single or double lines) with appropriate formatting.
+ 4. Adds optional newlines before and after the box.
+
+ Args:
+ message (str): The message to display inside the box.
+ type (str): Type of the message (e.g., "info", "warning", "error", "success"). Defaults to "info".
+ width (int): Width of the box. Defaults to 120.
+ add_newlines (bool): Whether to add newlines before and after the box. Defaults to True.
+ double_line (bool): Whether to use double lines for the box border. Defaults to False.
+
+ Returns:
+ str: A formatted string containing the styled message box.
+ """
+
+ init()
+
+ # Define border and text colors for different types
+ styles = {
+ "warning": (Fore.YELLOW, Fore.LIGHTYELLOW_EX, "⚠"),
+ "info": (Fore.BLUE, Fore.LIGHTBLUE_EX, "ℹ"),
+ "success": (Fore.GREEN, Fore.LIGHTGREEN_EX, "✓"),
+ "error": (Fore.RED, Fore.LIGHTRED_EX, "×"),
+ }
+
+ border_color, text_color, prefix = styles.get(type.lower(), styles["info"])
+
+ # Define box characters based on line style
+ box_chars = {
+ "single": ("─", "│", "┌", "┐", "└", "┘"),
+ "double": ("═", "║", "╔", "╗", "╚", "╝")
+ }
+ line_style = "double" if double_line else "single"
+ h_line, v_line, tl, tr, bl, br = box_chars[line_style]
+
+ # Process lines with lighter text color
+ formatted_lines = []
+ raw_lines = message.split('\n')
+
+ if raw_lines:
+ first_line = f"{prefix} {raw_lines[0].strip()}"
+ wrapped_first = textwrap.fill(first_line, width=width-4)
+ formatted_lines.extend(wrapped_first.split('\n'))
+
+ for line in raw_lines[1:]:
+ if line.strip():
+ wrapped = textwrap.fill(f" {line.strip()}", width=width-4)
+ formatted_lines.extend(wrapped.split('\n'))
+ else:
+ formatted_lines.append("")
+
+ # Create the box with colored borders and lighter text
+ horizontal_line = h_line * (width - 1)
+ box = [
+ f"{border_color}{tl}{horizontal_line}{tr}",
+ *[f"{border_color}{v_line}{text_color} {line:<{width-2}}{border_color}{v_line}" for line in formatted_lines],
+ f"{border_color}{bl}{horizontal_line}{br}{Style.RESET_ALL}"
+ ]
+
+ result = "\n".join(box)
+ if add_newlines:
+ result = f"\n{result}\n"
+
+ return result
+
+def calculate_semaphore_count():
+ """
+ Calculate the optimal semaphore count based on system resources.
+
+ How it works:
+ 1. Determines the number of CPU cores and total system memory.
+ 2. Sets a base count as half of the available CPU cores.
+ 3. Limits the count based on memory, assuming 2GB per semaphore instance.
+ 4. Returns the minimum value between CPU and memory-based limits.
+
+ Returns:
+ int: The calculated semaphore count.
+ """
+
+ cpu_count = os.cpu_count()
+ memory_gb = get_system_memory() / (1024 ** 3) # Convert to GB
+ base_count = max(1, cpu_count // 2)
+ memory_based_cap = int(memory_gb / 2) # Assume 2GB per instance
+ return min(base_count, memory_based_cap)
+
+def get_system_memory():
+ """
+ Get the total system memory in bytes.
+
+ How it works:
+ 1. Detects the operating system.
+ 2. Reads memory information from system-specific commands or files.
+ 3. Converts the memory to bytes for uniformity.
+
+ Returns:
+ int: The total system memory in bytes.
+
+ Raises:
+ OSError: If the operating system is unsupported.
+ """
+
+ system = platform.system()
+ if system == "Linux":
+ with open('/proc/meminfo', 'r') as mem:
+ for line in mem:
+ if line.startswith('MemTotal:'):
+ return int(line.split()[1]) * 1024 # Convert KB to bytes
+ elif system == "Darwin": # macOS
+ import subprocess
+ output = subprocess.check_output(['sysctl', '-n', 'hw.memsize']).decode('utf-8')
+ return int(output.strip())
+ elif system == "Windows":
+ import ctypes
+ kernel32 = ctypes.windll.kernel32
+ c_ulonglong = ctypes.c_ulonglong
+ class MEMORYSTATUSEX(ctypes.Structure):
+ _fields_ = [
+ ('dwLength', ctypes.c_ulong),
+ ('dwMemoryLoad', ctypes.c_ulong),
+ ('ullTotalPhys', c_ulonglong),
+ ('ullAvailPhys', c_ulonglong),
+ ('ullTotalPageFile', c_ulonglong),
+ ('ullAvailPageFile', c_ulonglong),
+ ('ullTotalVirtual', c_ulonglong),
+ ('ullAvailVirtual', c_ulonglong),
+ ('ullAvailExtendedVirtual', c_ulonglong),
+ ]
+ memoryStatus = MEMORYSTATUSEX()
+ memoryStatus.dwLength = ctypes.sizeof(MEMORYSTATUSEX)
+ kernel32.GlobalMemoryStatusEx(ctypes.byref(memoryStatus))
+ return memoryStatus.ullTotalPhys
+ else:
+ raise OSError("Unsupported operating system")
+
+def get_home_folder():
+ """
+ Get or create the home folder for Crawl4AI configuration and cache.
+
+ How it works:
+ 1. Uses environment variables or defaults to the user's home directory.
+ 2. Creates `.crawl4ai` and its subdirectories (`cache`, `models`) if they don't exist.
+ 3. Returns the path to the home folder.
+
+ Returns:
+ str: The path to the Crawl4AI home folder.
+ """
+
+ home_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home())), ".crawl4ai")
+ os.makedirs(home_folder, exist_ok=True)
+ os.makedirs(f"{home_folder}/cache", exist_ok=True)
+ os.makedirs(f"{home_folder}/models", exist_ok=True)
+ return home_folder
+
+def beautify_html(escaped_html):
+ """
+ Beautifies an escaped HTML string.
+
+ Parameters:
+ escaped_html (str): A string containing escaped HTML.
+
+ Returns:
+ str: A beautifully formatted HTML string.
+ """
+ # Unescape the HTML string
+ unescaped_html = html.unescape(escaped_html)
+
+ # Use BeautifulSoup to parse and prettify the HTML
+ soup = BeautifulSoup(unescaped_html, 'html.parser')
+ pretty_html = soup.prettify()
+
+ return pretty_html
+
+def split_and_parse_json_objects(json_string):
+ """
+ Splits a JSON string which is a list of objects and tries to parse each object.
+
+ Parameters:
+ json_string (str): A string representation of a list of JSON objects, e.g., '[{...}, {...}, ...]'.
+
+ Returns:
+ tuple: A tuple containing two lists:
+ - First list contains all successfully parsed JSON objects.
+ - Second list contains the string representations of all segments that couldn't be parsed.
+ """
+ # Trim the leading '[' and trailing ']'
+ if json_string.startswith('[') and json_string.endswith(']'):
+ json_string = json_string[1:-1].strip()
+
+ # Split the string into segments that look like individual JSON objects
+ segments = []
+ depth = 0
+ start_index = 0
+
+ for i, char in enumerate(json_string):
+ if char == '{':
+ if depth == 0:
+ start_index = i
+ depth += 1
+ elif char == '}':
+ depth -= 1
+ if depth == 0:
+ segments.append(json_string[start_index:i+1])
+
+ # Try parsing each segment
+ parsed_objects = []
+ unparsed_segments = []
+
+ for segment in segments:
+ try:
+ obj = json.loads(segment)
+ parsed_objects.append(obj)
+ except json.JSONDecodeError:
+ unparsed_segments.append(segment)
+
+ return parsed_objects, unparsed_segments
+
+def sanitize_html(html):
+ """
+ Sanitize an HTML string by escaping quotes.
+
+ How it works:
+ 1. Replaces all unwanted and special characters with an empty string.
+ 2. Escapes double and single quotes for safe usage.
+
+ Args:
+ html (str): The HTML string to sanitize.
+
+ Returns:
+ str: The sanitized HTML string.
+ """
+
+ # Replace all unwanted and special characters with an empty string
+ sanitized_html = html
+ # sanitized_html = re.sub(r'[^\w\s.,;:!?=\[\]{}()<>\/\\\-"]', '', html)
+
+ # Escape all double and single quotes
+ sanitized_html = sanitized_html.replace('"', '\\"').replace("'", "\\'")
+
+ return sanitized_html
+
+def sanitize_input_encode(text: str) -> str:
+ """Sanitize input to handle potential encoding issues."""
+ try:
+ try:
+ if not text:
+ return ''
+ # Attempt to encode and decode as UTF-8 to handle potential encoding issues
+ return text.encode('utf-8', errors='ignore').decode('utf-8')
+ except UnicodeEncodeError as e:
+ print(f"Warning: Encoding issue detected. Some characters may be lost. Error: {e}")
+ # Fall back to ASCII if UTF-8 fails
+ return text.encode('ascii', errors='ignore').decode('ascii')
+ except Exception as e:
+ raise ValueError(f"Error sanitizing input: {str(e)}") from e
+
+def escape_json_string(s):
+ """
+ Escapes characters in a string to be JSON safe.
+
+ Parameters:
+ s (str): The input string to be escaped.
+
+ Returns:
+ str: The escaped string, safe for JSON encoding.
+ """
+ # Replace problematic backslash first
+ s = s.replace('\\', '\\\\')
+
+ # Replace the double quote
+ s = s.replace('"', '\\"')
+
+ # Escape control characters
+ s = s.replace('\b', '\\b')
+ s = s.replace('\f', '\\f')
+ s = s.replace('\n', '\\n')
+ s = s.replace('\r', '\\r')
+ s = s.replace('\t', '\\t')
+
+ # Additional problematic characters
+ # Unicode control characters
+ s = re.sub(r'[\x00-\x1f\x7f-\x9f]', lambda x: '\\u{:04x}'.format(ord(x.group())), s)
+
+ return s
+
+def replace_inline_tags(soup, tags, only_text=False):
+ """
+ Replace inline HTML tags with Markdown-style equivalents.
+
+ How it works:
+ 1. Maps specific tags (e.g., , ) to Markdown syntax.
+ 2. Finds and replaces all occurrences of these tags in the provided BeautifulSoup object.
+ 3. Optionally replaces tags with their text content only.
+
+ Args:
+ soup (BeautifulSoup): Parsed HTML content.
+ tags (List[str]): List of tags to replace.
+ only_text (bool): Whether to replace tags with plain text. Defaults to False.
+
+ Returns:
+ BeautifulSoup: Updated BeautifulSoup object with replaced tags.
+ """
+
+ tag_replacements = {
+ 'b': lambda tag: f"**{tag.text}**",
+ 'i': lambda tag: f"*{tag.text}*",
+ 'u': lambda tag: f"__{tag.text}__",
+ 'span': lambda tag: f"{tag.text}",
+ 'del': lambda tag: f"~~{tag.text}~~",
+ 'ins': lambda tag: f"++{tag.text}++",
+ 'sub': lambda tag: f"~{tag.text}~",
+ 'sup': lambda tag: f"^^{tag.text}^^",
+ 'strong': lambda tag: f"**{tag.text}**",
+ 'em': lambda tag: f"*{tag.text}*",
+ 'code': lambda tag: f"`{tag.text}`",
+ 'kbd': lambda tag: f"`{tag.text}`",
+ 'var': lambda tag: f"_{tag.text}_",
+ 's': lambda tag: f"~~{tag.text}~~",
+ 'q': lambda tag: f'"{tag.text}"',
+ 'abbr': lambda tag: f"{tag.text} ({tag.get('title', '')})",
+ 'cite': lambda tag: f"_{tag.text}_",
+ 'dfn': lambda tag: f"_{tag.text}_",
+ 'time': lambda tag: f"{tag.text}",
+ 'small': lambda tag: f"{tag.text} ",
+ 'mark': lambda tag: f"=={tag.text}=="
+ }
+
+ replacement_data = [(tag, tag_replacements.get(tag, lambda t: t.text)) for tag in tags]
+
+ for tag_name, replacement_func in replacement_data:
+ for tag in soup.find_all(tag_name):
+ replacement_text = tag.text if only_text else replacement_func(tag)
+ tag.replace_with(replacement_text)
+
+ return soup
+
+ # for tag_name in tags:
+ # for tag in soup.find_all(tag_name):
+ # if not only_text:
+ # replacement_text = tag_replacements.get(tag_name, lambda t: t.text)(tag)
+ # tag.replace_with(replacement_text)
+ # else:
+ # tag.replace_with(tag.text)
+
+ # return soup
+
+def get_content_of_website(url, html, word_count_threshold = MIN_WORD_THRESHOLD, css_selector = None, **kwargs):
+ """
+ Extract structured content, media, and links from website HTML.
+
+ How it works:
+ 1. Parses the HTML content using BeautifulSoup.
+ 2. Extracts internal/external links and media (images, videos, audios).
+ 3. Cleans the content by removing unwanted tags and attributes.
+ 4. Converts cleaned HTML to Markdown.
+ 5. Collects metadata and returns the extracted information.
+
+ Args:
+ url (str): The website URL.
+ html (str): The HTML content of the website.
+ word_count_threshold (int): Minimum word count for content inclusion. Defaults to MIN_WORD_THRESHOLD.
+ css_selector (Optional[str]): CSS selector to extract specific content. Defaults to None.
+
+ Returns:
+ Dict[str, Any]: Extracted content including Markdown, cleaned HTML, media, links, and metadata.
+ """
+
+ try:
+ if not html:
+ return None
+ # Parse HTML content with BeautifulSoup
+ soup = BeautifulSoup(html, 'html.parser')
+
+ # Get the content within the tag
+ body = soup.body
+
+ # If css_selector is provided, extract content based on the selector
+ if css_selector:
+ selected_elements = body.select(css_selector)
+ if not selected_elements:
+ raise InvalidCSSSelectorError(f"Invalid CSS selector , No elements found for CSS selector: {css_selector}")
+ div_tag = soup.new_tag('div')
+ for el in selected_elements:
+ div_tag.append(el)
+ body = div_tag
+
+ links = {
+ 'internal': [],
+ 'external': []
+ }
+
+ # Extract all internal and external links
+ for a in body.find_all('a', href=True):
+ href = a['href']
+ url_base = url.split('/')[2]
+ if href.startswith('http') and url_base not in href:
+ links['external'].append({
+ 'href': href,
+ 'text': a.get_text()
+ })
+ else:
+ links['internal'].append(
+ {
+ 'href': href,
+ 'text': a.get_text()
+ }
+ )
+
+ # Remove script, style, and other tags that don't carry useful content from body
+ for tag in body.find_all(['script', 'style', 'link', 'meta', 'noscript']):
+ tag.decompose()
+
+ # Remove all attributes from remaining tags in body, except for img tags
+ for tag in body.find_all():
+ if tag.name != 'img':
+ tag.attrs = {}
+
+ # Extract all img tgas int0 [{src: '', alt: ''}]
+ media = {
+ 'images': [],
+ 'videos': [],
+ 'audios': []
+ }
+ for img in body.find_all('img'):
+ media['images'].append({
+ 'src': img.get('src'),
+ 'alt': img.get('alt'),
+ "type": "image"
+ })
+
+ # Extract all video tags into [{src: '', alt: ''}]
+ for video in body.find_all('video'):
+ media['videos'].append({
+ 'src': video.get('src'),
+ 'alt': video.get('alt'),
+ "type": "video"
+ })
+
+ # Extract all audio tags into [{src: '', alt: ''}]
+ for audio in body.find_all('audio'):
+ media['audios'].append({
+ 'src': audio.get('src'),
+ 'alt': audio.get('alt'),
+ "type": "audio"
+ })
+
+ # Replace images with their alt text or remove them if no alt text is available
+ for img in body.find_all('img'):
+ alt_text = img.get('alt')
+ if alt_text:
+ img.replace_with(soup.new_string(alt_text))
+ else:
+ img.decompose()
+
+
+ # Create a function that replace content of all"pre" tag with its inner text
+ def replace_pre_tags_with_text(node):
+ for child in node.find_all('pre'):
+ # set child inner html to its text
+ child.string = child.get_text()
+ return node
+
+ # Replace all "pre" tags with their inner text
+ body = replace_pre_tags_with_text(body)
+
+ # Replace inline tags with their text content
+ body = replace_inline_tags(
+ body,
+ ['b', 'i', 'u', 'span', 'del', 'ins', 'sub', 'sup', 'strong', 'em', 'code', 'kbd', 'var', 's', 'q', 'abbr', 'cite', 'dfn', 'time', 'small', 'mark'],
+ only_text=kwargs.get('only_text', False)
+ )
+
+ # Recursively remove empty elements, their parent elements, and elements with word count below threshold
+ def remove_empty_and_low_word_count_elements(node, word_count_threshold):
+ for child in node.contents:
+ if isinstance(child, element.Tag):
+ remove_empty_and_low_word_count_elements(child, word_count_threshold)
+ word_count = len(child.get_text(strip=True).split())
+ if (len(child.contents) == 0 and not child.get_text(strip=True)) or word_count < word_count_threshold:
+ child.decompose()
+ return node
+
+ body = remove_empty_and_low_word_count_elements(body, word_count_threshold)
+
+ def remove_small_text_tags(body: Tag, word_count_threshold: int = MIN_WORD_THRESHOLD):
+ # We'll use a list to collect all tags that don't meet the word count requirement
+ tags_to_remove = []
+
+ # Traverse all tags in the body
+ for tag in body.find_all(True): # True here means all tags
+ # Check if the tag contains text and if it's not just whitespace
+ if tag.string and tag.string.strip():
+ # Split the text by spaces and count the words
+ word_count = len(tag.string.strip().split())
+ # If the word count is less than the threshold, mark the tag for removal
+ if word_count < word_count_threshold:
+ tags_to_remove.append(tag)
+
+ # Remove all marked tags from the tree
+ for tag in tags_to_remove:
+ tag.decompose() # or tag.extract() to remove and get the element
+
+ return body
+
+
+ # Remove small text tags
+ body = remove_small_text_tags(body, word_count_threshold)
+
+ def is_empty_or_whitespace(tag: Tag):
+ if isinstance(tag, NavigableString):
+ return not tag.strip()
+ # Check if the tag itself is empty or all its children are empty/whitespace
+ if not tag.contents:
+ return True
+ return all(is_empty_or_whitespace(child) for child in tag.contents)
+
+ def remove_empty_tags(body: Tag):
+ # Continue processing until no more changes are made
+ changes = True
+ while changes:
+ changes = False
+ # Collect all tags that are empty or contain only whitespace
+ empty_tags = [tag for tag in body.find_all(True) if is_empty_or_whitespace(tag)]
+ for tag in empty_tags:
+ # If a tag is empty, decompose it
+ tag.decompose()
+ changes = True # Mark that a change was made
+
+ return body
+
+
+ # Remove empty tags
+ body = remove_empty_tags(body)
+
+ # Flatten nested elements with only one child of the same type
+ def flatten_nested_elements(node):
+ for child in node.contents:
+ if isinstance(child, element.Tag):
+ flatten_nested_elements(child)
+ if len(child.contents) == 1 and child.contents[0].name == child.name:
+ # print('Flattening:', child.name)
+ child_content = child.contents[0]
+ child.replace_with(child_content)
+
+ return node
+
+ body = flatten_nested_elements(body)
+
+
+
+ # Remove comments
+ for comment in soup.find_all(string=lambda text: isinstance(text, Comment)):
+ comment.extract()
+
+ # Remove consecutive empty newlines and replace multiple spaces with a single space
+ cleaned_html = str(body).replace('\n\n', '\n').replace(' ', ' ')
+
+ # Sanitize the cleaned HTML content
+ cleaned_html = sanitize_html(cleaned_html)
+ # sanitized_html = escape_json_string(cleaned_html)
+
+ # Convert cleaned HTML to Markdown
+ h = html2text.HTML2Text()
+ h = CustomHTML2Text()
+ h.ignore_links = True
+ markdown = h.handle(cleaned_html)
+ markdown = markdown.replace(' ```', '```')
+
+ try:
+ meta = extract_metadata(html, soup)
+ except Exception as e:
+ print('Error extracting metadata:', str(e))
+ meta = {}
+
+
+ # Return the Markdown content
+ return{
+ 'markdown': markdown,
+ 'cleaned_html': cleaned_html,
+ 'success': True,
+ 'media': media,
+ 'links': links,
+ 'metadata': meta
+ }
+
+ except Exception as e:
+ print('Error processing HTML content:', str(e))
+ raise InvalidCSSSelectorError(f"Invalid CSS selector: {css_selector}") from e
+
+def get_content_of_website_optimized(url: str, html: str, word_count_threshold: int = MIN_WORD_THRESHOLD, css_selector: str = None, **kwargs) -> Dict[str, Any]:
+ if not html:
+ return None
+
+ soup = BeautifulSoup(html, 'html.parser')
+ body = soup.body
+
+ image_description_min_word_threshold = kwargs.get('image_description_min_word_threshold', IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD)
+
+ for tag in kwargs.get('excluded_tags', []) or []:
+ for el in body.select(tag):
+ el.decompose()
+
+ if css_selector:
+ selected_elements = body.select(css_selector)
+ if not selected_elements:
+ raise InvalidCSSSelectorError(f"Invalid CSS selector, No elements found for CSS selector: {css_selector}")
+ body = soup.new_tag('div')
+ for el in selected_elements:
+ body.append(el)
+
+ links = {'internal': [], 'external': []}
+ media = {'images': [], 'videos': [], 'audios': []}
+
+ # Extract meaningful text for media files from closest parent
+ def find_closest_parent_with_useful_text(tag):
+ current_tag = tag
+ while current_tag:
+ current_tag = current_tag.parent
+ # Get the text content from the parent tag
+ if current_tag:
+ text_content = current_tag.get_text(separator=' ',strip=True)
+ # Check if the text content has at least word_count_threshold
+ if len(text_content.split()) >= image_description_min_word_threshold:
+ return text_content
+ return None
+
+ def process_image(img, url, index, total_images):
+ #Check if an image has valid display and inside undesired html elements
+ def is_valid_image(img, parent, parent_classes):
+ style = img.get('style', '')
+ src = img.get('src', '')
+ classes_to_check = ['button', 'icon', 'logo']
+ tags_to_check = ['button', 'input']
+ return all([
+ 'display:none' not in style,
+ src,
+ not any(s in var for var in [src, img.get('alt', ''), *parent_classes] for s in classes_to_check),
+ parent.name not in tags_to_check
+ ])
+
+ #Score an image for it's usefulness
+ def score_image_for_usefulness(img, base_url, index, images_count):
+ # Function to parse image height/width value and units
+ def parse_dimension(dimension):
+ if dimension:
+ match = re.match(r"(\d+)(\D*)", dimension)
+ if match:
+ number = int(match.group(1))
+ unit = match.group(2) or 'px' # Default unit is 'px' if not specified
+ return number, unit
+ return None, None
+
+ # Fetch image file metadata to extract size and extension
+ def fetch_image_file_size(img, base_url):
+ #If src is relative path construct full URL, if not it may be CDN URL
+ img_url = urljoin(base_url,img.get('src'))
+ try:
+ response = requests.head(img_url)
+ if response.status_code == 200:
+ return response.headers.get('Content-Length',None)
+ else:
+ print(f"Failed to retrieve file size for {img_url}")
+ return None
+ except InvalidSchema as e:
+ return None
+ finally:
+ return
+
+ image_height = img.get('height')
+ height_value, height_unit = parse_dimension(image_height)
+ image_width = img.get('width')
+ width_value, width_unit = parse_dimension(image_width)
+ image_size = 0 #int(fetch_image_file_size(img,base_url) or 0)
+ image_format = os.path.splitext(img.get('src',''))[1].lower()
+ # Remove . from format
+ image_format = image_format.strip('.')
+ score = 0
+ if height_value:
+ if height_unit == 'px' and height_value > 150:
+ score += 1
+ if height_unit in ['%','vh','vmin','vmax'] and height_value >30:
+ score += 1
+ if width_value:
+ if width_unit == 'px' and width_value > 150:
+ score += 1
+ if width_unit in ['%','vh','vmin','vmax'] and width_value >30:
+ score += 1
+ if image_size > 10000:
+ score += 1
+ if img.get('alt') != '':
+ score+=1
+ if any(image_format==format for format in ['jpg','png','webp']):
+ score+=1
+ if index/images_count<0.5:
+ score+=1
+ return score
+
+ if not is_valid_image(img, img.parent, img.parent.get('class', [])):
+ return None
+ score = score_image_for_usefulness(img, url, index, total_images)
+ if score <= IMAGE_SCORE_THRESHOLD:
+ return None
+ return {
+ 'src': img.get('src', '').replace('\\"', '"').strip(),
+ 'alt': img.get('alt', ''),
+ 'desc': find_closest_parent_with_useful_text(img),
+ 'score': score,
+ 'type': 'image'
+ }
+
+ def process_element(element: element.PageElement) -> bool:
+ try:
+ if isinstance(element, NavigableString):
+ if isinstance(element, Comment):
+ element.extract()
+ return False
+
+ if element.name in ['script', 'style', 'link', 'meta', 'noscript']:
+ element.decompose()
+ return False
+
+ keep_element = False
+
+ if element.name == 'a' and element.get('href'):
+ href = element['href']
+ url_base = url.split('/')[2]
+ link_data = {'href': href, 'text': element.get_text()}
+ if href.startswith('http') and url_base not in href:
+ links['external'].append(link_data)
+ else:
+ links['internal'].append(link_data)
+ keep_element = True
+
+ elif element.name == 'img':
+ return True # Always keep image elements
+
+ elif element.name in ['video', 'audio']:
+ media[f"{element.name}s"].append({
+ 'src': element.get('src'),
+ 'alt': element.get('alt'),
+ 'type': element.name,
+ 'description': find_closest_parent_with_useful_text(element)
+ })
+ source_tags = element.find_all('source')
+ for source_tag in source_tags:
+ media[f"{element.name}s"].append({
+ 'src': source_tag.get('src'),
+ 'alt': element.get('alt'),
+ 'type': element.name,
+ 'description': find_closest_parent_with_useful_text(element)
+ })
+ return True # Always keep video and audio elements
+
+ if element.name != 'pre':
+ if element.name in ['b', 'i', 'u', 'span', 'del', 'ins', 'sub', 'sup', 'strong', 'em', 'code', 'kbd', 'var', 's', 'q', 'abbr', 'cite', 'dfn', 'time', 'small', 'mark']:
+ if kwargs.get('only_text', False):
+ element.replace_with(element.get_text())
+ else:
+ element.unwrap()
+ elif element.name != 'img':
+ element.attrs = {}
+
+ # Process children
+ for child in list(element.children):
+ if isinstance(child, NavigableString) and not isinstance(child, Comment):
+ if len(child.strip()) > 0:
+ keep_element = True
+ else:
+ if process_element(child):
+ keep_element = True
+
+
+ # Check word count
+ if not keep_element:
+ word_count = len(element.get_text(strip=True).split())
+ keep_element = word_count >= word_count_threshold
+
+ if not keep_element:
+ element.decompose()
+
+ return keep_element
+ except Exception as e:
+ print('Error processing element:', str(e))
+ return False
+
+ #process images by filtering and extracting contextual text from the page
+ imgs = body.find_all('img')
+ media['images'] = [
+ result for result in
+ (process_image(img, url, i, len(imgs)) for i, img in enumerate(imgs))
+ if result is not None
+ ]
+
+ process_element(body)
+
+ def flatten_nested_elements(node):
+ if isinstance(node, NavigableString):
+ return node
+ if len(node.contents) == 1 and isinstance(node.contents[0], element.Tag) and node.contents[0].name == node.name:
+ return flatten_nested_elements(node.contents[0])
+ node.contents = [flatten_nested_elements(child) for child in node.contents]
+ return node
+
+ body = flatten_nested_elements(body)
+ base64_pattern = re.compile(r'data:image/[^;]+;base64,([^"]+)')
+ for img in imgs:
+ try:
+ src = img.get('src', '')
+ if base64_pattern.match(src):
+ img['src'] = base64_pattern.sub('', src)
+ except:
+ pass
+
+ cleaned_html = str(body).replace('\n\n', '\n').replace(' ', ' ')
+ cleaned_html = sanitize_html(cleaned_html)
+
+ h = CustomHTML2Text()
+ h.ignore_links = True
+ markdown = h.handle(cleaned_html)
+ markdown = markdown.replace(' ```', '```')
+
+ try:
+ meta = extract_metadata(html, soup)
+ except Exception as e:
+ print('Error extracting metadata:', str(e))
+ meta = {}
+
+ return {
+ 'markdown': markdown,
+ 'cleaned_html': cleaned_html,
+ 'success': True,
+ 'media': media,
+ 'links': links,
+ 'metadata': meta
+ }
+
+def extract_metadata(html, soup=None):
+ """
+ Extract optimized content, media, and links from website HTML.
+
+ How it works:
+ 1. Similar to `get_content_of_website`, but optimized for performance.
+ 2. Filters and scores images for usefulness.
+ 3. Extracts contextual descriptions for media files.
+ 4. Handles excluded tags and CSS selectors.
+ 5. Cleans HTML and converts it to Markdown.
+
+ Args:
+ url (str): The website URL.
+ html (str): The HTML content of the website.
+ word_count_threshold (int): Minimum word count for content inclusion. Defaults to MIN_WORD_THRESHOLD.
+ css_selector (Optional[str]): CSS selector to extract specific content. Defaults to None.
+ **kwargs: Additional options for customization.
+
+ Returns:
+ Dict[str, Any]: Extracted content including Markdown, cleaned HTML, media, links, and metadata.
+ """
+
+ metadata = {}
+
+ if not html and not soup:
+ return {}
+
+ if not soup:
+ soup = BeautifulSoup(html, 'lxml')
+
+ head = soup.head
+ if not head:
+ return metadata
+
+ # Title
+ title_tag = head.find('title')
+ metadata['title'] = title_tag.string.strip() if title_tag and title_tag.string else None
+
+ # Meta description
+ description_tag = head.find('meta', attrs={'name': 'description'})
+ metadata['description'] = description_tag.get('content', '').strip() if description_tag else None
+
+ # Meta keywords
+ keywords_tag = head.find('meta', attrs={'name': 'keywords'})
+ metadata['keywords'] = keywords_tag.get('content', '').strip() if keywords_tag else None
+
+ # Meta author
+ author_tag = head.find('meta', attrs={'name': 'author'})
+ metadata['author'] = author_tag.get('content', '').strip() if author_tag else None
+
+ # Open Graph metadata
+ og_tags = head.find_all('meta', attrs={'property': re.compile(r'^og:')})
+ for tag in og_tags:
+ property_name = tag.get('property', '').strip()
+ content = tag.get('content', '').strip()
+ if property_name and content:
+ metadata[property_name] = content
+
+ # Twitter Card metadata
+ twitter_tags = head.find_all('meta', attrs={'name': re.compile(r'^twitter:')})
+ for tag in twitter_tags:
+ property_name = tag.get('name', '').strip()
+ content = tag.get('content', '').strip()
+ if property_name and content:
+ metadata[property_name] = content
+
+ return metadata
+
+def extract_xml_tags(string):
+ """
+ Extracts XML tags from a string.
+
+ Args:
+ string (str): The input string containing XML tags.
+
+ Returns:
+ List[str]: A list of XML tags extracted from the input string.
+ """
+ tags = re.findall(r'<(\w+)>', string)
+ return list(set(tags))
+
+def extract_xml_data(tags, string):
+ """
+ Extract data for specified XML tags from a string.
+
+ How it works:
+ 1. Searches the string for each tag using regex.
+ 2. Extracts the content within the tags.
+ 3. Returns a dictionary of tag-content pairs.
+
+ Args:
+ tags (List[str]): The list of XML tags to extract.
+ string (str): The input string containing XML data.
+
+ Returns:
+ Dict[str, str]: A dictionary with tag names as keys and extracted content as values.
+ """
+
+ data = {}
+
+ for tag in tags:
+ pattern = f"<{tag}>(.*?)"
+ match = re.search(pattern, string, re.DOTALL)
+ if match:
+ data[tag] = match.group(1).strip()
+ else:
+ data[tag] = ""
+
+ return data
+
+def perform_completion_with_backoff(
+ provider,
+ prompt_with_variables,
+ api_token,
+ json_response = False,
+ base_url=None,
+ **kwargs
+ ):
+ """
+ Perform an API completion request with exponential backoff.
+
+ How it works:
+ 1. Sends a completion request to the API.
+ 2. Retries on rate-limit errors with exponential delays.
+ 3. Returns the API response or an error after all retries.
+
+ Args:
+ provider (str): The name of the API provider.
+ prompt_with_variables (str): The input prompt for the completion request.
+ api_token (str): The API token for authentication.
+ json_response (bool): Whether to request a JSON response. Defaults to False.
+ base_url (Optional[str]): The base URL for the API. Defaults to None.
+ **kwargs: Additional arguments for the API request.
+
+ Returns:
+ dict: The API response or an error message after all retries.
+ """
+
+ from litellm import completion
+ from litellm.exceptions import RateLimitError
+ max_attempts = 3
+ base_delay = 2 # Base delay in seconds, you can adjust this based on your needs
+
+ extra_args = {
+ "temperature": 0.01,
+ 'api_key': api_token,
+ 'base_url': base_url
+ }
+ if json_response:
+ extra_args["response_format"] = { "type": "json_object" }
+
+ if kwargs.get("extra_args"):
+ extra_args.update(kwargs["extra_args"])
+
+ for attempt in range(max_attempts):
+ try:
+
+ response =completion(
+ model=provider,
+ messages=[
+ {"role": "user", "content": prompt_with_variables}
+ ],
+ **extra_args
+ )
+ return response # Return the successful response
+ except RateLimitError as e:
+ print("Rate limit error:", str(e))
+
+ # Check if we have exhausted our max attempts
+ if attempt < max_attempts - 1:
+ # Calculate the delay and wait
+ delay = base_delay * (2 ** attempt) # Exponential backoff formula
+ print(f"Waiting for {delay} seconds before retrying...")
+ time.sleep(delay)
+ else:
+ # Return an error response after exhausting all retries
+ return [{
+ "index": 0,
+ "tags": ["error"],
+ "content": ["Rate limit error. Please try again later."]
+ }]
+
+def extract_blocks(url, html, provider = DEFAULT_PROVIDER, api_token = None, base_url = None):
+ """
+ Extract content blocks from website HTML using an AI provider.
+
+ How it works:
+ 1. Prepares a prompt by sanitizing and escaping HTML.
+ 2. Sends the prompt to an AI provider with optional retries.
+ 3. Parses the response to extract structured blocks or errors.
+
+ Args:
+ url (str): The website URL.
+ html (str): The HTML content of the website.
+ provider (str): The AI provider for content extraction. Defaults to DEFAULT_PROVIDER.
+ api_token (Optional[str]): The API token for authentication. Defaults to None.
+ base_url (Optional[str]): The base URL for the API. Defaults to None.
+
+ Returns:
+ List[dict]: A list of extracted content blocks.
+ """
+
+ # api_token = os.getenv('GROQ_API_KEY', None) if not api_token else api_token
+ api_token = PROVIDER_MODELS.get(provider, None) if not api_token else api_token
+
+ variable_values = {
+ "URL": url,
+ "HTML": escape_json_string(sanitize_html(html)),
+ }
+
+ prompt_with_variables = PROMPT_EXTRACT_BLOCKS
+ for variable in variable_values:
+ prompt_with_variables = prompt_with_variables.replace(
+ "{" + variable + "}", variable_values[variable]
+ )
+
+ response = perform_completion_with_backoff(provider, prompt_with_variables, api_token, base_url=base_url)
+
+ try:
+ blocks = extract_xml_data(["blocks"], response.choices[0].message.content)['blocks']
+ blocks = json.loads(blocks)
+ ## Add error: False to the blocks
+ for block in blocks:
+ block['error'] = False
+ except Exception as e:
+ parsed, unparsed = split_and_parse_json_objects(response.choices[0].message.content)
+ blocks = parsed
+ # Append all unparsed segments as onr error block and content is list of unparsed segments
+ if unparsed:
+ blocks.append({
+ "index": 0,
+ "error": True,
+ "tags": ["error"],
+ "content": unparsed
+ })
+ return blocks
+
+def extract_blocks_batch(batch_data, provider = "groq/llama3-70b-8192", api_token = None):
+ """
+ Extract content blocks from a batch of website HTMLs.
+
+ How it works:
+ 1. Prepares prompts for each URL and HTML pair.
+ 2. Sends the prompts to the AI provider in a batch request.
+ 3. Parses the responses to extract structured blocks or errors.
+
+ Args:
+ batch_data (List[Tuple[str, str]]): A list of (URL, HTML) pairs.
+ provider (str): The AI provider for content extraction. Defaults to "groq/llama3-70b-8192".
+ api_token (Optional[str]): The API token for authentication. Defaults to None.
+
+ Returns:
+ List[dict]: A list of extracted content blocks from all batch items.
+ """
+
+ api_token = os.getenv('GROQ_API_KEY', None) if not api_token else api_token
+ from litellm import batch_completion
+ messages = []
+
+ for url, html in batch_data:
+ variable_values = {
+ "URL": url,
+ "HTML": html,
+ }
+
+ prompt_with_variables = PROMPT_EXTRACT_BLOCKS
+ for variable in variable_values:
+ prompt_with_variables = prompt_with_variables.replace(
+ "{" + variable + "}", variable_values[variable]
+ )
+
+ messages.append([{"role": "user", "content": prompt_with_variables}])
+
+
+ responses = batch_completion(
+ model = provider,
+ messages = messages,
+ temperature = 0.01
+ )
+
+ all_blocks = []
+ for response in responses:
+ try:
+ blocks = extract_xml_data(["blocks"], response.choices[0].message.content)['blocks']
+ blocks = json.loads(blocks)
+
+ except Exception as e:
+ blocks = [{
+ "index": 0,
+ "tags": ["error"],
+ "content": ["Error extracting blocks from the HTML content. Choose another provider/model or try again."],
+ "questions": ["What went wrong during the block extraction process?"]
+ }]
+ all_blocks.append(blocks)
+
+ return sum(all_blocks, [])
+
+def merge_chunks_based_on_token_threshold(chunks, token_threshold):
+ """
+ Merges small chunks into larger ones based on the total token threshold.
+
+ :param chunks: List of text chunks to be merged based on token count.
+ :param token_threshold: Max number of tokens for each merged chunk.
+ :return: List of merged text chunks.
+ """
+ merged_sections = []
+ current_chunk = []
+ total_token_so_far = 0
+
+ for chunk in chunks:
+ chunk_token_count = len(chunk.split()) * 1.3 # Estimate token count with a factor
+ if total_token_so_far + chunk_token_count < token_threshold:
+ current_chunk.append(chunk)
+ total_token_so_far += chunk_token_count
+ else:
+ if current_chunk:
+ merged_sections.append('\n\n'.join(current_chunk))
+ current_chunk = [chunk]
+ total_token_so_far = chunk_token_count
+
+ # Add the last chunk if it exists
+ if current_chunk:
+ merged_sections.append('\n\n'.join(current_chunk))
+
+ return merged_sections
+
+def process_sections(url: str, sections: list, provider: str, api_token: str, base_url=None) -> list:
+ """
+ Process sections of HTML content sequentially or in parallel.
+
+ How it works:
+ 1. Sequentially processes sections with delays for "groq/" providers.
+ 2. Uses ThreadPoolExecutor for parallel processing with other providers.
+ 3. Extracts content blocks for each section.
+
+ Args:
+ url (str): The website URL.
+ sections (List[str]): The list of HTML sections to process.
+ provider (str): The AI provider for content extraction.
+ api_token (str): The API token for authentication.
+ base_url (Optional[str]): The base URL for the API. Defaults to None.
+
+ Returns:
+ List[dict]: The list of extracted content blocks from all sections.
+ """
+
+ extracted_content = []
+ if provider.startswith("groq/"):
+ # Sequential processing with a delay
+ for section in sections:
+ extracted_content.extend(extract_blocks(url, section, provider, api_token, base_url=base_url))
+ time.sleep(0.5) # 500 ms delay between each processing
+ else:
+ # Parallel processing using ThreadPoolExecutor
+ with ThreadPoolExecutor() as executor:
+ futures = [executor.submit(extract_blocks, url, section, provider, api_token, base_url=base_url) for section in sections]
+ for future in as_completed(futures):
+ extracted_content.extend(future.result())
+
+ return extracted_content
+
+def wrap_text(draw, text, font, max_width):
+ """
+ Wrap text to fit within a specified width for rendering.
+
+ How it works:
+ 1. Splits the text into words.
+ 2. Constructs lines that fit within the maximum width using the provided font.
+ 3. Returns the wrapped text as a single string.
+
+ Args:
+ draw (ImageDraw.Draw): The drawing context for measuring text size.
+ text (str): The text to wrap.
+ font (ImageFont.FreeTypeFont): The font to use for measuring text size.
+ max_width (int): The maximum width for each line.
+
+ Returns:
+ str: The wrapped text.
+ """
+
+ # Wrap the text to fit within the specified width
+ lines = []
+ words = text.split()
+ while words:
+ line = ''
+ while words and draw.textbbox((0, 0), line + words[0], font=font)[2] <= max_width:
+ line += (words.pop(0) + ' ')
+ lines.append(line)
+ return '\n'.join(lines)
+
+def format_html(html_string):
+ """
+ Prettify an HTML string using BeautifulSoup.
+
+ How it works:
+ 1. Parses the HTML string with BeautifulSoup.
+ 2. Formats the HTML with proper indentation.
+ 3. Returns the prettified HTML string.
+
+ Args:
+ html_string (str): The HTML string to format.
+
+ Returns:
+ str: The prettified HTML string.
+ """
+
+ soup = BeautifulSoup(html_string, 'lxml.parser')
+ return soup.prettify()
+
+def fast_format_html(html_string):
+ """
+ A fast HTML formatter that uses string operations instead of parsing.
+
+ Args:
+ html_string (str): The HTML string to format
+
+ Returns:
+ str: The formatted HTML string
+ """
+ # Initialize variables
+ indent = 0
+ indent_str = " " # Two spaces for indentation
+ formatted = []
+ in_content = False
+
+ # Split by < and > to separate tags and content
+ parts = html_string.replace('>', '>\n').replace('<', '\n<').split('\n')
+
+ for part in parts:
+ if not part.strip():
+ continue
+
+ # Handle closing tags
+ if part.startswith(''):
+ formatted.append(indent_str * indent + part)
+
+ # Handle opening tags
+ elif part.startswith('<'):
+ formatted.append(indent_str * indent + part)
+ indent += 1
+
+ # Handle content between tags
+ else:
+ content = part.strip()
+ if content:
+ formatted.append(indent_str * indent + content)
+
+ return '\n'.join(formatted)
+
+def normalize_url(href, base_url):
+ """Normalize URLs to ensure consistent format"""
+ from urllib.parse import urljoin, urlparse
+
+ # Parse base URL to get components
+ parsed_base = urlparse(base_url)
+ if not parsed_base.scheme or not parsed_base.netloc:
+ raise ValueError(f"Invalid base URL format: {base_url}")
+
+ # Use urljoin to handle all cases
+ normalized = urljoin(base_url, href.strip())
+ return normalized
+
+def normalize_url_tmp(href, base_url):
+ """Normalize URLs to ensure consistent format"""
+ # Extract protocol and domain from base URL
+ try:
+ base_parts = base_url.split('/')
+ protocol = base_parts[0]
+ domain = base_parts[2]
+ except IndexError:
+ raise ValueError(f"Invalid base URL format: {base_url}")
+
+ # Handle special protocols
+ special_protocols = {'mailto:', 'tel:', 'ftp:', 'file:', 'data:', 'javascript:'}
+ if any(href.lower().startswith(proto) for proto in special_protocols):
+ return href.strip()
+
+ # Handle anchor links
+ if href.startswith('#'):
+ return f"{base_url}{href}"
+
+ # Handle protocol-relative URLs
+ if href.startswith('//'):
+ return f"{protocol}{href}"
+
+ # Handle root-relative URLs
+ if href.startswith('/'):
+ return f"{protocol}//{domain}{href}"
+
+ # Handle relative URLs
+ if not href.startswith(('http://', 'https://')):
+ # Remove leading './' if present
+ href = href.lstrip('./')
+ return f"{protocol}//{domain}/{href}"
+
+ return href.strip()
+
+def get_base_domain(url: str) -> str:
+ """
+ Extract the base domain from a given URL, handling common edge cases.
+
+ How it works:
+ 1. Parses the URL to extract the domain.
+ 2. Removes the port number and 'www' prefix.
+ 3. Handles special domains (e.g., 'co.uk') to extract the correct base.
+
+ Args:
+ url (str): The URL to extract the base domain from.
+
+ Returns:
+ str: The extracted base domain or an empty string if parsing fails.
+ """
+ try:
+ # Get domain from URL
+ domain = urlparse(url).netloc.lower()
+ if not domain:
+ return ""
+
+ # Remove port if present
+ domain = domain.split(':')[0]
+
+ # Remove www
+ domain = re.sub(r'^www\.', '', domain)
+
+ # Extract last two parts of domain (handles co.uk etc)
+ parts = domain.split('.')
+ if len(parts) > 2 and parts[-2] in {
+ 'co', 'com', 'org', 'gov', 'edu', 'net',
+ 'mil', 'int', 'ac', 'ad', 'ae', 'af', 'ag'
+ }:
+ return '.'.join(parts[-3:])
+
+ return '.'.join(parts[-2:])
+ except Exception:
+ return ""
+
+def is_external_url(url: str, base_domain: str) -> bool:
+ """
+ Extract the base domain from a given URL, handling common edge cases.
+
+ How it works:
+ 1. Parses the URL to extract the domain.
+ 2. Removes the port number and 'www' prefix.
+ 3. Handles special domains (e.g., 'co.uk') to extract the correct base.
+
+ Args:
+ url (str): The URL to extract the base domain from.
+
+ Returns:
+ str: The extracted base domain or an empty string if parsing fails.
+ """
+ special = {'mailto:', 'tel:', 'ftp:', 'file:', 'data:', 'javascript:'}
+ if any(url.lower().startswith(p) for p in special):
+ return True
+
+ try:
+ parsed = urlparse(url)
+ if not parsed.netloc: # Relative URL
+ return False
+
+ # Strip 'www.' from both domains for comparison
+ url_domain = parsed.netloc.lower().replace('www.', '')
+ base = base_domain.lower().replace('www.', '')
+
+ # Check if URL domain ends with base domain
+ return not url_domain.endswith(base)
+ except Exception:
+ return False
+
+def clean_tokens(tokens: list[str]) -> list[str]:
+ """
+ Clean a list of tokens by removing noise, stop words, and short tokens.
+
+ How it works:
+ 1. Defines a set of noise words and stop words.
+ 2. Filters tokens based on length and exclusion criteria.
+ 3. Excludes tokens starting with certain symbols (e.g., "↑", "▲").
+
+ Args:
+ tokens (list[str]): The list of tokens to clean.
+
+ Returns:
+ list[str]: The cleaned list of tokens.
+ """
+
+ # Set of tokens to remove
+ noise = {'ccp', 'up', '↑', '▲', '⬆️', 'a', 'an', 'at', 'by', 'in', 'of', 'on', 'to', 'the'}
+
+ STOP_WORDS = {
+ 'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
+ 'has', 'he', 'in', 'is', 'it', 'its', 'of', 'on', 'that', 'the',
+ 'to', 'was', 'were', 'will', 'with',
+
+ # Pronouns
+ 'i', 'you', 'he', 'she', 'it', 'we', 'they',
+ 'me', 'him', 'her', 'us', 'them',
+ 'my', 'your', 'his', 'her', 'its', 'our', 'their',
+ 'mine', 'yours', 'hers', 'ours', 'theirs',
+ 'myself', 'yourself', 'himself', 'herself', 'itself', 'ourselves', 'themselves',
+
+ # Common verbs
+ 'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+ 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing',
+
+ # Prepositions
+ 'about', 'above', 'across', 'after', 'against', 'along', 'among', 'around',
+ 'at', 'before', 'behind', 'below', 'beneath', 'beside', 'between', 'beyond',
+ 'by', 'down', 'during', 'except', 'for', 'from', 'in', 'inside', 'into',
+ 'near', 'of', 'off', 'on', 'out', 'outside', 'over', 'past', 'through',
+ 'to', 'toward', 'under', 'underneath', 'until', 'up', 'upon', 'with', 'within',
+
+ # Conjunctions
+ 'and', 'but', 'or', 'nor', 'for', 'yet', 'so',
+ 'although', 'because', 'since', 'unless',
+
+ # Articles
+ 'a', 'an', 'the',
+
+ # Other common words
+ 'this', 'that', 'these', 'those',
+ 'what', 'which', 'who', 'whom', 'whose',
+ 'when', 'where', 'why', 'how',
+ 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', 'such',
+ 'can', 'cannot', "can't", 'could', "couldn't",
+ 'may', 'might', 'must', "mustn't",
+ 'shall', 'should', "shouldn't",
+ 'will', "won't", 'would', "wouldn't",
+ 'not', "n't", 'no', 'nor', 'none'
+ }
+
+ # Single comprehension, more efficient than multiple passes
+ return [token for token in tokens
+ if len(token) > 2
+ and token not in noise
+ and token not in STOP_WORDS
+ and not token.startswith('↑')
+ and not token.startswith('▲')
+ and not token.startswith('⬆')]
+
+def profile_and_time(func):
+ """
+ Decorator to profile a function's execution time and performance.
+
+ How it works:
+ 1. Records the start time before executing the function.
+ 2. Profiles the function's execution using `cProfile`.
+ 3. Prints the elapsed time and profiling statistics.
+
+ Args:
+ func (Callable): The function to decorate.
+
+ Returns:
+ Callable: The decorated function with profiling and timing enabled.
+ """
+
+ @wraps(func)
+ def wrapper(self, *args, **kwargs):
+ # Start timer
+ start_time = time.perf_counter()
+
+ # Setup profiler
+ profiler = cProfile.Profile()
+ profiler.enable()
+
+ # Run function
+ result = func(self, *args, **kwargs)
+
+ # Stop profiler
+ profiler.disable()
+
+ # Calculate elapsed time
+ elapsed_time = time.perf_counter() - start_time
+
+ # Print timing
+ print(f"[PROFILER] Scraping completed in {elapsed_time:.2f} seconds")
+
+ # Print profiling stats
+ stats = pstats.Stats(profiler)
+ stats.sort_stats('cumulative') # Sort by cumulative time
+ stats.print_stats(20) # Print top 20 time-consuming functions
+
+ return result
+ return wrapper
+
+def generate_content_hash(content: str) -> str:
+ """Generate a unique hash for content"""
+ return xxhash.xxh64(content.encode()).hexdigest()
+ # return hashlib.sha256(content.encode()).hexdigest()
+
+def ensure_content_dirs(base_path: str) -> Dict[str, str]:
+ """Create content directories if they don't exist"""
+ dirs = {
+ 'html': 'html_content',
+ 'cleaned': 'cleaned_html',
+ 'markdown': 'markdown_content',
+ 'extracted': 'extracted_content',
+ 'screenshots': 'screenshots',
+ 'screenshot': 'screenshots'
+ }
+
+ content_paths = {}
+ for key, dirname in dirs.items():
+ path = os.path.join(base_path, dirname)
+ os.makedirs(path, exist_ok=True)
+ content_paths[key] = path
+
+ return content_paths
+
+def configure_windows_event_loop():
+ """
+ Configure the Windows event loop to use ProactorEventLoop.
+ This resolves the NotImplementedError that occurs on Windows when using asyncio subprocesses.
+
+ This function should only be called on Windows systems and before any async operations.
+ On non-Windows systems, this function does nothing.
+
+ Example:
+ ```python
+ from crawl4ai.async_configs import configure_windows_event_loop
+
+ # Call this before any async operations if you're on Windows
+ configure_windows_event_loop()
+ ```
+ """
+ if platform.system() == 'Windows':
+ asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+
+def get_error_context(exc_info, context_lines: int = 5):
+ """
+ Extract error context with more reliable line number tracking.
+
+ Args:
+ exc_info: The exception info from sys.exc_info()
+ context_lines: Number of lines to show before and after the error
+
+ Returns:
+ dict: Error context information
+ """
+ import traceback
+ import linecache
+ import os
+
+ # Get the full traceback
+ tb = traceback.extract_tb(exc_info[2])
+
+ # Get the last frame (where the error occurred)
+ last_frame = tb[-1]
+ filename = last_frame.filename
+ line_no = last_frame.lineno
+ func_name = last_frame.name
+
+ # Get the source code context using linecache
+ # This is more reliable than inspect.getsourcelines
+ context_start = max(1, line_no - context_lines)
+ context_end = line_no + context_lines + 1
+
+ # Build the context lines with line numbers
+ context_lines = []
+ for i in range(context_start, context_end):
+ line = linecache.getline(filename, i)
+ if line:
+ # Remove any trailing whitespace/newlines and add the pointer for error line
+ line = line.rstrip()
+ pointer = '→' if i == line_no else ' '
+ context_lines.append(f"{i:4d} {pointer} {line}")
+
+ # Join the lines with newlines
+ code_context = '\n'.join(context_lines)
+
+ # Get relative path for cleaner output
+ try:
+ rel_path = os.path.relpath(filename)
+ except ValueError:
+ # Fallback if relpath fails (can happen on Windows with different drives)
+ rel_path = filename
+
+ return {
+ "filename": rel_path,
+ "line_no": line_no,
+ "function": func_name,
+ "code_context": code_context
+ }
+
+
+
\ No newline at end of file
diff --git a/crawl4ai/utils.scraping.py b/crawl4ai/utils.scraping.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/crawl4ai/version_manager.py b/crawl4ai/version_manager.py
new file mode 100644
index 0000000000000000000000000000000000000000..8ae2de2e937fae526351b8dfd586567d21bc7be3
--- /dev/null
+++ b/crawl4ai/version_manager.py
@@ -0,0 +1,30 @@
+# version_manager.py
+import os
+from pathlib import Path
+from packaging import version
+from . import __version__
+
+class VersionManager:
+ def __init__(self):
+ self.home_dir = Path.home() / ".crawl4ai"
+ self.version_file = self.home_dir / "version.txt"
+
+ def get_installed_version(self):
+ """Get the version recorded in home directory"""
+ if not self.version_file.exists():
+ return None
+ try:
+ return version.parse(self.version_file.read_text().strip())
+ except:
+ return None
+
+ def update_version(self):
+ """Update the version file to current library version"""
+ self.version_file.write_text(__version__.__version__)
+
+ def needs_update(self):
+ """Check if database needs update based on version"""
+ installed = self.get_installed_version()
+ current = version.parse(__version__.__version__)
+ return installed is None or installed < current
+
diff --git a/crawl4ai/web_crawler.py b/crawl4ai/web_crawler.py
new file mode 100644
index 0000000000000000000000000000000000000000..a32a988d70767397102c054a95391fc00d0f8145
--- /dev/null
+++ b/crawl4ai/web_crawler.py
@@ -0,0 +1,253 @@
+import os, time
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+from pathlib import Path
+
+from .models import UrlModel, CrawlResult
+from .database import init_db, get_cached_url, cache_url, DB_PATH, flush_db
+from .utils import *
+from .chunking_strategy import *
+from .extraction_strategy import *
+from .crawler_strategy import *
+from typing import List
+from concurrent.futures import ThreadPoolExecutor
+from .content_scraping_strategy import WebScrapingStrategy
+from .config import *
+import warnings
+import json
+warnings.filterwarnings("ignore", message='Field "model_name" has conflict with protected namespace "model_".')
+
+
+class WebCrawler:
+ def __init__(self, crawler_strategy: CrawlerStrategy = None, always_by_pass_cache: bool = False, verbose: bool = False):
+ self.crawler_strategy = crawler_strategy or LocalSeleniumCrawlerStrategy(verbose=verbose)
+ self.always_by_pass_cache = always_by_pass_cache
+ self.crawl4ai_folder = os.path.join(os.getenv("CRAWL4_AI_BASE_DIRECTORY", Path.home()), ".crawl4ai")
+ os.makedirs(self.crawl4ai_folder, exist_ok=True)
+ os.makedirs(f"{self.crawl4ai_folder}/cache", exist_ok=True)
+ init_db()
+ self.ready = False
+
+ def warmup(self):
+ print("[LOG] 🌤️ Warming up the WebCrawler")
+ self.run(
+ url='https://google.com/',
+ word_count_threshold=5,
+ extraction_strategy=NoExtractionStrategy(),
+ bypass_cache=False,
+ verbose=False
+ )
+ self.ready = True
+ print("[LOG] 🌞 WebCrawler is ready to crawl")
+
+ def fetch_page(
+ self,
+ url_model: UrlModel,
+ provider: str = DEFAULT_PROVIDER,
+ api_token: str = None,
+ extract_blocks_flag: bool = True,
+ word_count_threshold=MIN_WORD_THRESHOLD,
+ css_selector: str = None,
+ screenshot: bool = False,
+ use_cached_html: bool = False,
+ extraction_strategy: ExtractionStrategy = None,
+ chunking_strategy: ChunkingStrategy = RegexChunking(),
+ **kwargs,
+ ) -> CrawlResult:
+ return self.run(
+ url_model.url,
+ word_count_threshold,
+ extraction_strategy or NoExtractionStrategy(),
+ chunking_strategy,
+ bypass_cache=url_model.forced,
+ css_selector=css_selector,
+ screenshot=screenshot,
+ **kwargs,
+ )
+ pass
+
+ def fetch_pages(
+ self,
+ url_models: List[UrlModel],
+ provider: str = DEFAULT_PROVIDER,
+ api_token: str = None,
+ extract_blocks_flag: bool = True,
+ word_count_threshold=MIN_WORD_THRESHOLD,
+ use_cached_html: bool = False,
+ css_selector: str = None,
+ screenshot: bool = False,
+ extraction_strategy: ExtractionStrategy = None,
+ chunking_strategy: ChunkingStrategy = RegexChunking(),
+ **kwargs,
+ ) -> List[CrawlResult]:
+ extraction_strategy = extraction_strategy or NoExtractionStrategy()
+ def fetch_page_wrapper(url_model, *args, **kwargs):
+ return self.fetch_page(url_model, *args, **kwargs)
+
+ with ThreadPoolExecutor() as executor:
+ results = list(
+ executor.map(
+ fetch_page_wrapper,
+ url_models,
+ [provider] * len(url_models),
+ [api_token] * len(url_models),
+ [extract_blocks_flag] * len(url_models),
+ [word_count_threshold] * len(url_models),
+ [css_selector] * len(url_models),
+ [screenshot] * len(url_models),
+ [use_cached_html] * len(url_models),
+ [extraction_strategy] * len(url_models),
+ [chunking_strategy] * len(url_models),
+ *[kwargs] * len(url_models),
+ )
+ )
+
+ return results
+
+ def run(
+ self,
+ url: str,
+ word_count_threshold=MIN_WORD_THRESHOLD,
+ extraction_strategy: ExtractionStrategy = None,
+ chunking_strategy: ChunkingStrategy = RegexChunking(),
+ bypass_cache: bool = False,
+ css_selector: str = None,
+ screenshot: bool = False,
+ user_agent: str = None,
+ verbose=True,
+ **kwargs,
+ ) -> CrawlResult:
+ try:
+ extraction_strategy = extraction_strategy or NoExtractionStrategy()
+ extraction_strategy.verbose = verbose
+ if not isinstance(extraction_strategy, ExtractionStrategy):
+ raise ValueError("Unsupported extraction strategy")
+ if not isinstance(chunking_strategy, ChunkingStrategy):
+ raise ValueError("Unsupported chunking strategy")
+
+ word_count_threshold = max(word_count_threshold, MIN_WORD_THRESHOLD)
+
+ cached = None
+ screenshot_data = None
+ extracted_content = None
+ if not bypass_cache and not self.always_by_pass_cache:
+ cached = get_cached_url(url)
+
+ if kwargs.get("warmup", True) and not self.ready:
+ return None
+
+ if cached:
+ html = sanitize_input_encode(cached[1])
+ extracted_content = sanitize_input_encode(cached[4])
+ if screenshot:
+ screenshot_data = cached[9]
+ if not screenshot_data:
+ cached = None
+
+ if not cached or not html:
+ if user_agent:
+ self.crawler_strategy.update_user_agent(user_agent)
+ t1 = time.time()
+ html = sanitize_input_encode(self.crawler_strategy.crawl(url, **kwargs))
+ t2 = time.time()
+ if verbose:
+ print(f"[LOG] 🚀 Crawling done for {url}, success: {bool(html)}, time taken: {t2 - t1:.2f} seconds")
+ if screenshot:
+ screenshot_data = self.crawler_strategy.take_screenshot()
+
+
+ crawl_result = self.process_html(url, html, extracted_content, word_count_threshold, extraction_strategy, chunking_strategy, css_selector, screenshot_data, verbose, bool(cached), **kwargs)
+ crawl_result.success = bool(html)
+ return crawl_result
+ except Exception as e:
+ if not hasattr(e, "msg"):
+ e.msg = str(e)
+ print(f"[ERROR] 🚫 Failed to crawl {url}, error: {e.msg}")
+ return CrawlResult(url=url, html="", success=False, error_message=e.msg)
+
+ def process_html(
+ self,
+ url: str,
+ html: str,
+ extracted_content: str,
+ word_count_threshold: int,
+ extraction_strategy: ExtractionStrategy,
+ chunking_strategy: ChunkingStrategy,
+ css_selector: str,
+ screenshot: bool,
+ verbose: bool,
+ is_cached: bool,
+ **kwargs,
+ ) -> CrawlResult:
+ t = time.time()
+ # Extract content from HTML
+ try:
+ t1 = time.time()
+ scrapping_strategy = WebScrapingStrategy()
+ extra_params = {k: v for k, v in kwargs.items() if k not in ["only_text", "image_description_min_word_threshold"]}
+ result = scrapping_strategy.scrap(
+ url,
+ html,
+ word_count_threshold=word_count_threshold,
+ css_selector=css_selector,
+ only_text=kwargs.get("only_text", False),
+ image_description_min_word_threshold=kwargs.get(
+ "image_description_min_word_threshold", IMAGE_DESCRIPTION_MIN_WORD_THRESHOLD
+ ),
+ **extra_params,
+ )
+
+ # result = get_content_of_website_optimized(url, html, word_count_threshold, css_selector=css_selector, only_text=kwargs.get("only_text", False))
+ if verbose:
+ print(f"[LOG] 🚀 Content extracted for {url}, success: True, time taken: {time.time() - t1:.2f} seconds")
+
+ if result is None:
+ raise ValueError(f"Failed to extract content from the website: {url}")
+ except InvalidCSSSelectorError as e:
+ raise ValueError(str(e))
+
+ cleaned_html = sanitize_input_encode(result.get("cleaned_html", ""))
+ markdown = sanitize_input_encode(result.get("markdown", ""))
+ media = result.get("media", [])
+ links = result.get("links", [])
+ metadata = result.get("metadata", {})
+
+ if extracted_content is None:
+ if verbose:
+ print(f"[LOG] 🔥 Extracting semantic blocks for {url}, Strategy: {extraction_strategy.name}")
+
+ sections = chunking_strategy.chunk(markdown)
+ extracted_content = extraction_strategy.run(url, sections)
+ extracted_content = json.dumps(extracted_content, indent=4, default=str, ensure_ascii=False)
+
+ if verbose:
+ print(f"[LOG] 🚀 Extraction done for {url}, time taken: {time.time() - t:.2f} seconds.")
+
+ screenshot = None if not screenshot else screenshot
+
+ if not is_cached:
+ cache_url(
+ url,
+ html,
+ cleaned_html,
+ markdown,
+ extracted_content,
+ True,
+ json.dumps(media),
+ json.dumps(links),
+ json.dumps(metadata),
+ screenshot=screenshot,
+ )
+
+ return CrawlResult(
+ url=url,
+ html=html,
+ cleaned_html=format_html(cleaned_html),
+ markdown=markdown,
+ media=media,
+ links=links,
+ metadata=metadata,
+ screenshot=screenshot,
+ extracted_content=extracted_content,
+ success=True,
+ error_message="",
+ )
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 0000000000000000000000000000000000000000..4b22fd9846cc0185ffe281e85ae4378538de282f
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,67 @@
+services:
+ # Local build services for different platforms
+ crawl4ai-amd64:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ args:
+ PYTHON_VERSION: "3.10"
+ INSTALL_TYPE: ${INSTALL_TYPE:-basic}
+ ENABLE_GPU: false
+ platforms:
+ - linux/amd64
+ profiles: ["local-amd64"]
+ extends: &base-config
+ file: docker-compose.yml
+ service: base-config
+
+ crawl4ai-arm64:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ args:
+ PYTHON_VERSION: "3.10"
+ INSTALL_TYPE: ${INSTALL_TYPE:-basic}
+ ENABLE_GPU: false
+ platforms:
+ - linux/arm64
+ profiles: ["local-arm64"]
+ extends: *base-config
+
+ # Hub services for different platforms and versions
+ crawl4ai-hub-amd64:
+ image: unclecode/crawl4ai:${VERSION:-basic}-amd64
+ profiles: ["hub-amd64"]
+ extends: *base-config
+
+ crawl4ai-hub-arm64:
+ image: unclecode/crawl4ai:${VERSION:-basic}-arm64
+ profiles: ["hub-arm64"]
+ extends: *base-config
+
+ # Base configuration to be extended
+ base-config:
+ ports:
+ - "11235:11235"
+ - "8000:8000"
+ - "9222:9222"
+ - "8080:8080"
+ environment:
+ - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
+ - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+ - CLAUDE_API_KEY=${CLAUDE_API_KEY:-}
+ volumes:
+ - /dev/shm:/dev/shm
+ deploy:
+ resources:
+ limits:
+ memory: 4G
+ reservations:
+ memory: 1G
+ restart: unless-stopped
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:11235/health"]
+ interval: 30s
+ timeout: 10s
+ retries: 3
+ start_period: 40s
\ No newline at end of file
diff --git a/docs/assets/pitch-dark.png b/docs/assets/pitch-dark.png
new file mode 100644
index 0000000000000000000000000000000000000000..9b9b37b5dc2d7645a267b733a0d2b4916f5af203
Binary files /dev/null and b/docs/assets/pitch-dark.png differ
diff --git a/docs/assets/pitch-dark.svg b/docs/assets/pitch-dark.svg
new file mode 100644
index 0000000000000000000000000000000000000000..0913b2dad5f90ff96e73cd690ff315d0adb675d2
--- /dev/null
+++ b/docs/assets/pitch-dark.svg
@@ -0,0 +1,64 @@
+
+
+
+
+ Data Capitalization Opportunity
+
+ Transform digital footprints into assets
+ Personal data as capital
+ Enterprise knowledge valuation
+ New form of wealth creation
+
+
+
+ Authentic Data Potential
+
+ Vast reservoir of real insights
+ Enhanced AI development
+ Diverse human knowledge
+ Willing participation model
+
+
+
+
+
+
+ 1. Open-Source Foundation
+ Data extraction engine & community development
+
+
+ 2. Data Capitalization Platform
+ Tools to structure & value digital assets
+
+
+ 3. Shared Data Marketplace
+ Economic platform for data exchange
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Economic Vision: Shared Data Economy
+
+
\ No newline at end of file
diff --git a/docs/deprecated/docker-deployment.md b/docs/deprecated/docker-deployment.md
new file mode 100644
index 0000000000000000000000000000000000000000..db8446e324b4f76a8b919e39662ba2f5cb7e52c5
--- /dev/null
+++ b/docs/deprecated/docker-deployment.md
@@ -0,0 +1,189 @@
+# 🐳 Using Docker (Legacy)
+
+Crawl4AI is available as Docker images for easy deployment. You can either pull directly from Docker Hub (recommended) or build from the repository.
+
+---
+
+
+🐳 Option 1: Docker Hub (Recommended)
+
+Choose the appropriate image based on your platform and needs:
+
+### For AMD64 (Regular Linux/Windows):
+```bash
+# Basic version (recommended)
+docker pull unclecode/crawl4ai:basic-amd64
+docker run -p 11235:11235 unclecode/crawl4ai:basic-amd64
+
+# Full ML/LLM support
+docker pull unclecode/crawl4ai:all-amd64
+docker run -p 11235:11235 unclecode/crawl4ai:all-amd64
+
+# With GPU support
+docker pull unclecode/crawl4ai:gpu-amd64
+docker run -p 11235:11235 unclecode/crawl4ai:gpu-amd64
+```
+
+### For ARM64 (M1/M2 Macs, ARM servers):
+```bash
+# Basic version (recommended)
+docker pull unclecode/crawl4ai:basic-arm64
+docker run -p 11235:11235 unclecode/crawl4ai:basic-arm64
+
+# Full ML/LLM support
+docker pull unclecode/crawl4ai:all-arm64
+docker run -p 11235:11235 unclecode/crawl4ai:all-arm64
+
+# With GPU support
+docker pull unclecode/crawl4ai:gpu-arm64
+docker run -p 11235:11235 unclecode/crawl4ai:gpu-arm64
+```
+
+Need more memory? Add `--shm-size`:
+```bash
+docker run --shm-size=2gb -p 11235:11235 unclecode/crawl4ai:basic-amd64
+```
+
+Test the installation:
+```bash
+curl http://localhost:11235/health
+```
+
+### For Raspberry Pi (32-bit) (coming soon):
+```bash
+# Pull and run basic version (recommended for Raspberry Pi)
+docker pull unclecode/crawl4ai:basic-armv7
+docker run -p 11235:11235 unclecode/crawl4ai:basic-armv7
+
+# With increased shared memory if needed
+docker run --shm-size=2gb -p 11235:11235 unclecode/crawl4ai:basic-armv7
+```
+
+Note: Due to hardware constraints, only the basic version is recommended for Raspberry Pi.
+
+
+
+
+🐳 Option 2: Build from Repository
+
+Build the image locally based on your platform:
+
+```bash
+# Clone the repository
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai
+
+# For AMD64 (Regular Linux/Windows)
+docker build --platform linux/amd64 \
+ --tag crawl4ai:local \
+ --build-arg INSTALL_TYPE=basic \
+ .
+
+# For ARM64 (M1/M2 Macs, ARM servers)
+docker build --platform linux/arm64 \
+ --tag crawl4ai:local \
+ --build-arg INSTALL_TYPE=basic \
+ .
+```
+
+Build options:
+- INSTALL_TYPE=basic (default): Basic crawling features
+- INSTALL_TYPE=all: Full ML/LLM support
+- ENABLE_GPU=true: Add GPU support
+
+Example with all options:
+```bash
+docker build --platform linux/amd64 \
+ --tag crawl4ai:local \
+ --build-arg INSTALL_TYPE=all \
+ --build-arg ENABLE_GPU=true \
+ .
+```
+
+Run your local build:
+```bash
+# Regular run
+docker run -p 11235:11235 crawl4ai:local
+
+# With increased shared memory
+docker run --shm-size=2gb -p 11235:11235 crawl4ai:local
+```
+
+Test the installation:
+```bash
+curl http://localhost:11235/health
+```
+
+
+
+
+🐳 Option 3: Using Docker Compose
+
+Docker Compose provides a more structured way to run Crawl4AI, especially when dealing with environment variables and multiple configurations.
+
+```bash
+# Clone the repository
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai
+```
+
+### For AMD64 (Regular Linux/Windows):
+```bash
+# Build and run locally
+docker-compose --profile local-amd64 up
+
+# Run from Docker Hub
+VERSION=basic docker-compose --profile hub-amd64 up # Basic version
+VERSION=all docker-compose --profile hub-amd64 up # Full ML/LLM support
+VERSION=gpu docker-compose --profile hub-amd64 up # GPU support
+```
+
+### For ARM64 (M1/M2 Macs, ARM servers):
+```bash
+# Build and run locally
+docker-compose --profile local-arm64 up
+
+# Run from Docker Hub
+VERSION=basic docker-compose --profile hub-arm64 up # Basic version
+VERSION=all docker-compose --profile hub-arm64 up # Full ML/LLM support
+VERSION=gpu docker-compose --profile hub-arm64 up # GPU support
+```
+
+Environment variables (optional):
+```bash
+# Create a .env file
+CRAWL4AI_API_TOKEN=your_token
+OPENAI_API_KEY=your_openai_key
+CLAUDE_API_KEY=your_claude_key
+```
+
+The compose file includes:
+- Memory management (4GB limit, 1GB reserved)
+- Shared memory volume for browser support
+- Health checks
+- Auto-restart policy
+- All necessary port mappings
+
+Test the installation:
+```bash
+curl http://localhost:11235/health
+```
+
+
+
+
+🚀 One-Click Deployment
+
+Deploy your own instance of Crawl4AI with one click:
+
+[](https://www.digitalocean.com/?repo=https://github.com/unclecode/crawl4ai/tree/0.3.74&refcode=a0780f1bdb3d&utm_campaign=Referral_Invite&utm_medium=Referral_Program&utm_source=badge)
+
+> 💡 **Recommended specs**: 4GB RAM minimum. Select "professional-xs" or higher when deploying for stable operation.
+
+The deploy will:
+- Set up a Docker container with Crawl4AI
+- Configure Playwright and all dependencies
+- Start the FastAPI server on port `11235`
+- Set up health checks and auto-deployment
+
+
diff --git a/docs/examples/amazon_product_extraction_direct_url.py b/docs/examples/amazon_product_extraction_direct_url.py
new file mode 100644
index 0000000000000000000000000000000000000000..769c479e3f040e74c08dbdcc586fcaaddb92d4a4
--- /dev/null
+++ b/docs/examples/amazon_product_extraction_direct_url.py
@@ -0,0 +1,114 @@
+"""
+This example demonstrates how to use JSON CSS extraction to scrape product information
+from Amazon search results. It shows how to extract structured data like product titles,
+prices, ratings, and other details using CSS selectors.
+"""
+
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+import json
+
+async def extract_amazon_products():
+ # Initialize browser config
+ browser_config = BrowserConfig(
+ browser_type="chromium",
+ headless=True
+ )
+
+ # Initialize crawler config with JSON CSS extraction strategy
+ crawler_config = CrawlerRunConfig(
+ extraction_strategy=JsonCssExtractionStrategy(
+ schema={
+ "name": "Amazon Product Search Results",
+ "baseSelector": "[data-component-type='s-search-result']",
+ "fields": [
+ {
+ "name": "asin",
+ "selector": "",
+ "type": "attribute",
+ "attribute": "data-asin"
+ },
+ {
+ "name": "title",
+ "selector": "h2 a span",
+ "type": "text"
+ },
+ {
+ "name": "url",
+ "selector": "h2 a",
+ "type": "attribute",
+ "attribute": "href"
+ },
+ {
+ "name": "image",
+ "selector": ".s-image",
+ "type": "attribute",
+ "attribute": "src"
+ },
+ {
+ "name": "rating",
+ "selector": ".a-icon-star-small .a-icon-alt",
+ "type": "text"
+ },
+ {
+ "name": "reviews_count",
+ "selector": "[data-csa-c-func-deps='aui-da-a-popover'] ~ span span",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": ".a-price .a-offscreen",
+ "type": "text"
+ },
+ {
+ "name": "original_price",
+ "selector": ".a-price.a-text-price .a-offscreen",
+ "type": "text"
+ },
+ {
+ "name": "sponsored",
+ "selector": ".puis-sponsored-label-text",
+ "type": "exists"
+ },
+ {
+ "name": "delivery_info",
+ "selector": "[data-cy='delivery-recipe'] .a-color-base",
+ "type": "text",
+ "multiple": True
+ }
+ ]
+ }
+ )
+ )
+
+ # Example search URL (you should replace with your actual Amazon URL)
+ url = "https://www.amazon.com/s?k=Samsung+Galaxy+Tab"
+
+ # Use context manager for proper resource handling
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ # Extract the data
+ result = await crawler.arun(url=url, config=crawler_config)
+
+ # Process and print the results
+ if result and result.extracted_content:
+ # Parse the JSON string into a list of products
+ products = json.loads(result.extracted_content)
+
+ # Process each product in the list
+ for product in products:
+ print("\nProduct Details:")
+ print(f"ASIN: {product.get('asin')}")
+ print(f"Title: {product.get('title')}")
+ print(f"Price: {product.get('price')}")
+ print(f"Original Price: {product.get('original_price')}")
+ print(f"Rating: {product.get('rating')}")
+ print(f"Reviews: {product.get('reviews_count')}")
+ print(f"Sponsored: {'Yes' if product.get('sponsored') else 'No'}")
+ if product.get('delivery_info'):
+ print(f"Delivery: {' '.join(product['delivery_info'])}")
+ print("-" * 80)
+
+if __name__ == "__main__":
+ import asyncio
+ asyncio.run(extract_amazon_products())
diff --git a/docs/examples/amazon_product_extraction_using_hooks.py b/docs/examples/amazon_product_extraction_using_hooks.py
new file mode 100644
index 0000000000000000000000000000000000000000..a17d60c5944101364897a4eaf620fef634d96e27
--- /dev/null
+++ b/docs/examples/amazon_product_extraction_using_hooks.py
@@ -0,0 +1,145 @@
+"""
+This example demonstrates how to use JSON CSS extraction to scrape product information
+from Amazon search results. It shows how to extract structured data like product titles,
+prices, ratings, and other details using CSS selectors.
+"""
+
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+import json
+from playwright.async_api import Page, BrowserContext
+
+async def extract_amazon_products():
+ # Initialize browser config
+ browser_config = BrowserConfig(
+ # browser_type="chromium",
+ headless=True
+ )
+
+ # Initialize crawler config with JSON CSS extraction strategy nav-search-submit-button
+ crawler_config = CrawlerRunConfig(
+ cache_mode=CacheMode.BYPASS,
+
+ extraction_strategy=JsonCssExtractionStrategy(
+ schema={
+ "name": "Amazon Product Search Results",
+ "baseSelector": "[data-component-type='s-search-result']",
+ "fields": [
+ {
+ "name": "asin",
+ "selector": "",
+ "type": "attribute",
+ "attribute": "data-asin"
+ },
+ {
+ "name": "title",
+ "selector": "h2 a span",
+ "type": "text"
+ },
+ {
+ "name": "url",
+ "selector": "h2 a",
+ "type": "attribute",
+ "attribute": "href"
+ },
+ {
+ "name": "image",
+ "selector": ".s-image",
+ "type": "attribute",
+ "attribute": "src"
+ },
+ {
+ "name": "rating",
+ "selector": ".a-icon-star-small .a-icon-alt",
+ "type": "text"
+ },
+ {
+ "name": "reviews_count",
+ "selector": "[data-csa-c-func-deps='aui-da-a-popover'] ~ span span",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": ".a-price .a-offscreen",
+ "type": "text"
+ },
+ {
+ "name": "original_price",
+ "selector": ".a-price.a-text-price .a-offscreen",
+ "type": "text"
+ },
+ {
+ "name": "sponsored",
+ "selector": ".puis-sponsored-label-text",
+ "type": "exists"
+ },
+ {
+ "name": "delivery_info",
+ "selector": "[data-cy='delivery-recipe'] .a-color-base",
+ "type": "text",
+ "multiple": True
+ }
+ ]
+ }
+ )
+ )
+
+ url = "https://www.amazon.com/"
+
+ async def after_goto(page: Page, context: BrowserContext, url: str, response: dict, **kwargs):
+ """Hook called after navigating to each URL"""
+ print(f"[HOOK] after_goto - Successfully loaded: {url}")
+
+ try:
+ # Wait for search box to be available
+ search_box = await page.wait_for_selector('#twotabsearchtextbox', timeout=1000)
+
+ # Type the search query
+ await search_box.fill('Samsung Galaxy Tab')
+
+ # Get the search button and prepare for navigation
+ search_button = await page.wait_for_selector('#nav-search-submit-button', timeout=1000)
+
+ # Click with navigation waiting
+ await search_button.click()
+
+ # Wait for search results to load
+ await page.wait_for_selector('[data-component-type="s-search-result"]', timeout=10000)
+ print("[HOOK] Search completed and results loaded!")
+
+ except Exception as e:
+ print(f"[HOOK] Error during search operation: {str(e)}")
+
+ return page
+
+ # Use context manager for proper resource handling
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+
+ crawler.crawler_strategy.set_hook("after_goto", after_goto)
+
+ # Extract the data
+ result = await crawler.arun(url=url, config=crawler_config)
+
+ # Process and print the results
+ if result and result.extracted_content:
+ # Parse the JSON string into a list of products
+ products = json.loads(result.extracted_content)
+
+ # Process each product in the list
+ for product in products:
+ print("\nProduct Details:")
+ print(f"ASIN: {product.get('asin')}")
+ print(f"Title: {product.get('title')}")
+ print(f"Price: {product.get('price')}")
+ print(f"Original Price: {product.get('original_price')}")
+ print(f"Rating: {product.get('rating')}")
+ print(f"Reviews: {product.get('reviews_count')}")
+ print(f"Sponsored: {'Yes' if product.get('sponsored') else 'No'}")
+ if product.get('delivery_info'):
+ print(f"Delivery: {' '.join(product['delivery_info'])}")
+ print("-" * 80)
+
+if __name__ == "__main__":
+ import asyncio
+ asyncio.run(extract_amazon_products())
diff --git a/docs/examples/amazon_product_extraction_using_use_javascript.py b/docs/examples/amazon_product_extraction_using_use_javascript.py
new file mode 100644
index 0000000000000000000000000000000000000000..15e5d6f59a726852d38357a3ab16d523d3a108de
--- /dev/null
+++ b/docs/examples/amazon_product_extraction_using_use_javascript.py
@@ -0,0 +1,129 @@
+"""
+This example demonstrates how to use JSON CSS extraction to scrape product information
+from Amazon search results. It shows how to extract structured data like product titles,
+prices, ratings, and other details using CSS selectors.
+"""
+
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+import json
+from playwright.async_api import Page, BrowserContext
+
+async def extract_amazon_products():
+ # Initialize browser config
+ browser_config = BrowserConfig(
+ # browser_type="chromium",
+ headless=True
+ )
+
+ js_code_to_search = """
+ const task = async () => {
+ document.querySelector('#twotabsearchtextbox').value = 'Samsung Galaxy Tab';
+ document.querySelector('#nav-search-submit-button').click();
+ }
+ await task();
+ """
+ js_code_to_search_sync = """
+ document.querySelector('#twotabsearchtextbox').value = 'Samsung Galaxy Tab';
+ document.querySelector('#nav-search-submit-button').click();
+ """
+ crawler_config = CrawlerRunConfig(
+ cache_mode=CacheMode.BYPASS,
+ js_code = js_code_to_search,
+ wait_for='css:[data-component-type="s-search-result"]',
+ extraction_strategy=JsonCssExtractionStrategy(
+ schema={
+ "name": "Amazon Product Search Results",
+ "baseSelector": "[data-component-type='s-search-result']",
+ "fields": [
+ {
+ "name": "asin",
+ "selector": "",
+ "type": "attribute",
+ "attribute": "data-asin"
+ },
+ {
+ "name": "title",
+ "selector": "h2 a span",
+ "type": "text"
+ },
+ {
+ "name": "url",
+ "selector": "h2 a",
+ "type": "attribute",
+ "attribute": "href"
+ },
+ {
+ "name": "image",
+ "selector": ".s-image",
+ "type": "attribute",
+ "attribute": "src"
+ },
+ {
+ "name": "rating",
+ "selector": ".a-icon-star-small .a-icon-alt",
+ "type": "text"
+ },
+ {
+ "name": "reviews_count",
+ "selector": "[data-csa-c-func-deps='aui-da-a-popover'] ~ span span",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": ".a-price .a-offscreen",
+ "type": "text"
+ },
+ {
+ "name": "original_price",
+ "selector": ".a-price.a-text-price .a-offscreen",
+ "type": "text"
+ },
+ {
+ "name": "sponsored",
+ "selector": ".puis-sponsored-label-text",
+ "type": "exists"
+ },
+ {
+ "name": "delivery_info",
+ "selector": "[data-cy='delivery-recipe'] .a-color-base",
+ "type": "text",
+ "multiple": True
+ }
+ ]
+ }
+ )
+ )
+
+ # Example search URL (you should replace with your actual Amazon URL)
+ url = "https://www.amazon.com/"
+
+
+ # Use context manager for proper resource handling
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ # Extract the data
+ result = await crawler.arun(url=url, config=crawler_config)
+
+ # Process and print the results
+ if result and result.extracted_content:
+ # Parse the JSON string into a list of products
+ products = json.loads(result.extracted_content)
+
+ # Process each product in the list
+ for product in products:
+ print("\nProduct Details:")
+ print(f"ASIN: {product.get('asin')}")
+ print(f"Title: {product.get('title')}")
+ print(f"Price: {product.get('price')}")
+ print(f"Original Price: {product.get('original_price')}")
+ print(f"Rating: {product.get('rating')}")
+ print(f"Reviews: {product.get('reviews_count')}")
+ print(f"Sponsored: {'Yes' if product.get('sponsored') else 'No'}")
+ if product.get('delivery_info'):
+ print(f"Delivery: {' '.join(product['delivery_info'])}")
+ print("-" * 80)
+
+if __name__ == "__main__":
+ import asyncio
+ asyncio.run(extract_amazon_products())
diff --git a/docs/examples/assets/audio.mp3 b/docs/examples/assets/audio.mp3
new file mode 100644
index 0000000000000000000000000000000000000000..299149c6dec722f2a7274c2894bdde12d31107ef
Binary files /dev/null and b/docs/examples/assets/audio.mp3 differ
diff --git a/docs/examples/assets/basic.png b/docs/examples/assets/basic.png
new file mode 100644
index 0000000000000000000000000000000000000000..ea68852bb48ff94699e35e93becb045776d0085f
Binary files /dev/null and b/docs/examples/assets/basic.png differ
diff --git a/docs/examples/assets/cosine_extraction.png b/docs/examples/assets/cosine_extraction.png
new file mode 100644
index 0000000000000000000000000000000000000000..19252ad44da8dcefd409ce643e1f46f20e8d15a3
Binary files /dev/null and b/docs/examples/assets/cosine_extraction.png differ
diff --git a/docs/examples/assets/css_js.png b/docs/examples/assets/css_js.png
new file mode 100644
index 0000000000000000000000000000000000000000..9c0d2e60fef6a850badf251717c8bced2944ca36
Binary files /dev/null and b/docs/examples/assets/css_js.png differ
diff --git a/docs/examples/assets/css_selector.png b/docs/examples/assets/css_selector.png
new file mode 100644
index 0000000000000000000000000000000000000000..39357bb920182744dfce5509b5f289829eca2aba
Binary files /dev/null and b/docs/examples/assets/css_selector.png differ
diff --git a/docs/examples/assets/exec_script.png b/docs/examples/assets/exec_script.png
new file mode 100644
index 0000000000000000000000000000000000000000..c2e478f70984ba3445629a793a7424a1feab8a64
Binary files /dev/null and b/docs/examples/assets/exec_script.png differ
diff --git a/docs/examples/assets/llm_extraction.png b/docs/examples/assets/llm_extraction.png
new file mode 100644
index 0000000000000000000000000000000000000000..95d2accb9f1d66a6e6f614cf6c15f103962e6476
Binary files /dev/null and b/docs/examples/assets/llm_extraction.png differ
diff --git a/docs/examples/assets/semantic_extraction_cosine.png b/docs/examples/assets/semantic_extraction_cosine.png
new file mode 100644
index 0000000000000000000000000000000000000000..eace4cf502eda675abcc3be9bd790a210a344538
Binary files /dev/null and b/docs/examples/assets/semantic_extraction_cosine.png differ
diff --git a/docs/examples/assets/semantic_extraction_llm.png b/docs/examples/assets/semantic_extraction_llm.png
new file mode 100644
index 0000000000000000000000000000000000000000..1dba8bc6f73cb361942ac24faed85c042f5f474b
Binary files /dev/null and b/docs/examples/assets/semantic_extraction_llm.png differ
diff --git a/docs/examples/async_webcrawler_multiple_urls_example.py b/docs/examples/async_webcrawler_multiple_urls_example.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d63ac80f364c42ed37c24b902fe46dfd4113538
--- /dev/null
+++ b/docs/examples/async_webcrawler_multiple_urls_example.py
@@ -0,0 +1,48 @@
+# File: async_webcrawler_multiple_urls_example.py
+import os, sys
+# append 2 parent directories to sys.path to import crawl4ai
+parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(parent_dir)
+
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+ # Initialize the AsyncWebCrawler
+ async with AsyncWebCrawler(verbose=True) as crawler:
+ # List of URLs to crawl
+ urls = [
+ "https://example.com",
+ "https://python.org",
+ "https://github.com",
+ "https://stackoverflow.com",
+ "https://news.ycombinator.com"
+ ]
+
+ # Set up crawling parameters
+ word_count_threshold = 100
+
+ # Run the crawling process for multiple URLs
+ results = await crawler.arun_many(
+ urls=urls,
+ word_count_threshold=word_count_threshold,
+ bypass_cache=True,
+ verbose=True
+ )
+
+ # Process the results
+ for result in results:
+ if result.success:
+ print(f"Successfully crawled: {result.url}")
+ print(f"Title: {result.metadata.get('title', 'N/A')}")
+ print(f"Word count: {len(result.markdown.split())}")
+ print(f"Number of links: {len(result.links.get('internal', [])) + len(result.links.get('external', []))}")
+ print(f"Number of images: {len(result.media.get('images', []))}")
+ print("---")
+ else:
+ print(f"Failed to crawl: {result.url}")
+ print(f"Error: {result.error_message}")
+ print("---")
+
+if __name__ == "__main__":
+ asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/browser_optimization_example.py b/docs/examples/browser_optimization_example.py
new file mode 100644
index 0000000000000000000000000000000000000000..f57dc14782172e640fde80d2500d5a2aef9d2b26
--- /dev/null
+++ b/docs/examples/browser_optimization_example.py
@@ -0,0 +1,128 @@
+"""
+This example demonstrates optimal browser usage patterns in Crawl4AI:
+1. Sequential crawling with session reuse
+2. Parallel crawling with browser instance reuse
+3. Performance optimization settings
+"""
+
+import asyncio
+import os
+from typing import List
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+
+async def crawl_sequential(urls: List[str]):
+ """
+ Sequential crawling using session reuse - most efficient for moderate workloads
+ """
+ print("\n=== Sequential Crawling with Session Reuse ===")
+
+ # Configure browser with optimized settings
+ browser_config = BrowserConfig(
+ headless=True,
+ browser_args=[
+ "--disable-gpu", # Disable GPU acceleration
+ "--disable-dev-shm-usage", # Disable /dev/shm usage
+ "--no-sandbox", # Required for Docker
+ ],
+ viewport={
+ "width": 800,
+ "height": 600,
+ }, # Smaller viewport for better performance
+ )
+
+ # Configure crawl settings
+ crawl_config = CrawlerRunConfig(
+ markdown_generator=DefaultMarkdownGenerator(
+ # content_filter=PruningContentFilter(), In case you need fit_markdown
+ ),
+ )
+
+ # Create single crawler instance
+ crawler = AsyncWebCrawler(config=browser_config)
+ await crawler.start()
+
+ try:
+ session_id = "session1" # Use same session for all URLs
+ for url in urls:
+ result = await crawler.arun(
+ url=url,
+ config=crawl_config,
+ session_id=session_id, # Reuse same browser tab
+ )
+ if result.success:
+ print(f"Successfully crawled {url}")
+ print(f"Content length: {len(result.markdown_v2.raw_markdown)}")
+ finally:
+ await crawler.close()
+
+
+async def crawl_parallel(urls: List[str], max_concurrent: int = 3):
+ """
+ Parallel crawling while reusing browser instance - best for large workloads
+ """
+ print("\n=== Parallel Crawling with Browser Reuse ===")
+
+ browser_config = BrowserConfig(
+ headless=True,
+ browser_args=["--disable-gpu", "--disable-dev-shm-usage", "--no-sandbox"],
+ viewport={"width": 800, "height": 600},
+ )
+
+ crawl_config = CrawlerRunConfig(
+ markdown_generator=DefaultMarkdownGenerator(
+ # content_filter=PruningContentFilter(), In case you need fit_markdown
+ ),
+ )
+
+ # Create single crawler instance for all parallel tasks
+ crawler = AsyncWebCrawler(config=browser_config)
+ await crawler.start()
+
+ try:
+ # Create tasks in batches to control concurrency
+ for i in range(0, len(urls), max_concurrent):
+ batch = urls[i : i + max_concurrent]
+ tasks = []
+
+ for j, url in enumerate(batch):
+ session_id = (
+ f"parallel_session_{j}" # Different session per concurrent task
+ )
+ task = crawler.arun(url=url, config=crawl_config, session_id=session_id)
+ tasks.append(task)
+
+ # Wait for batch to complete
+ results = await asyncio.gather(*tasks, return_exceptions=True)
+
+ # Process results
+ for url, result in zip(batch, results):
+ if isinstance(result, Exception):
+ print(f"Error crawling {url}: {str(result)}")
+ elif result.success:
+ print(f"Successfully crawled {url}")
+ print(f"Content length: {len(result.markdown_v2.raw_markdown)}")
+ finally:
+ await crawler.close()
+
+
+async def main():
+ # Example URLs
+ urls = [
+ "https://example.com/page1",
+ "https://example.com/page2",
+ "https://example.com/page3",
+ "https://example.com/page4",
+ ]
+
+ # Demo sequential crawling
+ await crawl_sequential(urls)
+
+ # Demo parallel crawling
+ await crawl_parallel(urls, max_concurrent=2)
+
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/docs/examples/chainlit.md b/docs/examples/chainlit.md
new file mode 100644
index 0000000000000000000000000000000000000000..3b34b02f459a4e1bb038a0abc8cf5b59f8d80b76
--- /dev/null
+++ b/docs/examples/chainlit.md
@@ -0,0 +1,3 @@
+# Welcome to Crawl4AI! 🚀🤖
+
+Hi there, Developer! 👋 Here is an example of a research pipeline, where you can share a URL in your conversation with any LLM, and then the context of crawled pages will be used as the context.
\ No newline at end of file
diff --git a/docs/examples/crawlai_vs_firecrawl.py b/docs/examples/crawlai_vs_firecrawl.py
new file mode 100644
index 0000000000000000000000000000000000000000..b50b06dac8e1b7218ff63bd76fb28ee153d1c47e
--- /dev/null
+++ b/docs/examples/crawlai_vs_firecrawl.py
@@ -0,0 +1,67 @@
+import os, time
+# append the path to the root of the project
+import sys
+import asyncio
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..'))
+from firecrawl import FirecrawlApp
+from crawl4ai import AsyncWebCrawler
+__data__ = os.path.join(os.path.dirname(__file__), '..', '..') + '/.data'
+
+async def compare():
+ app = FirecrawlApp(api_key=os.environ['FIRECRAWL_API_KEY'])
+
+ # Tet Firecrawl with a simple crawl
+ start = time.time()
+ scrape_status = app.scrape_url(
+ 'https://www.nbcnews.com/business',
+ params={'formats': ['markdown', 'html']}
+ )
+ end = time.time()
+ print(f"Time taken: {end - start} seconds")
+ print(len(scrape_status['markdown']))
+ # save the markdown content with provider name
+ with open(f"{__data__}/firecrawl_simple.md", "w") as f:
+ f.write(scrape_status['markdown'])
+ # Count how many "cldnry.s-nbcnews.com" are in the markdown
+ print(scrape_status['markdown'].count("cldnry.s-nbcnews.com"))
+
+
+
+ async with AsyncWebCrawler() as crawler:
+ start = time.time()
+ result = await crawler.arun(
+ url="https://www.nbcnews.com/business",
+ # js_code=["const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"],
+ word_count_threshold=0,
+ bypass_cache=True,
+ verbose=False
+ )
+ end = time.time()
+ print(f"Time taken: {end - start} seconds")
+ print(len(result.markdown))
+ # save the markdown content with provider name
+ with open(f"{__data__}/crawl4ai_simple.md", "w") as f:
+ f.write(result.markdown)
+ # count how many "cldnry.s-nbcnews.com" are in the markdown
+ print(result.markdown.count("cldnry.s-nbcnews.com"))
+
+ start = time.time()
+ result = await crawler.arun(
+ url="https://www.nbcnews.com/business",
+ js_code=["const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"],
+ word_count_threshold=0,
+ bypass_cache=True,
+ verbose=False
+ )
+ end = time.time()
+ print(f"Time taken: {end - start} seconds")
+ print(len(result.markdown))
+ # save the markdown content with provider name
+ with open(f"{__data__}/crawl4ai_js.md", "w") as f:
+ f.write(result.markdown)
+ # count how many "cldnry.s-nbcnews.com" are in the markdown
+ print(result.markdown.count("cldnry.s-nbcnews.com"))
+
+if __name__ == "__main__":
+ asyncio.run(compare())
+
\ No newline at end of file
diff --git a/docs/examples/docker_example.py b/docs/examples/docker_example.py
new file mode 100644
index 0000000000000000000000000000000000000000..48acc80995c44493d66f4ccfc4df721543113066
--- /dev/null
+++ b/docs/examples/docker_example.py
@@ -0,0 +1,357 @@
+import requests
+import json
+import time
+import sys
+import base64
+import os
+from typing import Dict, Any
+
+class Crawl4AiTester:
+ def __init__(self, base_url: str = "http://localhost:11235", api_token: str = None):
+ self.base_url = base_url
+ self.api_token = api_token or os.getenv('CRAWL4AI_API_TOKEN') or "test_api_code" # Check environment variable as fallback
+ self.headers = {'Authorization': f'Bearer {self.api_token}'} if self.api_token else {}
+
+ def submit_and_wait(self, request_data: Dict[str, Any], timeout: int = 300) -> Dict[str, Any]:
+ # Submit crawl job
+ response = requests.post(f"{self.base_url}/crawl", json=request_data, headers=self.headers)
+ if response.status_code == 403:
+ raise Exception("API token is invalid or missing")
+ task_id = response.json()["task_id"]
+ print(f"Task ID: {task_id}")
+
+ # Poll for result
+ start_time = time.time()
+ while True:
+ if time.time() - start_time > timeout:
+ raise TimeoutError(f"Task {task_id} did not complete within {timeout} seconds")
+
+ result = requests.get(f"{self.base_url}/task/{task_id}", headers=self.headers)
+ status = result.json()
+
+ if status["status"] == "failed":
+ print("Task failed:", status.get("error"))
+ raise Exception(f"Task failed: {status.get('error')}")
+
+ if status["status"] == "completed":
+ return status
+
+ time.sleep(2)
+
+ def submit_sync(self, request_data: Dict[str, Any]) -> Dict[str, Any]:
+ response = requests.post(f"{self.base_url}/crawl_sync", json=request_data, headers=self.headers, timeout=60)
+ if response.status_code == 408:
+ raise TimeoutError("Task did not complete within server timeout")
+ response.raise_for_status()
+ return response.json()
+
+ def crawl_direct(self, request_data: Dict[str, Any]) -> Dict[str, Any]:
+ """Directly crawl without using task queue"""
+ response = requests.post(
+ f"{self.base_url}/crawl_direct",
+ json=request_data,
+ headers=self.headers
+ )
+ response.raise_for_status()
+ return response.json()
+
+def test_docker_deployment(version="basic"):
+ tester = Crawl4AiTester(
+ base_url="http://localhost:11235" ,
+ # base_url="https://api.crawl4ai.com" # just for example
+ # api_token="test" # just for example
+ )
+ print(f"Testing Crawl4AI Docker {version} version")
+
+ # Health check with timeout and retry
+ max_retries = 5
+ for i in range(max_retries):
+ try:
+ health = requests.get(f"{tester.base_url}/health", timeout=10)
+ print("Health check:", health.json())
+ break
+ except requests.exceptions.RequestException as e:
+ if i == max_retries - 1:
+ print(f"Failed to connect after {max_retries} attempts")
+ sys.exit(1)
+ print(f"Waiting for service to start (attempt {i+1}/{max_retries})...")
+ time.sleep(5)
+
+ # Test cases based on version
+ test_basic_crawl_direct(tester)
+ test_basic_crawl(tester)
+ test_basic_crawl(tester)
+ test_basic_crawl_sync(tester)
+
+ if version in ["full", "transformer"]:
+ test_cosine_extraction(tester)
+
+ test_js_execution(tester)
+ test_css_selector(tester)
+ test_structured_extraction(tester)
+ test_llm_extraction(tester)
+ test_llm_with_ollama(tester)
+ test_screenshot(tester)
+
+
+def test_basic_crawl(tester: Crawl4AiTester):
+ print("\n=== Testing Basic Crawl ===")
+ request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 10,
+ "session_id": "test"
+ }
+
+ result = tester.submit_and_wait(request)
+ print(f"Basic crawl result length: {len(result['result']['markdown'])}")
+ assert result["result"]["success"]
+ assert len(result["result"]["markdown"]) > 0
+
+def test_basic_crawl_sync(tester: Crawl4AiTester):
+ print("\n=== Testing Basic Crawl (Sync) ===")
+ request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 10,
+ "session_id": "test"
+ }
+
+ result = tester.submit_sync(request)
+ print(f"Basic crawl result length: {len(result['result']['markdown'])}")
+ assert result['status'] == 'completed'
+ assert result['result']['success']
+ assert len(result['result']['markdown']) > 0
+
+def test_basic_crawl_direct(tester: Crawl4AiTester):
+ print("\n=== Testing Basic Crawl (Direct) ===")
+ request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 10,
+ # "session_id": "test"
+ "cache_mode": "bypass" # or "enabled", "disabled", "read_only", "write_only"
+ }
+
+ result = tester.crawl_direct(request)
+ print(f"Basic crawl result length: {len(result['result']['markdown'])}")
+ assert result['result']['success']
+ assert len(result['result']['markdown']) > 0
+
+def test_js_execution(tester: Crawl4AiTester):
+ print("\n=== Testing JS Execution ===")
+ request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 8,
+ "js_code": [
+ "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+ ],
+ "wait_for": "article.tease-card:nth-child(10)",
+ "crawler_params": {
+ "headless": True
+ }
+ }
+
+ result = tester.submit_and_wait(request)
+ print(f"JS execution result length: {len(result['result']['markdown'])}")
+ assert result["result"]["success"]
+
+def test_css_selector(tester: Crawl4AiTester):
+ print("\n=== Testing CSS Selector ===")
+ request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 7,
+ "css_selector": ".wide-tease-item__description",
+ "crawler_params": {
+ "headless": True
+ },
+ "extra": {"word_count_threshold": 10}
+
+ }
+
+ result = tester.submit_and_wait(request)
+ print(f"CSS selector result length: {len(result['result']['markdown'])}")
+ assert result["result"]["success"]
+
+def test_structured_extraction(tester: Crawl4AiTester):
+ print("\n=== Testing Structured Extraction ===")
+ schema = {
+ "name": "Coinbase Crypto Prices",
+ "baseSelector": ".cds-tableRow-t45thuk",
+ "fields": [
+ {
+ "name": "crypto",
+ "selector": "td:nth-child(1) h2",
+ "type": "text",
+ },
+ {
+ "name": "symbol",
+ "selector": "td:nth-child(1) p",
+ "type": "text",
+ },
+ {
+ "name": "price",
+ "selector": "td:nth-child(2)",
+ "type": "text",
+ }
+ ],
+ }
+
+ request = {
+ "urls": "https://www.coinbase.com/explore",
+ "priority": 9,
+ "extraction_config": {
+ "type": "json_css",
+ "params": {
+ "schema": schema
+ }
+ }
+ }
+
+ result = tester.submit_and_wait(request)
+ extracted = json.loads(result["result"]["extracted_content"])
+ print(f"Extracted {len(extracted)} items")
+ print("Sample item:", json.dumps(extracted[0], indent=2))
+ assert result["result"]["success"]
+ assert len(extracted) > 0
+
+def test_llm_extraction(tester: Crawl4AiTester):
+ print("\n=== Testing LLM Extraction ===")
+ schema = {
+ "type": "object",
+ "properties": {
+ "model_name": {
+ "type": "string",
+ "description": "Name of the OpenAI model."
+ },
+ "input_fee": {
+ "type": "string",
+ "description": "Fee for input token for the OpenAI model."
+ },
+ "output_fee": {
+ "type": "string",
+ "description": "Fee for output token for the OpenAI model."
+ }
+ },
+ "required": ["model_name", "input_fee", "output_fee"]
+ }
+
+ request = {
+ "urls": "https://openai.com/api/pricing",
+ "priority": 8,
+ "extraction_config": {
+ "type": "llm",
+ "params": {
+ "provider": "openai/gpt-4o-mini",
+ "api_token": os.getenv("OPENAI_API_KEY"),
+ "schema": schema,
+ "extraction_type": "schema",
+ "instruction": """From the crawled content, extract all mentioned model names along with their fees for input and output tokens."""
+ }
+ },
+ "crawler_params": {"word_count_threshold": 1}
+ }
+
+ try:
+ result = tester.submit_and_wait(request)
+ extracted = json.loads(result["result"]["extracted_content"])
+ print(f"Extracted {len(extracted)} model pricing entries")
+ print("Sample entry:", json.dumps(extracted[0], indent=2))
+ assert result["result"]["success"]
+ except Exception as e:
+ print(f"LLM extraction test failed (might be due to missing API key): {str(e)}")
+
+def test_llm_with_ollama(tester: Crawl4AiTester):
+ print("\n=== Testing LLM with Ollama ===")
+ schema = {
+ "type": "object",
+ "properties": {
+ "article_title": {
+ "type": "string",
+ "description": "The main title of the news article"
+ },
+ "summary": {
+ "type": "string",
+ "description": "A brief summary of the article content"
+ },
+ "main_topics": {
+ "type": "array",
+ "items": {"type": "string"},
+ "description": "Main topics or themes discussed in the article"
+ }
+ }
+ }
+
+ request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 8,
+ "extraction_config": {
+ "type": "llm",
+ "params": {
+ "provider": "ollama/llama2",
+ "schema": schema,
+ "extraction_type": "schema",
+ "instruction": "Extract the main article information including title, summary, and main topics."
+ }
+ },
+ "extra": {"word_count_threshold": 1},
+ "crawler_params": {"verbose": True}
+ }
+
+ try:
+ result = tester.submit_and_wait(request)
+ extracted = json.loads(result["result"]["extracted_content"])
+ print("Extracted content:", json.dumps(extracted, indent=2))
+ assert result["result"]["success"]
+ except Exception as e:
+ print(f"Ollama extraction test failed: {str(e)}")
+
+def test_cosine_extraction(tester: Crawl4AiTester):
+ print("\n=== Testing Cosine Extraction ===")
+ request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 8,
+ "extraction_config": {
+ "type": "cosine",
+ "params": {
+ "semantic_filter": "business finance economy",
+ "word_count_threshold": 10,
+ "max_dist": 0.2,
+ "top_k": 3
+ }
+ }
+ }
+
+ try:
+ result = tester.submit_and_wait(request)
+ extracted = json.loads(result["result"]["extracted_content"])
+ print(f"Extracted {len(extracted)} text clusters")
+ print("First cluster tags:", extracted[0]["tags"])
+ assert result["result"]["success"]
+ except Exception as e:
+ print(f"Cosine extraction test failed: {str(e)}")
+
+def test_screenshot(tester: Crawl4AiTester):
+ print("\n=== Testing Screenshot ===")
+ request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 5,
+ "screenshot": True,
+ "crawler_params": {
+ "headless": True
+ }
+ }
+
+ result = tester.submit_and_wait(request)
+ print("Screenshot captured:", bool(result["result"]["screenshot"]))
+
+ if result["result"]["screenshot"]:
+ # Save screenshot
+ screenshot_data = base64.b64decode(result["result"]["screenshot"])
+ with open("test_screenshot.jpg", "wb") as f:
+ f.write(screenshot_data)
+ print("Screenshot saved as test_screenshot.jpg")
+
+ assert result["result"]["success"]
+
+if __name__ == "__main__":
+ version = sys.argv[1] if len(sys.argv) > 1 else "basic"
+ # version = "full"
+ test_docker_deployment(version)
\ No newline at end of file
diff --git a/docs/examples/extraction_strategies_example.py b/docs/examples/extraction_strategies_example.py
new file mode 100644
index 0000000000000000000000000000000000000000..348b891ee1290e8c5e10a8d1c53b49cd02ac7606
--- /dev/null
+++ b/docs/examples/extraction_strategies_example.py
@@ -0,0 +1,115 @@
+"""
+Example demonstrating different extraction strategies with various input formats.
+This example shows how to:
+1. Use different input formats (markdown, HTML, fit_markdown)
+2. Work with JSON-based extractors (CSS and XPath)
+3. Use LLM-based extraction with different input formats
+4. Configure browser and crawler settings properly
+"""
+
+import asyncio
+import os
+from typing import Dict, Any
+
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.extraction_strategy import (
+ LLMExtractionStrategy,
+ JsonCssExtractionStrategy,
+ JsonXPathExtractionStrategy
+)
+from crawl4ai.chunking_strategy import RegexChunking, IdentityChunking
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def run_extraction(crawler: AsyncWebCrawler, url: str, strategy, name: str):
+ """Helper function to run extraction with proper configuration"""
+ try:
+ # Configure the crawler run settings
+ config = CrawlerRunConfig(
+ cache_mode=CacheMode.BYPASS,
+ extraction_strategy=strategy,
+ markdown_generator=DefaultMarkdownGenerator(
+ content_filter=PruningContentFilter() # For fit_markdown support
+ )
+ )
+
+ # Run the crawler
+ result = await crawler.arun(url=url, config=config)
+
+ if result.success:
+ print(f"\n=== {name} Results ===")
+ print(f"Extracted Content: {result.extracted_content}")
+ print(f"Raw Markdown Length: {len(result.markdown_v2.raw_markdown)}")
+ print(f"Citations Markdown Length: {len(result.markdown_v2.markdown_with_citations)}")
+ else:
+ print(f"Error in {name}: Crawl failed")
+
+ except Exception as e:
+ print(f"Error in {name}: {str(e)}")
+
+async def main():
+ # Example URL (replace with actual URL)
+ url = "https://example.com/product-page"
+
+ # Configure browser settings
+ browser_config = BrowserConfig(
+ headless=True,
+ verbose=True
+ )
+
+ # Initialize extraction strategies
+
+ # 1. LLM Extraction with different input formats
+ markdown_strategy = LLMExtractionStrategy(
+ provider="openai/gpt-4o-mini",
+ api_token=os.getenv("OPENAI_API_KEY"),
+ instruction="Extract product information including name, price, and description"
+ )
+
+ html_strategy = LLMExtractionStrategy(
+ input_format="html",
+ provider="openai/gpt-4o-mini",
+ api_token=os.getenv("OPENAI_API_KEY"),
+ instruction="Extract product information from HTML including structured data"
+ )
+
+ fit_markdown_strategy = LLMExtractionStrategy(
+ input_format="fit_markdown",
+ provider="openai/gpt-4o-mini",
+ api_token=os.getenv("OPENAI_API_KEY"),
+ instruction="Extract product information from cleaned markdown"
+ )
+
+ # 2. JSON CSS Extraction (automatically uses HTML input)
+ css_schema = {
+ "baseSelector": ".product",
+ "fields": [
+ {"name": "title", "selector": "h1.product-title", "type": "text"},
+ {"name": "price", "selector": ".price", "type": "text"},
+ {"name": "description", "selector": ".description", "type": "text"}
+ ]
+ }
+ css_strategy = JsonCssExtractionStrategy(schema=css_schema)
+
+ # 3. JSON XPath Extraction (automatically uses HTML input)
+ xpath_schema = {
+ "baseSelector": "//div[@class='product']",
+ "fields": [
+ {"name": "title", "selector": ".//h1[@class='product-title']/text()", "type": "text"},
+ {"name": "price", "selector": ".//span[@class='price']/text()", "type": "text"},
+ {"name": "description", "selector": ".//div[@class='description']/text()", "type": "text"}
+ ]
+ }
+ xpath_strategy = JsonXPathExtractionStrategy(schema=xpath_schema)
+
+ # Use context manager for proper resource handling
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ # Run all strategies
+ await run_extraction(crawler, url, markdown_strategy, "Markdown LLM")
+ await run_extraction(crawler, url, html_strategy, "HTML LLM")
+ await run_extraction(crawler, url, fit_markdown_strategy, "Fit Markdown LLM")
+ await run_extraction(crawler, url, css_strategy, "CSS Extraction")
+ await run_extraction(crawler, url, xpath_strategy, "XPath Extraction")
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/docs/examples/full_page_screenshot_and_pdf_export.md b/docs/examples/full_page_screenshot_and_pdf_export.md
new file mode 100644
index 0000000000000000000000000000000000000000..8522675c3537b29fc1fbbbb5b8aa378e1974d78a
--- /dev/null
+++ b/docs/examples/full_page_screenshot_and_pdf_export.md
@@ -0,0 +1,58 @@
+# Capturing Full-Page Screenshots and PDFs from Massive Webpages with Crawl4AI
+
+When dealing with very long web pages, traditional full-page screenshots can be slow or fail entirely. For large pages (like extensive Wikipedia articles), generating a single massive screenshot often leads to delays, memory issues, or style differences.
+
+**The New Approach:**
+We’ve introduced a new feature that effortlessly handles even the biggest pages by first exporting them as a PDF, then converting that PDF into a high-quality image. This approach leverages the browser’s built-in PDF rendering, making it both stable and efficient for very long content. You also have the option to directly save the PDF for your own usage—no need for multiple passes or complex stitching logic.
+
+**Key Benefits:**
+- **Reliability:** The PDF export never times out and works regardless of page length.
+- **Versatility:** Get both the PDF and a screenshot in one crawl, without reloading or reprocessing.
+- **Performance:** Skips manual scrolling and stitching images, reducing complexity and runtime.
+
+**Simple Example:**
+```python
+import os, sys
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+
+# Adjust paths as needed
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+
+async def main():
+ async with AsyncWebCrawler() as crawler:
+ # Request both PDF and screenshot
+ result = await crawler.arun(
+ url='https://en.wikipedia.org/wiki/List_of_common_misconceptions',
+ cache_mode=CacheMode.BYPASS,
+ pdf=True,
+ screenshot=True
+ )
+
+ if result.success:
+ # Save screenshot
+ if result.screenshot:
+ from base64 import b64decode
+ with open(os.path.join(__location__, "screenshot.png"), "wb") as f:
+ f.write(b64decode(result.screenshot))
+
+ # Save PDF
+ if result.pdf:
+ pdf_bytes = b64decode(result.pdf)
+ with open(os.path.join(__location__, "page.pdf"), "wb") as f:
+ f.write(pdf_bytes)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**What Happens Under the Hood:**
+- Crawl4AI navigates to the target page.
+- If `pdf=True`, it exports the current page as a full PDF, capturing all of its content no matter the length.
+- If `screenshot=True`, and a PDF is already available, it directly converts the first page of that PDF to an image for you—no repeated loading or scrolling.
+- Finally, you get your PDF and/or screenshot ready to use.
+
+**Conclusion:**
+With this feature, Crawl4AI becomes even more robust and versatile for large-scale content extraction. Whether you need a PDF snapshot or a quick screenshot, you now have a reliable solution for even the most extensive webpages.
\ No newline at end of file
diff --git a/docs/examples/hello_world.py b/docs/examples/hello_world.py
new file mode 100644
index 0000000000000000000000000000000000000000..18534d0e09ad7c6054cfc91ccd4c694143691530
--- /dev/null
+++ b/docs/examples/hello_world.py
@@ -0,0 +1,20 @@
+import asyncio
+from crawl4ai import *
+
+async def main():
+ browser_config = BrowserConfig(headless=True, verbose=True)
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ crawler_config = CrawlerRunConfig(
+ cache_mode=CacheMode.BYPASS,
+ markdown_generator=DefaultMarkdownGenerator(
+ content_filter=PruningContentFilter(threshold=0.48, threshold_type="fixed", min_word_threshold=0)
+ )
+ )
+ result = await crawler.arun(
+ url="https://www.helloworld.org",
+ config=crawler_config
+ )
+ print(result.markdown_v2.raw_markdown[:500])
+
+if __name__ == "__main__":
+ asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/hooks_example.py b/docs/examples/hooks_example.py
new file mode 100644
index 0000000000000000000000000000000000000000..09e0bc17d204b0c9f14eea6fc1f559eebf11720a
--- /dev/null
+++ b/docs/examples/hooks_example.py
@@ -0,0 +1,107 @@
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from playwright.async_api import Page, BrowserContext
+
+async def main():
+ print("🔗 Hooks Example: Demonstrating different hook use cases")
+
+ # Configure browser settings
+ browser_config = BrowserConfig(
+ headless=True
+ )
+
+ # Configure crawler settings
+ crawler_run_config = CrawlerRunConfig(
+ js_code="window.scrollTo(0, document.body.scrollHeight);",
+ wait_for="body",
+ cache_mode=CacheMode.BYPASS
+ )
+
+ # Create crawler instance
+ crawler = AsyncWebCrawler(config=browser_config)
+
+ # Define and set hook functions
+ async def on_browser_created(browser, context: BrowserContext, **kwargs):
+ """Hook called after the browser is created"""
+ print("[HOOK] on_browser_created - Browser is ready!")
+ # Example: Set a cookie that will be used for all requests
+ return browser
+
+ async def on_page_context_created(page: Page, context: BrowserContext, **kwargs):
+ """Hook called after a new page and context are created"""
+ print("[HOOK] on_page_context_created - New page created!")
+ # Example: Set default viewport size
+ await context.add_cookies([{
+ 'name': 'session_id',
+ 'value': 'example_session',
+ 'domain': '.example.com',
+ 'path': '/'
+ }])
+ await page.set_viewport_size({"width": 1920, "height": 1080})
+ return page
+
+ async def on_user_agent_updated(page: Page, context: BrowserContext, user_agent: str, **kwargs):
+ """Hook called when the user agent is updated"""
+ print(f"[HOOK] on_user_agent_updated - New user agent: {user_agent}")
+ return page
+
+ async def on_execution_started(page: Page, context: BrowserContext, **kwargs):
+ """Hook called after custom JavaScript execution"""
+ print("[HOOK] on_execution_started - Custom JS executed!")
+ return page
+
+ async def before_goto(page: Page, context: BrowserContext, url: str, **kwargs):
+ """Hook called before navigating to each URL"""
+ print(f"[HOOK] before_goto - About to visit: {url}")
+ # Example: Add custom headers for the request
+ await page.set_extra_http_headers({
+ "Custom-Header": "my-value"
+ })
+ return page
+
+ async def after_goto(page: Page, context: BrowserContext, url: str, response: dict, **kwargs):
+ """Hook called after navigating to each URL"""
+ print(f"[HOOK] after_goto - Successfully loaded: {url}")
+ # Example: Wait for a specific element to be loaded
+ try:
+ await page.wait_for_selector('.content', timeout=1000)
+ print("Content element found!")
+ except:
+ print("Content element not found, continuing anyway")
+ return page
+
+ async def before_retrieve_html(page: Page, context: BrowserContext, **kwargs):
+ """Hook called before retrieving the HTML content"""
+ print("[HOOK] before_retrieve_html - About to get HTML content")
+ # Example: Scroll to bottom to trigger lazy loading
+ await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
+ return page
+
+ async def before_return_html(page: Page, context: BrowserContext, html:str, **kwargs):
+ """Hook called before returning the HTML content"""
+ print(f"[HOOK] before_return_html - Got HTML content (length: {len(html)})")
+ # Example: You could modify the HTML content here if needed
+ return page
+
+ # Set all the hooks
+ crawler.crawler_strategy.set_hook("on_browser_created", on_browser_created)
+ crawler.crawler_strategy.set_hook("on_page_context_created", on_page_context_created)
+ crawler.crawler_strategy.set_hook("on_user_agent_updated", on_user_agent_updated)
+ crawler.crawler_strategy.set_hook("on_execution_started", on_execution_started)
+ crawler.crawler_strategy.set_hook("before_goto", before_goto)
+ crawler.crawler_strategy.set_hook("after_goto", after_goto)
+ crawler.crawler_strategy.set_hook("before_retrieve_html", before_retrieve_html)
+ crawler.crawler_strategy.set_hook("before_return_html", before_return_html)
+
+ await crawler.start()
+
+ # Example usage: crawl a simple website
+ url = 'https://example.com'
+ result = await crawler.arun(url, config=crawler_run_config)
+ print(f"\nCrawled URL: {result.url}")
+ print(f"HTML length: {len(result.html)}")
+
+ await crawler.close()
+
+if __name__ == "__main__":
+ import asyncio
+ asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/language_support_example.py b/docs/examples/language_support_example.py
new file mode 100644
index 0000000000000000000000000000000000000000..b74a8402e81263edc83e2ecb04454bcd4774f52b
--- /dev/null
+++ b/docs/examples/language_support_example.py
@@ -0,0 +1,45 @@
+import asyncio
+from crawl4ai import AsyncWebCrawler, AsyncPlaywrightCrawlerStrategy
+
+async def main():
+ # Example 1: Setting language when creating the crawler
+ crawler1 = AsyncWebCrawler(
+ crawler_strategy=AsyncPlaywrightCrawlerStrategy(
+ headers={"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7"}
+ )
+ )
+ result1 = await crawler1.arun("https://www.example.com")
+ print("Example 1 result:", result1.extracted_content[:100]) # Print first 100 characters
+
+ # Example 2: Setting language before crawling
+ crawler2 = AsyncWebCrawler()
+ crawler2.crawler_strategy.headers["Accept-Language"] = "es-ES,es;q=0.9,en-US;q=0.8,en;q=0.7"
+ result2 = await crawler2.arun("https://www.example.com")
+ print("Example 2 result:", result2.extracted_content[:100])
+
+ # Example 3: Setting language when calling arun method
+ crawler3 = AsyncWebCrawler()
+ result3 = await crawler3.arun(
+ "https://www.example.com",
+ headers={"Accept-Language": "de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7"}
+ )
+ print("Example 3 result:", result3.extracted_content[:100])
+
+ # Example 4: Crawling multiple pages with different languages
+ urls = [
+ ("https://www.example.com", "fr-FR,fr;q=0.9"),
+ ("https://www.example.org", "es-ES,es;q=0.9"),
+ ("https://www.example.net", "de-DE,de;q=0.9"),
+ ]
+
+ crawler4 = AsyncWebCrawler()
+ results = await asyncio.gather(*[
+ crawler4.arun(url, headers={"Accept-Language": lang})
+ for url, lang in urls
+ ])
+
+ for url, result in zip([u for u, _ in urls], results):
+ print(f"Result for {url}:", result.extracted_content[:100])
+
+if __name__ == "__main__":
+ asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/llm_extraction_openai_pricing.py b/docs/examples/llm_extraction_openai_pricing.py
new file mode 100644
index 0000000000000000000000000000000000000000..5ae3d4d1f68e96d85483468bee24bd084599bf26
--- /dev/null
+++ b/docs/examples/llm_extraction_openai_pricing.py
@@ -0,0 +1,40 @@
+from crawl4ai.extraction_strategy import *
+from crawl4ai.crawler_strategy import *
+import asyncio
+from pydantic import BaseModel, Field
+
+url = r'https://openai.com/api/pricing/'
+
+class OpenAIModelFee(BaseModel):
+ model_name: str = Field(..., description="Name of the OpenAI model.")
+ input_fee: str = Field(..., description="Fee for input token for the OpenAI model.")
+ output_fee: str = Field(..., description="Fee for output token for the OpenAI model.")
+
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+ # Use AsyncWebCrawler
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(
+ url=url,
+ word_count_threshold=1,
+ extraction_strategy= LLMExtractionStrategy(
+ # provider= "openai/gpt-4o", api_token = os.getenv('OPENAI_API_KEY'),
+ provider= "groq/llama-3.1-70b-versatile", api_token = os.getenv('GROQ_API_KEY'),
+ schema=OpenAIModelFee.model_json_schema(),
+ extraction_type="schema",
+ instruction="From the crawled content, extract all mentioned model names along with their " \
+ "fees for input and output tokens. Make sure not to miss anything in the entire content. " \
+ 'One extracted model JSON format should look like this: ' \
+ '{ "model_name": "GPT-4", "input_fee": "US$10.00 / 1M tokens", "output_fee": "US$30.00 / 1M tokens" }'
+ ),
+
+ )
+ print("Success:", result.success)
+ model_fees = json.loads(result.extracted_content)
+ print(len(model_fees))
+
+ with open(".data/data.json", "w", encoding="utf-8") as f:
+ f.write(result.extracted_content)
+
+asyncio.run(main())
diff --git a/docs/examples/quickstart.ipynb b/docs/examples/quickstart.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..4751dec8b9c2c55df265d4f1440eec0a14c9ddb1
--- /dev/null
+++ b/docs/examples/quickstart.ipynb
@@ -0,0 +1,664 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "0cba38e5",
+ "metadata": {},
+ "source": [
+ "# Crawl4AI 🕷️🤖\n",
+ "\n{data["content"]}\n '
+ for ref, data in user_session["context"].items()
+ ]
+ if context_messages:
+ system_message = {
+ "role": "system",
+ "content": (
+ "You are a helpful bot. Use the following context for answering questions. "
+ "Refer to the sources using the REF number in square brackets, e.g., [1], only if the source is given in the appendices below.\n\n"
+ "If the question requires any information from the provided appendices or context, refer to the sources. "
+ "If not, there is no need to add a references section. "
+ "At the end of your response, provide a reference section listing the URLs and their REF numbers only if sources from the appendices were used.\n\n"
+ "\n\n".join(context_messages)
+ )
+ }
+ else:
+ system_message = {
+ "role": "system",
+ "content": "You are a helpful assistant."
+ }
+
+
+ msg = cl.Message(content="")
+ await msg.send()
+
+ # Get response from the LLM
+ stream = await client.chat.completions.create(
+ messages=[
+ system_message,
+ *user_session["history"]
+ ],
+ stream=True,
+ **settings
+ )
+
+ assistant_response = ""
+ async for part in stream:
+ if token := part.choices[0].delta.content:
+ assistant_response += token
+ await msg.stream_token(token)
+
+ # Add assistant message to the history
+ user_session["history"].append({
+ "role": "assistant",
+ "content": assistant_response
+ })
+ await msg.update()
+
+ # Append the reference section to the assistant's response
+ reference_section = "\n\nReferences:\n"
+ for ref, data in user_session["context"].items():
+ reference_section += f"[{ref.split('_')[1]}]: {data['url']}\n"
+
+ msg.content += reference_section
+ await msg.update()
+
+
+@cl.on_audio_chunk
+async def on_audio_chunk(chunk: cl.AudioChunk):
+ if chunk.isStart:
+ buffer = BytesIO()
+ # This is required for whisper to recognize the file type
+ buffer.name = f"input_audio.{chunk.mimeType.split('/')[1]}"
+ # Initialize the session for a new audio stream
+ cl.user_session.set("audio_buffer", buffer)
+ cl.user_session.set("audio_mime_type", chunk.mimeType)
+
+ # Write the chunks to a buffer and transcribe the whole audio at the end
+ cl.user_session.get("audio_buffer").write(chunk.data)
+
+ pass
+
+@cl.step(type="tool")
+async def speech_to_text(audio_file):
+ cli = Groq()
+
+ response = await client.audio.transcriptions.create(
+ model="whisper-large-v3", file=audio_file
+ )
+
+ return response.text
+
+
+@cl.on_audio_end
+async def on_audio_end(elements: list[ElementBased]):
+ # Get the audio buffer from the session
+ audio_buffer: BytesIO = cl.user_session.get("audio_buffer")
+ audio_buffer.seek(0) # Move the file pointer to the beginning
+ audio_file = audio_buffer.read()
+ audio_mime_type: str = cl.user_session.get("audio_mime_type")
+
+ start_time = time.time()
+ whisper_input = (audio_buffer.name, audio_file, audio_mime_type)
+ transcription = await speech_to_text(whisper_input)
+ end_time = time.time()
+ print(f"Transcription took {end_time - start_time} seconds")
+
+ user_msg = cl.Message(
+ author="You",
+ type="user_message",
+ content=transcription
+ )
+ await user_msg.send()
+ await on_message(user_msg)
+
+
+if __name__ == "__main__":
+ from chainlit.cli import run_chainlit
+ run_chainlit(__file__)
+
+
diff --git a/docs/examples/rest_call.py b/docs/examples/rest_call.py
new file mode 100644
index 0000000000000000000000000000000000000000..465c61142992d340e7275a6efd4b8c7627cefb17
--- /dev/null
+++ b/docs/examples/rest_call.py
@@ -0,0 +1,64 @@
+
+import requests, base64, os
+
+data = {
+ "urls": ["https://www.nbcnews.com/business"],
+ "screenshot": True,
+}
+
+response = requests.post("https://crawl4ai.com/crawl", json=data)
+result = response.json()['results'][0]
+print(result.keys())
+# dict_keys(['url', 'html', 'success', 'cleaned_html', 'media',
+# 'links', 'screenshot', 'markdown', 'extracted_content',
+# 'metadata', 'error_message'])
+with open("screenshot.png", "wb") as f:
+ f.write(base64.b64decode(result['screenshot']))
+
+# Example of filtering the content using CSS selectors
+data = {
+ "urls": [
+ "https://www.nbcnews.com/business"
+ ],
+ "css_selector": "article",
+ "screenshot": True,
+}
+
+# Example of executing a JS script on the page before extracting the content
+data = {
+ "urls": [
+ "https://www.nbcnews.com/business"
+ ],
+ "screenshot": True,
+ 'js' : ["""
+ const loadMoreButton = Array.from(document.querySelectorAll('button')).
+ find(button => button.textContent.includes('Load More'));
+ loadMoreButton && loadMoreButton.click();
+ """]
+}
+
+# Example of using a custom extraction strategy
+data = {
+ "urls": [
+ "https://www.nbcnews.com/business"
+ ],
+ "extraction_strategy": "CosineStrategy",
+ "extraction_strategy_args": {
+ "semantic_filter": "inflation rent prices"
+ },
+}
+
+# Example of using LLM to extract content
+data = {
+ "urls": [
+ "https://www.nbcnews.com/business"
+ ],
+ "extraction_strategy": "LLMExtractionStrategy",
+ "extraction_strategy_args": {
+ "provider": "groq/llama3-8b-8192",
+ "api_token": os.environ.get("GROQ_API_KEY"),
+ "instruction": """I am interested in only financial news,
+ and translate them in French."""
+ },
+}
+
diff --git a/docs/examples/sample_ecommerce.html b/docs/examples/sample_ecommerce.html
new file mode 100644
index 0000000000000000000000000000000000000000..4698d9c69ba6ea36fd4709b7daca3ab78ec7f0fb
--- /dev/null
+++ b/docs/examples/sample_ecommerce.html
@@ -0,0 +1,106 @@
+
+
+
+ Sample E-commerce Page for JsonCssExtractionStrategy Testing
+
+
+
+ Sample E-commerce Product Catalog
+
+
+
+
+
\ No newline at end of file
diff --git a/docs/examples/ssl_example.py b/docs/examples/ssl_example.py
new file mode 100644
index 0000000000000000000000000000000000000000..410e9485e4b17a1dd2af901f32eea283d6db24ea
--- /dev/null
+++ b/docs/examples/ssl_example.py
@@ -0,0 +1,46 @@
+"""Example showing how to work with SSL certificates in Crawl4AI."""
+
+import asyncio
+import os
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+
+# Create tmp directory if it doesn't exist
+parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+tmp_dir = os.path.join(parent_dir, "tmp")
+os.makedirs(tmp_dir, exist_ok=True)
+
+async def main():
+ # Configure crawler to fetch SSL certificate
+ config = CrawlerRunConfig(
+ fetch_ssl_certificate=True,
+ cache_mode=CacheMode.BYPASS # Bypass cache to always get fresh certificates
+ )
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(
+ url='https://example.com',
+ config=config
+ )
+
+ if result.success and result.ssl_certificate:
+ cert = result.ssl_certificate
+
+ # 1. Access certificate properties directly
+ print("\nCertificate Information:")
+ print(f"Issuer: {cert.issuer.get('CN', '')}")
+ print(f"Valid until: {cert.valid_until}")
+ print(f"Fingerprint: {cert.fingerprint}")
+
+ # 2. Export certificate in different formats
+ cert.to_json(os.path.join(tmp_dir, "certificate.json")) # For analysis
+ print("\nCertificate exported to:")
+ print(f"- JSON: {os.path.join(tmp_dir, 'certificate.json')}")
+
+ pem_data = cert.to_pem(os.path.join(tmp_dir, "certificate.pem")) # For web servers
+ print(f"- PEM: {os.path.join(tmp_dir, 'certificate.pem')}")
+
+ der_data = cert.to_der(os.path.join(tmp_dir, "certificate.der")) # For Java apps
+ print(f"- DER: {os.path.join(tmp_dir, 'certificate.der')}")
+
+if __name__ == "__main__":
+ asyncio.run(main())
diff --git a/docs/examples/storage_state_tutorial.md b/docs/examples/storage_state_tutorial.md
new file mode 100644
index 0000000000000000000000000000000000000000..304e6399ad906f66c103b9b5b7789fb76e369d00
--- /dev/null
+++ b/docs/examples/storage_state_tutorial.md
@@ -0,0 +1,225 @@
+### Using `storage_state` to Pre-Load Cookies and LocalStorage
+
+Crawl4ai’s `AsyncWebCrawler` lets you preserve and reuse session data, including cookies and localStorage, across multiple runs. By providing a `storage_state`, you can start your crawls already “logged in” or with any other necessary session data—no need to repeat the login flow every time.
+
+#### What is `storage_state`?
+
+`storage_state` can be:
+
+- A dictionary containing cookies and localStorage data.
+- A path to a JSON file that holds this information.
+
+When you pass `storage_state` to the crawler, it applies these cookies and localStorage entries before loading any pages. This means your crawler effectively starts in a known authenticated or pre-configured state.
+
+#### Example Structure
+
+Here’s an example storage state:
+
+```json
+{
+ "cookies": [
+ {
+ "name": "session",
+ "value": "abcd1234",
+ "domain": "example.com",
+ "path": "/",
+ "expires": 1675363572.037711,
+ "httpOnly": false,
+ "secure": false,
+ "sameSite": "None"
+ }
+ ],
+ "origins": [
+ {
+ "origin": "https://example.com",
+ "localStorage": [
+ { "name": "token", "value": "my_auth_token" },
+ { "name": "refreshToken", "value": "my_refresh_token" }
+ ]
+ }
+ ]
+}
+```
+
+This JSON sets a `session` cookie and two localStorage entries (`token` and `refreshToken`) for `https://example.com`.
+
+---
+
+### Passing `storage_state` as a Dictionary
+
+You can directly provide the data as a dictionary:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+ storage_dict = {
+ "cookies": [
+ {
+ "name": "session",
+ "value": "abcd1234",
+ "domain": "example.com",
+ "path": "/",
+ "expires": 1675363572.037711,
+ "httpOnly": False,
+ "secure": False,
+ "sameSite": "None"
+ }
+ ],
+ "origins": [
+ {
+ "origin": "https://example.com",
+ "localStorage": [
+ {"name": "token", "value": "my_auth_token"},
+ {"name": "refreshToken", "value": "my_refresh_token"}
+ ]
+ }
+ ]
+ }
+
+ async with AsyncWebCrawler(
+ headless=True,
+ storage_state=storage_dict
+ ) as crawler:
+ result = await crawler.arun(url='https://example.com/protected')
+ if result.success:
+ print("Crawl succeeded with pre-loaded session data!")
+ print("Page HTML length:", len(result.html))
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+### Passing `storage_state` as a File
+
+If you prefer a file-based approach, save the JSON above to `mystate.json` and reference it:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+ async with AsyncWebCrawler(
+ headless=True,
+ storage_state="mystate.json" # Uses a JSON file instead of a dictionary
+ ) as crawler:
+ result = await crawler.arun(url='https://example.com/protected')
+ if result.success:
+ print("Crawl succeeded with pre-loaded session data!")
+ print("Page HTML length:", len(result.html))
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+### Using `storage_state` to Avoid Repeated Logins (Sign In Once, Use Later)
+
+A common scenario is when you need to log in to a site (entering username/password, etc.) to access protected pages. Doing so every crawl is cumbersome. Instead, you can:
+
+1. Perform the login once in a hook.
+2. After login completes, export the resulting `storage_state` to a file.
+3. On subsequent runs, provide that `storage_state` to skip the login step.
+
+**Step-by-Step Example:**
+
+**First Run (Perform Login and Save State):**
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def on_browser_created_hook(browser):
+ # Access the default context and create a page
+ context = browser.contexts[0]
+ page = await context.new_page()
+
+ # Navigate to the login page
+ await page.goto("https://example.com/login", wait_until="domcontentloaded")
+
+ # Fill in credentials and submit
+ await page.fill("input[name='username']", "myuser")
+ await page.fill("input[name='password']", "mypassword")
+ await page.click("button[type='submit']")
+ await page.wait_for_load_state("networkidle")
+
+ # Now the site sets tokens in localStorage and cookies
+ # Export this state to a file so we can reuse it
+ await context.storage_state(path="my_storage_state.json")
+ await page.close()
+
+async def main():
+ # First run: perform login and export the storage_state
+ async with AsyncWebCrawler(
+ headless=True,
+ verbose=True,
+ hooks={"on_browser_created": on_browser_created_hook},
+ use_persistent_context=True,
+ user_data_dir="./my_user_data"
+ ) as crawler:
+
+ # After on_browser_created_hook runs, we have storage_state saved to my_storage_state.json
+ result = await crawler.arun(
+ url='https://example.com/protected-page',
+ cache_mode=CacheMode.BYPASS,
+ markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+ )
+ print("First run result success:", result.success)
+ if result.success:
+ print("Protected page HTML length:", len(result.html))
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**Second Run (Reuse Saved State, No Login Needed):**
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def main():
+ # Second run: no need to hook on_browser_created this time.
+ # Just provide the previously saved storage state.
+ async with AsyncWebCrawler(
+ headless=True,
+ verbose=True,
+ use_persistent_context=True,
+ user_data_dir="./my_user_data",
+ storage_state="my_storage_state.json" # Reuse previously exported state
+ ) as crawler:
+
+ # Now the crawler starts already logged in
+ result = await crawler.arun(
+ url='https://example.com/protected-page',
+ cache_mode=CacheMode.BYPASS,
+ markdown_generator=DefaultMarkdownGenerator(options={"ignore_links": True}),
+ )
+ print("Second run result success:", result.success)
+ if result.success:
+ print("Protected page HTML length:", len(result.html))
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**What’s Happening Here?**
+
+- During the first run, the `on_browser_created_hook` logs into the site.
+- After logging in, the crawler exports the current session (cookies, localStorage, etc.) to `my_storage_state.json`.
+- On subsequent runs, passing `storage_state="my_storage_state.json"` starts the browser context with these tokens already in place, skipping the login steps.
+
+**Sign Out Scenario:**
+If the website allows you to sign out by clearing tokens or by navigating to a sign-out URL, you can also run a script that uses `on_browser_created_hook` or `arun` to simulate signing out, then export the resulting `storage_state` again. That would give you a baseline “logged out” state to start fresh from next time.
+
+---
+
+### Conclusion
+
+By using `storage_state`, you can skip repetitive actions, like logging in, and jump straight into crawling protected content. Whether you provide a file path or a dictionary, this powerful feature helps maintain state between crawls, simplifying your data extraction pipelines.
\ No newline at end of file
diff --git a/docs/examples/summarize_page.py b/docs/examples/summarize_page.py
new file mode 100644
index 0000000000000000000000000000000000000000..8515899970a3d02301c62803dd99999a0cfa42be
--- /dev/null
+++ b/docs/examples/summarize_page.py
@@ -0,0 +1,46 @@
+import os
+import time
+import json
+from crawl4ai.web_crawler import WebCrawler
+from crawl4ai.chunking_strategy import *
+from crawl4ai.extraction_strategy import *
+from crawl4ai.crawler_strategy import *
+
+url = r'https://marketplace.visualstudio.com/items?itemName=Unclecode.groqopilot'
+
+crawler = WebCrawler()
+crawler.warmup()
+
+from pydantic import BaseModel, Field
+
+class PageSummary(BaseModel):
+ title: str = Field(..., description="Title of the page.")
+ summary: str = Field(..., description="Summary of the page.")
+ brief_summary: str = Field(..., description="Brief summary of the page.")
+ keywords: list = Field(..., description="Keywords assigned to the page.")
+
+result = crawler.run(
+ url=url,
+ word_count_threshold=1,
+ extraction_strategy= LLMExtractionStrategy(
+ provider= "openai/gpt-4o", api_token = os.getenv('OPENAI_API_KEY'),
+ schema=PageSummary.model_json_schema(),
+ extraction_type="schema",
+ apply_chunking =False,
+ instruction="From the crawled content, extract the following details: "\
+ "1. Title of the page "\
+ "2. Summary of the page, which is a detailed summary "\
+ "3. Brief summary of the page, which is a paragraph text "\
+ "4. Keywords assigned to the page, which is a list of keywords. "\
+ 'The extracted JSON format should look like this: '\
+ '{ "title": "Page Title", "summary": "Detailed summary of the page.", "brief_summary": "Brief summary in a paragraph.", "keywords": ["keyword1", "keyword2", "keyword3"] }'
+ ),
+ bypass_cache=True,
+)
+
+page_summary = json.loads(result.extracted_content)
+
+print(page_summary)
+
+with open(".data/page_summary.json", "w", encoding="utf-8") as f:
+ f.write(result.extracted_content)
diff --git a/docs/examples/tutorial_dynamic_clicks.md b/docs/examples/tutorial_dynamic_clicks.md
new file mode 100644
index 0000000000000000000000000000000000000000..d9669952b4a63e47d71ea71d9396d3d77ae3888b
--- /dev/null
+++ b/docs/examples/tutorial_dynamic_clicks.md
@@ -0,0 +1,117 @@
+# Tutorial: Clicking Buttons to Load More Content with Crawl4AI
+
+## Introduction
+
+When scraping dynamic websites, it’s common to encounter “Load More” or “Next” buttons that must be clicked to reveal new content. Crawl4AI provides a straightforward way to handle these situations using JavaScript execution and waiting conditions. In this tutorial, we’ll cover two approaches:
+
+1. **Step-by-step (Session-based) Approach:** Multiple calls to `arun()` to progressively load more content.
+2. **Single-call Approach:** Execute a more complex JavaScript snippet inside a single `arun()` call to handle all clicks at once before the extraction.
+
+## Prerequisites
+
+- A working installation of Crawl4AI
+- Basic familiarity with Python’s `async`/`await` syntax
+
+## Step-by-Step Approach
+
+Use a session ID to maintain state across multiple `arun()` calls:
+
+```python
+from crawl4ai import AsyncWebCrawler, CacheMode
+
+js_code = [
+ # This JS finds the “Next” button and clicks it
+ "const nextButton = document.querySelector('button.next'); nextButton && nextButton.click();"
+]
+
+wait_for_condition = "css:.new-content-class"
+
+async with AsyncWebCrawler(headless=True, verbose=True) as crawler:
+ # 1. Load the initial page
+ result_initial = await crawler.arun(
+ url="https://example.com",
+ cache_mode=CacheMode.BYPASS,
+ session_id="my_session"
+ )
+
+ # 2. Click the 'Next' button and wait for new content
+ result_next = await crawler.arun(
+ url="https://example.com",
+ session_id="my_session",
+ js_code=js_code,
+ wait_for=wait_for_condition,
+ js_only=True,
+ cache_mode=CacheMode.BYPASS
+ )
+
+# `result_next` now contains the updated HTML after clicking 'Next'
+```
+
+**Key Points:**
+- **`session_id`**: Keeps the same browser context open.
+- **`js_code`**: Executes JavaScript in the context of the already loaded page.
+- **`wait_for`**: Ensures the crawler waits until new content is fully loaded.
+- **`js_only=True`**: Runs the JS in the current session without reloading the page.
+
+By repeating the `arun()` call multiple times and modifying the `js_code` (e.g., clicking different modules or pages), you can iteratively load all the desired content.
+
+## Single-call Approach
+
+If the page allows it, you can run a single `arun()` call with a more elaborate JavaScript snippet that:
+- Iterates over all the modules or "Next" buttons
+- Clicks them one by one
+- Waits for content updates between each click
+- Once done, returns control to Crawl4AI for extraction.
+
+Example snippet:
+
+```python
+from crawl4ai import AsyncWebCrawler, CacheMode
+
+js_code = [
+ # Example JS that clicks multiple modules:
+ """
+ (async () => {
+ const modules = document.querySelectorAll('.module-item');
+ for (let i = 0; i < modules.length; i++) {
+ modules[i].scrollIntoView();
+ modules[i].click();
+ // Wait for each module’s content to load, adjust 100ms as needed
+ await new Promise(r => setTimeout(r, 100));
+ }
+ })();
+ """
+]
+
+async with AsyncWebCrawler(headless=True, verbose=True) as crawler:
+ result = await crawler.arun(
+ url="https://example.com",
+ js_code=js_code,
+ wait_for="css:.final-loaded-content-class",
+ cache_mode=CacheMode.BYPASS
+ )
+
+# `result` now contains all content after all modules have been clicked in one go.
+```
+
+**Key Points:**
+- All interactions (clicks and waits) happen before the extraction.
+- Ideal for pages where all steps can be done in a single pass.
+
+## Choosing the Right Approach
+
+- **Step-by-Step (Session-based)**:
+ - Good when you need fine-grained control or must dynamically check conditions before clicking the next page.
+ - Useful if the page requires multiple conditions checked at runtime.
+
+- **Single-call**:
+ - Perfect if the sequence of interactions is known in advance.
+ - Cleaner code if the page’s structure is consistent and predictable.
+
+## Conclusion
+
+Crawl4AI makes it easy to handle dynamic content:
+- Use session IDs and multiple `arun()` calls for stepwise crawling.
+- Or pack all actions into one `arun()` call if the interactions are well-defined upfront.
+
+This flexibility ensures you can handle a wide range of dynamic web pages efficiently.
diff --git a/docs/examples/v0.3.74.overview.py b/docs/examples/v0.3.74.overview.py
new file mode 100644
index 0000000000000000000000000000000000000000..362ae8fc4446a9173e0dd91c6f0f9d995c6b7d53
--- /dev/null
+++ b/docs/examples/v0.3.74.overview.py
@@ -0,0 +1,277 @@
+import os, sys
+# append the parent directory to the sys.path
+parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(parent_dir)
+parent_parent_dir = os.path.dirname(parent_dir)
+sys.path.append(parent_parent_dir)
+__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
+__data__ = os.path.join(__location__, "__data")
+import asyncio
+from pathlib import Path
+import aiohttp
+import json
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+# 1. File Download Processing Example
+async def download_example():
+ """Example of downloading files from Python.org"""
+ # downloads_path = os.path.join(os.getcwd(), "downloads")
+ downloads_path = os.path.join(Path.home(), ".crawl4ai", "downloads")
+ os.makedirs(downloads_path, exist_ok=True)
+
+ print(f"Downloads will be saved to: {downloads_path}")
+
+ async with AsyncWebCrawler(
+ accept_downloads=True,
+ downloads_path=downloads_path,
+ verbose=True
+ ) as crawler:
+ result = await crawler.arun(
+ url="https://www.python.org/downloads/",
+ js_code="""
+ // Find and click the first Windows installer link
+ const downloadLink = document.querySelector('a[href$=".exe"]');
+ if (downloadLink) {
+ console.log('Found download link:', downloadLink.href);
+ downloadLink.click();
+ } else {
+ console.log('No .exe download link found');
+ }
+ """,
+ delay_before_return_html=1, # Wait 5 seconds to ensure download starts
+ cache_mode=CacheMode.BYPASS
+ )
+
+ if result.downloaded_files:
+ print("\nDownload successful!")
+ print("Downloaded files:")
+ for file_path in result.downloaded_files:
+ print(f"- {file_path}")
+ print(f" File size: {os.path.getsize(file_path) / (1024*1024):.2f} MB")
+ else:
+ print("\nNo files were downloaded")
+
+# 2. Local File and Raw HTML Processing Example
+async def local_and_raw_html_example():
+ """Example of processing local files and raw HTML"""
+ # Create a sample HTML file
+ sample_file = os.path.join(__data__, "sample.html")
+ with open(sample_file, "w") as f:
+ f.write("""
+
+ Test Content
+ This is a test paragraph.
+
+ """)
+
+ async with AsyncWebCrawler(verbose=True) as crawler:
+ # Process local file
+ local_result = await crawler.arun(
+ url=f"file://{os.path.abspath(sample_file)}"
+ )
+
+ # Process raw HTML
+ raw_html = """
+
+ Raw HTML Test
+ This is a test of raw HTML processing.
+
+ """
+ raw_result = await crawler.arun(
+ url=f"raw:{raw_html}"
+ )
+
+ # Clean up
+ os.remove(sample_file)
+
+ print("Local file content:", local_result.markdown)
+ print("\nRaw HTML content:", raw_result.markdown)
+
+# 3. Enhanced Markdown Generation Example
+async def markdown_generation_example():
+ """Example of enhanced markdown generation with citations and LLM-friendly features"""
+ async with AsyncWebCrawler(verbose=True) as crawler:
+ # Create a content filter (optional)
+ content_filter = BM25ContentFilter(
+ # user_query="History and cultivation",
+ bm25_threshold=1.0
+ )
+
+ result = await crawler.arun(
+ url="https://en.wikipedia.org/wiki/Apple",
+ css_selector="main div#bodyContent",
+ content_filter=content_filter,
+ cache_mode=CacheMode.BYPASS
+ )
+
+ from crawl4ai import AsyncWebCrawler
+ from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+ result = await crawler.arun(
+ url="https://en.wikipedia.org/wiki/Apple",
+ css_selector="main div#bodyContent",
+ content_filter=BM25ContentFilter()
+ )
+ print(result.markdown_v2.fit_markdown)
+
+ print("\nMarkdown Generation Results:")
+ print(f"1. Original markdown length: {len(result.markdown)}")
+ print(f"2. New markdown versions (markdown_v2):")
+ print(f" - Raw markdown length: {len(result.markdown_v2.raw_markdown)}")
+ print(f" - Citations markdown length: {len(result.markdown_v2.markdown_with_citations)}")
+ print(f" - References section length: {len(result.markdown_v2.references_markdown)}")
+ if result.markdown_v2.fit_markdown:
+ print(f" - Filtered markdown length: {len(result.markdown_v2.fit_markdown)}")
+
+ # Save examples to files
+ output_dir = os.path.join(__data__, "markdown_examples")
+ os.makedirs(output_dir, exist_ok=True)
+
+ # Save different versions
+ with open(os.path.join(output_dir, "1_raw_markdown.md"), "w") as f:
+ f.write(result.markdown_v2.raw_markdown)
+
+ with open(os.path.join(output_dir, "2_citations_markdown.md"), "w") as f:
+ f.write(result.markdown_v2.markdown_with_citations)
+
+ with open(os.path.join(output_dir, "3_references.md"), "w") as f:
+ f.write(result.markdown_v2.references_markdown)
+
+ if result.markdown_v2.fit_markdown:
+ with open(os.path.join(output_dir, "4_filtered_markdown.md"), "w") as f:
+ f.write(result.markdown_v2.fit_markdown)
+
+ print(f"\nMarkdown examples saved to: {output_dir}")
+
+ # Show a sample of citations and references
+ print("\nSample of markdown with citations:")
+ print(result.markdown_v2.markdown_with_citations[:500] + "...\n")
+ print("Sample of references:")
+ print('\n'.join(result.markdown_v2.references_markdown.split('\n')[:10]) + "...")
+
+# 4. Browser Management Example
+async def browser_management_example():
+ """Example of using enhanced browser management features"""
+ # Use the specified user directory path
+ user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
+ os.makedirs(user_data_dir, exist_ok=True)
+
+ print(f"Browser profile will be saved to: {user_data_dir}")
+
+ async with AsyncWebCrawler(
+ use_managed_browser=True,
+ user_data_dir=user_data_dir,
+ headless=False,
+ verbose=True
+ ) as crawler:
+
+ result = await crawler.arun(
+ url="https://crawl4ai.com",
+ # session_id="persistent_session_1",
+ cache_mode=CacheMode.BYPASS
+ )
+ # Use GitHub as an example - it's a good test for browser management
+ # because it requires proper browser handling
+ result = await crawler.arun(
+ url="https://github.com/trending",
+ # session_id="persistent_session_1",
+ cache_mode=CacheMode.BYPASS
+ )
+
+ print("\nBrowser session result:", result.success)
+ if result.success:
+ print("Page title:", result.metadata.get('title', 'No title found'))
+
+# 5. API Usage Example
+async def api_example():
+ """Example of using the new API endpoints"""
+ api_token = os.getenv('CRAWL4AI_API_TOKEN') or "test_api_code"
+ headers = {'Authorization': f'Bearer {api_token}'}
+ async with aiohttp.ClientSession() as session:
+ # Submit crawl job
+ crawl_request = {
+ "urls": ["https://news.ycombinator.com"], # Hacker News as an example
+ "extraction_config": {
+ "type": "json_css",
+ "params": {
+ "schema": {
+ "name": "Hacker News Articles",
+ "baseSelector": ".athing",
+ "fields": [
+ {
+ "name": "title",
+ "selector": ".title a",
+ "type": "text"
+ },
+ {
+ "name": "score",
+ "selector": ".score",
+ "type": "text"
+ },
+ {
+ "name": "url",
+ "selector": ".title a",
+ "type": "attribute",
+ "attribute": "href"
+ }
+ ]
+ }
+ }
+ },
+ "crawler_params": {
+ "headless": True,
+ # "use_managed_browser": True
+ },
+ "cache_mode": "bypass",
+ # "screenshot": True,
+ # "magic": True
+ }
+
+ async with session.post(
+ "http://localhost:11235/crawl",
+ json=crawl_request,
+ headers=headers
+ ) as response:
+ task_data = await response.json()
+ task_id = task_data["task_id"]
+
+ # Check task status
+ while True:
+ async with session.get(
+ f"http://localhost:11235/task/{task_id}",
+ headers=headers
+ ) as status_response:
+ result = await status_response.json()
+ print(f"Task status: {result['status']}")
+
+ if result["status"] == "completed":
+ print("Task completed!")
+ print("Results:")
+ news = json.loads(result["results"][0]['extracted_content'])
+ print(json.dumps(news[:4], indent=2))
+ break
+ else:
+ await asyncio.sleep(1)
+
+# Main execution
+async def main():
+ # print("Running Crawl4AI feature examples...")
+
+ # print("\n1. Running Download Example:")
+ # await download_example()
+
+ # print("\n2. Running Markdown Generation Example:")
+ # await markdown_generation_example()
+
+ # # print("\n3. Running Local and Raw HTML Example:")
+ # await local_and_raw_html_example()
+
+ # # print("\n4. Running Browser Management Example:")
+ await browser_management_example()
+
+ # print("\n5. Running API Example:")
+ await api_example()
+
+if __name__ == "__main__":
+ asyncio.run(main())
\ No newline at end of file
diff --git a/docs/examples/v0_4_24_walkthrough.py b/docs/examples/v0_4_24_walkthrough.py
new file mode 100644
index 0000000000000000000000000000000000000000..135ac29c7ef9f4b75d328f3583675867d66367e0
--- /dev/null
+++ b/docs/examples/v0_4_24_walkthrough.py
@@ -0,0 +1,443 @@
+"""
+Crawl4AI v0.4.24 Feature Walkthrough
+===================================
+
+This script demonstrates the new features introduced in Crawl4AI v0.4.24.
+Each section includes detailed examples and explanations of the new capabilities.
+"""
+
+import asyncio
+import os
+import json
+import re
+from typing import List, Optional, Dict, Any
+from pydantic import BaseModel, Field
+from crawl4ai import (
+ AsyncWebCrawler,
+ BrowserConfig,
+ CrawlerRunConfig,
+ CacheMode,
+ LLMExtractionStrategy,
+ JsonCssExtractionStrategy
+)
+from crawl4ai.content_filter_strategy import RelevantContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from bs4 import BeautifulSoup
+
+# Sample HTML for demonstrations
+SAMPLE_HTML = """
+
+
+
+
+
+
First post content...
+
Read More
+
+
+
+
+
+
Second post content...
+
Read More
+
+
+
+
+
+
+
Technical Requirements
+
+
+ 5+ years experience in Machine Learning
+
+
+ Proficiency in Python and PyTorch/TensorFlow
+
+
+ Experience with distributed training systems
+
+
+
+
Professional Skills
+
+
+ Strong problem-solving abilities
+
+
+ Experience leading technical teams
+
+
+
+
+
+
+ Application Deadline: February 28, 2024
+
+
+
+
+
")}value(){return this.buffer}span(e){
+ this.buffer+=``}}const s=(e={})=>{const n={children:[]}
+ ;return Object.assign(n,e),n};class o{constructor(){
+ this.rootNode=s(),this.stack=[this.rootNode]}get top(){
+ return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){
+ this.top.children.push(e)}openNode(e){const n=s({scope:e})
+ ;this.add(n),this.stack.push(n)}closeNode(){
+ if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){
+ for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)}
+ walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,n){
+ return"string"==typeof n?e.addText(n):n.children&&(e.openNode(n),
+ n.children.forEach((n=>this._walk(e,n))),e.closeNode(n)),e}static _collapse(e){
+ "string"!=typeof e&&e.children&&(e.children.every((e=>"string"==typeof e))?e.children=[e.children.join("")]:e.children.forEach((e=>{
+ o._collapse(e)})))}}class l extends o{constructor(e){super(),this.options=e}
+ addText(e){""!==e&&this.add(e)}startScope(e){this.openNode(e)}endScope(){
+ this.closeNode()}__addSublanguage(e,n){const t=e.root
+ ;n&&(t.scope="language:"+n),this.add(t)}toHTML(){
+ return new r(this,this.options).value()}finalize(){
+ return this.closeAllNodes(),!0}}function c(e){
+ return e?"string"==typeof e?e:e.source:null}function d(e){return b("(?=",e,")")}
+ function g(e){return b("(?:",e,")*")}function u(e){return b("(?:",e,")?")}
+ function b(...e){return e.map((e=>c(e))).join("")}function m(...e){const n=(e=>{
+ const n=e[e.length-1]
+ ;return"object"==typeof n&&n.constructor===Object?(e.splice(e.length-1,1),n):{}
+ })(e);return"("+(n.capture?"":"?:")+e.map((e=>c(e))).join("|")+")"}
+ function p(e){return RegExp(e.toString()+"|").exec("").length-1}
+ const _=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./
+ ;function h(e,{joinWith:n}){let t=0;return e.map((e=>{t+=1;const n=t
+ ;let a=c(e),i="";for(;a.length>0;){const e=_.exec(a);if(!e){i+=a;break}
+ i+=a.substring(0,e.index),
+ a=a.substring(e.index+e[0].length),"\\"===e[0][0]&&e[1]?i+="\\"+(Number(e[1])+n):(i+=e[0],
+ "("===e[0]&&t++)}return i})).map((e=>`(${e})`)).join(n)}
+ const f="[a-zA-Z]\\w*",E="[a-zA-Z_]\\w*",y="\\b\\d+(\\.\\d+)?",N="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",w="\\b(0b[01]+)",v={
+ begin:"\\\\[\\s\\S]",relevance:0},O={scope:"string",begin:"'",end:"'",
+ illegal:"\\n",contains:[v]},k={scope:"string",begin:'"',end:'"',illegal:"\\n",
+ contains:[v]},x=(e,n,t={})=>{const i=a({scope:"comment",begin:e,end:n,
+ contains:[]},t);i.contains.push({scope:"doctag",
+ begin:"[ ]*(?=(TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):)",
+ end:/(TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):/,excludeBegin:!0,relevance:0})
+ ;const r=m("I","a","is","so","us","to","at","if","in","it","on",/[A-Za-z]+['](d|ve|re|ll|t|s|n)/,/[A-Za-z]+[-][a-z]+/,/[A-Za-z][a-z]{2,}/)
+ ;return i.contains.push({begin:b(/[ ]+/,"(",r,/[.]?[:]?([.][ ]|[ ])/,"){3}")}),i
+ },M=x("//","$"),S=x("/\\*","\\*/"),A=x("#","$");var C=Object.freeze({
+ __proto__:null,APOS_STRING_MODE:O,BACKSLASH_ESCAPE:v,BINARY_NUMBER_MODE:{
+ scope:"number",begin:w,relevance:0},BINARY_NUMBER_RE:w,COMMENT:x,
+ C_BLOCK_COMMENT_MODE:S,C_LINE_COMMENT_MODE:M,C_NUMBER_MODE:{scope:"number",
+ begin:N,relevance:0},C_NUMBER_RE:N,END_SAME_AS_BEGIN:e=>Object.assign(e,{
+ "on:begin":(e,n)=>{n.data._beginMatch=e[1]},"on:end":(e,n)=>{
+ n.data._beginMatch!==e[1]&&n.ignoreMatch()}}),HASH_COMMENT_MODE:A,IDENT_RE:f,
+ MATCH_NOTHING_RE:/\b\B/,METHOD_GUARD:{begin:"\\.\\s*"+E,relevance:0},
+ NUMBER_MODE:{scope:"number",begin:y,relevance:0},NUMBER_RE:y,
+ PHRASAL_WORDS_MODE:{
+ begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/
+ },QUOTE_STRING_MODE:k,REGEXP_MODE:{scope:"regexp",begin:/\/(?=[^/\n]*\/)/,
+ end:/\/[gimuy]*/,contains:[v,{begin:/\[/,end:/\]/,relevance:0,contains:[v]}]},
+ RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",
+ SHEBANG:(e={})=>{const n=/^#![ ]*\//
+ ;return e.binary&&(e.begin=b(n,/.*\b/,e.binary,/\b.*/)),a({scope:"meta",begin:n,
+ end:/$/,relevance:0,"on:begin":(e,n)=>{0!==e.index&&n.ignoreMatch()}},e)},
+ TITLE_MODE:{scope:"title",begin:f,relevance:0},UNDERSCORE_IDENT_RE:E,
+ UNDERSCORE_TITLE_MODE:{scope:"title",begin:E,relevance:0}});function T(e,n){
+ "."===e.input[e.index-1]&&n.ignoreMatch()}function R(e,n){
+ void 0!==e.className&&(e.scope=e.className,delete e.className)}function D(e,n){
+ n&&e.beginKeywords&&(e.begin="\\b("+e.beginKeywords.split(" ").join("|")+")(?!\\.)(?=\\b|\\s)",
+ e.__beforeBegin=T,e.keywords=e.keywords||e.beginKeywords,delete e.beginKeywords,
+ void 0===e.relevance&&(e.relevance=0))}function I(e,n){
+ Array.isArray(e.illegal)&&(e.illegal=m(...e.illegal))}function L(e,n){
+ if(e.match){
+ if(e.begin||e.end)throw Error("begin & end are not supported with match")
+ ;e.begin=e.match,delete e.match}}function B(e,n){
+ void 0===e.relevance&&(e.relevance=1)}const $=(e,n)=>{if(!e.beforeMatch)return
+ ;if(e.starts)throw Error("beforeMatch cannot be used with starts")
+ ;const t=Object.assign({},e);Object.keys(e).forEach((n=>{delete e[n]
+ })),e.keywords=t.keywords,e.begin=b(t.beforeMatch,d(t.begin)),e.starts={
+ relevance:0,contains:[Object.assign(t,{endsParent:!0})]
+ },e.relevance=0,delete t.beforeMatch
+ },z=["of","and","for","in","not","or","if","then","parent","list","value"],F="keyword"
+ ;function U(e,n,t=F){const a=Object.create(null)
+ ;return"string"==typeof e?i(t,e.split(" ")):Array.isArray(e)?i(t,e):Object.keys(e).forEach((t=>{
+ Object.assign(a,U(e[t],n,t))})),a;function i(e,t){
+ n&&(t=t.map((e=>e.toLowerCase()))),t.forEach((n=>{const t=n.split("|")
+ ;a[t[0]]=[e,j(t[0],t[1])]}))}}function j(e,n){
+ return n?Number(n):(e=>z.includes(e.toLowerCase()))(e)?0:1}const P={},K=e=>{
+ console.error(e)},H=(e,...n)=>{console.log("WARN: "+e,...n)},q=(e,n)=>{
+ P[`${e}/${n}`]||(console.log(`Deprecated as of ${e}. ${n}`),P[`${e}/${n}`]=!0)
+ },G=Error();function Z(e,n,{key:t}){let a=0;const i=e[t],r={},s={}
+ ;for(let e=1;e<=n.length;e++)s[e+a]=i[e],r[e+a]=!0,a+=p(n[e-1])
+ ;e[t]=s,e[t]._emit=r,e[t]._multi=!0}function W(e){(e=>{
+ e.scope&&"object"==typeof e.scope&&null!==e.scope&&(e.beginScope=e.scope,
+ delete e.scope)})(e),"string"==typeof e.beginScope&&(e.beginScope={
+ _wrap:e.beginScope}),"string"==typeof e.endScope&&(e.endScope={_wrap:e.endScope
+ }),(e=>{if(Array.isArray(e.begin)){
+ if(e.skip||e.excludeBegin||e.returnBegin)throw K("skip, excludeBegin, returnBegin not compatible with beginScope: {}"),
+ G
+ ;if("object"!=typeof e.beginScope||null===e.beginScope)throw K("beginScope must be object"),
+ G;Z(e,e.begin,{key:"beginScope"}),e.begin=h(e.begin,{joinWith:""})}})(e),(e=>{
+ if(Array.isArray(e.end)){
+ if(e.skip||e.excludeEnd||e.returnEnd)throw K("skip, excludeEnd, returnEnd not compatible with endScope: {}"),
+ G
+ ;if("object"!=typeof e.endScope||null===e.endScope)throw K("endScope must be object"),
+ G;Z(e,e.end,{key:"endScope"}),e.end=h(e.end,{joinWith:""})}})(e)}function Q(e){
+ function n(n,t){
+ return RegExp(c(n),"m"+(e.case_insensitive?"i":"")+(e.unicodeRegex?"u":"")+(t?"g":""))
+ }class t{constructor(){
+ this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0}
+ addRule(e,n){
+ n.position=this.position++,this.matchIndexes[this.matchAt]=n,this.regexes.push([n,e]),
+ this.matchAt+=p(e)+1}compile(){0===this.regexes.length&&(this.exec=()=>null)
+ ;const e=this.regexes.map((e=>e[1]));this.matcherRe=n(h(e,{joinWith:"|"
+ }),!0),this.lastIndex=0}exec(e){this.matcherRe.lastIndex=this.lastIndex
+ ;const n=this.matcherRe.exec(e);if(!n)return null
+ ;const t=n.findIndex(((e,n)=>n>0&&void 0!==e)),a=this.matchIndexes[t]
+ ;return n.splice(0,t),Object.assign(n,a)}}class i{constructor(){
+ this.rules=[],this.multiRegexes=[],
+ this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){
+ if(this.multiRegexes[e])return this.multiRegexes[e];const n=new t
+ ;return this.rules.slice(e).forEach((([e,t])=>n.addRule(e,t))),
+ n.compile(),this.multiRegexes[e]=n,n}resumingScanAtSamePosition(){
+ return 0!==this.regexIndex}considerAll(){this.regexIndex=0}addRule(e,n){
+ this.rules.push([e,n]),"begin"===n.type&&this.count++}exec(e){
+ const n=this.getMatcher(this.regexIndex);n.lastIndex=this.lastIndex
+ ;let t=n.exec(e)
+ ;if(this.resumingScanAtSamePosition())if(t&&t.index===this.lastIndex);else{
+ const n=this.getMatcher(0);n.lastIndex=this.lastIndex+1,t=n.exec(e)}
+ return t&&(this.regexIndex+=t.position+1,
+ this.regexIndex===this.count&&this.considerAll()),t}}
+ if(e.compilerExtensions||(e.compilerExtensions=[]),
+ e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.")
+ ;return e.classNameAliases=a(e.classNameAliases||{}),function t(r,s){const o=r
+ ;if(r.isCompiled)return o
+ ;[R,L,W,$].forEach((e=>e(r,s))),e.compilerExtensions.forEach((e=>e(r,s))),
+ r.__beforeBegin=null,[D,I,B].forEach((e=>e(r,s))),r.isCompiled=!0;let l=null
+ ;return"object"==typeof r.keywords&&r.keywords.$pattern&&(r.keywords=Object.assign({},r.keywords),
+ l=r.keywords.$pattern,
+ delete r.keywords.$pattern),l=l||/\w+/,r.keywords&&(r.keywords=U(r.keywords,e.case_insensitive)),
+ o.keywordPatternRe=n(l,!0),
+ s&&(r.begin||(r.begin=/\B|\b/),o.beginRe=n(o.begin),r.end||r.endsWithParent||(r.end=/\B|\b/),
+ r.end&&(o.endRe=n(o.end)),
+ o.terminatorEnd=c(o.end)||"",r.endsWithParent&&s.terminatorEnd&&(o.terminatorEnd+=(r.end?"|":"")+s.terminatorEnd)),
+ r.illegal&&(o.illegalRe=n(r.illegal)),
+ r.contains||(r.contains=[]),r.contains=[].concat(...r.contains.map((e=>(e=>(e.variants&&!e.cachedVariants&&(e.cachedVariants=e.variants.map((n=>a(e,{
+ variants:null},n)))),e.cachedVariants?e.cachedVariants:X(e)?a(e,{
+ starts:e.starts?a(e.starts):null
+ }):Object.isFrozen(e)?a(e):e))("self"===e?r:e)))),r.contains.forEach((e=>{t(e,o)
+ })),r.starts&&t(r.starts,s),o.matcher=(e=>{const n=new i
+ ;return e.contains.forEach((e=>n.addRule(e.begin,{rule:e,type:"begin"
+ }))),e.terminatorEnd&&n.addRule(e.terminatorEnd,{type:"end"
+ }),e.illegal&&n.addRule(e.illegal,{type:"illegal"}),n})(o),o}(e)}function X(e){
+ return!!e&&(e.endsWithParent||X(e.starts))}class V extends Error{
+ constructor(e,n){super(e),this.name="HTMLInjectionError",this.html=n}}
+ const J=t,Y=a,ee=Symbol("nomatch"),ne=t=>{
+ const a=Object.create(null),i=Object.create(null),r=[];let s=!0
+ ;const o="Could not find the language '{}', did you forget to load/include a language module?",c={
+ disableAutodetect:!0,name:"Plain text",contains:[]};let p={
+ ignoreUnescapedHTML:!1,throwUnescapedHTML:!1,noHighlightRe:/^(no-?highlight)$/i,
+ languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-",
+ cssSelector:"pre code",languages:null,__emitter:l};function _(e){
+ return p.noHighlightRe.test(e)}function h(e,n,t){let a="",i=""
+ ;"object"==typeof n?(a=e,
+ t=n.ignoreIllegals,i=n.language):(q("10.7.0","highlight(lang, code, ...args) has been deprecated."),
+ q("10.7.0","Please use highlight(code, options) instead.\nhttps://github.com/highlightjs/highlight.js/issues/2277"),
+ i=e,a=n),void 0===t&&(t=!0);const r={code:a,language:i};x("before:highlight",r)
+ ;const s=r.result?r.result:f(r.language,r.code,t)
+ ;return s.code=r.code,x("after:highlight",s),s}function f(e,t,i,r){
+ const l=Object.create(null);function c(){if(!x.keywords)return void S.addText(A)
+ ;let e=0;x.keywordPatternRe.lastIndex=0;let n=x.keywordPatternRe.exec(A),t=""
+ ;for(;n;){t+=A.substring(e,n.index)
+ ;const i=w.case_insensitive?n[0].toLowerCase():n[0],r=(a=i,x.keywords[a]);if(r){
+ const[e,a]=r
+ ;if(S.addText(t),t="",l[i]=(l[i]||0)+1,l[i]<=7&&(C+=a),e.startsWith("_"))t+=n[0];else{
+ const t=w.classNameAliases[e]||e;g(n[0],t)}}else t+=n[0]
+ ;e=x.keywordPatternRe.lastIndex,n=x.keywordPatternRe.exec(A)}var a
+ ;t+=A.substring(e),S.addText(t)}function d(){null!=x.subLanguage?(()=>{
+ if(""===A)return;let e=null;if("string"==typeof x.subLanguage){
+ if(!a[x.subLanguage])return void S.addText(A)
+ ;e=f(x.subLanguage,A,!0,M[x.subLanguage]),M[x.subLanguage]=e._top
+ }else e=E(A,x.subLanguage.length?x.subLanguage:null)
+ ;x.relevance>0&&(C+=e.relevance),S.__addSublanguage(e._emitter,e.language)
+ })():c(),A=""}function g(e,n){
+ ""!==e&&(S.startScope(n),S.addText(e),S.endScope())}function u(e,n){let t=1
+ ;const a=n.length-1;for(;t<=a;){if(!e._emit[t]){t++;continue}
+ const a=w.classNameAliases[e[t]]||e[t],i=n[t];a?g(i,a):(A=i,c(),A=""),t++}}
+ function b(e,n){
+ return e.scope&&"string"==typeof e.scope&&S.openNode(w.classNameAliases[e.scope]||e.scope),
+ e.beginScope&&(e.beginScope._wrap?(g(A,w.classNameAliases[e.beginScope._wrap]||e.beginScope._wrap),
+ A=""):e.beginScope._multi&&(u(e.beginScope,n),A="")),x=Object.create(e,{parent:{
+ value:x}}),x}function m(e,t,a){let i=((e,n)=>{const t=e&&e.exec(n)
+ ;return t&&0===t.index})(e.endRe,a);if(i){if(e["on:end"]){const a=new n(e)
+ ;e["on:end"](t,a),a.isMatchIgnored&&(i=!1)}if(i){
+ for(;e.endsParent&&e.parent;)e=e.parent;return e}}
+ if(e.endsWithParent)return m(e.parent,t,a)}function _(e){
+ return 0===x.matcher.regexIndex?(A+=e[0],1):(D=!0,0)}function h(e){
+ const n=e[0],a=t.substring(e.index),i=m(x,e,a);if(!i)return ee;const r=x
+ ;x.endScope&&x.endScope._wrap?(d(),
+ g(n,x.endScope._wrap)):x.endScope&&x.endScope._multi?(d(),
+ u(x.endScope,e)):r.skip?A+=n:(r.returnEnd||r.excludeEnd||(A+=n),
+ d(),r.excludeEnd&&(A=n));do{
+ x.scope&&S.closeNode(),x.skip||x.subLanguage||(C+=x.relevance),x=x.parent
+ }while(x!==i.parent);return i.starts&&b(i.starts,e),r.returnEnd?0:n.length}
+ let y={};function N(a,r){const o=r&&r[0];if(A+=a,null==o)return d(),0
+ ;if("begin"===y.type&&"end"===r.type&&y.index===r.index&&""===o){
+ if(A+=t.slice(r.index,r.index+1),!s){const n=Error(`0 width match regex (${e})`)
+ ;throw n.languageName=e,n.badRule=y.rule,n}return 1}
+ if(y=r,"begin"===r.type)return(e=>{
+ const t=e[0],a=e.rule,i=new n(a),r=[a.__beforeBegin,a["on:begin"]]
+ ;for(const n of r)if(n&&(n(e,i),i.isMatchIgnored))return _(t)
+ ;return a.skip?A+=t:(a.excludeBegin&&(A+=t),
+ d(),a.returnBegin||a.excludeBegin||(A=t)),b(a,e),a.returnBegin?0:t.length})(r)
+ ;if("illegal"===r.type&&!i){
+ const e=Error('Illegal lexeme "'+o+'" for mode "'+(x.scope||"")+'"')
+ ;throw e.mode=x,e}if("end"===r.type){const e=h(r);if(e!==ee)return e}
+ if("illegal"===r.type&&""===o)return 1
+ ;if(R>1e5&&R>3*r.index)throw Error("potential infinite loop, way more iterations than matches")
+ ;return A+=o,o.length}const w=v(e)
+ ;if(!w)throw K(o.replace("{}",e)),Error('Unknown language: "'+e+'"')
+ ;const O=Q(w);let k="",x=r||O;const M={},S=new p.__emitter(p);(()=>{const e=[]
+ ;for(let n=x;n!==w;n=n.parent)n.scope&&e.unshift(n.scope)
+ ;e.forEach((e=>S.openNode(e)))})();let A="",C=0,T=0,R=0,D=!1;try{
+ if(w.__emitTokens)w.__emitTokens(t,S);else{for(x.matcher.considerAll();;){
+ R++,D?D=!1:x.matcher.considerAll(),x.matcher.lastIndex=T
+ ;const e=x.matcher.exec(t);if(!e)break;const n=N(t.substring(T,e.index),e)
+ ;T=e.index+n}N(t.substring(T))}return S.finalize(),k=S.toHTML(),{language:e,
+ value:k,relevance:C,illegal:!1,_emitter:S,_top:x}}catch(n){
+ if(n.message&&n.message.includes("Illegal"))return{language:e,value:J(t),
+ illegal:!0,relevance:0,_illegalBy:{message:n.message,index:T,
+ context:t.slice(T-100,T+100),mode:n.mode,resultSoFar:k},_emitter:S};if(s)return{
+ language:e,value:J(t),illegal:!1,relevance:0,errorRaised:n,_emitter:S,_top:x}
+ ;throw n}}function E(e,n){n=n||p.languages||Object.keys(a);const t=(e=>{
+ const n={value:J(e),illegal:!1,relevance:0,_top:c,_emitter:new p.__emitter(p)}
+ ;return n._emitter.addText(e),n})(e),i=n.filter(v).filter(k).map((n=>f(n,e,!1)))
+ ;i.unshift(t);const r=i.sort(((e,n)=>{
+ if(e.relevance!==n.relevance)return n.relevance-e.relevance
+ ;if(e.language&&n.language){if(v(e.language).supersetOf===n.language)return 1
+ ;if(v(n.language).supersetOf===e.language)return-1}return 0})),[s,o]=r,l=s
+ ;return l.secondBest=o,l}function y(e){let n=null;const t=(e=>{
+ let n=e.className+" ";n+=e.parentNode?e.parentNode.className:""
+ ;const t=p.languageDetectRe.exec(n);if(t){const n=v(t[1])
+ ;return n||(H(o.replace("{}",t[1])),
+ H("Falling back to no-highlight mode for this block.",e)),n?t[1]:"no-highlight"}
+ return n.split(/\s+/).find((e=>_(e)||v(e)))})(e);if(_(t))return
+ ;if(x("before:highlightElement",{el:e,language:t
+ }),e.dataset.highlighted)return void console.log("Element previously highlighted. To highlight again, first unset `dataset.highlighted`.",e)
+ ;if(e.children.length>0&&(p.ignoreUnescapedHTML||(console.warn("One of your code blocks includes unescaped HTML. This is a potentially serious security risk."),
+ console.warn("https://github.com/highlightjs/highlight.js/wiki/security"),
+ console.warn("The element with unescaped HTML:"),
+ console.warn(e)),p.throwUnescapedHTML))throw new V("One of your code blocks includes unescaped HTML.",e.innerHTML)
+ ;n=e;const a=n.textContent,r=t?h(a,{language:t,ignoreIllegals:!0}):E(a)
+ ;e.innerHTML=r.value,e.dataset.highlighted="yes",((e,n,t)=>{const a=n&&i[n]||t
+ ;e.classList.add("hljs"),e.classList.add("language-"+a)
+ })(e,t,r.language),e.result={language:r.language,re:r.relevance,
+ relevance:r.relevance},r.secondBest&&(e.secondBest={
+ language:r.secondBest.language,relevance:r.secondBest.relevance
+ }),x("after:highlightElement",{el:e,result:r,text:a})}let N=!1;function w(){
+ "loading"!==document.readyState?document.querySelectorAll(p.cssSelector).forEach(y):N=!0
+ }function v(e){return e=(e||"").toLowerCase(),a[e]||a[i[e]]}
+ function O(e,{languageName:n}){"string"==typeof e&&(e=[e]),e.forEach((e=>{
+ i[e.toLowerCase()]=n}))}function k(e){const n=v(e)
+ ;return n&&!n.disableAutodetect}function x(e,n){const t=e;r.forEach((e=>{
+ e[t]&&e[t](n)}))}
+ "undefined"!=typeof window&&window.addEventListener&&window.addEventListener("DOMContentLoaded",(()=>{
+ N&&w()}),!1),Object.assign(t,{highlight:h,highlightAuto:E,highlightAll:w,
+ highlightElement:y,
+ highlightBlock:e=>(q("10.7.0","highlightBlock will be removed entirely in v12.0"),
+ q("10.7.0","Please use highlightElement now."),y(e)),configure:e=>{p=Y(p,e)},
+ initHighlighting:()=>{
+ w(),q("10.6.0","initHighlighting() deprecated. Use highlightAll() now.")},
+ initHighlightingOnLoad:()=>{
+ w(),q("10.6.0","initHighlightingOnLoad() deprecated. Use highlightAll() now.")
+ },registerLanguage:(e,n)=>{let i=null;try{i=n(t)}catch(n){
+ if(K("Language definition for '{}' could not be registered.".replace("{}",e)),
+ !s)throw n;K(n),i=c}
+ i.name||(i.name=e),a[e]=i,i.rawDefinition=n.bind(null,t),i.aliases&&O(i.aliases,{
+ languageName:e})},unregisterLanguage:e=>{delete a[e]
+ ;for(const n of Object.keys(i))i[n]===e&&delete i[n]},
+ listLanguages:()=>Object.keys(a),getLanguage:v,registerAliases:O,
+ autoDetection:k,inherit:Y,addPlugin:e=>{(e=>{
+ e["before:highlightBlock"]&&!e["before:highlightElement"]&&(e["before:highlightElement"]=n=>{
+ e["before:highlightBlock"](Object.assign({block:n.el},n))
+ }),e["after:highlightBlock"]&&!e["after:highlightElement"]&&(e["after:highlightElement"]=n=>{
+ e["after:highlightBlock"](Object.assign({block:n.el},n))})})(e),r.push(e)},
+ removePlugin:e=>{const n=r.indexOf(e);-1!==n&&r.splice(n,1)}}),t.debugMode=()=>{
+ s=!1},t.safeMode=()=>{s=!0},t.versionString="11.9.0",t.regex={concat:b,
+ lookahead:d,either:m,optional:u,anyNumberOfTimes:g}
+ ;for(const n in C)"object"==typeof C[n]&&e(C[n]);return Object.assign(t,C),t
+ },te=ne({});te.newInstance=()=>ne({});var ae=te;const ie=e=>({IMPORTANT:{
+ scope:"meta",begin:"!important"},BLOCK_COMMENT:e.C_BLOCK_COMMENT_MODE,HEXCOLOR:{
+ scope:"number",begin:/#(([0-9a-fA-F]{3,4})|(([0-9a-fA-F]{2}){3,4}))\b/},
+ FUNCTION_DISPATCH:{className:"built_in",begin:/[\w-]+(?=\()/},
+ ATTRIBUTE_SELECTOR_MODE:{scope:"selector-attr",begin:/\[/,end:/\]/,illegal:"$",
+ contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]},CSS_NUMBER_MODE:{
+ scope:"number",
+ begin:e.NUMBER_RE+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",
+ relevance:0},CSS_VARIABLE:{className:"attr",begin:/--[A-Za-z_][A-Za-z0-9_-]*/}
+ }),re=["a","abbr","address","article","aside","audio","b","blockquote","body","button","canvas","caption","cite","code","dd","del","details","dfn","div","dl","dt","em","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","html","i","iframe","img","input","ins","kbd","label","legend","li","main","mark","menu","nav","object","ol","p","q","quote","samp","section","span","strong","summary","sup","table","tbody","td","textarea","tfoot","th","thead","time","tr","ul","var","video"],se=["any-hover","any-pointer","aspect-ratio","color","color-gamut","color-index","device-aspect-ratio","device-height","device-width","display-mode","forced-colors","grid","height","hover","inverted-colors","monochrome","orientation","overflow-block","overflow-inline","pointer","prefers-color-scheme","prefers-contrast","prefers-reduced-motion","prefers-reduced-transparency","resolution","scan","scripting","update","width","min-width","max-width","min-height","max-height"],oe=["active","any-link","blank","checked","current","default","defined","dir","disabled","drop","empty","enabled","first","first-child","first-of-type","fullscreen","future","focus","focus-visible","focus-within","has","host","host-context","hover","indeterminate","in-range","invalid","is","lang","last-child","last-of-type","left","link","local-link","not","nth-child","nth-col","nth-last-child","nth-last-col","nth-last-of-type","nth-of-type","only-child","only-of-type","optional","out-of-range","past","placeholder-shown","read-only","read-write","required","right","root","scope","target","target-within","user-invalid","valid","visited","where"],le=["after","backdrop","before","cue","cue-region","first-letter","first-line","grammar-error","marker","part","placeholder","selection","slotted","spelling-error"],ce=["align-content","align-items","align-self","all","animation","animation-delay","animation-direction","animation-duration","animation-fill-mode","animation-iteration-count","animation-name","animation-play-state","animation-timing-function","backface-visibility","background","background-attachment","background-blend-mode","background-clip","background-color","background-image","background-origin","background-position","background-repeat","background-size","block-size","border","border-block","border-block-color","border-block-end","border-block-end-color","border-block-end-style","border-block-end-width","border-block-start","border-block-start-color","border-block-start-style","border-block-start-width","border-block-style","border-block-width","border-bottom","border-bottom-color","border-bottom-left-radius","border-bottom-right-radius","border-bottom-style","border-bottom-width","border-collapse","border-color","border-image","border-image-outset","border-image-repeat","border-image-slice","border-image-source","border-image-width","border-inline","border-inline-color","border-inline-end","border-inline-end-color","border-inline-end-style","border-inline-end-width","border-inline-start","border-inline-start-color","border-inline-start-style","border-inline-start-width","border-inline-style","border-inline-width","border-left","border-left-color","border-left-style","border-left-width","border-radius","border-right","border-right-color","border-right-style","border-right-width","border-spacing","border-style","border-top","border-top-color","border-top-left-radius","border-top-right-radius","border-top-style","border-top-width","border-width","bottom","box-decoration-break","box-shadow","box-sizing","break-after","break-before","break-inside","caption-side","caret-color","clear","clip","clip-path","clip-rule","color","column-count","column-fill","column-gap","column-rule","column-rule-color","column-rule-style","column-rule-width","column-span","column-width","columns","contain","content","content-visibility","counter-increment","counter-reset","cue","cue-after","cue-before","cursor","direction","display","empty-cells","filter","flex","flex-basis","flex-direction","flex-flow","flex-grow","flex-shrink","flex-wrap","float","flow","font","font-display","font-family","font-feature-settings","font-kerning","font-language-override","font-size","font-size-adjust","font-smoothing","font-stretch","font-style","font-synthesis","font-variant","font-variant-caps","font-variant-east-asian","font-variant-ligatures","font-variant-numeric","font-variant-position","font-variation-settings","font-weight","gap","glyph-orientation-vertical","grid","grid-area","grid-auto-columns","grid-auto-flow","grid-auto-rows","grid-column","grid-column-end","grid-column-start","grid-gap","grid-row","grid-row-end","grid-row-start","grid-template","grid-template-areas","grid-template-columns","grid-template-rows","hanging-punctuation","height","hyphens","icon","image-orientation","image-rendering","image-resolution","ime-mode","inline-size","isolation","justify-content","left","letter-spacing","line-break","line-height","list-style","list-style-image","list-style-position","list-style-type","margin","margin-block","margin-block-end","margin-block-start","margin-bottom","margin-inline","margin-inline-end","margin-inline-start","margin-left","margin-right","margin-top","marks","mask","mask-border","mask-border-mode","mask-border-outset","mask-border-repeat","mask-border-slice","mask-border-source","mask-border-width","mask-clip","mask-composite","mask-image","mask-mode","mask-origin","mask-position","mask-repeat","mask-size","mask-type","max-block-size","max-height","max-inline-size","max-width","min-block-size","min-height","min-inline-size","min-width","mix-blend-mode","nav-down","nav-index","nav-left","nav-right","nav-up","none","normal","object-fit","object-position","opacity","order","orphans","outline","outline-color","outline-offset","outline-style","outline-width","overflow","overflow-wrap","overflow-x","overflow-y","padding","padding-block","padding-block-end","padding-block-start","padding-bottom","padding-inline","padding-inline-end","padding-inline-start","padding-left","padding-right","padding-top","page-break-after","page-break-before","page-break-inside","pause","pause-after","pause-before","perspective","perspective-origin","pointer-events","position","quotes","resize","rest","rest-after","rest-before","right","row-gap","scroll-margin","scroll-margin-block","scroll-margin-block-end","scroll-margin-block-start","scroll-margin-bottom","scroll-margin-inline","scroll-margin-inline-end","scroll-margin-inline-start","scroll-margin-left","scroll-margin-right","scroll-margin-top","scroll-padding","scroll-padding-block","scroll-padding-block-end","scroll-padding-block-start","scroll-padding-bottom","scroll-padding-inline","scroll-padding-inline-end","scroll-padding-inline-start","scroll-padding-left","scroll-padding-right","scroll-padding-top","scroll-snap-align","scroll-snap-stop","scroll-snap-type","scrollbar-color","scrollbar-gutter","scrollbar-width","shape-image-threshold","shape-margin","shape-outside","speak","speak-as","src","tab-size","table-layout","text-align","text-align-all","text-align-last","text-combine-upright","text-decoration","text-decoration-color","text-decoration-line","text-decoration-style","text-emphasis","text-emphasis-color","text-emphasis-position","text-emphasis-style","text-indent","text-justify","text-orientation","text-overflow","text-rendering","text-shadow","text-transform","text-underline-position","top","transform","transform-box","transform-origin","transform-style","transition","transition-delay","transition-duration","transition-property","transition-timing-function","unicode-bidi","vertical-align","visibility","voice-balance","voice-duration","voice-family","voice-pitch","voice-range","voice-rate","voice-stress","voice-volume","white-space","widows","width","will-change","word-break","word-spacing","word-wrap","writing-mode","z-index"].reverse(),de=oe.concat(le)
+ ;var ge="[0-9](_*[0-9])*",ue=`\\.(${ge})`,be="[0-9a-fA-F](_*[0-9a-fA-F])*",me={
+ className:"number",variants:[{
+ begin:`(\\b(${ge})((${ue})|\\.)?|(${ue}))[eE][+-]?(${ge})[fFdD]?\\b`},{
+ begin:`\\b(${ge})((${ue})[fFdD]?\\b|\\.([fFdD]\\b)?)`},{
+ begin:`(${ue})[fFdD]?\\b`},{begin:`\\b(${ge})[fFdD]\\b`},{
+ begin:`\\b0[xX]((${be})\\.?|(${be})?\\.(${be}))[pP][+-]?(${ge})[fFdD]?\\b`},{
+ begin:"\\b(0|[1-9](_*[0-9])*)[lL]?\\b"},{begin:`\\b0[xX](${be})[lL]?\\b`},{
+ begin:"\\b0(_*[0-7])*[lL]?\\b"},{begin:"\\b0[bB][01](_*[01])*[lL]?\\b"}],
+ relevance:0};function pe(e,n,t){return-1===t?"":e.replace(n,(a=>pe(e,n,t-1)))}
+ const _e="[A-Za-z$_][0-9A-Za-z$_]*",he=["as","in","of","if","for","while","finally","var","new","function","do","return","void","else","break","catch","instanceof","with","throw","case","default","try","switch","continue","typeof","delete","let","yield","const","class","debugger","async","await","static","import","from","export","extends"],fe=["true","false","null","undefined","NaN","Infinity"],Ee=["Object","Function","Boolean","Symbol","Math","Date","Number","BigInt","String","RegExp","Array","Float32Array","Float64Array","Int8Array","Uint8Array","Uint8ClampedArray","Int16Array","Int32Array","Uint16Array","Uint32Array","BigInt64Array","BigUint64Array","Set","Map","WeakSet","WeakMap","ArrayBuffer","SharedArrayBuffer","Atomics","DataView","JSON","Promise","Generator","GeneratorFunction","AsyncFunction","Reflect","Proxy","Intl","WebAssembly"],ye=["Error","EvalError","InternalError","RangeError","ReferenceError","SyntaxError","TypeError","URIError"],Ne=["setInterval","setTimeout","clearInterval","clearTimeout","require","exports","eval","isFinite","isNaN","parseFloat","parseInt","decodeURI","decodeURIComponent","encodeURI","encodeURIComponent","escape","unescape"],we=["arguments","this","super","console","window","document","localStorage","sessionStorage","module","global"],ve=[].concat(Ne,Ee,ye)
+ ;function Oe(e){const n=e.regex,t=_e,a={begin:/<[A-Za-z0-9\\._:-]+/,
+ end:/\/[A-Za-z0-9\\._:-]+>|\/>/,isTrulyOpeningTag:(e,n)=>{
+ const t=e[0].length+e.index,a=e.input[t]
+ ;if("<"===a||","===a)return void n.ignoreMatch();let i
+ ;">"===a&&(((e,{after:n})=>{const t="",M={
+ match:[/const|var|let/,/\s+/,t,/\s*/,/=\s*/,/(async\s*)?/,n.lookahead(x)],
+ keywords:"async",className:{1:"keyword",3:"title.function"},contains:[f]}
+ ;return{name:"JavaScript",aliases:["js","jsx","mjs","cjs"],keywords:i,exports:{
+ PARAMS_CONTAINS:h,CLASS_REFERENCE:y},illegal:/#(?![$_A-z])/,
+ contains:[e.SHEBANG({label:"shebang",binary:"node",relevance:5}),{
+ label:"use_strict",className:"meta",relevance:10,
+ begin:/^\s*['"]use (strict|asm)['"]/
+ },e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,d,g,u,b,m,{match:/\$\d+/},l,y,{
+ className:"attr",begin:t+n.lookahead(":"),relevance:0},M,{
+ begin:"("+e.RE_STARTERS_RE+"|\\b(case|return|throw)\\b)\\s*",
+ keywords:"return throw case",relevance:0,contains:[m,e.REGEXP_MODE,{
+ className:"function",begin:x,returnBegin:!0,end:"\\s*=>",contains:[{
+ className:"params",variants:[{begin:e.UNDERSCORE_IDENT_RE,relevance:0},{
+ className:null,begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,
+ excludeEnd:!0,keywords:i,contains:h}]}]},{begin:/,/,relevance:0},{match:/\s+/,
+ relevance:0},{variants:[{begin:"<>",end:""},{
+ match:/<[A-Za-z0-9\\._:-]+\s*\/>/},{begin:a.begin,
+ "on:begin":a.isTrulyOpeningTag,end:a.end}],subLanguage:"xml",contains:[{
+ begin:a.begin,end:a.end,skip:!0,contains:["self"]}]}]},N,{
+ beginKeywords:"while if switch catch for"},{
+ begin:"\\b(?!function)"+e.UNDERSCORE_IDENT_RE+"\\([^()]*(\\([^()]*(\\([^()]*\\)[^()]*)*\\)[^()]*)*\\)\\s*\\{",
+ returnBegin:!0,label:"func.def",contains:[f,e.inherit(e.TITLE_MODE,{begin:t,
+ className:"title.function"})]},{match:/\.\.\./,relevance:0},O,{match:"\\$"+t,
+ relevance:0},{match:[/\bconstructor(?=\s*\()/],className:{1:"title.function"},
+ contains:[f]},w,{relevance:0,match:/\b[A-Z][A-Z_0-9]+\b/,
+ className:"variable.constant"},E,k,{match:/\$[(.]/}]}}
+ const ke=e=>b(/\b/,e,/\w$/.test(e)?/\b/:/\B/),xe=["Protocol","Type"].map(ke),Me=["init","self"].map(ke),Se=["Any","Self"],Ae=["actor","any","associatedtype","async","await",/as\?/,/as!/,"as","borrowing","break","case","catch","class","consume","consuming","continue","convenience","copy","default","defer","deinit","didSet","distributed","do","dynamic","each","else","enum","extension","fallthrough",/fileprivate\(set\)/,"fileprivate","final","for","func","get","guard","if","import","indirect","infix",/init\?/,/init!/,"inout",/internal\(set\)/,"internal","in","is","isolated","nonisolated","lazy","let","macro","mutating","nonmutating",/open\(set\)/,"open","operator","optional","override","postfix","precedencegroup","prefix",/private\(set\)/,"private","protocol",/public\(set\)/,"public","repeat","required","rethrows","return","set","some","static","struct","subscript","super","switch","throws","throw",/try\?/,/try!/,"try","typealias",/unowned\(safe\)/,/unowned\(unsafe\)/,"unowned","var","weak","where","while","willSet"],Ce=["false","nil","true"],Te=["assignment","associativity","higherThan","left","lowerThan","none","right"],Re=["#colorLiteral","#column","#dsohandle","#else","#elseif","#endif","#error","#file","#fileID","#fileLiteral","#filePath","#function","#if","#imageLiteral","#keyPath","#line","#selector","#sourceLocation","#warning"],De=["abs","all","any","assert","assertionFailure","debugPrint","dump","fatalError","getVaList","isKnownUniquelyReferenced","max","min","numericCast","pointwiseMax","pointwiseMin","precondition","preconditionFailure","print","readLine","repeatElement","sequence","stride","swap","swift_unboxFromSwiftValueWithType","transcode","type","unsafeBitCast","unsafeDowncast","withExtendedLifetime","withUnsafeMutablePointer","withUnsafePointer","withVaList","withoutActuallyEscaping","zip"],Ie=m(/[/=\-+!*%<>&|^~?]/,/[\u00A1-\u00A7]/,/[\u00A9\u00AB]/,/[\u00AC\u00AE]/,/[\u00B0\u00B1]/,/[\u00B6\u00BB\u00BF\u00D7\u00F7]/,/[\u2016-\u2017]/,/[\u2020-\u2027]/,/[\u2030-\u203E]/,/[\u2041-\u2053]/,/[\u2055-\u205E]/,/[\u2190-\u23FF]/,/[\u2500-\u2775]/,/[\u2794-\u2BFF]/,/[\u2E00-\u2E7F]/,/[\u3001-\u3003]/,/[\u3008-\u3020]/,/[\u3030]/),Le=m(Ie,/[\u0300-\u036F]/,/[\u1DC0-\u1DFF]/,/[\u20D0-\u20FF]/,/[\uFE00-\uFE0F]/,/[\uFE20-\uFE2F]/),Be=b(Ie,Le,"*"),$e=m(/[a-zA-Z_]/,/[\u00A8\u00AA\u00AD\u00AF\u00B2-\u00B5\u00B7-\u00BA]/,/[\u00BC-\u00BE\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u00FF]/,/[\u0100-\u02FF\u0370-\u167F\u1681-\u180D\u180F-\u1DBF]/,/[\u1E00-\u1FFF]/,/[\u200B-\u200D\u202A-\u202E\u203F-\u2040\u2054\u2060-\u206F]/,/[\u2070-\u20CF\u2100-\u218F\u2460-\u24FF\u2776-\u2793]/,/[\u2C00-\u2DFF\u2E80-\u2FFF]/,/[\u3004-\u3007\u3021-\u302F\u3031-\u303F\u3040-\uD7FF]/,/[\uF900-\uFD3D\uFD40-\uFDCF\uFDF0-\uFE1F\uFE30-\uFE44]/,/[\uFE47-\uFEFE\uFF00-\uFFFD]/),ze=m($e,/\d/,/[\u0300-\u036F\u1DC0-\u1DFF\u20D0-\u20FF\uFE20-\uFE2F]/),Fe=b($e,ze,"*"),Ue=b(/[A-Z]/,ze,"*"),je=["attached","autoclosure",b(/convention\(/,m("swift","block","c"),/\)/),"discardableResult","dynamicCallable","dynamicMemberLookup","escaping","freestanding","frozen","GKInspectable","IBAction","IBDesignable","IBInspectable","IBOutlet","IBSegueAction","inlinable","main","nonobjc","NSApplicationMain","NSCopying","NSManaged",b(/objc\(/,Fe,/\)/),"objc","objcMembers","propertyWrapper","requires_stored_property_inits","resultBuilder","Sendable","testable","UIApplicationMain","unchecked","unknown","usableFromInline","warn_unqualified_access"],Pe=["iOS","iOSApplicationExtension","macOS","macOSApplicationExtension","macCatalyst","macCatalystApplicationExtension","watchOS","watchOSApplicationExtension","tvOS","tvOSApplicationExtension","swift"]
+ ;var Ke=Object.freeze({__proto__:null,grmr_bash:e=>{const n=e.regex,t={},a={
+ begin:/\$\{/,end:/\}/,contains:["self",{begin:/:-/,contains:[t]}]}
+ ;Object.assign(t,{className:"variable",variants:[{
+ begin:n.concat(/\$[\w\d#@][\w\d_]*/,"(?![\\w\\d])(?![$])")},a]});const i={
+ className:"subst",begin:/\$\(/,end:/\)/,contains:[e.BACKSLASH_ESCAPE]},r={
+ begin:/<<-?\s*(?=\w+)/,starts:{contains:[e.END_SAME_AS_BEGIN({begin:/(\w+)/,
+ end:/(\w+)/,className:"string"})]}},s={className:"string",begin:/"/,end:/"/,
+ contains:[e.BACKSLASH_ESCAPE,t,i]};i.contains.push(s);const o={begin:/\$?\(\(/,
+ end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},e.NUMBER_MODE,t]
+ },l=e.SHEBANG({binary:"(fish|bash|zsh|sh|csh|ksh|tcsh|dash|scsh)",relevance:10
+ }),c={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0,
+ contains:[e.inherit(e.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{
+ name:"Bash",aliases:["sh"],keywords:{$pattern:/\b[a-z][a-z0-9._-]+\b/,
+ keyword:["if","then","else","elif","fi","for","while","until","in","do","done","case","esac","function","select"],
+ literal:["true","false"],
+ built_in:["break","cd","continue","eval","exec","exit","export","getopts","hash","pwd","readonly","return","shift","test","times","trap","umask","unset","alias","bind","builtin","caller","command","declare","echo","enable","help","let","local","logout","mapfile","printf","read","readarray","source","type","typeset","ulimit","unalias","set","shopt","autoload","bg","bindkey","bye","cap","chdir","clone","comparguments","compcall","compctl","compdescribe","compfiles","compgroups","compquote","comptags","comptry","compvalues","dirs","disable","disown","echotc","echoti","emulate","fc","fg","float","functions","getcap","getln","history","integer","jobs","kill","limit","log","noglob","popd","print","pushd","pushln","rehash","sched","setcap","setopt","stat","suspend","ttyctl","unfunction","unhash","unlimit","unsetopt","vared","wait","whence","where","which","zcompile","zformat","zftp","zle","zmodload","zparseopts","zprof","zpty","zregexparse","zsocket","zstyle","ztcp","chcon","chgrp","chown","chmod","cp","dd","df","dir","dircolors","ln","ls","mkdir","mkfifo","mknod","mktemp","mv","realpath","rm","rmdir","shred","sync","touch","truncate","vdir","b2sum","base32","base64","cat","cksum","comm","csplit","cut","expand","fmt","fold","head","join","md5sum","nl","numfmt","od","paste","ptx","pr","sha1sum","sha224sum","sha256sum","sha384sum","sha512sum","shuf","sort","split","sum","tac","tail","tr","tsort","unexpand","uniq","wc","arch","basename","chroot","date","dirname","du","echo","env","expr","factor","groups","hostid","id","link","logname","nice","nohup","nproc","pathchk","pinky","printenv","printf","pwd","readlink","runcon","seq","sleep","stat","stdbuf","stty","tee","test","timeout","tty","uname","unlink","uptime","users","who","whoami","yes"]
+ },contains:[l,e.SHEBANG(),c,o,e.HASH_COMMENT_MODE,r,{match:/(\/[a-z._-]+)+/},s,{
+ match:/\\"/},{className:"string",begin:/'/,end:/'/},{match:/\\'/},t]}},
+ grmr_c:e=>{const n=e.regex,t=e.COMMENT("//","$",{contains:[{begin:/\\\n/}]
+ }),a="decltype\\(auto\\)",i="[a-zA-Z_]\\w*::",r="("+a+"|"+n.optional(i)+"[a-zA-Z_]\\w*"+n.optional("<[^<>]+>")+")",s={
+ className:"type",variants:[{begin:"\\b[a-z\\d_]*_t\\b"},{
+ match:/\batomic_[a-z]{3,6}\b/}]},o={className:"string",variants:[{
+ begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{
+ begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",
+ end:"'",illegal:"."},e.END_SAME_AS_BEGIN({
+ begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},l={
+ className:"number",variants:[{begin:"\\b(0b[01']+)"},{
+ begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)((ll|LL|l|L)(u|U)?|(u|U)(ll|LL|l|L)?|f|F|b|B)"
+ },{
+ begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"
+ }],relevance:0},c={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{
+ keyword:"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"
+ },contains:[{begin:/\\\n/,relevance:0},e.inherit(o,{className:"string"}),{
+ className:"string",begin:/<.*?>/},t,e.C_BLOCK_COMMENT_MODE]},d={
+ className:"title",begin:n.optional(i)+e.IDENT_RE,relevance:0
+ },g=n.optional(i)+e.IDENT_RE+"\\s*\\(",u={
+ keyword:["asm","auto","break","case","continue","default","do","else","enum","extern","for","fortran","goto","if","inline","register","restrict","return","sizeof","struct","switch","typedef","union","volatile","while","_Alignas","_Alignof","_Atomic","_Generic","_Noreturn","_Static_assert","_Thread_local","alignas","alignof","noreturn","static_assert","thread_local","_Pragma"],
+ type:["float","double","signed","unsigned","int","short","long","char","void","_Bool","_Complex","_Imaginary","_Decimal32","_Decimal64","_Decimal128","const","static","complex","bool","imaginary"],
+ literal:"true false NULL",
+ built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set pair bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap priority_queue make_pair array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr"
+ },b=[c,s,t,e.C_BLOCK_COMMENT_MODE,l,o],m={variants:[{begin:/=/,end:/;/},{
+ begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],
+ keywords:u,contains:b.concat([{begin:/\(/,end:/\)/,keywords:u,
+ contains:b.concat(["self"]),relevance:0}]),relevance:0},p={
+ begin:"("+r+"[\\*&\\s]+)+"+g,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,
+ keywords:u,illegal:/[^\w\s\*&:<>.]/,contains:[{begin:a,keywords:u,relevance:0},{
+ begin:g,returnBegin:!0,contains:[e.inherit(d,{className:"title.function"})],
+ relevance:0},{relevance:0,match:/,/},{className:"params",begin:/\(/,end:/\)/,
+ keywords:u,relevance:0,contains:[t,e.C_BLOCK_COMMENT_MODE,o,l,s,{begin:/\(/,
+ end:/\)/,keywords:u,relevance:0,contains:["self",t,e.C_BLOCK_COMMENT_MODE,o,l,s]
+ }]},s,t,e.C_BLOCK_COMMENT_MODE,c]};return{name:"C",aliases:["h"],keywords:u,
+ disableAutodetect:!0,illegal:"=]/,contains:[{
+ beginKeywords:"final class struct"},e.TITLE_MODE]}]),exports:{preprocessor:c,
+ strings:o,keywords:u}}},grmr_cpp:e=>{const n=e.regex,t=e.COMMENT("//","$",{
+ contains:[{begin:/\\\n/}]
+ }),a="decltype\\(auto\\)",i="[a-zA-Z_]\\w*::",r="(?!struct)("+a+"|"+n.optional(i)+"[a-zA-Z_]\\w*"+n.optional("<[^<>]+>")+")",s={
+ className:"type",begin:"\\b[a-z\\d_]*_t\\b"},o={className:"string",variants:[{
+ begin:'(u8?|U|L)?"',end:'"',illegal:"\\n",contains:[e.BACKSLASH_ESCAPE]},{
+ begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)",
+ end:"'",illegal:"."},e.END_SAME_AS_BEGIN({
+ begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},l={
+ className:"number",variants:[{begin:"\\b(0b[01']+)"},{
+ begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)((ll|LL|l|L)(u|U)?|(u|U)(ll|LL|l|L)?|f|F|b|B)"
+ },{
+ begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"
+ }],relevance:0},c={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{
+ keyword:"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include"
+ },contains:[{begin:/\\\n/,relevance:0},e.inherit(o,{className:"string"}),{
+ className:"string",begin:/<.*?>/},t,e.C_BLOCK_COMMENT_MODE]},d={
+ className:"title",begin:n.optional(i)+e.IDENT_RE,relevance:0
+ },g=n.optional(i)+e.IDENT_RE+"\\s*\\(",u={
+ type:["bool","char","char16_t","char32_t","char8_t","double","float","int","long","short","void","wchar_t","unsigned","signed","const","static"],
+ keyword:["alignas","alignof","and","and_eq","asm","atomic_cancel","atomic_commit","atomic_noexcept","auto","bitand","bitor","break","case","catch","class","co_await","co_return","co_yield","compl","concept","const_cast|10","consteval","constexpr","constinit","continue","decltype","default","delete","do","dynamic_cast|10","else","enum","explicit","export","extern","false","final","for","friend","goto","if","import","inline","module","mutable","namespace","new","noexcept","not","not_eq","nullptr","operator","or","or_eq","override","private","protected","public","reflexpr","register","reinterpret_cast|10","requires","return","sizeof","static_assert","static_cast|10","struct","switch","synchronized","template","this","thread_local","throw","transaction_safe","transaction_safe_dynamic","true","try","typedef","typeid","typename","union","using","virtual","volatile","while","xor","xor_eq"],
+ literal:["NULL","false","nullopt","nullptr","true"],built_in:["_Pragma"],
+ _type_hints:["any","auto_ptr","barrier","binary_semaphore","bitset","complex","condition_variable","condition_variable_any","counting_semaphore","deque","false_type","future","imaginary","initializer_list","istringstream","jthread","latch","lock_guard","multimap","multiset","mutex","optional","ostringstream","packaged_task","pair","promise","priority_queue","queue","recursive_mutex","recursive_timed_mutex","scoped_lock","set","shared_future","shared_lock","shared_mutex","shared_timed_mutex","shared_ptr","stack","string_view","stringstream","timed_mutex","thread","true_type","tuple","unique_lock","unique_ptr","unordered_map","unordered_multimap","unordered_multiset","unordered_set","variant","vector","weak_ptr","wstring","wstring_view"]
+ },b={className:"function.dispatch",relevance:0,keywords:{
+ _hint:["abort","abs","acos","apply","as_const","asin","atan","atan2","calloc","ceil","cerr","cin","clog","cos","cosh","cout","declval","endl","exchange","exit","exp","fabs","floor","fmod","forward","fprintf","fputs","free","frexp","fscanf","future","invoke","isalnum","isalpha","iscntrl","isdigit","isgraph","islower","isprint","ispunct","isspace","isupper","isxdigit","labs","launder","ldexp","log","log10","make_pair","make_shared","make_shared_for_overwrite","make_tuple","make_unique","malloc","memchr","memcmp","memcpy","memset","modf","move","pow","printf","putchar","puts","realloc","scanf","sin","sinh","snprintf","sprintf","sqrt","sscanf","std","stderr","stdin","stdout","strcat","strchr","strcmp","strcpy","strcspn","strlen","strncat","strncmp","strncpy","strpbrk","strrchr","strspn","strstr","swap","tan","tanh","terminate","to_underlying","tolower","toupper","vfprintf","visit","vprintf","vsprintf"]
+ },
+ begin:n.concat(/\b/,/(?!decltype)/,/(?!if)/,/(?!for)/,/(?!switch)/,/(?!while)/,e.IDENT_RE,n.lookahead(/(<[^<>]+>|)\s*\(/))
+ },m=[b,c,s,t,e.C_BLOCK_COMMENT_MODE,l,o],p={variants:[{begin:/=/,end:/;/},{
+ begin:/\(/,end:/\)/},{beginKeywords:"new throw return else",end:/;/}],
+ keywords:u,contains:m.concat([{begin:/\(/,end:/\)/,keywords:u,
+ contains:m.concat(["self"]),relevance:0}]),relevance:0},_={className:"function",
+ begin:"("+r+"[\\*&\\s]+)+"+g,returnBegin:!0,end:/[{;=]/,excludeEnd:!0,
+ keywords:u,illegal:/[^\w\s\*&:<>.]/,contains:[{begin:a,keywords:u,relevance:0},{
+ begin:g,returnBegin:!0,contains:[d],relevance:0},{begin:/::/,relevance:0},{
+ begin:/:/,endsWithParent:!0,contains:[o,l]},{relevance:0,match:/,/},{
+ className:"params",begin:/\(/,end:/\)/,keywords:u,relevance:0,
+ contains:[t,e.C_BLOCK_COMMENT_MODE,o,l,s,{begin:/\(/,end:/\)/,keywords:u,
+ relevance:0,contains:["self",t,e.C_BLOCK_COMMENT_MODE,o,l,s]}]
+ },s,t,e.C_BLOCK_COMMENT_MODE,c]};return{name:"C++",
+ aliases:["cc","c++","h++","hpp","hh","hxx","cxx"],keywords:u,illegal:"",keywords:u,contains:["self",s]},{begin:e.IDENT_RE+"::",keywords:u},{
+ match:[/\b(?:enum(?:\s+(?:class|struct))?|class|struct|union)/,/\s+/,/\w+/],
+ className:{1:"keyword",3:"title.class"}}])}},grmr_csharp:e=>{const n={
+ keyword:["abstract","as","base","break","case","catch","class","const","continue","do","else","event","explicit","extern","finally","fixed","for","foreach","goto","if","implicit","in","interface","internal","is","lock","namespace","new","operator","out","override","params","private","protected","public","readonly","record","ref","return","scoped","sealed","sizeof","stackalloc","static","struct","switch","this","throw","try","typeof","unchecked","unsafe","using","virtual","void","volatile","while"].concat(["add","alias","and","ascending","async","await","by","descending","equals","from","get","global","group","init","into","join","let","nameof","not","notnull","on","or","orderby","partial","remove","select","set","unmanaged","value|0","var","when","where","with","yield"]),
+ built_in:["bool","byte","char","decimal","delegate","double","dynamic","enum","float","int","long","nint","nuint","object","sbyte","short","string","ulong","uint","ushort"],
+ literal:["default","false","null","true"]},t=e.inherit(e.TITLE_MODE,{
+ begin:"[a-zA-Z](\\.?\\w)*"}),a={className:"number",variants:[{
+ begin:"\\b(0b[01']+)"},{
+ begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{
+ begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"
+ }],relevance:0},i={className:"string",begin:'@"',end:'"',contains:[{begin:'""'}]
+ },r=e.inherit(i,{illegal:/\n/}),s={className:"subst",begin:/\{/,end:/\}/,
+ keywords:n},o=e.inherit(s,{illegal:/\n/}),l={className:"string",begin:/\$"/,
+ end:'"',illegal:/\n/,contains:[{begin:/\{\{/},{begin:/\}\}/
+ },e.BACKSLASH_ESCAPE,o]},c={className:"string",begin:/\$@"/,end:'"',contains:[{
+ begin:/\{\{/},{begin:/\}\}/},{begin:'""'},s]},d=e.inherit(c,{illegal:/\n/,
+ contains:[{begin:/\{\{/},{begin:/\}\}/},{begin:'""'},o]})
+ ;s.contains=[c,l,i,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.C_BLOCK_COMMENT_MODE],
+ o.contains=[d,l,r,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,a,e.inherit(e.C_BLOCK_COMMENT_MODE,{
+ illegal:/\n/})];const g={variants:[c,l,i,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]
+ },u={begin:"<",end:">",contains:[{beginKeywords:"in out"},t]
+ },b=e.IDENT_RE+"(<"+e.IDENT_RE+"(\\s*,\\s*"+e.IDENT_RE+")*>)?(\\[\\])?",m={
+ begin:"@"+e.IDENT_RE,relevance:0};return{name:"C#",aliases:["cs","c#"],
+ keywords:n,illegal:/::/,contains:[e.COMMENT("///","$",{returnBegin:!0,
+ contains:[{className:"doctag",variants:[{begin:"///",relevance:0},{
+ begin:"\x3c!--|--\x3e"},{begin:""}]}]
+ }),e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{className:"meta",begin:"#",
+ end:"$",keywords:{
+ keyword:"if else elif endif define undef warning error line region endregion pragma checksum"
+ }},g,a,{beginKeywords:"class interface",relevance:0,end:/[{;=]/,
+ illegal:/[^\s:,]/,contains:[{beginKeywords:"where class"
+ },t,u,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{beginKeywords:"namespace",
+ relevance:0,end:/[{;=]/,illegal:/[^\s:]/,
+ contains:[t,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{
+ beginKeywords:"record",relevance:0,end:/[{;=]/,illegal:/[^\s:]/,
+ contains:[t,u,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"meta",
+ begin:"^\\s*\\[(?=[\\w])",excludeBegin:!0,end:"\\]",excludeEnd:!0,contains:[{
+ className:"string",begin:/"/,end:/"/}]},{
+ beginKeywords:"new return throw await else",relevance:0},{className:"function",
+ begin:"("+b+"\\s+)+"+e.IDENT_RE+"\\s*(<[^=]+>\\s*)?\\(",returnBegin:!0,
+ end:/\s*[{;=]/,excludeEnd:!0,keywords:n,contains:[{
+ beginKeywords:"public private protected static internal protected abstract async extern override unsafe virtual new sealed partial",
+ relevance:0},{begin:e.IDENT_RE+"\\s*(<[^=]+>\\s*)?\\(",returnBegin:!0,
+ contains:[e.TITLE_MODE,u],relevance:0},{match:/\(\)/},{className:"params",
+ begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:n,relevance:0,
+ contains:[g,a,e.C_BLOCK_COMMENT_MODE]
+ },e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},m]}},grmr_css:e=>{
+ const n=e.regex,t=ie(e),a=[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE];return{
+ name:"CSS",case_insensitive:!0,illegal:/[=|'\$]/,keywords:{
+ keyframePosition:"from to"},classNameAliases:{keyframePosition:"selector-tag"},
+ contains:[t.BLOCK_COMMENT,{begin:/-(webkit|moz|ms|o)-(?=[a-z])/
+ },t.CSS_NUMBER_MODE,{className:"selector-id",begin:/#[A-Za-z0-9_-]+/,relevance:0
+ },{className:"selector-class",begin:"\\.[a-zA-Z-][a-zA-Z0-9_-]*",relevance:0
+ },t.ATTRIBUTE_SELECTOR_MODE,{className:"selector-pseudo",variants:[{
+ begin:":("+oe.join("|")+")"},{begin:":(:)?("+le.join("|")+")"}]
+ },t.CSS_VARIABLE,{className:"attribute",begin:"\\b("+ce.join("|")+")\\b"},{
+ begin:/:/,end:/[;}{]/,
+ contains:[t.BLOCK_COMMENT,t.HEXCOLOR,t.IMPORTANT,t.CSS_NUMBER_MODE,...a,{
+ begin:/(url|data-uri)\(/,end:/\)/,relevance:0,keywords:{built_in:"url data-uri"
+ },contains:[...a,{className:"string",begin:/[^)]/,endsWithParent:!0,
+ excludeEnd:!0}]},t.FUNCTION_DISPATCH]},{begin:n.lookahead(/@/),end:"[{;]",
+ relevance:0,illegal:/:/,contains:[{className:"keyword",begin:/@-?\w[\w]*(-\w+)*/
+ },{begin:/\s/,endsWithParent:!0,excludeEnd:!0,relevance:0,keywords:{
+ $pattern:/[a-z-]+/,keyword:"and or not only",attribute:se.join(" ")},contains:[{
+ begin:/[a-z-]+(?=:)/,className:"attribute"},...a,t.CSS_NUMBER_MODE]}]},{
+ className:"selector-tag",begin:"\\b("+re.join("|")+")\\b"}]}},grmr_diff:e=>{
+ const n=e.regex;return{name:"Diff",aliases:["patch"],contains:[{
+ className:"meta",relevance:10,
+ match:n.either(/^@@ +-\d+,\d+ +\+\d+,\d+ +@@/,/^\*\*\* +\d+,\d+ +\*\*\*\*$/,/^--- +\d+,\d+ +----$/)
+ },{className:"comment",variants:[{
+ begin:n.either(/Index: /,/^index/,/={3,}/,/^-{3}/,/^\*{3} /,/^\+{3}/,/^diff --git/),
+ end:/$/},{match:/^\*{15}$/}]},{className:"addition",begin:/^\+/,end:/$/},{
+ className:"deletion",begin:/^-/,end:/$/},{className:"addition",begin:/^!/,
+ end:/$/}]}},grmr_go:e=>{const n={
+ keyword:["break","case","chan","const","continue","default","defer","else","fallthrough","for","func","go","goto","if","import","interface","map","package","range","return","select","struct","switch","type","var"],
+ type:["bool","byte","complex64","complex128","error","float32","float64","int8","int16","int32","int64","string","uint8","uint16","uint32","uint64","int","uint","uintptr","rune"],
+ literal:["true","false","iota","nil"],
+ built_in:["append","cap","close","complex","copy","imag","len","make","new","panic","print","println","real","recover","delete"]
+ };return{name:"Go",aliases:["golang"],keywords:n,illegal:"{const n=e.regex;return{name:"GraphQL",aliases:["gql"],
+ case_insensitive:!0,disableAutodetect:!1,keywords:{
+ keyword:["query","mutation","subscription","type","input","schema","directive","interface","union","scalar","fragment","enum","on"],
+ literal:["true","false","null"]},
+ contains:[e.HASH_COMMENT_MODE,e.QUOTE_STRING_MODE,e.NUMBER_MODE,{
+ scope:"punctuation",match:/[.]{3}/,relevance:0},{scope:"punctuation",
+ begin:/[\!\(\)\:\=\[\]\{\|\}]{1}/,relevance:0},{scope:"variable",begin:/\$/,
+ end:/\W/,excludeEnd:!0,relevance:0},{scope:"meta",match:/@\w+/,excludeEnd:!0},{
+ scope:"symbol",begin:n.concat(/[_A-Za-z][_0-9A-Za-z]*/,n.lookahead(/\s*:/)),
+ relevance:0}],illegal:[/[;<']/,/BEGIN/]}},grmr_ini:e=>{const n=e.regex,t={
+ className:"number",relevance:0,variants:[{begin:/([+-]+)?[\d]+_[\d_]+/},{
+ begin:e.NUMBER_RE}]},a=e.COMMENT();a.variants=[{begin:/;/,end:/$/},{begin:/#/,
+ end:/$/}];const i={className:"variable",variants:[{begin:/\$[\w\d"][\w\d_]*/},{
+ begin:/\$\{(.*?)\}/}]},r={className:"literal",
+ begin:/\bon|off|true|false|yes|no\b/},s={className:"string",
+ contains:[e.BACKSLASH_ESCAPE],variants:[{begin:"'''",end:"'''",relevance:10},{
+ begin:'"""',end:'"""',relevance:10},{begin:'"',end:'"'},{begin:"'",end:"'"}]
+ },o={begin:/\[/,end:/\]/,contains:[a,r,i,s,t,"self"],relevance:0
+ },l=n.either(/[A-Za-z0-9_-]+/,/"(\\"|[^"])*"/,/'[^']*'/);return{
+ name:"TOML, also INI",aliases:["toml"],case_insensitive:!0,illegal:/\S/,
+ contains:[a,{className:"section",begin:/\[+/,end:/\]+/},{
+ begin:n.concat(l,"(\\s*\\.\\s*",l,")*",n.lookahead(/\s*=\s*[^#\s]/)),
+ className:"attr",starts:{end:/$/,contains:[a,o,r,i,s,t]}}]}},grmr_java:e=>{
+ const n=e.regex,t="[\xc0-\u02b8a-zA-Z_$][\xc0-\u02b8a-zA-Z_$0-9]*",a=t+pe("(?:<"+t+"~~~(?:\\s*,\\s*"+t+"~~~)*>)?",/~~~/g,2),i={
+ keyword:["synchronized","abstract","private","var","static","if","const ","for","while","strictfp","finally","protected","import","native","final","void","enum","else","break","transient","catch","instanceof","volatile","case","assert","package","default","public","try","switch","continue","throws","protected","public","private","module","requires","exports","do","sealed","yield","permits"],
+ literal:["false","true","null"],
+ type:["char","boolean","long","float","int","byte","short","double"],
+ built_in:["super","this"]},r={className:"meta",begin:"@"+t,contains:[{
+ begin:/\(/,end:/\)/,contains:["self"]}]},s={className:"params",begin:/\(/,
+ end:/\)/,keywords:i,relevance:0,contains:[e.C_BLOCK_COMMENT_MODE],endsParent:!0}
+ ;return{name:"Java",aliases:["jsp"],keywords:i,illegal:/<\/|#/,
+ contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{begin:/\w+@/,
+ relevance:0},{className:"doctag",begin:"@[A-Za-z]+"}]}),{
+ begin:/import java\.[a-z]+\./,keywords:"import",relevance:2
+ },e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,{begin:/"""/,end:/"""/,
+ className:"string",contains:[e.BACKSLASH_ESCAPE]
+ },e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{
+ match:[/\b(?:class|interface|enum|extends|implements|new)/,/\s+/,t],className:{
+ 1:"keyword",3:"title.class"}},{match:/non-sealed/,scope:"keyword"},{
+ begin:[n.concat(/(?!else)/,t),/\s+/,t,/\s+/,/=(?!=)/],className:{1:"type",
+ 3:"variable",5:"operator"}},{begin:[/record/,/\s+/,t],className:{1:"keyword",
+ 3:"title.class"},contains:[s,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{
+ beginKeywords:"new throw return else",relevance:0},{
+ begin:["(?:"+a+"\\s+)",e.UNDERSCORE_IDENT_RE,/\s*(?=\()/],className:{
+ 2:"title.function"},keywords:i,contains:[{className:"params",begin:/\(/,
+ end:/\)/,keywords:i,relevance:0,
+ contains:[r,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,me,e.C_BLOCK_COMMENT_MODE]
+ },e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},me,r]}},grmr_javascript:Oe,
+ grmr_json:e=>{const n=["true","false","null"],t={scope:"literal",
+ beginKeywords:n.join(" ")};return{name:"JSON",keywords:{literal:n},contains:[{
+ className:"attr",begin:/"(\\.|[^\\"\r\n])*"(?=\s*:)/,relevance:1.01},{
+ match:/[{}[\],:]/,className:"punctuation",relevance:0
+ },e.QUOTE_STRING_MODE,t,e.C_NUMBER_MODE,e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE],
+ illegal:"\\S"}},grmr_kotlin:e=>{const n={
+ keyword:"abstract as val var vararg get set class object open private protected public noinline crossinline dynamic final enum if else do while for when throw try catch finally import package is in fun override companion reified inline lateinit init interface annotation data sealed internal infix operator out by constructor super tailrec where const inner suspend typealias external expect actual",
+ built_in:"Byte Short Char Int Long Boolean Float Double Void Unit Nothing",
+ literal:"true false null"},t={className:"symbol",begin:e.UNDERSCORE_IDENT_RE+"@"
+ },a={className:"subst",begin:/\$\{/,end:/\}/,contains:[e.C_NUMBER_MODE]},i={
+ className:"variable",begin:"\\$"+e.UNDERSCORE_IDENT_RE},r={className:"string",
+ variants:[{begin:'"""',end:'"""(?=[^"])',contains:[i,a]},{begin:"'",end:"'",
+ illegal:/\n/,contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"',illegal:/\n/,
+ contains:[e.BACKSLASH_ESCAPE,i,a]}]};a.contains.push(r);const s={
+ className:"meta",
+ begin:"@(?:file|property|field|get|set|receiver|param|setparam|delegate)\\s*:(?:\\s*"+e.UNDERSCORE_IDENT_RE+")?"
+ },o={className:"meta",begin:"@"+e.UNDERSCORE_IDENT_RE,contains:[{begin:/\(/,
+ end:/\)/,contains:[e.inherit(r,{className:"string"}),"self"]}]
+ },l=me,c=e.COMMENT("/\\*","\\*/",{contains:[e.C_BLOCK_COMMENT_MODE]}),d={
+ variants:[{className:"type",begin:e.UNDERSCORE_IDENT_RE},{begin:/\(/,end:/\)/,
+ contains:[]}]},g=d;return g.variants[1].contains=[d],d.variants[1].contains=[g],
+ {name:"Kotlin",aliases:["kt","kts"],keywords:n,
+ contains:[e.COMMENT("/\\*\\*","\\*/",{relevance:0,contains:[{className:"doctag",
+ begin:"@[A-Za-z]+"}]}),e.C_LINE_COMMENT_MODE,c,{className:"keyword",
+ begin:/\b(break|continue|return|this)\b/,starts:{contains:[{className:"symbol",
+ begin:/@\w+/}]}},t,s,o,{className:"function",beginKeywords:"fun",end:"[(]|$",
+ returnBegin:!0,excludeEnd:!0,keywords:n,relevance:5,contains:[{
+ begin:e.UNDERSCORE_IDENT_RE+"\\s*\\(",returnBegin:!0,relevance:0,
+ contains:[e.UNDERSCORE_TITLE_MODE]},{className:"type",begin://,
+ keywords:"reified",relevance:0},{className:"params",begin:/\(/,end:/\)/,
+ endsParent:!0,keywords:n,relevance:0,contains:[{begin:/:/,end:/[=,\/]/,
+ endsWithParent:!0,contains:[d,e.C_LINE_COMMENT_MODE,c],relevance:0
+ },e.C_LINE_COMMENT_MODE,c,s,o,r,e.C_NUMBER_MODE]},c]},{
+ begin:[/class|interface|trait/,/\s+/,e.UNDERSCORE_IDENT_RE],beginScope:{
+ 3:"title.class"},keywords:"class interface trait",end:/[:\{(]|$/,excludeEnd:!0,
+ illegal:"extends implements",contains:[{
+ beginKeywords:"public protected internal private constructor"
+ },e.UNDERSCORE_TITLE_MODE,{className:"type",begin://,excludeBegin:!0,
+ excludeEnd:!0,relevance:0},{className:"type",begin:/[,:]\s*/,end:/[<\(,){\s]|$/,
+ excludeBegin:!0,returnEnd:!0},s,o]},r,{className:"meta",begin:"^#!/usr/bin/env",
+ end:"$",illegal:"\n"},l]}},grmr_less:e=>{
+ const n=ie(e),t=de,a="[\\w-]+",i="("+a+"|@\\{"+a+"\\})",r=[],s=[],o=e=>({
+ className:"string",begin:"~?"+e+".*?"+e}),l=(e,n,t)=>({className:e,begin:n,
+ relevance:t}),c={$pattern:/[a-z-]+/,keyword:"and or not only",
+ attribute:se.join(" ")},d={begin:"\\(",end:"\\)",contains:s,keywords:c,
+ relevance:0}
+ ;s.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,o("'"),o('"'),n.CSS_NUMBER_MODE,{
+ begin:"(url|data-uri)\\(",starts:{className:"string",end:"[\\)\\n]",
+ excludeEnd:!0}
+ },n.HEXCOLOR,d,l("variable","@@?"+a,10),l("variable","@\\{"+a+"\\}"),l("built_in","~?`[^`]*?`"),{
+ className:"attribute",begin:a+"\\s*:",end:":",returnBegin:!0,excludeEnd:!0
+ },n.IMPORTANT,{beginKeywords:"and not"},n.FUNCTION_DISPATCH);const g=s.concat({
+ begin:/\{/,end:/\}/,contains:r}),u={beginKeywords:"when",endsWithParent:!0,
+ contains:[{beginKeywords:"and not"}].concat(s)},b={begin:i+"\\s*:",
+ returnBegin:!0,end:/[;}]/,relevance:0,contains:[{begin:/-(webkit|moz|ms|o)-/
+ },n.CSS_VARIABLE,{className:"attribute",begin:"\\b("+ce.join("|")+")\\b",
+ end:/(?=:)/,starts:{endsWithParent:!0,illegal:"[<=$]",relevance:0,contains:s}}]
+ },m={className:"keyword",
+ begin:"@(import|media|charset|font-face|(-[a-z]+-)?keyframes|supports|document|namespace|page|viewport|host)\\b",
+ starts:{end:"[;{}]",keywords:c,returnEnd:!0,contains:s,relevance:0}},p={
+ className:"variable",variants:[{begin:"@"+a+"\\s*:",relevance:15},{begin:"@"+a
+ }],starts:{end:"[;}]",returnEnd:!0,contains:g}},_={variants:[{
+ begin:"[\\.#:&\\[>]",end:"[;{}]"},{begin:i,end:/\{/}],returnBegin:!0,
+ returnEnd:!0,illegal:"[<='$\"]",relevance:0,
+ contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,u,l("keyword","all\\b"),l("variable","@\\{"+a+"\\}"),{
+ begin:"\\b("+re.join("|")+")\\b",className:"selector-tag"
+ },n.CSS_NUMBER_MODE,l("selector-tag",i,0),l("selector-id","#"+i),l("selector-class","\\."+i,0),l("selector-tag","&",0),n.ATTRIBUTE_SELECTOR_MODE,{
+ className:"selector-pseudo",begin:":("+oe.join("|")+")"},{
+ className:"selector-pseudo",begin:":(:)?("+le.join("|")+")"},{begin:/\(/,
+ end:/\)/,relevance:0,contains:g},{begin:"!important"},n.FUNCTION_DISPATCH]},h={
+ begin:a+":(:)?"+`(${t.join("|")})`,returnBegin:!0,contains:[_]}
+ ;return r.push(e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,m,p,h,b,_,u,n.FUNCTION_DISPATCH),
+ {name:"Less",case_insensitive:!0,illegal:"[=>'/<($\"]",contains:r}},
+ grmr_lua:e=>{const n="\\[=*\\[",t="\\]=*\\]",a={begin:n,end:t,contains:["self"]
+ },i=[e.COMMENT("--(?!"+n+")","$"),e.COMMENT("--"+n,t,{contains:[a],relevance:10
+ })];return{name:"Lua",keywords:{$pattern:e.UNDERSCORE_IDENT_RE,
+ literal:"true false nil",
+ keyword:"and break do else elseif end for goto if in local not or repeat return then until while",
+ built_in:"_G _ENV _VERSION __index __newindex __mode __call __metatable __tostring __len __gc __add __sub __mul __div __mod __pow __concat __unm __eq __lt __le assert collectgarbage dofile error getfenv getmetatable ipairs load loadfile loadstring module next pairs pcall print rawequal rawget rawset require select setfenv setmetatable tonumber tostring type unpack xpcall arg self coroutine resume yield status wrap create running debug getupvalue debug sethook getmetatable gethook setmetatable setlocal traceback setfenv getinfo setupvalue getlocal getregistry getfenv io lines write close flush open output type read stderr stdin input stdout popen tmpfile math log max acos huge ldexp pi cos tanh pow deg tan cosh sinh random randomseed frexp ceil floor rad abs sqrt modf asin min mod fmod log10 atan2 exp sin atan os exit setlocale date getenv difftime remove time clock tmpname rename execute package preload loadlib loaded loaders cpath config path seeall string sub upper len gfind rep find match char dump gmatch reverse byte format gsub lower table setn insert getn foreachi maxn foreach concat sort remove"
+ },contains:i.concat([{className:"function",beginKeywords:"function",end:"\\)",
+ contains:[e.inherit(e.TITLE_MODE,{
+ begin:"([_a-zA-Z]\\w*\\.)*([_a-zA-Z]\\w*:)?[_a-zA-Z]\\w*"}),{className:"params",
+ begin:"\\(",endsWithParent:!0,contains:i}].concat(i)
+ },e.C_NUMBER_MODE,e.APOS_STRING_MODE,e.QUOTE_STRING_MODE,{className:"string",
+ begin:n,end:t,contains:[a],relevance:5}])}},grmr_makefile:e=>{const n={
+ className:"variable",variants:[{begin:"\\$\\("+e.UNDERSCORE_IDENT_RE+"\\)",
+ contains:[e.BACKSLASH_ESCAPE]},{begin:/\$[@%{
+ const n={begin:/<\/?[A-Za-z_]/,end:">",subLanguage:"xml",relevance:0},t={
+ variants:[{begin:/\[.+?\]\[.*?\]/,relevance:0},{
+ begin:/\[.+?\]\(((data|javascript|mailto):|(?:http|ftp)s?:\/\/).*?\)/,
+ relevance:2},{
+ begin:e.regex.concat(/\[.+?\]\(/,/[A-Za-z][A-Za-z0-9+.-]*/,/:\/\/.*?\)/),
+ relevance:2},{begin:/\[.+?\]\([./?&#].*?\)/,relevance:1},{
+ begin:/\[.*?\]\(.*?\)/,relevance:0}],returnBegin:!0,contains:[{match:/\[(?=\])/
+ },{className:"string",relevance:0,begin:"\\[",end:"\\]",excludeBegin:!0,
+ returnEnd:!0},{className:"link",relevance:0,begin:"\\]\\(",end:"\\)",
+ excludeBegin:!0,excludeEnd:!0},{className:"symbol",relevance:0,begin:"\\]\\[",
+ end:"\\]",excludeBegin:!0,excludeEnd:!0}]},a={className:"strong",contains:[],
+ variants:[{begin:/_{2}(?!\s)/,end:/_{2}/},{begin:/\*{2}(?!\s)/,end:/\*{2}/}]
+ },i={className:"emphasis",contains:[],variants:[{begin:/\*(?![*\s])/,end:/\*/},{
+ begin:/_(?![_\s])/,end:/_/,relevance:0}]},r=e.inherit(a,{contains:[]
+ }),s=e.inherit(i,{contains:[]});a.contains.push(s),i.contains.push(r)
+ ;let o=[n,t];return[a,i,r,s].forEach((e=>{e.contains=e.contains.concat(o)
+ })),o=o.concat(a,i),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{
+ className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:o},{
+ begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n",
+ contains:o}]}]},n,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)",
+ end:"\\s+",excludeEnd:!0},a,i,{className:"quote",begin:"^>\\s+",contains:o,
+ end:"$"},{className:"code",variants:[{begin:"(`{3,})[^`](.|\\n)*?\\1`*[ ]*"},{
+ begin:"(~{3,})[^~](.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{
+ begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))",
+ contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{
+ begin:"^[-\\*]{3,}",end:"$"},t,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{
+ className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{
+ className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}},grmr_objectivec:e=>{
+ const n=/[a-zA-Z@][a-zA-Z0-9_]*/,t={$pattern:n,
+ keyword:["@interface","@class","@protocol","@implementation"]};return{
+ name:"Objective-C",aliases:["mm","objc","obj-c","obj-c++","objective-c++"],
+ keywords:{"variable.language":["this","super"],$pattern:n,
+ keyword:["while","export","sizeof","typedef","const","struct","for","union","volatile","static","mutable","if","do","return","goto","enum","else","break","extern","asm","case","default","register","explicit","typename","switch","continue","inline","readonly","assign","readwrite","self","@synchronized","id","typeof","nonatomic","IBOutlet","IBAction","strong","weak","copy","in","out","inout","bycopy","byref","oneway","__strong","__weak","__block","__autoreleasing","@private","@protected","@public","@try","@property","@end","@throw","@catch","@finally","@autoreleasepool","@synthesize","@dynamic","@selector","@optional","@required","@encode","@package","@import","@defs","@compatibility_alias","__bridge","__bridge_transfer","__bridge_retained","__bridge_retain","__covariant","__contravariant","__kindof","_Nonnull","_Nullable","_Null_unspecified","__FUNCTION__","__PRETTY_FUNCTION__","__attribute__","getter","setter","retain","unsafe_unretained","nonnull","nullable","null_unspecified","null_resettable","class","instancetype","NS_DESIGNATED_INITIALIZER","NS_UNAVAILABLE","NS_REQUIRES_SUPER","NS_RETURNS_INNER_POINTER","NS_INLINE","NS_AVAILABLE","NS_DEPRECATED","NS_ENUM","NS_OPTIONS","NS_SWIFT_UNAVAILABLE","NS_ASSUME_NONNULL_BEGIN","NS_ASSUME_NONNULL_END","NS_REFINED_FOR_SWIFT","NS_SWIFT_NAME","NS_SWIFT_NOTHROW","NS_DURING","NS_HANDLER","NS_ENDHANDLER","NS_VALUERETURN","NS_VOIDRETURN"],
+ literal:["false","true","FALSE","TRUE","nil","YES","NO","NULL"],
+ built_in:["dispatch_once_t","dispatch_queue_t","dispatch_sync","dispatch_async","dispatch_once"],
+ type:["int","float","char","unsigned","signed","short","long","double","wchar_t","unichar","void","bool","BOOL","id|0","_Bool"]
+ },illegal:"/,end:/$/,illegal:"\\n"
+ },e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE]},{className:"class",
+ begin:"("+t.keyword.join("|")+")\\b",end:/(\{|$)/,excludeEnd:!0,keywords:t,
+ contains:[e.UNDERSCORE_TITLE_MODE]},{begin:"\\."+e.UNDERSCORE_IDENT_RE,
+ relevance:0}]}},grmr_perl:e=>{const n=e.regex,t=/[dualxmsipngr]{0,12}/,a={
+ $pattern:/[\w.]+/,
+ keyword:"abs accept alarm and atan2 bind binmode bless break caller chdir chmod chomp chop chown chr chroot close closedir connect continue cos crypt dbmclose dbmopen defined delete die do dump each else elsif endgrent endhostent endnetent endprotoent endpwent endservent eof eval exec exists exit exp fcntl fileno flock for foreach fork format formline getc getgrent getgrgid getgrnam gethostbyaddr gethostbyname gethostent getlogin getnetbyaddr getnetbyname getnetent getpeername getpgrp getpriority getprotobyname getprotobynumber getprotoent getpwent getpwnam getpwuid getservbyname getservbyport getservent getsockname getsockopt given glob gmtime goto grep gt hex if index int ioctl join keys kill last lc lcfirst length link listen local localtime log lstat lt ma map mkdir msgctl msgget msgrcv msgsnd my ne next no not oct open opendir or ord our pack package pipe pop pos print printf prototype push q|0 qq quotemeta qw qx rand read readdir readline readlink readpipe recv redo ref rename require reset return reverse rewinddir rindex rmdir say scalar seek seekdir select semctl semget semop send setgrent sethostent setnetent setpgrp setpriority setprotoent setpwent setservent setsockopt shift shmctl shmget shmread shmwrite shutdown sin sleep socket socketpair sort splice split sprintf sqrt srand stat state study sub substr symlink syscall sysopen sysread sysseek system syswrite tell telldir tie tied time times tr truncate uc ucfirst umask undef unless unlink unpack unshift untie until use utime values vec wait waitpid wantarray warn when while write x|0 xor y|0"
+ },i={className:"subst",begin:"[$@]\\{",end:"\\}",keywords:a},r={begin:/->\{/,
+ end:/\}/},s={variants:[{begin:/\$\d/},{
+ begin:n.concat(/[$%@](\^\w\b|#\w+(::\w+)*|\{\w+\}|\w+(::\w*)*)/,"(?![A-Za-z])(?![@$%])")
+ },{begin:/[$%@][^\s\w{]/,relevance:0}]
+ },o=[e.BACKSLASH_ESCAPE,i,s],l=[/!/,/\//,/\|/,/\?/,/'/,/"/,/#/],c=(e,a,i="\\1")=>{
+ const r="\\1"===i?i:n.concat(i,a)
+ ;return n.concat(n.concat("(?:",e,")"),a,/(?:\\.|[^\\\/])*?/,r,/(?:\\.|[^\\\/])*?/,i,t)
+ },d=(e,a,i)=>n.concat(n.concat("(?:",e,")"),a,/(?:\\.|[^\\\/])*?/,i,t),g=[s,e.HASH_COMMENT_MODE,e.COMMENT(/^=\w/,/=cut/,{
+ endsWithParent:!0}),r,{className:"string",contains:o,variants:[{
+ begin:"q[qwxr]?\\s*\\(",end:"\\)",relevance:5},{begin:"q[qwxr]?\\s*\\[",
+ end:"\\]",relevance:5},{begin:"q[qwxr]?\\s*\\{",end:"\\}",relevance:5},{
+ begin:"q[qwxr]?\\s*\\|",end:"\\|",relevance:5},{begin:"q[qwxr]?\\s*<",end:">",
+ relevance:5},{begin:"qw\\s+q",end:"q",relevance:5},{begin:"'",end:"'",
+ contains:[e.BACKSLASH_ESCAPE]},{begin:'"',end:'"'},{begin:"`",end:"`",
+ contains:[e.BACKSLASH_ESCAPE]},{begin:/\{\w+\}/,relevance:0},{
+ begin:"-?\\w+\\s*=>",relevance:0}]},{className:"number",
+ begin:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",
+ relevance:0},{
+ begin:"(\\/\\/|"+e.RE_STARTERS_RE+"|\\b(split|return|print|reverse|grep)\\b)\\s*",
+ keywords:"split return print reverse grep",relevance:0,
+ contains:[e.HASH_COMMENT_MODE,{className:"regexp",variants:[{
+ begin:c("s|tr|y",n.either(...l,{capture:!0}))},{begin:c("s|tr|y","\\(","\\)")},{
+ begin:c("s|tr|y","\\[","\\]")},{begin:c("s|tr|y","\\{","\\}")}],relevance:2},{
+ className:"regexp",variants:[{begin:/(m|qr)\/\//,relevance:0},{
+ begin:d("(?:m|qr)?",/\//,/\//)},{begin:d("m|qr",n.either(...l,{capture:!0
+ }),/\1/)},{begin:d("m|qr",/\(/,/\)/)},{begin:d("m|qr",/\[/,/\]/)},{
+ begin:d("m|qr",/\{/,/\}/)}]}]},{className:"function",beginKeywords:"sub",
+ end:"(\\s*\\(.*?\\))?[;{]",excludeEnd:!0,relevance:5,contains:[e.TITLE_MODE]},{
+ begin:"-\\w\\b",relevance:0},{begin:"^__DATA__$",end:"^__END__$",
+ subLanguage:"mojolicious",contains:[{begin:"^@@.*",end:"$",className:"comment"}]
+ }];return i.contains=g,r.contains=g,{name:"Perl",aliases:["pl","pm"],keywords:a,
+ contains:g}},grmr_php:e=>{
+ const n=e.regex,t=/(?![A-Za-z0-9])(?![$])/,a=n.concat(/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/,t),i=n.concat(/(\\?[A-Z][a-z0-9_\x7f-\xff]+|\\?[A-Z]+(?=[A-Z][a-z0-9_\x7f-\xff])){1,}/,t),r={
+ scope:"variable",match:"\\$+"+a},s={scope:"subst",variants:[{begin:/\$\w+/},{
+ begin:/\{\$/,end:/\}/}]},o=e.inherit(e.APOS_STRING_MODE,{illegal:null
+ }),l="[ \t\n]",c={scope:"string",variants:[e.inherit(e.QUOTE_STRING_MODE,{
+ illegal:null,contains:e.QUOTE_STRING_MODE.contains.concat(s)}),o,{
+ begin:/<<<[ \t]*(?:(\w+)|"(\w+)")\n/,end:/[ \t]*(\w+)\b/,
+ contains:e.QUOTE_STRING_MODE.contains.concat(s),"on:begin":(e,n)=>{
+ n.data._beginMatch=e[1]||e[2]},"on:end":(e,n)=>{
+ n.data._beginMatch!==e[1]&&n.ignoreMatch()}},e.END_SAME_AS_BEGIN({
+ begin:/<<<[ \t]*'(\w+)'\n/,end:/[ \t]*(\w+)\b/})]},d={scope:"number",variants:[{
+ begin:"\\b0[bB][01]+(?:_[01]+)*\\b"},{begin:"\\b0[oO][0-7]+(?:_[0-7]+)*\\b"},{
+ begin:"\\b0[xX][\\da-fA-F]+(?:_[\\da-fA-F]+)*\\b"},{
+ begin:"(?:\\b\\d+(?:_\\d+)*(\\.(?:\\d+(?:_\\d+)*))?|\\B\\.\\d+)(?:[eE][+-]?\\d+)?"
+ }],relevance:0
+ },g=["false","null","true"],u=["__CLASS__","__DIR__","__FILE__","__FUNCTION__","__COMPILER_HALT_OFFSET__","__LINE__","__METHOD__","__NAMESPACE__","__TRAIT__","die","echo","exit","include","include_once","print","require","require_once","array","abstract","and","as","binary","bool","boolean","break","callable","case","catch","class","clone","const","continue","declare","default","do","double","else","elseif","empty","enddeclare","endfor","endforeach","endif","endswitch","endwhile","enum","eval","extends","final","finally","float","for","foreach","from","global","goto","if","implements","instanceof","insteadof","int","integer","interface","isset","iterable","list","match|0","mixed","new","never","object","or","private","protected","public","readonly","real","return","string","switch","throw","trait","try","unset","use","var","void","while","xor","yield"],b=["Error|0","AppendIterator","ArgumentCountError","ArithmeticError","ArrayIterator","ArrayObject","AssertionError","BadFunctionCallException","BadMethodCallException","CachingIterator","CallbackFilterIterator","CompileError","Countable","DirectoryIterator","DivisionByZeroError","DomainException","EmptyIterator","ErrorException","Exception","FilesystemIterator","FilterIterator","GlobIterator","InfiniteIterator","InvalidArgumentException","IteratorIterator","LengthException","LimitIterator","LogicException","MultipleIterator","NoRewindIterator","OutOfBoundsException","OutOfRangeException","OuterIterator","OverflowException","ParentIterator","ParseError","RangeException","RecursiveArrayIterator","RecursiveCachingIterator","RecursiveCallbackFilterIterator","RecursiveDirectoryIterator","RecursiveFilterIterator","RecursiveIterator","RecursiveIteratorIterator","RecursiveRegexIterator","RecursiveTreeIterator","RegexIterator","RuntimeException","SeekableIterator","SplDoublyLinkedList","SplFileInfo","SplFileObject","SplFixedArray","SplHeap","SplMaxHeap","SplMinHeap","SplObjectStorage","SplObserver","SplPriorityQueue","SplQueue","SplStack","SplSubject","SplTempFileObject","TypeError","UnderflowException","UnexpectedValueException","UnhandledMatchError","ArrayAccess","BackedEnum","Closure","Fiber","Generator","Iterator","IteratorAggregate","Serializable","Stringable","Throwable","Traversable","UnitEnum","WeakReference","WeakMap","Directory","__PHP_Incomplete_Class","parent","php_user_filter","self","static","stdClass"],m={
+ keyword:u,literal:(e=>{const n=[];return e.forEach((e=>{
+ n.push(e),e.toLowerCase()===e?n.push(e.toUpperCase()):n.push(e.toLowerCase())
+ })),n})(g),built_in:b},p=e=>e.map((e=>e.replace(/\|\d+$/,""))),_={variants:[{
+ match:[/new/,n.concat(l,"+"),n.concat("(?!",p(b).join("\\b|"),"\\b)"),i],scope:{
+ 1:"keyword",4:"title.class"}}]},h=n.concat(a,"\\b(?!\\()"),f={variants:[{
+ match:[n.concat(/::/,n.lookahead(/(?!class\b)/)),h],scope:{2:"variable.constant"
+ }},{match:[/::/,/class/],scope:{2:"variable.language"}},{
+ match:[i,n.concat(/::/,n.lookahead(/(?!class\b)/)),h],scope:{1:"title.class",
+ 3:"variable.constant"}},{match:[i,n.concat("::",n.lookahead(/(?!class\b)/))],
+ scope:{1:"title.class"}},{match:[i,/::/,/class/],scope:{1:"title.class",
+ 3:"variable.language"}}]},E={scope:"attr",
+ match:n.concat(a,n.lookahead(":"),n.lookahead(/(?!::)/))},y={relevance:0,
+ begin:/\(/,end:/\)/,keywords:m,contains:[E,r,f,e.C_BLOCK_COMMENT_MODE,c,d,_]
+ },N={relevance:0,
+ match:[/\b/,n.concat("(?!fn\\b|function\\b|",p(u).join("\\b|"),"|",p(b).join("\\b|"),"\\b)"),a,n.concat(l,"*"),n.lookahead(/(?=\()/)],
+ scope:{3:"title.function.invoke"},contains:[y]};y.contains.push(N)
+ ;const w=[E,f,e.C_BLOCK_COMMENT_MODE,c,d,_];return{case_insensitive:!1,
+ keywords:m,contains:[{begin:n.concat(/#\[\s*/,i),beginScope:"meta",end:/]/,
+ endScope:"meta",keywords:{literal:g,keyword:["new","array"]},contains:[{
+ begin:/\[/,end:/]/,keywords:{literal:g,keyword:["new","array"]},
+ contains:["self",...w]},...w,{scope:"meta",match:i}]
+ },e.HASH_COMMENT_MODE,e.COMMENT("//","$"),e.COMMENT("/\\*","\\*/",{contains:[{
+ scope:"doctag",match:"@[A-Za-z]+"}]}),{match:/__halt_compiler\(\);/,
+ keywords:"__halt_compiler",starts:{scope:"comment",end:e.MATCH_NOTHING_RE,
+ contains:[{match:/\?>/,scope:"meta",endsParent:!0}]}},{scope:"meta",variants:[{
+ begin:/<\?php/,relevance:10},{begin:/<\?=/},{begin:/<\?/,relevance:.1},{
+ begin:/\?>/}]},{scope:"variable.language",match:/\$this\b/},r,N,f,{
+ match:[/const/,/\s/,a],scope:{1:"keyword",3:"variable.constant"}},_,{
+ scope:"function",relevance:0,beginKeywords:"fn function",end:/[;{]/,
+ excludeEnd:!0,illegal:"[$%\\[]",contains:[{beginKeywords:"use"
+ },e.UNDERSCORE_TITLE_MODE,{begin:"=>",endsParent:!0},{scope:"params",
+ begin:"\\(",end:"\\)",excludeBegin:!0,excludeEnd:!0,keywords:m,
+ contains:["self",r,f,e.C_BLOCK_COMMENT_MODE,c,d]}]},{scope:"class",variants:[{
+ beginKeywords:"enum",illegal:/[($"]/},{beginKeywords:"class interface trait",
+ illegal:/[:($"]/}],relevance:0,end:/\{/,excludeEnd:!0,contains:[{
+ beginKeywords:"extends implements"},e.UNDERSCORE_TITLE_MODE]},{
+ beginKeywords:"namespace",relevance:0,end:";",illegal:/[.']/,
+ contains:[e.inherit(e.UNDERSCORE_TITLE_MODE,{scope:"title.class"})]},{
+ beginKeywords:"use",relevance:0,end:";",contains:[{
+ match:/\b(as|const|function)\b/,scope:"keyword"},e.UNDERSCORE_TITLE_MODE]},c,d]}
+ },grmr_php_template:e=>({name:"PHP template",subLanguage:"xml",contains:[{
+ begin:/<\?(php|=)?/,end:/\?>/,subLanguage:"php",contains:[{begin:"/\\*",
+ end:"\\*/",skip:!0},{begin:'b"',end:'"',skip:!0},{begin:"b'",end:"'",skip:!0
+ },e.inherit(e.APOS_STRING_MODE,{illegal:null,className:null,contains:null,
+ skip:!0}),e.inherit(e.QUOTE_STRING_MODE,{illegal:null,className:null,
+ contains:null,skip:!0})]}]}),grmr_plaintext:e=>({name:"Plain text",
+ aliases:["text","txt"],disableAutodetect:!0}),grmr_python:e=>{
+ const n=e.regex,t=/[\p{XID_Start}_]\p{XID_Continue}*/u,a=["and","as","assert","async","await","break","case","class","continue","def","del","elif","else","except","finally","for","from","global","if","import","in","is","lambda","match","nonlocal|10","not","or","pass","raise","return","try","while","with","yield"],i={
+ $pattern:/[A-Za-z]\w+|__\w+__/,keyword:a,
+ built_in:["__import__","abs","all","any","ascii","bin","bool","breakpoint","bytearray","bytes","callable","chr","classmethod","compile","complex","delattr","dict","dir","divmod","enumerate","eval","exec","filter","float","format","frozenset","getattr","globals","hasattr","hash","help","hex","id","input","int","isinstance","issubclass","iter","len","list","locals","map","max","memoryview","min","next","object","oct","open","ord","pow","print","property","range","repr","reversed","round","set","setattr","slice","sorted","staticmethod","str","sum","super","tuple","type","vars","zip"],
+ literal:["__debug__","Ellipsis","False","None","NotImplemented","True"],
+ type:["Any","Callable","Coroutine","Dict","List","Literal","Generic","Optional","Sequence","Set","Tuple","Type","Union"]
+ },r={className:"meta",begin:/^(>>>|\.\.\.) /},s={className:"subst",begin:/\{/,
+ end:/\}/,keywords:i,illegal:/#/},o={begin:/\{\{/,relevance:0},l={
+ className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{
+ begin:/([uU]|[bB]|[rR]|[bB][rR]|[rR][bB])?'''/,end:/'''/,
+ contains:[e.BACKSLASH_ESCAPE,r],relevance:10},{
+ begin:/([uU]|[bB]|[rR]|[bB][rR]|[rR][bB])?"""/,end:/"""/,
+ contains:[e.BACKSLASH_ESCAPE,r],relevance:10},{
+ begin:/([fF][rR]|[rR][fF]|[fF])'''/,end:/'''/,
+ contains:[e.BACKSLASH_ESCAPE,r,o,s]},{begin:/([fF][rR]|[rR][fF]|[fF])"""/,
+ end:/"""/,contains:[e.BACKSLASH_ESCAPE,r,o,s]},{begin:/([uU]|[rR])'/,end:/'/,
+ relevance:10},{begin:/([uU]|[rR])"/,end:/"/,relevance:10},{
+ begin:/([bB]|[bB][rR]|[rR][bB])'/,end:/'/},{begin:/([bB]|[bB][rR]|[rR][bB])"/,
+ end:/"/},{begin:/([fF][rR]|[rR][fF]|[fF])'/,end:/'/,
+ contains:[e.BACKSLASH_ESCAPE,o,s]},{begin:/([fF][rR]|[rR][fF]|[fF])"/,end:/"/,
+ contains:[e.BACKSLASH_ESCAPE,o,s]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]
+ },c="[0-9](_?[0-9])*",d=`(\\b(${c}))?\\.(${c})|\\b(${c})\\.`,g="\\b|"+a.join("|"),u={
+ className:"number",relevance:0,variants:[{
+ begin:`(\\b(${c})|(${d}))[eE][+-]?(${c})[jJ]?(?=${g})`},{begin:`(${d})[jJ]?`},{
+ begin:`\\b([1-9](_?[0-9])*|0+(_?0)*)[lLjJ]?(?=${g})`},{
+ begin:`\\b0[bB](_?[01])+[lL]?(?=${g})`},{begin:`\\b0[oO](_?[0-7])+[lL]?(?=${g})`
+ },{begin:`\\b0[xX](_?[0-9a-fA-F])+[lL]?(?=${g})`},{begin:`\\b(${c})[jJ](?=${g})`
+ }]},b={className:"comment",begin:n.lookahead(/# type:/),end:/$/,keywords:i,
+ contains:[{begin:/# type:/},{begin:/#/,end:/\b\B/,endsWithParent:!0}]},m={
+ className:"params",variants:[{className:"",begin:/\(\s*\)/,skip:!0},{begin:/\(/,
+ end:/\)/,excludeBegin:!0,excludeEnd:!0,keywords:i,
+ contains:["self",r,u,l,e.HASH_COMMENT_MODE]}]};return s.contains=[l,u,r],{
+ name:"Python",aliases:["py","gyp","ipython"],unicodeRegex:!0,keywords:i,
+ illegal:/(<\/|\?)|=>/,contains:[r,u,{begin:/\bself\b/},{beginKeywords:"if",
+ relevance:0},l,b,e.HASH_COMMENT_MODE,{match:[/\bdef/,/\s+/,t],scope:{
+ 1:"keyword",3:"title.function"},contains:[m]},{variants:[{
+ match:[/\bclass/,/\s+/,t,/\s*/,/\(\s*/,t,/\s*\)/]},{match:[/\bclass/,/\s+/,t]}],
+ scope:{1:"keyword",3:"title.class",6:"title.class.inherited"}},{
+ className:"meta",begin:/^[\t ]*@/,end:/(?=#)|$/,contains:[u,m,l]}]}},
+ grmr_python_repl:e=>({aliases:["pycon"],contains:[{className:"meta.prompt",
+ starts:{end:/ |$/,starts:{end:"$",subLanguage:"python"}},variants:[{
+ begin:/^>>>(?=[ ]|$)/},{begin:/^\.\.\.(?=[ ]|$)/}]}]}),grmr_r:e=>{
+ const n=e.regex,t=/(?:(?:[a-zA-Z]|\.[._a-zA-Z])[._a-zA-Z0-9]*)|\.(?!\d)/,a=n.either(/0[xX][0-9a-fA-F]+\.[0-9a-fA-F]*[pP][+-]?\d+i?/,/0[xX][0-9a-fA-F]+(?:[pP][+-]?\d+)?[Li]?/,/(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?[Li]?/),i=/[=!<>:]=|\|\||&&|:::?|<-|<<-|->>|->|\|>|[-+*\/?!$&|:<=>@^~]|\*\*/,r=n.either(/[()]/,/[{}]/,/\[\[/,/[[\]]/,/\\/,/,/)
+ ;return{name:"R",keywords:{$pattern:t,
+ keyword:"function if in break next repeat else for while",
+ literal:"NULL NA TRUE FALSE Inf NaN NA_integer_|10 NA_real_|10 NA_character_|10 NA_complex_|10",
+ built_in:"LETTERS letters month.abb month.name pi T F abs acos acosh all any anyNA Arg as.call as.character as.complex as.double as.environment as.integer as.logical as.null.default as.numeric as.raw asin asinh atan atanh attr attributes baseenv browser c call ceiling class Conj cos cosh cospi cummax cummin cumprod cumsum digamma dim dimnames emptyenv exp expression floor forceAndCall gamma gc.time globalenv Im interactive invisible is.array is.atomic is.call is.character is.complex is.double is.environment is.expression is.finite is.function is.infinite is.integer is.language is.list is.logical is.matrix is.na is.name is.nan is.null is.numeric is.object is.pairlist is.raw is.recursive is.single is.symbol lazyLoadDBfetch length lgamma list log max min missing Mod names nargs nzchar oldClass on.exit pos.to.env proc.time prod quote range Re rep retracemem return round seq_along seq_len seq.int sign signif sin sinh sinpi sqrt standardGeneric substitute sum switch tan tanh tanpi tracemem trigamma trunc unclass untracemem UseMethod xtfrm"
+ },contains:[e.COMMENT(/#'/,/$/,{contains:[{scope:"doctag",match:/@examples/,
+ starts:{end:n.lookahead(n.either(/\n^#'\s*(?=@[a-zA-Z]+)/,/\n^(?!#')/)),
+ endsParent:!0}},{scope:"doctag",begin:"@param",end:/$/,contains:[{
+ scope:"variable",variants:[{match:t},{match:/`(?:\\.|[^`\\])+`/}],endsParent:!0
+ }]},{scope:"doctag",match:/@[a-zA-Z]+/},{scope:"keyword",match:/\\[a-zA-Z]+/}]
+ }),e.HASH_COMMENT_MODE,{scope:"string",contains:[e.BACKSLASH_ESCAPE],
+ variants:[e.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\(/,end:/\)(-*)"/
+ }),e.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\{/,end:/\}(-*)"/
+ }),e.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\[/,end:/\](-*)"/
+ }),e.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\(/,end:/\)(-*)'/
+ }),e.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\{/,end:/\}(-*)'/
+ }),e.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\[/,end:/\](-*)'/}),{begin:'"',end:'"',
+ relevance:0},{begin:"'",end:"'",relevance:0}]},{relevance:0,variants:[{scope:{
+ 1:"operator",2:"number"},match:[i,a]},{scope:{1:"operator",2:"number"},
+ match:[/%[^%]*%/,a]},{scope:{1:"punctuation",2:"number"},match:[r,a]},{scope:{
+ 2:"number"},match:[/[^a-zA-Z0-9._]|^/,a]}]},{scope:{3:"operator"},
+ match:[t,/\s+/,/<-/,/\s+/]},{scope:"operator",relevance:0,variants:[{match:i},{
+ match:/%[^%]*%/}]},{scope:"punctuation",relevance:0,match:r},{begin:"`",end:"`",
+ contains:[{begin:/\\./}]}]}},grmr_ruby:e=>{
+ const n=e.regex,t="([a-zA-Z_]\\w*[!?=]?|[-+~]@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?)",a=n.either(/\b([A-Z]+[a-z0-9]+)+/,/\b([A-Z]+[a-z0-9]+)+[A-Z]+/),i=n.concat(a,/(::\w+)*/),r={
+ "variable.constant":["__FILE__","__LINE__","__ENCODING__"],
+ "variable.language":["self","super"],
+ keyword:["alias","and","begin","BEGIN","break","case","class","defined","do","else","elsif","end","END","ensure","for","if","in","module","next","not","or","redo","require","rescue","retry","return","then","undef","unless","until","when","while","yield","include","extend","prepend","public","private","protected","raise","throw"],
+ built_in:["proc","lambda","attr_accessor","attr_reader","attr_writer","define_method","private_constant","module_function"],
+ literal:["true","false","nil"]},s={className:"doctag",begin:"@[A-Za-z]+"},o={
+ begin:"#<",end:">"},l=[e.COMMENT("#","$",{contains:[s]
+ }),e.COMMENT("^=begin","^=end",{contains:[s],relevance:10
+ }),e.COMMENT("^__END__",e.MATCH_NOTHING_RE)],c={className:"subst",begin:/#\{/,
+ end:/\}/,keywords:r},d={className:"string",contains:[e.BACKSLASH_ESCAPE,c],
+ variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/`/,end:/`/},{
+ begin:/%[qQwWx]?\(/,end:/\)/},{begin:/%[qQwWx]?\[/,end:/\]/},{
+ begin:/%[qQwWx]?\{/,end:/\}/},{begin:/%[qQwWx]?/},{begin:/%[qQwWx]?\//,
+ end:/\//},{begin:/%[qQwWx]?%/,end:/%/},{begin:/%[qQwWx]?-/,end:/-/},{
+ begin:/%[qQwWx]?\|/,end:/\|/},{begin:/\B\?(\\\d{1,3})/},{
+ begin:/\B\?(\\x[A-Fa-f0-9]{1,2})/},{begin:/\B\?(\\u\{?[A-Fa-f0-9]{1,6}\}?)/},{
+ begin:/\B\?(\\M-\\C-|\\M-\\c|\\c\\M-|\\M-|\\C-\\M-)[\x20-\x7e]/},{
+ begin:/\B\?\\(c|C-)[\x20-\x7e]/},{begin:/\B\?\\?\S/},{
+ begin:n.concat(/<<[-~]?'?/,n.lookahead(/(\w+)(?=\W)[^\n]*\n(?:[^\n]*\n)*?\s*\1\b/)),
+ contains:[e.END_SAME_AS_BEGIN({begin:/(\w+)/,end:/(\w+)/,
+ contains:[e.BACKSLASH_ESCAPE,c]})]}]},g="[0-9](_?[0-9])*",u={className:"number",
+ relevance:0,variants:[{
+ begin:`\\b([1-9](_?[0-9])*|0)(\\.(${g}))?([eE][+-]?(${g})|r)?i?\\b`},{
+ begin:"\\b0[dD][0-9](_?[0-9])*r?i?\\b"},{begin:"\\b0[bB][0-1](_?[0-1])*r?i?\\b"
+ },{begin:"\\b0[oO][0-7](_?[0-7])*r?i?\\b"},{
+ begin:"\\b0[xX][0-9a-fA-F](_?[0-9a-fA-F])*r?i?\\b"},{
+ begin:"\\b0(_?[0-7])+r?i?\\b"}]},b={variants:[{match:/\(\)/},{
+ className:"params",begin:/\(/,end:/(?=\))/,excludeBegin:!0,endsParent:!0,
+ keywords:r}]},m=[d,{variants:[{match:[/class\s+/,i,/\s+<\s+/,i]},{
+ match:[/\b(class|module)\s+/,i]}],scope:{2:"title.class",
+ 4:"title.class.inherited"},keywords:r},{match:[/(include|extend)\s+/,i],scope:{
+ 2:"title.class"},keywords:r},{relevance:0,match:[i,/\.new[. (]/],scope:{
+ 1:"title.class"}},{relevance:0,match:/\b[A-Z][A-Z_0-9]+\b/,
+ className:"variable.constant"},{relevance:0,match:a,scope:"title.class"},{
+ match:[/def/,/\s+/,t],scope:{1:"keyword",3:"title.function"},contains:[b]},{
+ begin:e.IDENT_RE+"::"},{className:"symbol",
+ begin:e.UNDERSCORE_IDENT_RE+"(!|\\?)?:",relevance:0},{className:"symbol",
+ begin:":(?!\\s)",contains:[d,{begin:t}],relevance:0},u,{className:"variable",
+ begin:"(\\$\\W)|((\\$|@@?)(\\w+))(?=[^@$?])(?![A-Za-z])(?![@$?'])"},{
+ className:"params",begin:/\|/,end:/\|/,excludeBegin:!0,excludeEnd:!0,
+ relevance:0,keywords:r},{begin:"("+e.RE_STARTERS_RE+"|unless)\\s*",
+ keywords:"unless",contains:[{className:"regexp",contains:[e.BACKSLASH_ESCAPE,c],
+ illegal:/\n/,variants:[{begin:"/",end:"/[a-z]*"},{begin:/%r\{/,end:/\}[a-z]*/},{
+ begin:"%r\\(",end:"\\)[a-z]*"},{begin:"%r!",end:"![a-z]*"},{begin:"%r\\[",
+ end:"\\][a-z]*"}]}].concat(o,l),relevance:0}].concat(o,l)
+ ;c.contains=m,b.contains=m;const p=[{begin:/^\s*=>/,starts:{end:"$",contains:m}
+ },{className:"meta.prompt",
+ begin:"^([>?]>|[\\w#]+\\(\\w+\\):\\d+:\\d+[>*]|(\\w+-)?\\d+\\.\\d+\\.\\d+(p\\d+)?[^\\d][^>]+>)(?=[ ])",
+ starts:{end:"$",keywords:r,contains:m}}];return l.unshift(o),{name:"Ruby",
+ aliases:["rb","gemspec","podspec","thor","irb"],keywords:r,illegal:/\/\*/,
+ contains:[e.SHEBANG({binary:"ruby"})].concat(p).concat(l).concat(m)}},
+ grmr_rust:e=>{const n=e.regex,t={className:"title.function.invoke",relevance:0,
+ begin:n.concat(/\b/,/(?!let|for|while|if|else|match\b)/,e.IDENT_RE,n.lookahead(/\s*\(/))
+ },a="([ui](8|16|32|64|128|size)|f(32|64))?",i=["drop ","Copy","Send","Sized","Sync","Drop","Fn","FnMut","FnOnce","ToOwned","Clone","Debug","PartialEq","PartialOrd","Eq","Ord","AsRef","AsMut","Into","From","Default","Iterator","Extend","IntoIterator","DoubleEndedIterator","ExactSizeIterator","SliceConcatExt","ToString","assert!","assert_eq!","bitflags!","bytes!","cfg!","col!","concat!","concat_idents!","debug_assert!","debug_assert_eq!","env!","eprintln!","panic!","file!","format!","format_args!","include_bytes!","include_str!","line!","local_data_key!","module_path!","option_env!","print!","println!","select!","stringify!","try!","unimplemented!","unreachable!","vec!","write!","writeln!","macro_rules!","assert_ne!","debug_assert_ne!"],r=["i8","i16","i32","i64","i128","isize","u8","u16","u32","u64","u128","usize","f32","f64","str","char","bool","Box","Option","Result","String","Vec"]
+ ;return{name:"Rust",aliases:["rs"],keywords:{$pattern:e.IDENT_RE+"!?",type:r,
+ keyword:["abstract","as","async","await","become","box","break","const","continue","crate","do","dyn","else","enum","extern","false","final","fn","for","if","impl","in","let","loop","macro","match","mod","move","mut","override","priv","pub","ref","return","self","Self","static","struct","super","trait","true","try","type","typeof","unsafe","unsized","use","virtual","where","while","yield"],
+ literal:["true","false","Some","None","Ok","Err"],built_in:i},illegal:""},t]}},
+ grmr_scss:e=>{const n=ie(e),t=le,a=oe,i="@[a-z-]+",r={className:"variable",
+ begin:"(\\$[a-zA-Z-][a-zA-Z0-9_-]*)\\b",relevance:0};return{name:"SCSS",
+ case_insensitive:!0,illegal:"[=/|']",
+ contains:[e.C_LINE_COMMENT_MODE,e.C_BLOCK_COMMENT_MODE,n.CSS_NUMBER_MODE,{
+ className:"selector-id",begin:"#[A-Za-z0-9_-]+",relevance:0},{
+ className:"selector-class",begin:"\\.[A-Za-z0-9_-]+",relevance:0
+ },n.ATTRIBUTE_SELECTOR_MODE,{className:"selector-tag",
+ begin:"\\b("+re.join("|")+")\\b",relevance:0},{className:"selector-pseudo",
+ begin:":("+a.join("|")+")"},{className:"selector-pseudo",
+ begin:":(:)?("+t.join("|")+")"},r,{begin:/\(/,end:/\)/,
+ contains:[n.CSS_NUMBER_MODE]},n.CSS_VARIABLE,{className:"attribute",
+ begin:"\\b("+ce.join("|")+")\\b"},{
+ begin:"\\b(whitespace|wait|w-resize|visible|vertical-text|vertical-ideographic|uppercase|upper-roman|upper-alpha|underline|transparent|top|thin|thick|text|text-top|text-bottom|tb-rl|table-header-group|table-footer-group|sw-resize|super|strict|static|square|solid|small-caps|separate|se-resize|scroll|s-resize|rtl|row-resize|ridge|right|repeat|repeat-y|repeat-x|relative|progress|pointer|overline|outside|outset|oblique|nowrap|not-allowed|normal|none|nw-resize|no-repeat|no-drop|newspaper|ne-resize|n-resize|move|middle|medium|ltr|lr-tb|lowercase|lower-roman|lower-alpha|loose|list-item|line|line-through|line-edge|lighter|left|keep-all|justify|italic|inter-word|inter-ideograph|inside|inset|inline|inline-block|inherit|inactive|ideograph-space|ideograph-parenthesis|ideograph-numeric|ideograph-alpha|horizontal|hidden|help|hand|groove|fixed|ellipsis|e-resize|double|dotted|distribute|distribute-space|distribute-letter|distribute-all-lines|disc|disabled|default|decimal|dashed|crosshair|collapse|col-resize|circle|char|center|capitalize|break-word|break-all|bottom|both|bolder|bold|block|bidi-override|below|baseline|auto|always|all-scroll|absolute|table|table-cell)\\b"
+ },{begin:/:/,end:/[;}{]/,relevance:0,
+ contains:[n.BLOCK_COMMENT,r,n.HEXCOLOR,n.CSS_NUMBER_MODE,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,n.IMPORTANT,n.FUNCTION_DISPATCH]
+ },{begin:"@(page|font-face)",keywords:{$pattern:i,keyword:"@page @font-face"}},{
+ begin:"@",end:"[{;]",returnBegin:!0,keywords:{$pattern:/[a-z-]+/,
+ keyword:"and or not only",attribute:se.join(" ")},contains:[{begin:i,
+ className:"keyword"},{begin:/[a-z-]+(?=:)/,className:"attribute"
+ },r,e.QUOTE_STRING_MODE,e.APOS_STRING_MODE,n.HEXCOLOR,n.CSS_NUMBER_MODE]
+ },n.FUNCTION_DISPATCH]}},grmr_shell:e=>({name:"Shell Session",
+ aliases:["console","shellsession"],contains:[{className:"meta.prompt",
+ begin:/^\s{0,3}[/~\w\d[\]()@-]*[>%$#][ ]?/,starts:{end:/[^\\](?=\s*$)/,
+ subLanguage:"bash"}}]}),grmr_sql:e=>{
+ const n=e.regex,t=e.COMMENT("--","$"),a=["true","false","unknown"],i=["bigint","binary","blob","boolean","char","character","clob","date","dec","decfloat","decimal","float","int","integer","interval","nchar","nclob","national","numeric","real","row","smallint","time","timestamp","varchar","varying","varbinary"],r=["abs","acos","array_agg","asin","atan","avg","cast","ceil","ceiling","coalesce","corr","cos","cosh","count","covar_pop","covar_samp","cume_dist","dense_rank","deref","element","exp","extract","first_value","floor","json_array","json_arrayagg","json_exists","json_object","json_objectagg","json_query","json_table","json_table_primitive","json_value","lag","last_value","lead","listagg","ln","log","log10","lower","max","min","mod","nth_value","ntile","nullif","percent_rank","percentile_cont","percentile_disc","position","position_regex","power","rank","regr_avgx","regr_avgy","regr_count","regr_intercept","regr_r2","regr_slope","regr_sxx","regr_sxy","regr_syy","row_number","sin","sinh","sqrt","stddev_pop","stddev_samp","substring","substring_regex","sum","tan","tanh","translate","translate_regex","treat","trim","trim_array","unnest","upper","value_of","var_pop","var_samp","width_bucket"],s=["create table","insert into","primary key","foreign key","not null","alter table","add constraint","grouping sets","on overflow","character set","respect nulls","ignore nulls","nulls first","nulls last","depth first","breadth first"],o=r,l=["abs","acos","all","allocate","alter","and","any","are","array","array_agg","array_max_cardinality","as","asensitive","asin","asymmetric","at","atan","atomic","authorization","avg","begin","begin_frame","begin_partition","between","bigint","binary","blob","boolean","both","by","call","called","cardinality","cascaded","case","cast","ceil","ceiling","char","char_length","character","character_length","check","classifier","clob","close","coalesce","collate","collect","column","commit","condition","connect","constraint","contains","convert","copy","corr","corresponding","cos","cosh","count","covar_pop","covar_samp","create","cross","cube","cume_dist","current","current_catalog","current_date","current_default_transform_group","current_path","current_role","current_row","current_schema","current_time","current_timestamp","current_path","current_role","current_transform_group_for_type","current_user","cursor","cycle","date","day","deallocate","dec","decimal","decfloat","declare","default","define","delete","dense_rank","deref","describe","deterministic","disconnect","distinct","double","drop","dynamic","each","element","else","empty","end","end_frame","end_partition","end-exec","equals","escape","every","except","exec","execute","exists","exp","external","extract","false","fetch","filter","first_value","float","floor","for","foreign","frame_row","free","from","full","function","fusion","get","global","grant","group","grouping","groups","having","hold","hour","identity","in","indicator","initial","inner","inout","insensitive","insert","int","integer","intersect","intersection","interval","into","is","join","json_array","json_arrayagg","json_exists","json_object","json_objectagg","json_query","json_table","json_table_primitive","json_value","lag","language","large","last_value","lateral","lead","leading","left","like","like_regex","listagg","ln","local","localtime","localtimestamp","log","log10","lower","match","match_number","match_recognize","matches","max","member","merge","method","min","minute","mod","modifies","module","month","multiset","national","natural","nchar","nclob","new","no","none","normalize","not","nth_value","ntile","null","nullif","numeric","octet_length","occurrences_regex","of","offset","old","omit","on","one","only","open","or","order","out","outer","over","overlaps","overlay","parameter","partition","pattern","per","percent","percent_rank","percentile_cont","percentile_disc","period","portion","position","position_regex","power","precedes","precision","prepare","primary","procedure","ptf","range","rank","reads","real","recursive","ref","references","referencing","regr_avgx","regr_avgy","regr_count","regr_intercept","regr_r2","regr_slope","regr_sxx","regr_sxy","regr_syy","release","result","return","returns","revoke","right","rollback","rollup","row","row_number","rows","running","savepoint","scope","scroll","search","second","seek","select","sensitive","session_user","set","show","similar","sin","sinh","skip","smallint","some","specific","specifictype","sql","sqlexception","sqlstate","sqlwarning","sqrt","start","static","stddev_pop","stddev_samp","submultiset","subset","substring","substring_regex","succeeds","sum","symmetric","system","system_time","system_user","table","tablesample","tan","tanh","then","time","timestamp","timezone_hour","timezone_minute","to","trailing","translate","translate_regex","translation","treat","trigger","trim","trim_array","true","truncate","uescape","union","unique","unknown","unnest","update","upper","user","using","value","values","value_of","var_pop","var_samp","varbinary","varchar","varying","versioning","when","whenever","where","width_bucket","window","with","within","without","year","add","asc","collation","desc","final","first","last","view"].filter((e=>!r.includes(e))),c={
+ begin:n.concat(/\b/,n.either(...o),/\s*\(/),relevance:0,keywords:{built_in:o}}
+ ;return{name:"SQL",case_insensitive:!0,illegal:/[{}]|<\//,keywords:{
+ $pattern:/\b[\w\.]+/,keyword:((e,{exceptions:n,when:t}={})=>{const a=t
+ ;return n=n||[],e.map((e=>e.match(/\|\d+$/)||n.includes(e)?e:a(e)?e+"|0":e))
+ })(l,{when:e=>e.length<3}),literal:a,type:i,
+ built_in:["current_catalog","current_date","current_default_transform_group","current_path","current_role","current_schema","current_transform_group_for_type","current_user","session_user","system_time","system_user","current_time","localtime","current_timestamp","localtimestamp"]
+ },contains:[{begin:n.either(...s),relevance:0,keywords:{$pattern:/[\w\.]+/,
+ keyword:l.concat(s),literal:a,type:i}},{className:"type",
+ begin:n.either("double precision","large object","with timezone","without timezone")
+ },c,{className:"variable",begin:/@[a-z0-9][a-z0-9_]*/},{className:"string",
+ variants:[{begin:/'/,end:/'/,contains:[{begin:/''/}]}]},{begin:/"/,end:/"/,
+ contains:[{begin:/""/}]},e.C_NUMBER_MODE,e.C_BLOCK_COMMENT_MODE,t,{
+ className:"operator",begin:/[-+*/=%^~]|&&?|\|\|?|!=?|<(?:=>?|<|>)?|>[>=]?/,
+ relevance:0}]}},grmr_swift:e=>{const n={match:/\s+/,relevance:0
+ },t=e.COMMENT("/\\*","\\*/",{contains:["self"]}),a=[e.C_LINE_COMMENT_MODE,t],i={
+ match:[/\./,m(...xe,...Me)],className:{2:"keyword"}},r={match:b(/\./,m(...Ae)),
+ relevance:0},s=Ae.filter((e=>"string"==typeof e)).concat(["_|0"]),o={variants:[{
+ className:"keyword",
+ match:m(...Ae.filter((e=>"string"!=typeof e)).concat(Se).map(ke),...Me)}]},l={
+ $pattern:m(/\b\w+/,/#\w+/),keyword:s.concat(Re),literal:Ce},c=[i,r,o],g=[{
+ match:b(/\./,m(...De)),relevance:0},{className:"built_in",
+ match:b(/\b/,m(...De),/(?=\()/)}],u={match:/->/,relevance:0},p=[u,{
+ className:"operator",relevance:0,variants:[{match:Be},{match:`\\.(\\.|${Le})+`}]
+ }],_="([0-9]_*)+",h="([0-9a-fA-F]_*)+",f={className:"number",relevance:0,
+ variants:[{match:`\\b(${_})(\\.(${_}))?([eE][+-]?(${_}))?\\b`},{
+ match:`\\b0x(${h})(\\.(${h}))?([pP][+-]?(${_}))?\\b`},{match:/\b0o([0-7]_*)+\b/
+ },{match:/\b0b([01]_*)+\b/}]},E=(e="")=>({className:"subst",variants:[{
+ match:b(/\\/,e,/[0\\tnr"']/)},{match:b(/\\/,e,/u\{[0-9a-fA-F]{1,8}\}/)}]
+ }),y=(e="")=>({className:"subst",match:b(/\\/,e,/[\t ]*(?:[\r\n]|\r\n)/)
+ }),N=(e="")=>({className:"subst",label:"interpol",begin:b(/\\/,e,/\(/),end:/\)/
+ }),w=(e="")=>({begin:b(e,/"""/),end:b(/"""/,e),contains:[E(e),y(e),N(e)]
+ }),v=(e="")=>({begin:b(e,/"/),end:b(/"/,e),contains:[E(e),N(e)]}),O={
+ className:"string",
+ variants:[w(),w("#"),w("##"),w("###"),v(),v("#"),v("##"),v("###")]
+ },k=[e.BACKSLASH_ESCAPE,{begin:/\[/,end:/\]/,relevance:0,
+ contains:[e.BACKSLASH_ESCAPE]}],x={begin:/\/[^\s](?=[^/\n]*\/)/,end:/\//,
+ contains:k},M=e=>{const n=b(e,/\//),t=b(/\//,e);return{begin:n,end:t,
+ contains:[...k,{scope:"comment",begin:`#(?!.*${t})`,end:/$/}]}},S={
+ scope:"regexp",variants:[M("###"),M("##"),M("#"),x]},A={match:b(/`/,Fe,/`/)
+ },C=[A,{className:"variable",match:/\$\d+/},{className:"variable",
+ match:`\\$${ze}+`}],T=[{match:/(@|#(un)?)available/,scope:"keyword",starts:{
+ contains:[{begin:/\(/,end:/\)/,keywords:Pe,contains:[...p,f,O]}]}},{
+ scope:"keyword",match:b(/@/,m(...je))},{scope:"meta",match:b(/@/,Fe)}],R={
+ match:d(/\b[A-Z]/),relevance:0,contains:[{className:"type",
+ match:b(/(AV|CA|CF|CG|CI|CL|CM|CN|CT|MK|MP|MTK|MTL|NS|SCN|SK|UI|WK|XC)/,ze,"+")
+ },{className:"type",match:Ue,relevance:0},{match:/[?!]+/,relevance:0},{
+ match:/\.\.\./,relevance:0},{match:b(/\s+&\s+/,d(Ue)),relevance:0}]},D={
+ begin://,keywords:l,contains:[...a,...c,...T,u,R]};R.contains.push(D)
+ ;const I={begin:/\(/,end:/\)/,relevance:0,keywords:l,contains:["self",{
+ match:b(Fe,/\s*:/),keywords:"_|0",relevance:0
+ },...a,S,...c,...g,...p,f,O,...C,...T,R]},L={begin://,
+ keywords:"repeat each",contains:[...a,R]},B={begin:/\(/,end:/\)/,keywords:l,
+ contains:[{begin:m(d(b(Fe,/\s*:/)),d(b(Fe,/\s+/,Fe,/\s*:/))),end:/:/,
+ relevance:0,contains:[{className:"keyword",match:/\b_\b/},{className:"params",
+ match:Fe}]},...a,...c,...p,f,O,...T,R,I],endsParent:!0,illegal:/["']/},$={
+ match:[/(func|macro)/,/\s+/,m(A.match,Fe,Be)],className:{1:"keyword",
+ 3:"title.function"},contains:[L,B,n],illegal:[/\[/,/%/]},z={
+ match:[/\b(?:subscript|init[?!]?)/,/\s*(?=[<(])/],className:{1:"keyword"},
+ contains:[L,B,n],illegal:/\[|%/},F={match:[/operator/,/\s+/,Be],className:{
+ 1:"keyword",3:"title"}},U={begin:[/precedencegroup/,/\s+/,Ue],className:{
+ 1:"keyword",3:"title"},contains:[R],keywords:[...Te,...Ce],end:/}/}
+ ;for(const e of O.variants){const n=e.contains.find((e=>"interpol"===e.label))
+ ;n.keywords=l;const t=[...c,...g,...p,f,O,...C];n.contains=[...t,{begin:/\(/,
+ end:/\)/,contains:["self",...t]}]}return{name:"Swift",keywords:l,
+ contains:[...a,$,z,{beginKeywords:"struct protocol class extension enum actor",
+ end:"\\{",excludeEnd:!0,keywords:l,contains:[e.inherit(e.TITLE_MODE,{
+ className:"title.class",begin:/[A-Za-z$_][\u00C0-\u02B80-9A-Za-z$_]*/}),...c]
+ },F,U,{beginKeywords:"import",end:/$/,contains:[...a],relevance:0
+ },S,...c,...g,...p,f,O,...C,...T,R,I]}},grmr_typescript:e=>{
+ const n=Oe(e),t=_e,a=["any","void","number","boolean","string","object","never","symbol","bigint","unknown"],i={
+ beginKeywords:"namespace",end:/\{/,excludeEnd:!0,
+ contains:[n.exports.CLASS_REFERENCE]},r={beginKeywords:"interface",end:/\{/,
+ excludeEnd:!0,keywords:{keyword:"interface extends",built_in:a},
+ contains:[n.exports.CLASS_REFERENCE]},s={$pattern:_e,
+ keyword:he.concat(["type","namespace","interface","public","private","protected","implements","declare","abstract","readonly","enum","override"]),
+ literal:fe,built_in:ve.concat(a),"variable.language":we},o={className:"meta",
+ begin:"@"+t},l=(e,n,t)=>{const a=e.contains.findIndex((e=>e.label===n))
+ ;if(-1===a)throw Error("can not find mode to replace");e.contains.splice(a,1,t)}
+ ;return Object.assign(n.keywords,s),
+ n.exports.PARAMS_CONTAINS.push(o),n.contains=n.contains.concat([o,i,r]),
+ l(n,"shebang",e.SHEBANG()),l(n,"use_strict",{className:"meta",relevance:10,
+ begin:/^\s*['"]use strict['"]/
+ }),n.contains.find((e=>"func.def"===e.label)).relevance=0,Object.assign(n,{
+ name:"TypeScript",aliases:["ts","tsx","mts","cts"]}),n},grmr_vbnet:e=>{
+ const n=e.regex,t=/\d{1,2}\/\d{1,2}\/\d{4}/,a=/\d{4}-\d{1,2}-\d{1,2}/,i=/(\d|1[012])(:\d+){0,2} *(AM|PM)/,r=/\d{1,2}(:\d{1,2}){1,2}/,s={
+ className:"literal",variants:[{begin:n.concat(/# */,n.either(a,t),/ *#/)},{
+ begin:n.concat(/# */,r,/ *#/)},{begin:n.concat(/# */,i,/ *#/)},{
+ begin:n.concat(/# */,n.either(a,t),/ +/,n.either(i,r),/ *#/)}]
+ },o=e.COMMENT(/'''/,/$/,{contains:[{className:"doctag",begin:/<\/?/,end:/>/}]
+ }),l=e.COMMENT(null,/$/,{variants:[{begin:/'/},{begin:/([\t ]|^)REM(?=\s)/}]})
+ ;return{name:"Visual Basic .NET",aliases:["vb"],case_insensitive:!0,
+ classNameAliases:{label:"symbol"},keywords:{
+ keyword:"addhandler alias aggregate ansi as async assembly auto binary by byref byval call case catch class compare const continue custom declare default delegate dim distinct do each equals else elseif end enum erase error event exit explicit finally for friend from function get global goto group handles if implements imports in inherits interface into iterator join key let lib loop me mid module mustinherit mustoverride mybase myclass namespace narrowing new next notinheritable notoverridable of off on operator option optional order overloads overridable overrides paramarray partial preserve private property protected public raiseevent readonly redim removehandler resume return select set shadows shared skip static step stop structure strict sub synclock take text then throw to try unicode until using when where while widening with withevents writeonly yield",
+ built_in:"addressof and andalso await directcast gettype getxmlnamespace is isfalse isnot istrue like mod nameof new not or orelse trycast typeof xor cbool cbyte cchar cdate cdbl cdec cint clng cobj csbyte cshort csng cstr cuint culng cushort",
+ type:"boolean byte char date decimal double integer long object sbyte short single string uinteger ulong ushort",
+ literal:"true false nothing"},
+ illegal:"//|\\{|\\}|endif|gosub|variant|wend|^\\$ ",contains:[{
+ className:"string",begin:/"(""|[^/n])"C\b/},{className:"string",begin:/"/,
+ end:/"/,illegal:/\n/,contains:[{begin:/""/}]},s,{className:"number",relevance:0,
+ variants:[{begin:/\b\d[\d_]*((\.[\d_]+(E[+-]?[\d_]+)?)|(E[+-]?[\d_]+))[RFD@!#]?/
+ },{begin:/\b\d[\d_]*((U?[SIL])|[%&])?/},{begin:/&H[\dA-F_]+((U?[SIL])|[%&])?/},{
+ begin:/&O[0-7_]+((U?[SIL])|[%&])?/},{begin:/&B[01_]+((U?[SIL])|[%&])?/}]},{
+ className:"label",begin:/^\w+:/},o,l,{className:"meta",
+ begin:/[\t ]*#(const|disable|else|elseif|enable|end|externalsource|if|region)\b/,
+ end:/$/,keywords:{
+ keyword:"const disable else elseif enable end externalsource if region then"},
+ contains:[l]}]}},grmr_wasm:e=>{e.regex;const n=e.COMMENT(/\(;/,/;\)/)
+ ;return n.contains.push("self"),{name:"WebAssembly",keywords:{$pattern:/[\w.]+/,
+ keyword:["anyfunc","block","br","br_if","br_table","call","call_indirect","data","drop","elem","else","end","export","func","global.get","global.set","local.get","local.set","local.tee","get_global","get_local","global","if","import","local","loop","memory","memory.grow","memory.size","module","mut","nop","offset","param","result","return","select","set_global","set_local","start","table","tee_local","then","type","unreachable"]
+ },contains:[e.COMMENT(/;;/,/$/),n,{match:[/(?:offset|align)/,/\s*/,/=/],
+ className:{1:"keyword",3:"operator"}},{className:"variable",begin:/\$[\w_]+/},{
+ match:/(\((?!;)|\))+/,className:"punctuation",relevance:0},{
+ begin:[/(?:func|call|call_indirect)/,/\s+/,/\$[^\s)]+/],className:{1:"keyword",
+ 3:"title.function"}},e.QUOTE_STRING_MODE,{match:/(i32|i64|f32|f64)(?!\.)/,
+ className:"type"},{className:"keyword",
+ match:/\b(f32|f64|i32|i64)(?:\.(?:abs|add|and|ceil|clz|const|convert_[su]\/i(?:32|64)|copysign|ctz|demote\/f64|div(?:_[su])?|eqz?|extend_[su]\/i32|floor|ge(?:_[su])?|gt(?:_[su])?|le(?:_[su])?|load(?:(?:8|16|32)_[su])?|lt(?:_[su])?|max|min|mul|nearest|neg?|or|popcnt|promote\/f32|reinterpret\/[fi](?:32|64)|rem_[su]|rot[lr]|shl|shr_[su]|store(?:8|16|32)?|sqrt|sub|trunc(?:_[su]\/f(?:32|64))?|wrap\/i64|xor))\b/
+ },{className:"number",relevance:0,
+ match:/[+-]?\b(?:\d(?:_?\d)*(?:\.\d(?:_?\d)*)?(?:[eE][+-]?\d(?:_?\d)*)?|0x[\da-fA-F](?:_?[\da-fA-F])*(?:\.[\da-fA-F](?:_?[\da-fA-D])*)?(?:[pP][+-]?\d(?:_?\d)*)?)\b|\binf\b|\bnan(?::0x[\da-fA-F](?:_?[\da-fA-D])*)?\b/
+ }]}},grmr_xml:e=>{
+ const n=e.regex,t=n.concat(/[\p{L}_]/u,n.optional(/[\p{L}0-9_.-]*:/u),/[\p{L}0-9_.-]*/u),a={
+ className:"symbol",begin:/&[a-z]+;|&#[0-9]+;|&#x[a-f0-9]+;/},i={begin:/\s/,
+ contains:[{className:"keyword",begin:/#?[a-z_][a-z1-9_-]+/,illegal:/\n/}]
+ },r=e.inherit(i,{begin:/\(/,end:/\)/}),s=e.inherit(e.APOS_STRING_MODE,{
+ className:"string"}),o=e.inherit(e.QUOTE_STRING_MODE,{className:"string"}),l={
+ endsWithParent:!0,illegal:/`]+/}]}]}]};return{
+ name:"HTML, XML",
+ aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"],
+ case_insensitive:!0,unicodeRegex:!0,contains:[{className:"meta",begin://,relevance:10,contains:[i,o,s,r,{begin:/\[/,end:/\]/,contains:[{
+ className:"meta",begin://,contains:[i,r,o,s]}]}]
+ },e.COMMENT(//,{relevance:10}),{begin://,
+ relevance:10},a,{className:"meta",end:/\?>/,variants:[{begin:/<\?xml/,
+ relevance:10,contains:[o]},{begin:/<\?[a-z][a-z0-9]+/}]},{className:"tag",
+ begin:/)/,end:/>/,keywords:{name:"style"},contains:[l],starts:{
+ end:/<\/style>/,returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag",
+ begin:/)/,end:/>/,keywords:{name:"script"},contains:[l],starts:{
+ end:/<\/script>/,returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{
+ className:"tag",begin:/<>|<\/>/},{className:"tag",
+ begin:n.concat(//,/>/,/\s/)))),
+ end:/\/?>/,contains:[{className:"name",begin:t,relevance:0,starts:l}]},{
+ className:"tag",begin:n.concat(/<\//,n.lookahead(n.concat(t,/>/))),contains:[{
+ className:"name",begin:t,relevance:0},{begin:/>/,relevance:0,endsParent:!0}]}]}
+ },grmr_yaml:e=>{
+ const n="true false yes no null",t="[\\w#;/?:@&=+$,.~*'()[\\]]+",a={
+ className:"string",relevance:0,variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/
+ },{begin:/\S+/}],contains:[e.BACKSLASH_ESCAPE,{className:"template-variable",
+ variants:[{begin:/\{\{/,end:/\}\}/},{begin:/%\{/,end:/\}/}]}]},i=e.inherit(a,{
+ variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/[^\s,{}[\]]+/}]}),r={
+ end:",",endsWithParent:!0,excludeEnd:!0,keywords:n,relevance:0},s={begin:/\{/,
+ end:/\}/,contains:[r],illegal:"\\n",relevance:0},o={begin:"\\[",end:"\\]",
+ contains:[r],illegal:"\\n",relevance:0},l=[{className:"attr",variants:[{
+ begin:"\\w[\\w :\\/.-]*:(?=[ \t]|$)"},{begin:'"\\w[\\w :\\/.-]*":(?=[ \t]|$)'},{
+ begin:"'\\w[\\w :\\/.-]*':(?=[ \t]|$)"}]},{className:"meta",begin:"^---\\s*$",
+ relevance:10},{className:"string",
+ begin:"[\\|>]([1-9]?[+-])?[ ]*\\n( +)[^ ][^\\n]*\\n(\\2[^\\n]+\\n?)*"},{
+ begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0,
+ relevance:0},{className:"type",begin:"!\\w+!"+t},{className:"type",
+ begin:"!<"+t+">"},{className:"type",begin:"!"+t},{className:"type",begin:"!!"+t
+ },{className:"meta",begin:"&"+e.UNDERSCORE_IDENT_RE+"$"},{className:"meta",
+ begin:"\\*"+e.UNDERSCORE_IDENT_RE+"$"},{className:"bullet",begin:"-(?=[ ]|$)",
+ relevance:0},e.HASH_COMMENT_MODE,{beginKeywords:n,keywords:{literal:n}},{
+ className:"number",
+ begin:"\\b[0-9]{4}(-[0-9][0-9]){0,2}([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?(\\.[0-9]*)?([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?\\b"
+ },{className:"number",begin:e.C_NUMBER_RE+"\\b",relevance:0},s,o,a],c=[...l]
+ ;return c.pop(),c.push(i),r.contains=c,{name:"YAML",case_insensitive:!0,
+ aliases:["yml"],contains:l}}});const He=ae;for(const e of Object.keys(Ke)){
+ const n=e.replace("grmr_","").replace("_","-");He.registerLanguage(n,Ke[e])}
+ return He}()
+ ;"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs);
\ No newline at end of file
diff --git a/docs/md_v2/assets/highlight_init.js b/docs/md_v2/assets/highlight_init.js
new file mode 100644
index 0000000000000000000000000000000000000000..e237927864de512708ffc76fe5f7b94d28959328
--- /dev/null
+++ b/docs/md_v2/assets/highlight_init.js
@@ -0,0 +1,6 @@
+document.addEventListener('DOMContentLoaded', (event) => {
+ document.querySelectorAll('pre code').forEach((block) => {
+ hljs.highlightBlock(block);
+ });
+ });
+
\ No newline at end of file
diff --git a/docs/md_v2/assets/styles.css b/docs/md_v2/assets/styles.css
new file mode 100644
index 0000000000000000000000000000000000000000..68a93f5d18788db5d1e015e6aebb1fdebca82cc4
--- /dev/null
+++ b/docs/md_v2/assets/styles.css
@@ -0,0 +1,160 @@
+@font-face {
+ font-family: "Monaco";
+ font-style: normal;
+ font-weight: normal;
+ src: local("Monaco"), url("Monaco.woff") format("woff");
+}
+
+:root {
+ --global-font-size: 16px;
+ --global-line-height: 1.5em;
+ --global-space: 10px;
+ --font-stack: Menlo, Monaco, Lucida Console, Liberation Mono, DejaVu Sans Mono, Bitstream Vera Sans Mono,
+ Courier New, monospace, serif;
+ --font-stack: dm, Monaco, Courier New, monospace, serif;
+ --mono-font-stack: Menlo, Monaco, Lucida Console, Liberation Mono, DejaVu Sans Mono, Bitstream Vera Sans Mono,
+ Courier New, monospace, serif;
+
+ --background-color: #151515; /* Dark background */
+ --font-color: #eaeaea; /* Light font color for contrast */
+ --invert-font-color: #151515; /* Dark color for inverted elements */
+ --primary-color: #1a95e0; /* Primary color can remain the same or be adjusted for better contrast */
+ --secondary-color: #727578; /* Secondary color for less important text */
+ --error-color: #ff5555; /* Bright color for errors */
+ --progress-bar-background: #444; /* Darker background for progress bar */
+ --progress-bar-fill: #1a95e0; /* Bright color for progress bar fill */
+ --code-bg-color: #1e1e1e; /* Darker background for code blocks */
+ --input-style: solid; /* Keeping input style solid */
+ --block-background-color: #202020; /* Darker background for block elements */
+ --global-font-color: #eaeaea; /* Light font color for global elements */
+
+ --background-color: #222225;
+
+ --background-color: #070708;
+ --page-width: 70em;
+ --font-color: #e8e9ed;
+ --invert-font-color: #222225;
+ --secondary-color: #a3abba;
+ --secondary-color: #d5cec0;
+ --tertiary-color: #a3abba;
+ --primary-color: #09b5a5; /* Updated to the brand color */
+ --primary-color: #50ffff; /* Updated to the brand color */
+ --error-color: #ff3c74;
+ --progress-bar-background: #3f3f44;
+ --progress-bar-fill: #09b5a5; /* Updated to the brand color */
+ --code-bg-color: #3f3f44;
+ --input-style: solid;
+ --display-h1-decoration: none;
+
+ --display-h1-decoration: none;
+}
+
+/* body {
+ background-color: var(--background-color);
+ color: var(--font-color);
+}
+
+a {
+ color: var(--primary-color);
+}
+
+a:hover {
+ background-color: var(--primary-color);
+ color: var(--invert-font-color);
+}
+
+blockquote::after {
+ color: #444;
+}
+
+pre, code {
+ background-color: var(--code-bg-color);
+ color: var(--font-color);
+}
+
+.terminal-nav:first-child {
+ border-bottom: 1px dashed var(--secondary-color);
+} */
+
+.terminal-mkdocs-main-content {
+ line-height: var(--global-line-height);
+}
+
+strong,
+.highlight {
+ /* background: url(//s2.svgbox.net/pen-brushes.svg?ic=brush-1&color=50ffff); */
+ background-color: #50ffff33;
+}
+
+.terminal-card > header {
+ color: var(--font-color);
+ text-align: center;
+ background-color: var(--progress-bar-background);
+ padding: 0.3em 0.5em;
+}
+.btn.btn-sm {
+ color: var(--font-color);
+ padding: 0.2em 0.5em;
+ font-size: 0.8em;
+}
+
+.loading-message {
+ display: none;
+ margin-top: 20px;
+}
+
+.response-section {
+ display: none;
+ padding-top: 20px;
+}
+
+.tabs {
+ display: flex;
+ flex-direction: column;
+}
+.tab-list {
+ display: flex;
+ padding: 0;
+ margin: 0;
+ list-style-type: none;
+ border-bottom: 1px solid var(--font-color);
+}
+.tab-item {
+ cursor: pointer;
+ padding: 10px;
+ border: 1px solid var(--font-color);
+ margin-right: -1px;
+ border-bottom: none;
+}
+.tab-item:hover,
+.tab-item:focus,
+.tab-item:active {
+ background-color: var(--progress-bar-background);
+}
+.tab-content {
+ display: none;
+ border: 1px solid var(--font-color);
+ border-top: none;
+}
+.tab-content:first-of-type {
+ display: block;
+}
+
+.tab-content header {
+ padding: 0.5em;
+ display: flex;
+ justify-content: end;
+ align-items: center;
+ background-color: var(--progress-bar-background);
+}
+.tab-content pre {
+ margin: 0;
+ max-height: 300px; overflow: auto; border:none;
+}
+
+ol li::before {
+ content: counters(item, ".") ". ";
+ counter-increment: item;
+ /* float: left; */
+ /* padding-right: 5px; */
+}
\ No newline at end of file
diff --git a/docs/md_v2/basic/browser-config.md b/docs/md_v2/basic/browser-config.md
new file mode 100644
index 0000000000000000000000000000000000000000..7df4a97bbed715f21221f3973adf3310d267296b
--- /dev/null
+++ b/docs/md_v2/basic/browser-config.md
@@ -0,0 +1,208 @@
+# Browser Configuration
+
+Crawl4AI supports multiple browser engines and offers extensive configuration options for browser behavior.
+
+## Browser Types
+
+Choose from three browser engines:
+
+```python
+# Chromium (default)
+async with AsyncWebCrawler(browser_type="chromium") as crawler:
+ result = await crawler.arun(url="https://example.com")
+
+# Firefox
+async with AsyncWebCrawler(browser_type="firefox") as crawler:
+ result = await crawler.arun(url="https://example.com")
+
+# WebKit
+async with AsyncWebCrawler(browser_type="webkit") as crawler:
+ result = await crawler.arun(url="https://example.com")
+```
+
+## Basic Configuration
+
+Common browser settings:
+
+```python
+async with AsyncWebCrawler(
+ headless=True, # Run in headless mode (no GUI)
+ verbose=True, # Enable detailed logging
+ sleep_on_close=False # No delay when closing browser
+) as crawler:
+ result = await crawler.arun(url="https://example.com")
+```
+
+## Identity Management
+
+Control how your crawler appears to websites:
+
+```python
+# Custom user agent
+async with AsyncWebCrawler(
+ user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+) as crawler:
+ result = await crawler.arun(url="https://example.com")
+
+# Custom headers
+headers = {
+ "Accept-Language": "en-US,en;q=0.9",
+ "Cache-Control": "no-cache"
+}
+async with AsyncWebCrawler(headers=headers) as crawler:
+ result = await crawler.arun(url="https://example.com")
+```
+
+## Screenshot Capabilities
+
+Capture page screenshots with enhanced error handling:
+
+```python
+result = await crawler.arun(
+ url="https://example.com",
+ screenshot=True, # Enable screenshot
+ screenshot_wait_for=2.0 # Wait 2 seconds before capture
+)
+
+if result.screenshot: # Base64 encoded image
+ import base64
+ with open("screenshot.png", "wb") as f:
+ f.write(base64.b64decode(result.screenshot))
+```
+
+## Timeouts and Waiting
+
+Control page loading behavior:
+
+```python
+result = await crawler.arun(
+ url="https://example.com",
+ page_timeout=60000, # Page load timeout (ms)
+ delay_before_return_html=2.0, # Wait before content capture
+ wait_for="css:.dynamic-content" # Wait for specific element
+)
+```
+
+## JavaScript Execution
+
+Execute custom JavaScript before crawling:
+
+```python
+# Single JavaScript command
+result = await crawler.arun(
+ url="https://example.com",
+ js_code="window.scrollTo(0, document.body.scrollHeight);"
+)
+
+# Multiple commands
+js_commands = [
+ "window.scrollTo(0, document.body.scrollHeight);",
+ "document.querySelector('.load-more').click();"
+]
+result = await crawler.arun(
+ url="https://example.com",
+ js_code=js_commands
+)
+```
+
+## Proxy Configuration
+
+Use proxies for enhanced access:
+
+```python
+# Simple proxy
+async with AsyncWebCrawler(
+ proxy="http://proxy.example.com:8080"
+) as crawler:
+ result = await crawler.arun(url="https://example.com")
+
+# Proxy with authentication
+proxy_config = {
+ "server": "http://proxy.example.com:8080",
+ "username": "user",
+ "password": "pass"
+}
+async with AsyncWebCrawler(proxy_config=proxy_config) as crawler:
+ result = await crawler.arun(url="https://example.com")
+```
+
+## Anti-Detection Features
+
+Enable stealth features to avoid bot detection:
+
+```python
+result = await crawler.arun(
+ url="https://example.com",
+ simulate_user=True, # Simulate human behavior
+ override_navigator=True, # Mask automation signals
+ magic=True # Enable all anti-detection features
+)
+```
+
+## Handling Dynamic Content
+
+Configure browser to handle dynamic content:
+
+```python
+# Wait for dynamic content
+result = await crawler.arun(
+ url="https://example.com",
+ wait_for="js:() => document.querySelector('.content').children.length > 10",
+ process_iframes=True # Process iframe content
+)
+
+# Handle lazy-loaded images
+result = await crawler.arun(
+ url="https://example.com",
+ js_code="window.scrollTo(0, document.body.scrollHeight);",
+ delay_before_return_html=2.0 # Wait for images to load
+)
+```
+
+## Comprehensive Example
+
+Here's how to combine various browser configurations:
+
+```python
+async def crawl_with_advanced_config(url: str):
+ async with AsyncWebCrawler(
+ # Browser setup
+ browser_type="chromium",
+ headless=True,
+ verbose=True,
+
+ # Identity
+ user_agent="Custom User Agent",
+ headers={"Accept-Language": "en-US"},
+
+ # Proxy setup
+ proxy="http://proxy.example.com:8080"
+ ) as crawler:
+ result = await crawler.arun(
+ url=url,
+ # Content handling
+ process_iframes=True,
+ screenshot=True,
+
+ # Timing
+ page_timeout=60000,
+ delay_before_return_html=2.0,
+
+ # Anti-detection
+ magic=True,
+ simulate_user=True,
+
+ # Dynamic content
+ js_code=[
+ "window.scrollTo(0, document.body.scrollHeight);",
+ "document.querySelector('.load-more')?.click();"
+ ],
+ wait_for="css:.dynamic-content"
+ )
+
+ return {
+ "content": result.markdown,
+ "screenshot": result.screenshot,
+ "success": result.success
+ }
+```
\ No newline at end of file
diff --git a/docs/md_v2/basic/cache-modes.md b/docs/md_v2/basic/cache-modes.md
new file mode 100644
index 0000000000000000000000000000000000000000..73460e571b7989d375e5e88792990d1a61af742a
--- /dev/null
+++ b/docs/md_v2/basic/cache-modes.md
@@ -0,0 +1,81 @@
+# Crawl4AI Cache System and Migration Guide
+
+## Overview
+Starting from version 0.5.0, Crawl4AI introduces a new caching system that replaces the old boolean flags with a more intuitive `CacheMode` enum. This change simplifies cache control and makes the behavior more predictable.
+
+## Old vs New Approach
+
+### Old Way (Deprecated)
+The old system used multiple boolean flags:
+- `bypass_cache`: Skip cache entirely
+- `disable_cache`: Disable all caching
+- `no_cache_read`: Don't read from cache
+- `no_cache_write`: Don't write to cache
+
+### New Way (Recommended)
+The new system uses a single `CacheMode` enum:
+- `CacheMode.ENABLED`: Normal caching (read/write)
+- `CacheMode.DISABLED`: No caching at all
+- `CacheMode.READ_ONLY`: Only read from cache
+- `CacheMode.WRITE_ONLY`: Only write to cache
+- `CacheMode.BYPASS`: Skip cache for this operation
+
+## Migration Example
+
+### Old Code (Deprecated)
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def use_proxy():
+ async with AsyncWebCrawler(verbose=True) as crawler:
+ result = await crawler.arun(
+ url="https://www.nbcnews.com/business",
+ bypass_cache=True # Old way
+ )
+ print(len(result.markdown))
+
+async def main():
+ await use_proxy()
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+### New Code (Recommended)
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CacheMode
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def use_proxy():
+ config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS) # Use CacheMode in CrawlerRunConfig
+ async with AsyncWebCrawler(verbose=True) as crawler:
+ result = await crawler.arun(
+ url="https://www.nbcnews.com/business",
+ config=config # Pass the configuration object
+ )
+ print(len(result.markdown))
+
+async def main():
+ await use_proxy()
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+## Common Migration Patterns
+
+| Old Flag | New Mode |
+|-----------------------|---------------------------------|
+| `bypass_cache=True` | `cache_mode=CacheMode.BYPASS` |
+| `disable_cache=True` | `cache_mode=CacheMode.DISABLED`|
+| `no_cache_read=True` | `cache_mode=CacheMode.WRITE_ONLY` |
+| `no_cache_write=True` | `cache_mode=CacheMode.READ_ONLY` |
+
+## Suppressing Deprecation Warnings
+If you need time to migrate, you can temporarily suppress deprecation warnings:
+```python
+# In your config.py
+SHOW_DEPRECATION_WARNINGS = False
+```
diff --git a/docs/md_v2/basic/content-selection.md b/docs/md_v2/basic/content-selection.md
new file mode 100644
index 0000000000000000000000000000000000000000..ec838f2d9870f5a1a68208284daf51b4a40622a9
--- /dev/null
+++ b/docs/md_v2/basic/content-selection.md
@@ -0,0 +1,135 @@
+### Content Selection
+
+Crawl4AI provides multiple ways to select and filter specific content from webpages. Learn how to precisely target the content you need.
+
+#### CSS Selectors
+
+Extract specific content using a `CrawlerRunConfig` with CSS selectors:
+
+```python
+from crawl4ai.async_configs import CrawlerRunConfig
+
+config = CrawlerRunConfig(css_selector=".main-article") # Target main article content
+result = await crawler.arun(url="https://crawl4ai.com", config=config)
+
+config = CrawlerRunConfig(css_selector="article h1, article .content") # Target heading and content
+result = await crawler.arun(url="https://crawl4ai.com", config=config)
+```
+
+#### Content Filtering
+
+Control content inclusion or exclusion with `CrawlerRunConfig`:
+
+```python
+config = CrawlerRunConfig(
+ word_count_threshold=10, # Minimum words per block
+ excluded_tags=['form', 'header', 'footer', 'nav'], # Excluded tags
+ exclude_external_links=True, # Remove external links
+ exclude_social_media_links=True, # Remove social media links
+ exclude_external_images=True # Remove external images
+)
+
+result = await crawler.arun(url="https://crawl4ai.com", config=config)
+```
+
+#### Iframe Content
+
+Process iframe content by enabling specific options in `CrawlerRunConfig`:
+
+```python
+config = CrawlerRunConfig(
+ process_iframes=True, # Extract iframe content
+ remove_overlay_elements=True # Remove popups/modals that might block iframes
+)
+
+result = await crawler.arun(url="https://crawl4ai.com", config=config)
+```
+
+#### Structured Content Selection Using LLMs
+
+Leverage LLMs for intelligent content extraction:
+
+```python
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+from pydantic import BaseModel
+from typing import List
+
+class ArticleContent(BaseModel):
+ title: str
+ main_points: List[str]
+ conclusion: str
+
+strategy = LLMExtractionStrategy(
+ provider="ollama/nemotron",
+ schema=ArticleContent.schema(),
+ instruction="Extract the main article title, key points, and conclusion"
+)
+
+config = CrawlerRunConfig(extraction_strategy=strategy)
+
+result = await crawler.arun(url="https://crawl4ai.com", config=config)
+article = json.loads(result.extracted_content)
+```
+
+#### Pattern-Based Selection
+
+Extract content matching repetitive patterns:
+
+```python
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+
+schema = {
+ "name": "News Articles",
+ "baseSelector": "article.news-item",
+ "fields": [
+ {"name": "headline", "selector": "h2", "type": "text"},
+ {"name": "summary", "selector": ".summary", "type": "text"},
+ {"name": "category", "selector": ".category", "type": "text"},
+ {
+ "name": "metadata",
+ "type": "nested",
+ "fields": [
+ {"name": "author", "selector": ".author", "type": "text"},
+ {"name": "date", "selector": ".date", "type": "text"}
+ ]
+ }
+ ]
+}
+
+strategy = JsonCssExtractionStrategy(schema)
+config = CrawlerRunConfig(extraction_strategy=strategy)
+
+result = await crawler.arun(url="https://crawl4ai.com", config=config)
+articles = json.loads(result.extracted_content)
+```
+
+#### Comprehensive Example
+
+Combine different selection methods using `CrawlerRunConfig`:
+
+```python
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+async def extract_article_content(url: str):
+ # Define structured extraction
+ article_schema = {
+ "name": "Article",
+ "baseSelector": "article.main",
+ "fields": [
+ {"name": "title", "selector": "h1", "type": "text"},
+ {"name": "content", "selector": ".content", "type": "text"}
+ ]
+ }
+
+ # Define configuration
+ config = CrawlerRunConfig(
+ extraction_strategy=JsonCssExtractionStrategy(article_schema),
+ word_count_threshold=10,
+ excluded_tags=['nav', 'footer'],
+ exclude_external_links=True
+ )
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(url=url, config=config)
+ return json.loads(result.extracted_content)
+```
diff --git a/docs/md_v2/basic/content_filtering.md b/docs/md_v2/basic/content_filtering.md
new file mode 100644
index 0000000000000000000000000000000000000000..14f48ec6bf51aa1ebe6f18068c2a518409bf766c
--- /dev/null
+++ b/docs/md_v2/basic/content_filtering.md
@@ -0,0 +1,83 @@
+# Content Filtering in Crawl4AI
+
+This guide explains how to use content filtering strategies in Crawl4AI to extract the most relevant information from crawled web pages. You'll learn how to use the built-in `BM25ContentFilter` and how to create your own custom content filtering strategies.
+
+## Relevance Content Filter
+
+The `RelevanceContentFilter` is an abstract class providing a common interface for content filtering strategies. Specific algorithms, like `PruningContentFilter` or `BM25ContentFilter`, inherit from this class and implement the `filter_content` method. This method takes the HTML content as input and returns a list of filtered text blocks.
+
+## Pruning Content Filter
+
+The `PruningContentFilter` removes less relevant nodes based on metrics like text density, link density, and tag importance. Nodes that fall below a defined threshold are pruned, leaving only high-value content.
+
+### Usage
+
+```python
+from crawl4ai.async_configs import CrawlerRunConfig
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+config = CrawlerRunConfig(
+ content_filter=PruningContentFilter(
+ min_word_threshold=5,
+ threshold_type='dynamic',
+ threshold=0.45
+ ),
+ fit_markdown=True # Activates markdown fitting
+)
+
+result = await crawler.arun(url="https://example.com", config=config)
+
+if result.success:
+ print(f"Cleaned Markdown:\n{result.fit_markdown}")
+```
+
+### Parameters
+
+- **`min_word_threshold`**: (Optional) Minimum number of words a node must contain to be considered relevant. Nodes with fewer words are automatically pruned.
+- **`threshold_type`**: (Optional, default 'fixed') Controls how pruning thresholds are calculated:
+ - `'fixed'`: Uses a constant threshold value for all nodes.
+ - `'dynamic'`: Adjusts thresholds based on node properties (e.g., tag importance, text/link ratios).
+- **`threshold`**: (Optional, default 0.48) Base threshold for pruning:
+ - Fixed: Nodes scoring below this value are removed.
+ - Dynamic: This value adjusts based on node characteristics.
+
+### How It Works
+
+The algorithm evaluates each node using:
+- **Text density**: Ratio of text to overall content.
+- **Link density**: Proportion of text within links.
+- **Tag importance**: Weights based on HTML tag type (e.g., ``, ``, `
`).
+- **Content quality**: Metrics like text length and structural importance.
+
+## BM25 Algorithm
+
+The `BM25ContentFilter` uses the BM25 algorithm to rank and extract text chunks based on relevance to a search query or page metadata.
+
+### Usage
+
+```python
+from crawl4ai.async_configs import CrawlerRunConfig
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+config = CrawlerRunConfig(
+ content_filter=BM25ContentFilter(user_query="fruit nutrition health"),
+ fit_markdown=True # Activates markdown fitting
+)
+
+result = await crawler.arun(url="https://example.com", config=config)
+
+if result.success:
+ print(f"Filtered Content:\n{result.extracted_content}")
+ print(f"\nFiltered Markdown:\n{result.fit_markdown}")
+ print(f"\nFiltered HTML:\n{result.fit_html}")
+else:
+ print("Error:", result.error_message)
+```
+
+### Parameters
+
+- **`user_query`**: (Optional) A string representing the search query. If not provided, the filter extracts metadata (title, description, keywords) and uses it as the query.
+- **`bm25_threshold`**: (Optional, default 1.0) Threshold controlling relevance:
+ - Higher values return stricter, more relevant results.
+ - Lower values include more lenient filtering.
+
diff --git a/docs/md_v2/basic/docker-deploymeny.md b/docs/md_v2/basic/docker-deploymeny.md
new file mode 100644
index 0000000000000000000000000000000000000000..31d33e8ce82f349a938f7c94b9860a35d6351574
--- /dev/null
+++ b/docs/md_v2/basic/docker-deploymeny.md
@@ -0,0 +1,702 @@
+# Docker Deployment
+
+Crawl4AI provides official Docker images for easy deployment and scalability. This guide covers installation, configuration, and usage of Crawl4AI in Docker environments.
+
+## Quick Start 🚀
+
+Pull and run the basic version:
+
+```bash
+# Basic run without security
+docker pull unclecode/crawl4ai:basic
+docker run -p 11235:11235 unclecode/crawl4ai:basic
+
+# Run with API security enabled
+docker run -p 11235:11235 -e CRAWL4AI_API_TOKEN=your_secret_token unclecode/crawl4ai:basic
+```
+
+## Running with Docker Compose 🐳
+
+### Use Docker Compose (From Local Dockerfile or Docker Hub)
+
+Crawl4AI provides flexibility to use Docker Compose for managing your containerized services. You can either build the image locally from the provided `Dockerfile` or use the pre-built image from Docker Hub.
+
+### **Option 1: Using Docker Compose to Build Locally**
+If you want to build the image locally, use the provided `docker-compose.local.yml` file.
+
+```bash
+docker-compose -f docker-compose.local.yml up -d
+```
+
+This will:
+1. Build the Docker image from the provided `Dockerfile`.
+2. Start the container and expose it on `http://localhost:11235`.
+
+---
+
+### **Option 2: Using Docker Compose with Pre-Built Image from Hub**
+If you prefer using the pre-built image on Docker Hub, use the `docker-compose.hub.yml` file.
+
+```bash
+docker-compose -f docker-compose.hub.yml up -d
+```
+
+This will:
+1. Pull the pre-built image `unclecode/crawl4ai:basic` (or `all`, depending on your configuration).
+2. Start the container and expose it on `http://localhost:11235`.
+
+---
+
+### **Stopping the Running Services**
+
+To stop the services started via Docker Compose, you can use:
+
+```bash
+docker-compose -f docker-compose.local.yml down
+# OR
+docker-compose -f docker-compose.hub.yml down
+```
+
+If the containers don’t stop and the application is still running, check the running containers:
+
+```bash
+docker ps
+```
+
+Find the `CONTAINER ID` of the running service and stop it forcefully:
+
+```bash
+docker stop
+```
+
+---
+
+### **Debugging with Docker Compose**
+
+- **Check Logs**: To view the container logs:
+ ```bash
+ docker-compose -f docker-compose.local.yml logs -f
+ ```
+
+- **Remove Orphaned Containers**: If the service is still running unexpectedly:
+ ```bash
+ docker-compose -f docker-compose.local.yml down --remove-orphans
+ ```
+
+- **Manually Remove Network**: If the network is still in use:
+ ```bash
+ docker network ls
+ docker network rm crawl4ai_default
+ ```
+
+---
+
+### Why Use Docker Compose?
+
+Docker Compose is the recommended way to deploy Crawl4AI because:
+1. It simplifies multi-container setups.
+2. Allows you to define environment variables, resources, and ports in a single file.
+3. Makes it easier to switch between local development and production-ready images.
+
+For example, your `docker-compose.yml` could include API keys, token settings, and memory limits, making deployment quick and consistent.
+
+
+
+
+## API Security 🔒
+
+### Understanding CRAWL4AI_API_TOKEN
+
+The `CRAWL4AI_API_TOKEN` provides optional security for your Crawl4AI instance:
+
+- If `CRAWL4AI_API_TOKEN` is set: All API endpoints (except `/health`) require authentication
+- If `CRAWL4AI_API_TOKEN` is not set: The API is publicly accessible
+
+```bash
+# Secured Instance
+docker run -p 11235:11235 -e CRAWL4AI_API_TOKEN=your_secret_token unclecode/crawl4ai:all
+
+# Unsecured Instance
+docker run -p 11235:11235 unclecode/crawl4ai:all
+```
+
+### Making API Calls
+
+For secured instances, include the token in all requests:
+
+```python
+import requests
+
+# Setup headers if token is being used
+api_token = "your_secret_token" # Same token set in CRAWL4AI_API_TOKEN
+headers = {"Authorization": f"Bearer {api_token}"} if api_token else {}
+
+# Making authenticated requests
+response = requests.post(
+ "http://localhost:11235/crawl",
+ headers=headers,
+ json={
+ "urls": "https://example.com",
+ "priority": 10
+ }
+)
+
+# Checking task status
+task_id = response.json()["task_id"]
+status = requests.get(
+ f"http://localhost:11235/task/{task_id}",
+ headers=headers
+)
+```
+
+### Using with Docker Compose
+
+In your `docker-compose.yml`:
+```yaml
+services:
+ crawl4ai:
+ image: unclecode/crawl4ai:all
+ environment:
+ - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-} # Optional
+ # ... other configuration
+```
+
+Then either:
+1. Set in `.env` file:
+```env
+CRAWL4AI_API_TOKEN=your_secret_token
+```
+
+2. Or set via command line:
+```bash
+CRAWL4AI_API_TOKEN=your_secret_token docker-compose up
+```
+
+> **Security Note**: If you enable the API token, make sure to keep it secure and never commit it to version control. The token will be required for all API endpoints except the health check endpoint (`/health`).
+
+## Configuration Options 🔧
+
+### Environment Variables
+
+You can configure the service using environment variables:
+
+```bash
+# Basic configuration
+docker run -p 11235:11235 \
+ -e MAX_CONCURRENT_TASKS=5 \
+ unclecode/crawl4ai:all
+
+# With security and LLM support
+docker run -p 11235:11235 \
+ -e CRAWL4AI_API_TOKEN=your_secret_token \
+ -e OPENAI_API_KEY=sk-... \
+ -e ANTHROPIC_API_KEY=sk-ant-... \
+ unclecode/crawl4ai:all
+```
+
+### Using Docker Compose (Recommended) 🐳
+
+Create a `docker-compose.yml`:
+
+```yaml
+version: '3.8'
+
+services:
+ crawl4ai:
+ image: unclecode/crawl4ai:all
+ ports:
+ - "11235:11235"
+ environment:
+ - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-} # Optional API security
+ - MAX_CONCURRENT_TASKS=5
+ # LLM Provider Keys
+ - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+ - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
+ volumes:
+ - /dev/shm:/dev/shm
+ deploy:
+ resources:
+ limits:
+ memory: 4G
+ reservations:
+ memory: 1G
+```
+
+You can run it in two ways:
+
+1. Using environment variables directly:
+```bash
+CRAWL4AI_API_TOKEN=secret123 OPENAI_API_KEY=sk-... docker-compose up
+```
+
+2. Using a `.env` file (recommended):
+Create a `.env` file in the same directory:
+```env
+# API Security (optional)
+CRAWL4AI_API_TOKEN=your_secret_token
+
+# LLM Provider Keys
+OPENAI_API_KEY=sk-...
+ANTHROPIC_API_KEY=sk-ant-...
+
+# Other Configuration
+MAX_CONCURRENT_TASKS=5
+```
+
+Then simply run:
+```bash
+docker-compose up
+```
+
+### Testing the Deployment 🧪
+
+```python
+import requests
+
+# For unsecured instances
+def test_unsecured():
+ # Health check
+ health = requests.get("http://localhost:11235/health")
+ print("Health check:", health.json())
+
+ # Basic crawl
+ response = requests.post(
+ "http://localhost:11235/crawl",
+ json={
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 10
+ }
+ )
+ task_id = response.json()["task_id"]
+ print("Task ID:", task_id)
+
+# For secured instances
+def test_secured(api_token):
+ headers = {"Authorization": f"Bearer {api_token}"}
+
+ # Basic crawl with authentication
+ response = requests.post(
+ "http://localhost:11235/crawl",
+ headers=headers,
+ json={
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 10
+ }
+ )
+ task_id = response.json()["task_id"]
+ print("Task ID:", task_id)
+```
+
+### LLM Extraction Example 🤖
+
+When you've configured your LLM provider keys (via environment variables or `.env`), you can use LLM extraction:
+
+```python
+request = {
+ "urls": "https://example.com",
+ "extraction_config": {
+ "type": "llm",
+ "params": {
+ "provider": "openai/gpt-4",
+ "instruction": "Extract main topics from the page"
+ }
+ }
+}
+
+# Make the request (add headers if using API security)
+response = requests.post("http://localhost:11235/crawl", json=request)
+```
+
+> **Note**: Remember to add `.env` to your `.gitignore` to keep your API keys secure!
+
+
+## Usage Examples 📝
+
+### Basic Crawling
+
+```python
+request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 10
+}
+
+response = requests.post("http://localhost:11235/crawl", json=request)
+task_id = response.json()["task_id"]
+
+# Get results
+result = requests.get(f"http://localhost:11235/task/{task_id}")
+```
+
+### Structured Data Extraction
+
+```python
+schema = {
+ "name": "Crypto Prices",
+ "baseSelector": ".cds-tableRow-t45thuk",
+ "fields": [
+ {
+ "name": "crypto",
+ "selector": "td:nth-child(1) h2",
+ "type": "text",
+ },
+ {
+ "name": "price",
+ "selector": "td:nth-child(2)",
+ "type": "text",
+ }
+ ],
+}
+
+request = {
+ "urls": "https://www.coinbase.com/explore",
+ "extraction_config": {
+ "type": "json_css",
+ "params": {"schema": schema}
+ }
+}
+```
+
+### Dynamic Content Handling
+
+```python
+request = {
+ "urls": "https://www.nbcnews.com/business",
+ "js_code": [
+ "const loadMoreButton = Array.from(document.querySelectorAll('button')).find(button => button.textContent.includes('Load More')); loadMoreButton && loadMoreButton.click();"
+ ],
+ "wait_for": "article.tease-card:nth-child(10)"
+}
+```
+
+### AI-Powered Extraction (Full Version)
+
+```python
+request = {
+ "urls": "https://www.nbcnews.com/business",
+ "extraction_config": {
+ "type": "cosine",
+ "params": {
+ "semantic_filter": "business finance economy",
+ "word_count_threshold": 10,
+ "max_dist": 0.2,
+ "top_k": 3
+ }
+ }
+}
+```
+
+## Platform-Specific Instructions 💻
+
+### macOS
+```bash
+docker pull unclecode/crawl4ai:basic
+docker run -p 11235:11235 unclecode/crawl4ai:basic
+```
+
+### Ubuntu
+```bash
+# Basic version
+docker pull unclecode/crawl4ai:basic
+docker run -p 11235:11235 unclecode/crawl4ai:basic
+
+# With GPU support
+docker pull unclecode/crawl4ai:gpu
+docker run --gpus all -p 11235:11235 unclecode/crawl4ai:gpu
+```
+
+### Windows (PowerShell)
+```powershell
+docker pull unclecode/crawl4ai:basic
+docker run -p 11235:11235 unclecode/crawl4ai:basic
+```
+
+## Testing 🧪
+
+Save this as `test_docker.py`:
+
+```python
+import requests
+import json
+import time
+import sys
+
+class Crawl4AiTester:
+ def __init__(self, base_url: str = "http://localhost:11235"):
+ self.base_url = base_url
+
+ def submit_and_wait(self, request_data: dict, timeout: int = 300) -> dict:
+ # Submit crawl job
+ response = requests.post(f"{self.base_url}/crawl", json=request_data)
+ task_id = response.json()["task_id"]
+ print(f"Task ID: {task_id}")
+
+ # Poll for result
+ start_time = time.time()
+ while True:
+ if time.time() - start_time > timeout:
+ raise TimeoutError(f"Task {task_id} timeout")
+
+ result = requests.get(f"{self.base_url}/task/{task_id}")
+ status = result.json()
+
+ if status["status"] == "completed":
+ return status
+
+ time.sleep(2)
+
+def test_deployment():
+ tester = Crawl4AiTester()
+
+ # Test basic crawl
+ request = {
+ "urls": "https://www.nbcnews.com/business",
+ "priority": 10
+ }
+
+ result = tester.submit_and_wait(request)
+ print("Basic crawl successful!")
+ print(f"Content length: {len(result['result']['markdown'])}")
+
+if __name__ == "__main__":
+ test_deployment()
+```
+
+## Advanced Configuration ⚙️
+
+### Crawler Parameters
+
+The `crawler_params` field allows you to configure the browser instance and crawling behavior. Here are key parameters you can use:
+
+```python
+request = {
+ "urls": "https://example.com",
+ "crawler_params": {
+ # Browser Configuration
+ "headless": True, # Run in headless mode
+ "browser_type": "chromium", # chromium/firefox/webkit
+ "user_agent": "custom-agent", # Custom user agent
+ "proxy": "http://proxy:8080", # Proxy configuration
+
+ # Performance & Behavior
+ "page_timeout": 30000, # Page load timeout (ms)
+ "verbose": True, # Enable detailed logging
+ "semaphore_count": 5, # Concurrent request limit
+
+ # Anti-Detection Features
+ "simulate_user": True, # Simulate human behavior
+ "magic": True, # Advanced anti-detection
+ "override_navigator": True, # Override navigator properties
+
+ # Session Management
+ "user_data_dir": "./browser-data", # Browser profile location
+ "use_managed_browser": True, # Use persistent browser
+ }
+}
+```
+
+### Extra Parameters
+
+The `extra` field allows passing additional parameters directly to the crawler's `arun` function:
+
+```python
+request = {
+ "urls": "https://example.com",
+ "extra": {
+ "word_count_threshold": 10, # Min words per block
+ "only_text": True, # Extract only text
+ "bypass_cache": True, # Force fresh crawl
+ "process_iframes": True, # Include iframe content
+ }
+}
+```
+
+### Complete Examples
+
+1. **Advanced News Crawling**
+```python
+request = {
+ "urls": "https://www.nbcnews.com/business",
+ "crawler_params": {
+ "headless": True,
+ "page_timeout": 30000,
+ "remove_overlay_elements": True # Remove popups
+ },
+ "extra": {
+ "word_count_threshold": 50, # Longer content blocks
+ "bypass_cache": True # Fresh content
+ },
+ "css_selector": ".article-body"
+}
+```
+
+2. **Anti-Detection Configuration**
+```python
+request = {
+ "urls": "https://example.com",
+ "crawler_params": {
+ "simulate_user": True,
+ "magic": True,
+ "override_navigator": True,
+ "user_agent": "Mozilla/5.0 ...",
+ "headers": {
+ "Accept-Language": "en-US,en;q=0.9"
+ }
+ }
+}
+```
+
+3. **LLM Extraction with Custom Parameters**
+```python
+request = {
+ "urls": "https://openai.com/pricing",
+ "extraction_config": {
+ "type": "llm",
+ "params": {
+ "provider": "openai/gpt-4",
+ "schema": pricing_schema
+ }
+ },
+ "crawler_params": {
+ "verbose": True,
+ "page_timeout": 60000
+ },
+ "extra": {
+ "word_count_threshold": 1,
+ "only_text": True
+ }
+}
+```
+
+4. **Session-Based Dynamic Content**
+```python
+request = {
+ "urls": "https://example.com",
+ "crawler_params": {
+ "session_id": "dynamic_session",
+ "headless": False,
+ "page_timeout": 60000
+ },
+ "js_code": ["window.scrollTo(0, document.body.scrollHeight);"],
+ "wait_for": "js:() => document.querySelectorAll('.item').length > 10",
+ "extra": {
+ "delay_before_return_html": 2.0
+ }
+}
+```
+
+5. **Screenshot with Custom Timing**
+```python
+request = {
+ "urls": "https://example.com",
+ "screenshot": True,
+ "crawler_params": {
+ "headless": True,
+ "screenshot_wait_for": ".main-content"
+ },
+ "extra": {
+ "delay_before_return_html": 3.0
+ }
+}
+```
+
+### Parameter Reference Table
+
+| Category | Parameter | Type | Description |
+|----------|-----------|------|-------------|
+| Browser | headless | bool | Run browser in headless mode |
+| Browser | browser_type | str | Browser engine selection |
+| Browser | user_agent | str | Custom user agent string |
+| Network | proxy | str | Proxy server URL |
+| Network | headers | dict | Custom HTTP headers |
+| Timing | page_timeout | int | Page load timeout (ms) |
+| Timing | delay_before_return_html | float | Wait before capture |
+| Anti-Detection | simulate_user | bool | Human behavior simulation |
+| Anti-Detection | magic | bool | Advanced protection |
+| Session | session_id | str | Browser session ID |
+| Session | user_data_dir | str | Profile directory |
+| Content | word_count_threshold | int | Minimum words per block |
+| Content | only_text | bool | Text-only extraction |
+| Content | process_iframes | bool | Include iframe content |
+| Debug | verbose | bool | Detailed logging |
+| Debug | log_console | bool | Browser console logs |
+
+## Troubleshooting 🔍
+
+### Common Issues
+
+1. **Connection Refused**
+ ```
+ Error: Connection refused at localhost:11235
+ ```
+ Solution: Ensure the container is running and ports are properly mapped.
+
+2. **Resource Limits**
+ ```
+ Error: No available slots
+ ```
+ Solution: Increase MAX_CONCURRENT_TASKS or container resources.
+
+3. **GPU Access**
+ ```
+ Error: GPU not found
+ ```
+ Solution: Ensure proper NVIDIA drivers and use `--gpus all` flag.
+
+### Debug Mode
+
+Access container for debugging:
+```bash
+docker run -it --entrypoint /bin/bash unclecode/crawl4ai:all
+```
+
+View container logs:
+```bash
+docker logs [container_id]
+```
+
+## Best Practices 🌟
+
+1. **Resource Management**
+ - Set appropriate memory and CPU limits
+ - Monitor resource usage via health endpoint
+ - Use basic version for simple crawling tasks
+
+2. **Scaling**
+ - Use multiple containers for high load
+ - Implement proper load balancing
+ - Monitor performance metrics
+
+3. **Security**
+ - Use environment variables for sensitive data
+ - Implement proper network isolation
+ - Regular security updates
+
+## API Reference 📚
+
+### Health Check
+```http
+GET /health
+```
+
+### Submit Crawl Task
+```http
+POST /crawl
+Content-Type: application/json
+
+{
+ "urls": "string or array",
+ "extraction_config": {
+ "type": "basic|llm|cosine|json_css",
+ "params": {}
+ },
+ "priority": 1-10,
+ "ttl": 3600
+}
+```
+
+### Get Task Status
+```http
+GET /task/{task_id}
+```
+
+For more details, visit the [official documentation](https://crawl4ai.com/mkdocs/).
\ No newline at end of file
diff --git a/docs/md_v2/basic/file-download.md b/docs/md_v2/basic/file-download.md
new file mode 100644
index 0000000000000000000000000000000000000000..eac0f5cb6534f91a737d10cff9004b989c7ba75c
--- /dev/null
+++ b/docs/md_v2/basic/file-download.md
@@ -0,0 +1,129 @@
+# Download Handling in Crawl4AI
+
+This guide explains how to use Crawl4AI to handle file downloads during crawling. You'll learn how to trigger downloads, specify download locations, and access downloaded files.
+
+## Enabling Downloads
+
+To enable downloads, set the `accept_downloads` parameter in the `BrowserConfig` object and pass it to the crawler.
+
+```python
+from crawl4ai.async_configs import BrowserConfig, AsyncWebCrawler
+
+async def main():
+ config = BrowserConfig(accept_downloads=True) # Enable downloads globally
+ async with AsyncWebCrawler(config=config) as crawler:
+ # ... your crawling logic ...
+
+asyncio.run(main())
+```
+
+Or, enable it for a specific crawl by using `CrawlerRunConfig`:
+
+```python
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def main():
+ async with AsyncWebCrawler() as crawler:
+ config = CrawlerRunConfig(accept_downloads=True)
+ result = await crawler.arun(url="https://example.com", config=config)
+ # ...
+```
+
+## Specifying Download Location
+
+Specify the download directory using the `downloads_path` attribute in the `BrowserConfig` object. If not provided, Crawl4AI defaults to creating a "downloads" directory inside the `.crawl4ai` folder in your home directory.
+
+```python
+from crawl4ai.async_configs import BrowserConfig
+import os
+
+downloads_path = os.path.join(os.getcwd(), "my_downloads") # Custom download path
+os.makedirs(downloads_path, exist_ok=True)
+
+config = BrowserConfig(accept_downloads=True, downloads_path=downloads_path)
+
+async def main():
+ async with AsyncWebCrawler(config=config) as crawler:
+ result = await crawler.arun(url="https://example.com")
+ # ...
+```
+
+## Triggering Downloads
+
+Downloads are typically triggered by user interactions on a web page, such as clicking a download button. Use `js_code` in `CrawlerRunConfig` to simulate these actions and `wait_for` to allow sufficient time for downloads to start.
+
+```python
+from crawl4ai.async_configs import CrawlerRunConfig
+
+config = CrawlerRunConfig(
+ js_code="""
+ const downloadLink = document.querySelector('a[href$=".exe"]');
+ if (downloadLink) {
+ downloadLink.click();
+ }
+ """,
+ wait_for=5 # Wait 5 seconds for the download to start
+)
+
+result = await crawler.arun(url="https://www.python.org/downloads/", config=config)
+```
+
+## Accessing Downloaded Files
+
+The `downloaded_files` attribute of the `CrawlResult` object contains paths to downloaded files.
+
+```python
+if result.downloaded_files:
+ print("Downloaded files:")
+ for file_path in result.downloaded_files:
+ print(f"- {file_path}")
+ file_size = os.path.getsize(file_path)
+ print(f"- File size: {file_size} bytes")
+else:
+ print("No files downloaded.")
+```
+
+## Example: Downloading Multiple Files
+
+```python
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+import os
+from pathlib import Path
+
+async def download_multiple_files(url: str, download_path: str):
+ config = BrowserConfig(accept_downloads=True, downloads_path=download_path)
+ async with AsyncWebCrawler(config=config) as crawler:
+ run_config = CrawlerRunConfig(
+ js_code="""
+ const downloadLinks = document.querySelectorAll('a[download]');
+ for (const link of downloadLinks) {
+ link.click();
+ await new Promise(r => setTimeout(r, 2000)); // Delay between clicks
+ }
+ """,
+ wait_for=10 # Wait for all downloads to start
+ )
+ result = await crawler.arun(url=url, config=run_config)
+
+ if result.downloaded_files:
+ print("Downloaded files:")
+ for file in result.downloaded_files:
+ print(f"- {file}")
+ else:
+ print("No files downloaded.")
+
+# Usage
+download_path = os.path.join(Path.home(), ".crawl4ai", "downloads")
+os.makedirs(download_path, exist_ok=True)
+
+asyncio.run(download_multiple_files("https://www.python.org/downloads/windows/", download_path))
+```
+
+## Important Considerations
+
+- **Browser Context:** Downloads are managed within the browser context. Ensure `js_code` correctly targets the download triggers on the webpage.
+- **Timing:** Use `wait_for` in `CrawlerRunConfig` to manage download timing.
+- **Error Handling:** Handle errors to manage failed downloads or incorrect paths gracefully.
+- **Security:** Scan downloaded files for potential security threats before use.
+
+This revised guide ensures consistency with the `Crawl4AI` codebase by using `BrowserConfig` and `CrawlerRunConfig` for all download-related configurations. Let me know if further adjustments are needed!
\ No newline at end of file
diff --git a/docs/md_v2/basic/installation.md b/docs/md_v2/basic/installation.md
new file mode 100644
index 0000000000000000000000000000000000000000..de8aeafa501af77ca78da5e47017e32a771bd2e7
--- /dev/null
+++ b/docs/md_v2/basic/installation.md
@@ -0,0 +1,137 @@
+# Installation 💻
+
+Crawl4AI offers flexible installation options to suit various use cases. You can install it as a Python package, use it with Docker, or run it as a local server.
+
+## Option 1: Python Package Installation (Recommended)
+
+Crawl4AI is now available on PyPI, making installation easier than ever. Choose the option that best fits your needs:
+
+### Basic Installation
+
+For basic web crawling and scraping tasks:
+
+```bash
+pip install crawl4ai
+playwright install # Install Playwright dependencies
+```
+
+### Installation with PyTorch
+
+For advanced text clustering (includes CosineSimilarity cluster strategy):
+
+```bash
+pip install crawl4ai[torch]
+```
+
+### Installation with Transformers
+
+For text summarization and Hugging Face models:
+
+```bash
+pip install crawl4ai[transformer]
+```
+
+### Full Installation
+
+For all features:
+
+```bash
+pip install crawl4ai[all]
+```
+
+### Development Installation
+
+For contributors who plan to modify the source code:
+
+```bash
+git clone https://github.com/unclecode/crawl4ai.git
+cd crawl4ai
+pip install -e ".[all]"
+playwright install # Install Playwright dependencies
+```
+
+💡 After installation with "torch", "transformer", or "all" options, it's recommended to run the following CLI command to load the required models:
+
+```bash
+crawl4ai-download-models
+```
+
+This is optional but will boost the performance and speed of the crawler. You only need to do this once after installation.
+
+## Playwright Installation Note for Ubuntu
+
+If you encounter issues with Playwright installation on Ubuntu, you may need to install additional dependencies:
+
+```bash
+sudo apt-get install -y \
+ libwoff1 \
+ libopus0 \
+ libwebp7 \
+ libwebpdemux2 \
+ libenchant-2-2 \
+ libgudev-1.0-0 \
+ libsecret-1-0 \
+ libhyphen0 \
+ libgdk-pixbuf2.0-0 \
+ libegl1 \
+ libnotify4 \
+ libxslt1.1 \
+ libevent-2.1-7 \
+ libgles2 \
+ libxcomposite1 \
+ libatk1.0-0 \
+ libatk-bridge2.0-0 \
+ libepoxy0 \
+ libgtk-3-0 \
+ libharfbuzz-icu0 \
+ libgstreamer-gl1.0-0 \
+ libgstreamer-plugins-bad1.0-0 \
+ gstreamer1.0-plugins-good \
+ gstreamer1.0-plugins-bad \
+ libxt6 \
+ libxaw7 \
+ xvfb \
+ fonts-noto-color-emoji \
+ libfontconfig \
+ libfreetype6 \
+ xfonts-cyrillic \
+ xfonts-scalable \
+ fonts-liberation \
+ fonts-ipafont-gothic \
+ fonts-wqy-zenhei \
+ fonts-tlwg-loma-otf \
+ fonts-freefont-ttf
+```
+
+## Option 2: Using Docker (Coming Soon)
+
+Docker support for Crawl4AI is currently in progress and will be available soon. This will allow you to run Crawl4AI in a containerized environment, ensuring consistency across different systems.
+
+## Option 3: Local Server Installation
+
+For those who prefer to run Crawl4AI as a local server, instructions will be provided once the Docker implementation is complete.
+
+## Verifying Your Installation
+
+After installation, you can verify that Crawl4AI is working correctly by running a simple Python script:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+ async with AsyncWebCrawler(verbose=True) as crawler:
+ result = await crawler.arun(url="https://www.example.com")
+ print(result.markdown[:500]) # Print first 500 characters
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+This script should successfully crawl the example website and print the first 500 characters of the extracted content.
+
+## Getting Help
+
+If you encounter any issues during installation or usage, please check the [documentation](https://crawl4ai.com/mkdocs/) or raise an issue on the [GitHub repository](https://github.com/unclecode/crawl4ai/issues).
+
+Happy crawling! 🕷️🤖
\ No newline at end of file
diff --git a/docs/md_v2/basic/output-formats.md b/docs/md_v2/basic/output-formats.md
new file mode 100644
index 0000000000000000000000000000000000000000..3686c23cacd59b0a3040aa3f0ea8adbfa1df32b4
--- /dev/null
+++ b/docs/md_v2/basic/output-formats.md
@@ -0,0 +1,102 @@
+# Output Formats
+
+Crawl4AI provides multiple output formats to suit different needs, ranging from raw HTML to structured data using LLM or pattern-based extraction, and versatile markdown outputs.
+
+## Basic Formats
+
+```python
+result = await crawler.arun(url="https://example.com")
+
+# Access different formats
+raw_html = result.html # Original HTML
+clean_html = result.cleaned_html # Sanitized HTML
+markdown_v2 = result.markdown_v2 # Detailed markdown generation results
+fit_md = result.markdown_v2.fit_markdown # Most relevant content in markdown
+```
+
+> **Note**: The `markdown_v2` property will soon be replaced by `markdown`. It is recommended to start transitioning to using `markdown` for new implementations.
+
+## Raw HTML
+
+Original, unmodified HTML from the webpage. Useful when you need to:
+- Preserve the exact page structure.
+- Process HTML with your own tools.
+- Debug page issues.
+
+```python
+result = await crawler.arun(url="https://example.com")
+print(result.html) # Complete HTML including headers, scripts, etc.
+```
+
+## Cleaned HTML
+
+Sanitized HTML with unnecessary elements removed. Automatically:
+- Removes scripts and styles.
+- Cleans up formatting.
+- Preserves semantic structure.
+
+```python
+config = CrawlerRunConfig(
+ excluded_tags=['form', 'header', 'footer'], # Additional tags to remove
+ keep_data_attributes=False # Remove data-* attributes
+)
+result = await crawler.arun(url="https://example.com", config=config)
+print(result.cleaned_html)
+```
+
+## Standard Markdown
+
+HTML converted to clean markdown format. This output is useful for:
+- Content analysis.
+- Documentation.
+- Readability.
+
+```python
+config = CrawlerRunConfig(
+ markdown_generator=DefaultMarkdownGenerator(
+ options={"include_links": True} # Include links in markdown
+ )
+)
+result = await crawler.arun(url="https://example.com", config=config)
+print(result.markdown_v2.raw_markdown) # Standard markdown with links
+```
+
+## Fit Markdown
+
+Extract and convert only the most relevant content into markdown format. Best suited for:
+- Article extraction.
+- Focusing on the main content.
+- Removing boilerplate.
+
+To generate `fit_markdown`, use a content filter like `PruningContentFilter`:
+
+```python
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+config = CrawlerRunConfig(
+ content_filter=PruningContentFilter(
+ threshold=0.7,
+ threshold_type="dynamic",
+ min_word_threshold=100
+ )
+)
+result = await crawler.arun(url="https://example.com", config=config)
+print(result.markdown_v2.fit_markdown) # Extracted main content in markdown
+```
+
+## Markdown with Citations
+
+Generate markdown that includes citations for links. This format is ideal for:
+- Creating structured documentation.
+- Including references for extracted content.
+
+```python
+config = CrawlerRunConfig(
+ markdown_generator=DefaultMarkdownGenerator(
+ options={"citations": True} # Enable citations
+ )
+)
+result = await crawler.arun(url="https://example.com", config=config)
+print(result.markdown_v2.markdown_with_citations)
+print(result.markdown_v2.references_markdown) # Citations section
+```
diff --git a/docs/md_v2/basic/page-interaction.md b/docs/md_v2/basic/page-interaction.md
new file mode 100644
index 0000000000000000000000000000000000000000..07a2c9cd6858385c36da633618a317209bd0903b
--- /dev/null
+++ b/docs/md_v2/basic/page-interaction.md
@@ -0,0 +1,190 @@
+# Page Interaction
+
+Crawl4AI provides powerful features for interacting with dynamic webpages, handling JavaScript execution, and managing page events.
+
+## JavaScript Execution
+
+### Basic Execution
+
+```python
+from crawl4ai.async_configs import CrawlerRunConfig
+
+# Single JavaScript command
+config = CrawlerRunConfig(
+ js_code="window.scrollTo(0, document.body.scrollHeight);"
+)
+result = await crawler.arun(url="https://example.com", config=config)
+
+# Multiple commands
+js_commands = [
+ "window.scrollTo(0, document.body.scrollHeight);",
+ "document.querySelector('.load-more').click();",
+ "document.querySelector('#consent-button').click();"
+]
+config = CrawlerRunConfig(js_code=js_commands)
+result = await crawler.arun(url="https://example.com", config=config)
+```
+
+## Wait Conditions
+
+### CSS-Based Waiting
+
+Wait for elements to appear:
+
+```python
+config = CrawlerRunConfig(wait_for="css:.dynamic-content") # Wait for element with class 'dynamic-content'
+result = await crawler.arun(url="https://example.com", config=config)
+```
+
+### JavaScript-Based Waiting
+
+Wait for custom conditions:
+
+```python
+# Wait for number of elements
+wait_condition = """() => {
+ return document.querySelectorAll('.item').length > 10;
+}"""
+
+config = CrawlerRunConfig(wait_for=f"js:{wait_condition}")
+result = await crawler.arun(url="https://example.com", config=config)
+
+# Wait for dynamic content to load
+wait_for_content = """() => {
+ const content = document.querySelector('.content');
+ return content && content.innerText.length > 100;
+}"""
+
+config = CrawlerRunConfig(wait_for=f"js:{wait_for_content}")
+result = await crawler.arun(url="https://example.com", config=config)
+```
+
+## Handling Dynamic Content
+
+### Load More Content
+
+Handle infinite scroll or load more buttons:
+
+```python
+config = CrawlerRunConfig(
+ js_code=[
+ "window.scrollTo(0, document.body.scrollHeight);", # Scroll to bottom
+ "const loadMore = document.querySelector('.load-more'); if(loadMore) loadMore.click();" # Click load more
+ ],
+ wait_for="js:() => document.querySelectorAll('.item').length > previousCount" # Wait for new content
+)
+result = await crawler.arun(url="https://example.com", config=config)
+```
+
+### Form Interaction
+
+Handle forms and inputs:
+
+```python
+js_form_interaction = """
+ document.querySelector('#search').value = 'search term'; // Fill form fields
+ document.querySelector('form').submit(); // Submit form
+"""
+
+config = CrawlerRunConfig(
+ js_code=js_form_interaction,
+ wait_for="css:.results" # Wait for results to load
+)
+result = await crawler.arun(url="https://example.com", config=config)
+```
+
+## Timing Control
+
+### Delays and Timeouts
+
+Control timing of interactions:
+
+```python
+config = CrawlerRunConfig(
+ page_timeout=60000, # Page load timeout (ms)
+ delay_before_return_html=2.0 # Wait before capturing content
+)
+result = await crawler.arun(url="https://example.com", config=config)
+```
+
+## Complex Interactions Example
+
+Here's an example of handling a dynamic page with multiple interactions:
+
+```python
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+async def crawl_dynamic_content():
+ async with AsyncWebCrawler() as crawler:
+ # Initial page load
+ config = CrawlerRunConfig(
+ js_code="document.querySelector('.cookie-accept')?.click();", # Handle cookie consent
+ wait_for="css:.main-content"
+ )
+ result = await crawler.arun(url="https://example.com", config=config)
+
+ # Load more content
+ session_id = "dynamic_session" # Keep session for multiple interactions
+
+ for page in range(3): # Load 3 pages of content
+ config = CrawlerRunConfig(
+ session_id=session_id,
+ js_code=[
+ "window.scrollTo(0, document.body.scrollHeight);", # Scroll to bottom
+ "window.previousCount = document.querySelectorAll('.item').length;", # Store item count
+ "document.querySelector('.load-more')?.click();" # Click load more
+ ],
+ wait_for="""() => {
+ const currentCount = document.querySelectorAll('.item').length;
+ return currentCount > window.previousCount;
+ }""",
+ js_only=(page > 0) # Execute JS without reloading page for subsequent interactions
+ )
+ result = await crawler.arun(url="https://example.com", config=config)
+ print(f"Page {page + 1} items:", len(result.cleaned_html))
+
+ # Clean up session
+ await crawler.crawler_strategy.kill_session(session_id)
+```
+
+## Using with Extraction Strategies
+
+Combine page interaction with structured extraction:
+
+```python
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy, LLMExtractionStrategy
+from crawl4ai.async_configs import CrawlerRunConfig
+
+# Pattern-based extraction after interaction
+schema = {
+ "name": "Dynamic Items",
+ "baseSelector": ".item",
+ "fields": [
+ {"name": "title", "selector": "h2", "type": "text"},
+ {"name": "description", "selector": ".desc", "type": "text"}
+ ]
+}
+
+config = CrawlerRunConfig(
+ js_code="window.scrollTo(0, document.body.scrollHeight);",
+ wait_for="css:.item:nth-child(10)", # Wait for 10 items
+ extraction_strategy=JsonCssExtractionStrategy(schema)
+)
+result = await crawler.arun(url="https://example.com", config=config)
+
+# Or use LLM to analyze dynamic content
+class ContentAnalysis(BaseModel):
+ topics: List[str]
+ summary: str
+
+config = CrawlerRunConfig(
+ js_code="document.querySelector('.show-more').click();",
+ wait_for="css:.full-content",
+ extraction_strategy=LLMExtractionStrategy(
+ provider="ollama/nemotron",
+ schema=ContentAnalysis.schema(),
+ instruction="Analyze the full content"
+ )
+)
+result = await crawler.arun(url="https://example.com", config=config)
+```
diff --git a/docs/md_v2/basic/prefix-based-input.md b/docs/md_v2/basic/prefix-based-input.md
new file mode 100644
index 0000000000000000000000000000000000000000..6dfae9d45eeed59748fcc8c62844f55eba70ab75
--- /dev/null
+++ b/docs/md_v2/basic/prefix-based-input.md
@@ -0,0 +1,158 @@
+# Prefix-Based Input Handling in Crawl4AI
+
+This guide will walk you through using the Crawl4AI library to crawl web pages, local HTML files, and raw HTML strings. We'll demonstrate these capabilities using a Wikipedia page as an example.
+
+## Crawling a Web URL
+
+To crawl a live web page, provide the URL starting with `http://` or `https://`, using a `CrawlerRunConfig` object:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def crawl_web():
+ config = CrawlerRunConfig(bypass_cache=True)
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(url="https://en.wikipedia.org/wiki/apple", config=config)
+ if result.success:
+ print("Markdown Content:")
+ print(result.markdown)
+ else:
+ print(f"Failed to crawl: {result.error_message}")
+
+asyncio.run(crawl_web())
+```
+
+## Crawling a Local HTML File
+
+To crawl a local HTML file, prefix the file path with `file://`.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def crawl_local_file():
+ local_file_path = "/path/to/apple.html" # Replace with your file path
+ file_url = f"file://{local_file_path}"
+ config = CrawlerRunConfig(bypass_cache=True)
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(url=file_url, config=config)
+ if result.success:
+ print("Markdown Content from Local File:")
+ print(result.markdown)
+ else:
+ print(f"Failed to crawl local file: {result.error_message}")
+
+asyncio.run(crawl_local_file())
+```
+
+## Crawling Raw HTML Content
+
+To crawl raw HTML content, prefix the HTML string with `raw:`.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def crawl_raw_html():
+ raw_html = "Hello, World! "
+ raw_html_url = f"raw:{raw_html}"
+ config = CrawlerRunConfig(bypass_cache=True)
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(url=raw_html_url, config=config)
+ if result.success:
+ print("Markdown Content from Raw HTML:")
+ print(result.markdown)
+ else:
+ print(f"Failed to crawl raw HTML: {result.error_message}")
+
+asyncio.run(crawl_raw_html())
+```
+
+---
+
+# Complete Example
+
+Below is a comprehensive script that:
+
+1. Crawls the Wikipedia page for "Apple."
+2. Saves the HTML content to a local file (`apple.html`).
+3. Crawls the local HTML file and verifies the markdown length matches the original crawl.
+4. Crawls the raw HTML content from the saved file and verifies consistency.
+
+```python
+import os
+import sys
+import asyncio
+from pathlib import Path
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def main():
+ wikipedia_url = "https://en.wikipedia.org/wiki/apple"
+ script_dir = Path(__file__).parent
+ html_file_path = script_dir / "apple.html"
+
+ async with AsyncWebCrawler() as crawler:
+ # Step 1: Crawl the Web URL
+ print("\n=== Step 1: Crawling the Wikipedia URL ===")
+ web_config = CrawlerRunConfig(bypass_cache=True)
+ result = await crawler.arun(url=wikipedia_url, config=web_config)
+
+ if not result.success:
+ print(f"Failed to crawl {wikipedia_url}: {result.error_message}")
+ return
+
+ with open(html_file_path, 'w', encoding='utf-8') as f:
+ f.write(result.html)
+ web_crawl_length = len(result.markdown)
+ print(f"Length of markdown from web crawl: {web_crawl_length}\n")
+
+ # Step 2: Crawl from the Local HTML File
+ print("=== Step 2: Crawling from the Local HTML File ===")
+ file_url = f"file://{html_file_path.resolve()}"
+ file_config = CrawlerRunConfig(bypass_cache=True)
+ local_result = await crawler.arun(url=file_url, config=file_config)
+
+ if not local_result.success:
+ print(f"Failed to crawl local file {file_url}: {local_result.error_message}")
+ return
+
+ local_crawl_length = len(local_result.markdown)
+ assert web_crawl_length == local_crawl_length, "Markdown length mismatch"
+ print("✅ Markdown length matches between web and local file crawl.\n")
+
+ # Step 3: Crawl Using Raw HTML Content
+ print("=== Step 3: Crawling Using Raw HTML Content ===")
+ with open(html_file_path, 'r', encoding='utf-8') as f:
+ raw_html_content = f.read()
+ raw_html_url = f"raw:{raw_html_content}"
+ raw_config = CrawlerRunConfig(bypass_cache=True)
+ raw_result = await crawler.arun(url=raw_html_url, config=raw_config)
+
+ if not raw_result.success:
+ print(f"Failed to crawl raw HTML content: {raw_result.error_message}")
+ return
+
+ raw_crawl_length = len(raw_result.markdown)
+ assert web_crawl_length == raw_crawl_length, "Markdown length mismatch"
+ print("✅ Markdown length matches between web and raw HTML crawl.\n")
+
+ print("All tests passed successfully!")
+ if html_file_path.exists():
+ os.remove(html_file_path)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+# Conclusion
+
+With the unified `url` parameter and prefix-based handling in **Crawl4AI**, you can seamlessly handle web URLs, local HTML files, and raw HTML content. Use `CrawlerRunConfig` for flexible and consistent configuration in all scenarios.
\ No newline at end of file
diff --git a/docs/md_v2/basic/quickstart.md b/docs/md_v2/basic/quickstart.md
new file mode 100644
index 0000000000000000000000000000000000000000..ffc35986cc36f7a5d3cb1b823d5eeca193085cbd
--- /dev/null
+++ b/docs/md_v2/basic/quickstart.md
@@ -0,0 +1,172 @@
+# Quick Start Guide 🚀
+
+Welcome to the Crawl4AI Quickstart Guide! In this tutorial, we'll walk you through the basic usage of Crawl4AI, covering everything from initial setup to advanced features like chunking and extraction strategies, using asynchronous programming. Let's dive in! 🌟
+
+---
+
+## Getting Started 🛠️
+
+Set up your environment with `BrowserConfig` and create an `AsyncWebCrawler` instance.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import BrowserConfig
+
+async def main():
+ browser_config = BrowserConfig(verbose=True)
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ # Add your crawling logic here
+ pass
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+### Basic Usage
+
+Provide a URL and let Crawl4AI do the work!
+
+```python
+from crawl4ai.async_configs import CrawlerRunConfig
+
+async def main():
+ browser_config = BrowserConfig(verbose=True)
+ crawl_config = CrawlerRunConfig(url="https://www.nbcnews.com/business")
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ result = await crawler.arun(config=crawl_config)
+ print(f"Basic crawl result: {result.markdown[:500]}") # Print first 500 characters
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+### Taking Screenshots 📸
+
+Capture and save webpage screenshots with `CrawlerRunConfig`:
+
+```python
+from crawl4ai.async_configs import CacheMode
+
+async def capture_and_save_screenshot(url: str, output_path: str):
+ browser_config = BrowserConfig(verbose=True)
+ crawl_config = CrawlerRunConfig(
+ url=url,
+ screenshot=True,
+ cache_mode=CacheMode.BYPASS
+ )
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ result = await crawler.arun(config=crawl_config)
+
+ if result.success and result.screenshot:
+ import base64
+ screenshot_data = base64.b64decode(result.screenshot)
+ with open(output_path, 'wb') as f:
+ f.write(screenshot_data)
+ print(f"Screenshot saved successfully to {output_path}")
+ else:
+ print("Failed to capture screenshot")
+```
+
+---
+
+### Browser Selection 🌐
+
+Choose from multiple browser engines using `BrowserConfig`:
+
+```python
+from crawl4ai.async_configs import BrowserConfig
+
+# Use Firefox
+firefox_config = BrowserConfig(browser_type="firefox", verbose=True, headless=True)
+async with AsyncWebCrawler(config=firefox_config) as crawler:
+ result = await crawler.arun(config=CrawlerRunConfig(url="https://www.example.com"))
+
+# Use WebKit
+webkit_config = BrowserConfig(browser_type="webkit", verbose=True, headless=True)
+async with AsyncWebCrawler(config=webkit_config) as crawler:
+ result = await crawler.arun(config=CrawlerRunConfig(url="https://www.example.com"))
+
+# Use Chromium (default)
+chromium_config = BrowserConfig(verbose=True, headless=True)
+async with AsyncWebCrawler(config=chromium_config) as crawler:
+ result = await crawler.arun(config=CrawlerRunConfig(url="https://www.example.com"))
+```
+
+---
+
+### User Simulation 🎭
+
+Simulate real user behavior to bypass detection:
+
+```python
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+browser_config = BrowserConfig(verbose=True, headless=True)
+crawl_config = CrawlerRunConfig(
+ url="YOUR-URL-HERE",
+ cache_mode=CacheMode.BYPASS,
+ simulate_user=True, # Random mouse movements and clicks
+ override_navigator=True # Makes the browser appear like a real user
+)
+async with AsyncWebCrawler(config=browser_config) as crawler:
+ result = await crawler.arun(config=crawl_config)
+```
+
+---
+
+### Understanding Parameters 🧠
+
+Explore caching and forcing fresh crawls:
+
+```python
+async def main():
+ browser_config = BrowserConfig(verbose=True)
+
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ # First crawl (uses cache)
+ result1 = await crawler.arun(config=CrawlerRunConfig(url="https://www.nbcnews.com/business"))
+ print(f"First crawl result: {result1.markdown[:100]}...")
+
+ # Force fresh crawl
+ result2 = await crawler.arun(
+ config=CrawlerRunConfig(url="https://www.nbcnews.com/business", cache_mode=CacheMode.BYPASS)
+ )
+ print(f"Second crawl result: {result2.markdown[:100]}...")
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+### Adding a Chunking Strategy 🧩
+
+Split content into chunks using `RegexChunking`:
+
+```python
+from crawl4ai.chunking_strategy import RegexChunking
+
+async def main():
+ browser_config = BrowserConfig(verbose=True)
+ crawl_config = CrawlerRunConfig(
+ url="https://www.nbcnews.com/business",
+ chunking_strategy=RegexChunking(patterns=["\n\n"])
+ )
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ result = await crawler.arun(config=crawl_config)
+ print(f"RegexChunking result: {result.extracted_content[:200]}...")
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+### Advanced Features and Configurations
+
+For advanced examples (LLM strategies, knowledge graphs, pagination handling), ensure all code aligns with the `BrowserConfig` and `CrawlerRunConfig` pattern shown above.
diff --git a/docs/md_v2/basic/simple-crawling.md b/docs/md_v2/basic/simple-crawling.md
new file mode 100644
index 0000000000000000000000000000000000000000..ec63984c7fdff61de71e4a92e58a10736136b61e
--- /dev/null
+++ b/docs/md_v2/basic/simple-crawling.md
@@ -0,0 +1,145 @@
+# Simple Crawling
+
+This guide covers the basics of web crawling with Crawl4AI. You'll learn how to set up a crawler, make your first request, and understand the response.
+
+## Basic Usage
+
+Set up a simple crawl using `BrowserConfig` and `CrawlerRunConfig`:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+async def main():
+ browser_config = BrowserConfig() # Default browser configuration
+ run_config = CrawlerRunConfig() # Default crawl run configuration
+
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ result = await crawler.arun(
+ url="https://example.com",
+ config=run_config
+ )
+ print(result.markdown) # Print clean markdown content
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+## Understanding the Response
+
+The `arun()` method returns a `CrawlResult` object with several useful properties. Here's a quick overview (see [CrawlResult](../api/crawl-result.md) for complete details):
+
+```python
+result = await crawler.arun(
+ url="https://example.com",
+ config=CrawlerRunConfig(fit_markdown=True)
+)
+
+# Different content formats
+print(result.html) # Raw HTML
+print(result.cleaned_html) # Cleaned HTML
+print(result.markdown) # Markdown version
+print(result.fit_markdown) # Most relevant content in markdown
+
+# Check success status
+print(result.success) # True if crawl succeeded
+print(result.status_code) # HTTP status code (e.g., 200, 404)
+
+# Access extracted media and links
+print(result.media) # Dictionary of found media (images, videos, audio)
+print(result.links) # Dictionary of internal and external links
+```
+
+## Adding Basic Options
+
+Customize your crawl using `CrawlerRunConfig`:
+
+```python
+run_config = CrawlerRunConfig(
+ word_count_threshold=10, # Minimum words per content block
+ exclude_external_links=True, # Remove external links
+ remove_overlay_elements=True, # Remove popups/modals
+ process_iframes=True # Process iframe content
+)
+
+result = await crawler.arun(
+ url="https://example.com",
+ config=run_config
+)
+```
+
+## Handling Errors
+
+Always check if the crawl was successful:
+
+```python
+run_config = CrawlerRunConfig()
+result = await crawler.arun(url="https://example.com", config=run_config)
+
+if not result.success:
+ print(f"Crawl failed: {result.error_message}")
+ print(f"Status code: {result.status_code}")
+```
+
+## Logging and Debugging
+
+Enable verbose logging in `BrowserConfig`:
+
+```python
+browser_config = BrowserConfig(verbose=True)
+
+async with AsyncWebCrawler(config=browser_config) as crawler:
+ run_config = CrawlerRunConfig()
+ result = await crawler.arun(url="https://example.com", config=run_config)
+```
+
+## Complete Example
+
+Here's a more comprehensive example demonstrating common usage patterns:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig, CacheMode
+
+async def main():
+ browser_config = BrowserConfig(verbose=True)
+ run_config = CrawlerRunConfig(
+ # Content filtering
+ word_count_threshold=10,
+ excluded_tags=['form', 'header'],
+ exclude_external_links=True,
+
+ # Content processing
+ process_iframes=True,
+ remove_overlay_elements=True,
+
+ # Cache control
+ cache_mode=CacheMode.ENABLED # Use cache if available
+ )
+
+ async with AsyncWebCrawler(config=browser_config) as crawler:
+ result = await crawler.arun(
+ url="https://example.com",
+ config=run_config
+ )
+
+ if result.success:
+ # Print clean content
+ print("Content:", result.markdown[:500]) # First 500 chars
+
+ # Process images
+ for image in result.media["images"]:
+ print(f"Found image: {image['src']}")
+
+ # Process links
+ for link in result.links["internal"]:
+ print(f"Internal link: {link['href']}")
+
+ else:
+ print(f"Crawl failed: {result.error_message}")
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
diff --git a/docs/md_v2/blog/articles/dockerize_hooks.md b/docs/md_v2/blog/articles/dockerize_hooks.md
new file mode 100644
index 0000000000000000000000000000000000000000..965388ee4def925d47aaf2541f2fcea77bcb91ed
--- /dev/null
+++ b/docs/md_v2/blog/articles/dockerize_hooks.md
@@ -0,0 +1,46 @@
+## Introducing Event Streams and Interactive Hooks in Crawl4AI
+
+
+
+In the near future, I’m planning to enhance Crawl4AI’s capabilities by introducing an event stream mechanism that will give clients deeper, real-time insights into the crawling process. Today, hooks are a powerful feature at the code level—they let developers define custom logic at key points in the crawl. However, when using Crawl4AI as a service (e.g., through a Dockerized API), there isn’t an easy way to interact with these hooks at runtime.
+
+**What’s Changing?**
+
+I’m working on a solution that will allow the crawler to emit a continuous stream of events, updating clients on the current crawling stage, encountered pages, and any decision points. This event stream could be exposed over a standardized protocol like Server-Sent Events (SSE) or WebSockets, enabling clients to “subscribe” and listen as the crawler works.
+
+**Interactivity Through Process IDs**
+
+A key part of this new design is the concept of a unique process ID for each crawl session. Imagine you’re listening to an event stream that informs you:
+- The crawler just hit a certain page
+- It triggered a hook and is now pausing for instructions
+
+With the event stream in place, you can send a follow-up request back to the server—referencing the unique process ID—to provide extra data, instructions, or parameters. This might include selecting which links to follow next, adjusting extraction strategies, or providing authentication tokens for a protected API. Once the crawler receives these instructions, it resumes execution with the updated context.
+
+```mermaid
+sequenceDiagram
+ participant Client
+ participant Server
+ participant Crawler
+
+ Client->>Server: Start crawl request
+ Server->>Crawler: Initiate crawl with Process ID
+ Crawler-->>Server: Event: Page hit
+ Server-->>Client: Stream: Page hit event
+ Client->>Server: Instruction for Process ID
+ Server->>Crawler: Update crawl with new instructions
+ Crawler-->>Server: Event: Crawl completed
+ Server-->>Client: Stream: Crawl completed
+```
+
+**Benefits for Developers and Users**
+
+1. **Fine-Grained Control**: Instead of predefining all logic upfront, you can dynamically guide the crawler in response to actual data and conditions encountered mid-crawl.
+2. **Real-Time Insights**: Monitor progress, errors, or network bottlenecks as they happen, without waiting for the entire crawl to finish.
+3. **Enhanced Collaboration**: Different team members or automated systems can watch the same crawl events and provide input, making the crawling process more adaptive and intelligent.
+
+**Next Steps**
+
+I’m currently exploring the best APIs, technologies, and patterns to make this vision a reality. My goal is to deliver a seamless developer experience—one that integrates with existing Crawl4AI workflows while offering new flexibility and power.
+
+Stay tuned for more updates as I continue building this feature out. In the meantime, I’d love to hear any feedback or suggestions you might have to help shape this interactive, event-driven future of web crawling with Crawl4AI.
+
diff --git a/docs/md_v2/blog/index.md b/docs/md_v2/blog/index.md
new file mode 100644
index 0000000000000000000000000000000000000000..f7c8494da226ff85029bf84144d7c6ce4b7e38f6
--- /dev/null
+++ b/docs/md_v2/blog/index.md
@@ -0,0 +1,47 @@
+# Crawl4AI Blog
+
+Welcome to the Crawl4AI blog! Here you'll find detailed release notes, technical insights, and updates about the project. Whether you're looking for the latest improvements or want to dive deep into web crawling techniques, this is the place.
+
+## Latest Release
+
+### [0.4.2 - Configurable Crawlers, Session Management, and Smarter Screenshots](releases/0.4.2.md)
+*December 12, 2024*
+
+The 0.4.2 update brings massive improvements to configuration, making crawlers and browsers easier to manage with dedicated objects. You can now import/export local storage for seamless session management. Plus, long-page screenshots are faster and cleaner, and full-page PDF exports are now possible. Check out all the new features to make your crawling experience even smoother.
+
+[Read full release notes →](releases/0.4.2.md)
+
+---
+
+### [0.4.1 - Smarter Crawling with Lazy-Load Handling, Text-Only Mode, and More](releases/0.4.1.md)
+*December 8, 2024*
+
+This release brings major improvements to handling lazy-loaded images, a blazing-fast Text-Only Mode, full-page scanning for infinite scrolls, dynamic viewport adjustments, and session reuse for efficient crawling. If you're looking to improve speed, reliability, or handle dynamic content with ease, this update has you covered.
+
+[Read full release notes →](releases/0.4.1.md)
+
+---
+
+### [0.4.0 - Major Content Filtering Update](releases/0.4.0.md)
+*December 1, 2024*
+
+Introduced significant improvements to content filtering, multi-threaded environment handling, and user-agent generation. This release features the new PruningContentFilter, enhanced thread safety, and improved test coverage.
+
+[Read full release notes →](releases/0.4.0.md)
+
+## Project History
+
+Curious about how Crawl4AI has evolved? Check out our [complete changelog](https://github.com/unclecode/crawl4ai/blob/main/CHANGELOG.md) for a detailed history of all versions and updates.
+
+## Categories
+
+- [Technical Deep Dives](/blog/technical) - Coming soon
+- [Tutorials & Guides](/blog/tutorials) - Coming soon
+- [Community Updates](/blog/community) - Coming soon
+
+## Stay Updated
+
+- Star us on [GitHub](https://github.com/unclecode/crawl4ai)
+- Follow [@unclecode](https://twitter.com/unclecode) on Twitter
+- Join our community discussions on GitHub
+
diff --git a/docs/md_v2/blog/releases/0.4.0.md b/docs/md_v2/blog/releases/0.4.0.md
new file mode 100644
index 0000000000000000000000000000000000000000..0e7ee5df0e2bbf059cf6f57404a6c83d898df7bc
--- /dev/null
+++ b/docs/md_v2/blog/releases/0.4.0.md
@@ -0,0 +1,62 @@
+# Release Summary for Version 0.4.0 (December 1, 2024)
+
+## Overview
+The 0.4.0 release introduces significant improvements to content filtering, multi-threaded environment handling, user-agent generation, and test coverage. Key highlights include the introduction of the PruningContentFilter, designed to automatically identify and extract the most valuable parts of an HTML document, as well as enhancements to the BM25ContentFilter to extend its versatility and effectiveness.
+
+## Major Features and Enhancements
+
+### 1. PruningContentFilter
+- Introduced a new unsupervised content filtering strategy that scores and prunes less relevant nodes in an HTML document based on metrics like text and link density.
+- Focuses on retaining the most valuable parts of the content, making it highly effective for extracting relevant information from complex web pages.
+- Fully documented with updated README and expanded user guides.
+
+### 2. User-Agent Generator
+- Added a user-agent generator utility that resolves compatibility issues and supports customizable user-agent strings.
+- By default, the generator randomizes user agents for each request, adding diversity, but users can customize it for tailored scenarios.
+
+### 3. Enhanced Thread Safety
+- Improved handling of multi-threaded environments by adding better thread locks for parallel processing, ensuring consistency and stability when running multiple threads.
+
+### 4. Extended Content Filtering Strategies
+- Users now have access to both the PruningContentFilter for unsupervised extraction and the BM25ContentFilter for supervised filtering based on user queries.
+- Enhanced BM25ContentFilter with improved capabilities to process page titles, meta tags, and descriptions, allowing for more effective classification and clustering of text chunks.
+
+### 5. Documentation Updates
+- Updated examples and tutorials to promote the use of the PruningContentFilter alongside the BM25ContentFilter, providing clear instructions for selecting the appropriate filter for each use case.
+
+### 6. Unit Test Enhancements
+- Added unit tests for PruningContentFilter to ensure accuracy and reliability.
+- Enhanced BM25ContentFilter tests to cover additional edge cases and performance metrics, particularly for malformed HTML inputs.
+
+## Revised Change Logs for Version 0.4.0
+
+### PruningContentFilter (Dec 01, 2024)
+- Introduced the PruningContentFilter to optimize content extraction by pruning less relevant HTML nodes.
+ - **Affected Files:**
+ - **crawl4ai/content_filter_strategy.py**: Added a scoring-based pruning algorithm.
+ - **README.md**: Updated to include PruningContentFilter usage.
+ - **docs/md_v2/basic/content_filtering.md**: Expanded user documentation, detailing the use and benefits of PruningContentFilter.
+
+### Unit Tests for PruningContentFilter (Dec 01, 2024)
+- Added comprehensive unit tests for PruningContentFilter to ensure correctness and efficiency.
+ - **Affected Files:**
+ - **tests/async/test_content_filter_prune.py**: Created tests covering different pruning scenarios to ensure stability and correctness.
+
+### Enhanced BM25ContentFilter Tests (Dec 01, 2024)
+- Expanded tests to cover additional extraction scenarios and performance metrics, improving robustness.
+ - **Affected Files:**
+ - **tests/async/test_content_filter_bm25.py**: Added tests for edge cases, including malformed HTML inputs.
+
+### Documentation and Example Updates (Dec 01, 2024)
+- Revised examples to illustrate the use of PruningContentFilter alongside existing content filtering methods.
+ - **Affected Files:**
+ - **docs/examples/quickstart_async.py**: Enhanced example clarity and usability for new users.
+
+## Experimental Features
+- The PruningContentFilter is still under experimental development, and we continue to gather feedback for further refinements.
+
+## Conclusion
+This release significantly enhances the content extraction capabilities of Crawl4ai with the introduction of the PruningContentFilter, improved supervised filtering with BM25ContentFilter, and robust multi-threaded handling. Additionally, the user-agent generator provides much-needed versatility, resolving compatibility issues faced by many users.
+
+Users are encouraged to experiment with the new content filtering methods to determine which best suits their needs.
+
diff --git a/docs/md_v2/blog/releases/0.4.1.md b/docs/md_v2/blog/releases/0.4.1.md
new file mode 100644
index 0000000000000000000000000000000000000000..e770d0b21ade2fd2cfd625b1910deada85425c33
--- /dev/null
+++ b/docs/md_v2/blog/releases/0.4.1.md
@@ -0,0 +1,145 @@
+# Release Summary for Version 0.4.1 (December 8, 2024): Major Efficiency Boosts with New Features!
+
+_This post was generated with the help of ChatGPT, take everything with a grain of salt. 🧂_
+
+Hi everyone,
+
+I just finished putting together version 0.4.1 of Crawl4AI, and there are a few changes in here that I think you’ll find really helpful. I’ll explain what’s new, why it matters, and exactly how you can use these features (with the code to back it up). Let’s get into it.
+
+---
+
+### Handling Lazy Loading Better (Images Included)
+
+One thing that always bugged me with crawlers is how often they miss lazy-loaded content, especially images. In this version, I made sure Crawl4AI **waits for all images to load** before moving forward. This is useful because many modern websites only load images when they’re in the viewport or after some JavaScript executes.
+
+Here’s how to enable it:
+
+```python
+await crawler.crawl(
+ url="https://example.com",
+ wait_for_images=True # Add this argument to ensure images are fully loaded
+)
+```
+
+What this does is:
+1. Waits for the page to reach a "network idle" state.
+2. Ensures all images on the page have been completely loaded.
+
+This single change handles the majority of lazy-loading cases you’re likely to encounter.
+
+---
+
+### Text-Only Mode (Fast, Lightweight Crawling)
+
+Sometimes, you don’t need to download images or process JavaScript at all. For example, if you’re crawling to extract text data, you can enable **text-only mode** to speed things up. By disabling images, JavaScript, and other heavy resources, this mode makes crawling **3-4 times faster** in most cases.
+
+Here’s how to turn it on:
+
+```python
+crawler = AsyncPlaywrightCrawlerStrategy(
+ text_mode=True # Set this to True to enable text-only crawling
+)
+```
+
+When `text_mode=True`, the crawler automatically:
+- Disables GPU processing.
+- Blocks image and JavaScript resources.
+- Reduces the viewport size to 800x600 (you can override this with `viewport_width` and `viewport_height`).
+
+If you need to crawl thousands of pages where you only care about text, this mode will save you a ton of time and resources.
+
+---
+
+### Adjusting the Viewport Dynamically
+
+Another useful addition is the ability to **dynamically adjust the viewport size** to match the content on the page. This is particularly helpful when you’re working with responsive layouts or want to ensure all parts of the page load properly.
+
+Here’s how it works:
+1. The crawler calculates the page’s width and height after it loads.
+2. It adjusts the viewport to fit the content dimensions.
+3. (Optional) It uses Chrome DevTools Protocol (CDP) to simulate zooming out so everything fits in the viewport.
+
+To enable this, use:
+
+```python
+await crawler.crawl(
+ url="https://example.com",
+ adjust_viewport_to_content=True # Dynamically adjusts the viewport
+)
+```
+
+This approach makes sure the entire page gets loaded into the viewport, especially for layouts that load content based on visibility.
+
+---
+
+### Simulating Full-Page Scrolling
+
+Some websites load data dynamically as you scroll down the page. To handle these cases, I added support for **full-page scanning**. It simulates scrolling to the bottom of the page, checking for new content, and capturing it all.
+
+Here’s an example:
+
+```python
+await crawler.crawl(
+ url="https://example.com",
+ scan_full_page=True, # Enables scrolling
+ scroll_delay=0.2 # Waits 200ms between scrolls (optional)
+)
+```
+
+What happens here:
+1. The crawler scrolls down in increments, waiting for content to load after each scroll.
+2. It stops when no new content appears (i.e., dynamic elements stop loading).
+3. It scrolls back to the top before finishing (if necessary).
+
+If you’ve ever had to deal with infinite scroll pages, this is going to save you a lot of headaches.
+
+---
+
+### Reusing Browser Sessions (Save Time on Setup)
+
+By default, every time you crawl a page, a new browser context (or tab) is created. That’s fine for small crawls, but if you’re working on a large dataset, it’s more efficient to reuse the same session.
+
+I added a method called `create_session` for this:
+
+```python
+session_id = await crawler.create_session()
+
+# Use the same session for multiple crawls
+await crawler.crawl(
+ url="https://example.com/page1",
+ session_id=session_id # Reuse the session
+)
+await crawler.crawl(
+ url="https://example.com/page2",
+ session_id=session_id
+)
+```
+
+This avoids creating a new tab for every page, speeding up the crawl and reducing memory usage.
+
+---
+
+### Other Updates
+
+Here are a few smaller updates I’ve made:
+- **Light Mode**: Use `light_mode=True` to disable background processes, extensions, and other unnecessary features, making the browser more efficient.
+- **Logging**: Improved logs to make debugging easier.
+- **Defaults**: Added sensible defaults for things like `delay_before_return_html` (now set to 0.1 seconds).
+
+---
+
+### How to Get the Update
+
+You can install or upgrade to version `0.4.1` like this:
+
+```bash
+pip install crawl4ai --upgrade
+```
+
+As always, I’d love to hear your thoughts. If there’s something you think could be improved or if you have suggestions for future versions, let me know!
+
+Enjoy the new features, and happy crawling! 🕷️
+
+---
+
+
diff --git a/docs/md_v2/blog/releases/0.4.2.md b/docs/md_v2/blog/releases/0.4.2.md
new file mode 100644
index 0000000000000000000000000000000000000000..6f8f39e9f2bf37e9e3086de7a7a7c46acf42181d
--- /dev/null
+++ b/docs/md_v2/blog/releases/0.4.2.md
@@ -0,0 +1,86 @@
+## 🚀 Crawl4AI 0.4.2 Update: Smarter Crawling Just Got Easier (Dec 12, 2024)
+
+### Hey Developers,
+
+I’m excited to share Crawl4AI 0.4.2—a major upgrade that makes crawling smarter, faster, and a whole lot more intuitive. I’ve packed in a bunch of new features to simplify your workflows and improve your experience. Let’s cut to the chase!
+
+---
+
+### 🔧 **Configurable Browser and Crawler Behavior**
+
+You’ve asked for better control over how browsers and crawlers are configured, and now you’ve got it. With the new `BrowserConfig` and `CrawlerRunConfig` objects, you can set up your browser and crawling behavior exactly how you want. No more cluttering `arun` with a dozen arguments—just pass in your configs and go.
+
+**Example:**
+```python
+from crawl4ai import BrowserConfig, CrawlerRunConfig, AsyncWebCrawler
+
+browser_config = BrowserConfig(headless=True, viewport_width=1920, viewport_height=1080)
+crawler_config = CrawlerRunConfig(cache_mode="BYPASS")
+
+async with AsyncWebCrawler(config=browser_config) as crawler:
+ result = await crawler.arun(url="https://example.com", config=crawler_config)
+ print(result.markdown[:500])
+```
+
+This setup is a game-changer for scalability, keeping your code clean and flexible as we add more parameters in the future.
+
+Remember: If you like to use the old way, you can still pass arguments directly to `arun` as before, no worries!
+
+---
+
+### 🔐 **Streamlined Session Management**
+
+Here’s the big one: You can now pass local storage and cookies directly. Whether it’s setting values programmatically or importing a saved JSON state, managing sessions has never been easier. This is a must-have for authenticated crawls—just export your storage state once and reuse it effortlessly across runs.
+
+**Example:**
+1. Open a browser, log in manually, and export the storage state.
+2. Import the JSON file for seamless authenticated crawling:
+
+```python
+result = await crawler.arun(
+ url="https://example.com/protected",
+ storage_state="my_storage_state.json"
+)
+```
+
+---
+
+### 🔢 **Handling Large Pages: Supercharged Screenshots and PDF Conversion**
+
+Two big upgrades here:
+
+- **Blazing-fast long-page screenshots**: Turn extremely long web pages into clean, high-quality screenshots—without breaking a sweat. It’s optimized to handle large content without lag.
+
+- **Full-page PDF exports**: Now, you can also convert any page into a PDF with all the details intact. Perfect for archiving or sharing complex layouts.
+
+---
+
+### 🔧 **Other Cool Stuff**
+
+- **Anti-bot enhancements**: Magic mode now handles overlays, user simulation, and anti-detection features like a pro.
+- **JavaScript execution**: Execute custom JS snippets to handle dynamic content. No more wrestling with endless page interactions.
+
+---
+
+### 📊 **Performance Boosts and Dev-friendly Updates**
+
+- Faster rendering and viewport adjustments for better performance.
+- Improved cookie and local storage handling for seamless authentication.
+- Better debugging with detailed logs and actionable error messages.
+
+---
+
+### 🔠 **Use Cases You’ll Love**
+
+1. **Authenticated Crawls**: Login once, export your storage state, and reuse it across multiple requests without the headache.
+2. **Long-page Screenshots**: Perfect for blogs, e-commerce pages, or any endless-scroll website.
+3. **PDF Export**: Create professional-looking page PDFs in seconds.
+
+---
+
+### Let’s Get Crawling
+
+Crawl4AI 0.4.2 is ready for you to download and try. I’m always looking for ways to improve, so don’t hold back—share your thoughts and feedback.
+
+Happy Crawling! 🚀
+
diff --git a/docs/md_v2/extraction/chunking.md b/docs/md_v2/extraction/chunking.md
new file mode 100644
index 0000000000000000000000000000000000000000..f429310f822b63a413b9b36c2db6081f631e04b3
--- /dev/null
+++ b/docs/md_v2/extraction/chunking.md
@@ -0,0 +1,133 @@
+## Chunking Strategies 📚
+
+Crawl4AI provides several powerful chunking strategies to divide text into manageable parts for further processing. Each strategy has unique characteristics and is suitable for different scenarios. Let's explore them one by one.
+
+### RegexChunking
+
+`RegexChunking` splits text using regular expressions. This is ideal for creating chunks based on specific patterns like paragraphs or sentences.
+
+#### When to Use
+- Great for structured text with consistent delimiters.
+- Suitable for documents where specific patterns (e.g., double newlines, periods) indicate logical chunks.
+
+#### Parameters
+- `patterns` (list, optional): Regular expressions used to split the text. Default is to split by double newlines (`['\n\n']`).
+
+#### Example
+```python
+from crawl4ai.chunking_strategy import RegexChunking
+
+# Define patterns for splitting text
+patterns = [r'\n\n', r'\. ']
+chunker = RegexChunking(patterns=patterns)
+
+# Sample text
+text = "This is a sample text. It will be split into chunks.\n\nThis is another paragraph."
+
+# Chunk the text
+chunks = chunker.chunk(text)
+print(chunks)
+```
+
+### NlpSentenceChunking
+
+`NlpSentenceChunking` uses NLP models to split text into sentences, ensuring accurate sentence boundaries.
+
+#### When to Use
+- Ideal for texts where sentence boundaries are crucial.
+- Useful for creating chunks that preserve grammatical structures.
+
+#### Parameters
+- None.
+
+#### Example
+```python
+from crawl4ai.chunking_strategy import NlpSentenceChunking
+
+chunker = NlpSentenceChunking()
+
+# Sample text
+text = "This is a sample text. It will be split into sentences. Here's another sentence."
+
+# Chunk the text
+chunks = chunker.chunk(text)
+print(chunks)
+```
+
+### TopicSegmentationChunking
+
+`TopicSegmentationChunking` employs the TextTiling algorithm to segment text into topic-based chunks. This method identifies thematic boundaries.
+
+#### When to Use
+- Perfect for long documents with distinct topics.
+- Useful when preserving topic continuity is more important than maintaining text order.
+
+#### Parameters
+- `num_keywords` (int, optional): Number of keywords for each topic segment. Default is `3`.
+
+#### Example
+```python
+from crawl4ai.chunking_strategy import TopicSegmentationChunking
+
+chunker = TopicSegmentationChunking(num_keywords=3)
+
+# Sample text
+text = "This document contains several topics. Topic one discusses AI. Topic two covers machine learning."
+
+# Chunk the text
+chunks = chunker.chunk(text)
+print(chunks)
+```
+
+### FixedLengthWordChunking
+
+`FixedLengthWordChunking` splits text into chunks based on a fixed number of words. This ensures each chunk has approximately the same length.
+
+#### When to Use
+- Suitable for processing large texts where uniform chunk size is important.
+- Useful when the number of words per chunk needs to be controlled.
+
+#### Parameters
+- `chunk_size` (int, optional): Number of words per chunk. Default is `100`.
+
+#### Example
+```python
+from crawl4ai.chunking_strategy import FixedLengthWordChunking
+
+chunker = FixedLengthWordChunking(chunk_size=10)
+
+# Sample text
+text = "This is a sample text. It will be split into chunks of fixed length."
+
+# Chunk the text
+chunks = chunker.chunk(text)
+print(chunks)
+```
+
+### SlidingWindowChunking
+
+`SlidingWindowChunking` uses a sliding window approach to create overlapping chunks. Each chunk has a fixed length, and the window slides by a specified step size.
+
+#### When to Use
+- Ideal for creating overlapping chunks to preserve context.
+- Useful for tasks where context from adjacent chunks is needed.
+
+#### Parameters
+- `window_size` (int, optional): Number of words in each chunk. Default is `100`.
+- `step` (int, optional): Number of words to slide the window. Default is `50`.
+
+#### Example
+```python
+from crawl4ai.chunking_strategy import SlidingWindowChunking
+
+chunker = SlidingWindowChunking(window_size=10, step=5)
+
+# Sample text
+text = "This is a sample text. It will be split using a sliding window approach to preserve context."
+
+# Chunk the text
+chunks = chunker.chunk(text)
+print(chunks)
+```
+
+With these chunking strategies, you can choose the best method to divide your text based on your specific needs. Whether you need precise sentence boundaries, topic-based segmentation, or uniform chunk sizes, Crawl4AI has you covered. Happy chunking! 📝✨
diff --git a/docs/md_v2/extraction/cosine.md b/docs/md_v2/extraction/cosine.md
new file mode 100644
index 0000000000000000000000000000000000000000..9ce49e405e8e6bfef820e8a3b7e1dc26ee575150
--- /dev/null
+++ b/docs/md_v2/extraction/cosine.md
@@ -0,0 +1,222 @@
+# Cosine Strategy
+
+The Cosine Strategy in Crawl4AI uses similarity-based clustering to identify and extract relevant content sections from web pages. This strategy is particularly useful when you need to find and extract content based on semantic similarity rather than structural patterns.
+
+## How It Works
+
+The Cosine Strategy:
+1. Breaks down page content into meaningful chunks
+2. Converts text into vector representations
+3. Calculates similarity between chunks
+4. Clusters similar content together
+5. Ranks and filters content based on relevance
+
+## Basic Usage
+
+```python
+from crawl4ai.extraction_strategy import CosineStrategy
+
+strategy = CosineStrategy(
+ semantic_filter="product reviews", # Target content type
+ word_count_threshold=10, # Minimum words per cluster
+ sim_threshold=0.3 # Similarity threshold
+)
+
+async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(
+ url="https://example.com/reviews",
+ extraction_strategy=strategy
+ )
+
+ content = result.extracted_content
+```
+
+## Configuration Options
+
+### Core Parameters
+
+```python
+CosineStrategy(
+ # Content Filtering
+ semantic_filter: str = None, # Keywords/topic for content filtering
+ word_count_threshold: int = 10, # Minimum words per cluster
+ sim_threshold: float = 0.3, # Similarity threshold (0.0 to 1.0)
+
+ # Clustering Parameters
+ max_dist: float = 0.2, # Maximum distance for clustering
+ linkage_method: str = 'ward', # Clustering linkage method
+ top_k: int = 3, # Number of top categories to extract
+
+ # Model Configuration
+ model_name: str = 'sentence-transformers/all-MiniLM-L6-v2', # Embedding model
+
+ verbose: bool = False # Enable logging
+)
+```
+
+### Parameter Details
+
+1. **semantic_filter**
+ - Sets the target topic or content type
+ - Use keywords relevant to your desired content
+ - Example: "technical specifications", "user reviews", "pricing information"
+
+2. **sim_threshold**
+ - Controls how similar content must be to be grouped together
+ - Higher values (e.g., 0.8) mean stricter matching
+ - Lower values (e.g., 0.3) allow more variation
+ ```python
+ # Strict matching
+ strategy = CosineStrategy(sim_threshold=0.8)
+
+ # Loose matching
+ strategy = CosineStrategy(sim_threshold=0.3)
+ ```
+
+3. **word_count_threshold**
+ - Filters out short content blocks
+ - Helps eliminate noise and irrelevant content
+ ```python
+ # Only consider substantial paragraphs
+ strategy = CosineStrategy(word_count_threshold=50)
+ ```
+
+4. **top_k**
+ - Number of top content clusters to return
+ - Higher values return more diverse content
+ ```python
+ # Get top 5 most relevant content clusters
+ strategy = CosineStrategy(top_k=5)
+ ```
+
+## Use Cases
+
+### 1. Article Content Extraction
+```python
+strategy = CosineStrategy(
+ semantic_filter="main article content",
+ word_count_threshold=100, # Longer blocks for articles
+ top_k=1 # Usually want single main content
+)
+
+result = await crawler.arun(
+ url="https://example.com/blog/post",
+ extraction_strategy=strategy
+)
+```
+
+### 2. Product Review Analysis
+```python
+strategy = CosineStrategy(
+ semantic_filter="customer reviews and ratings",
+ word_count_threshold=20, # Reviews can be shorter
+ top_k=10, # Get multiple reviews
+ sim_threshold=0.4 # Allow variety in review content
+)
+```
+
+### 3. Technical Documentation
+```python
+strategy = CosineStrategy(
+ semantic_filter="technical specifications documentation",
+ word_count_threshold=30,
+ sim_threshold=0.6, # Stricter matching for technical content
+ max_dist=0.3 # Allow related technical sections
+)
+```
+
+## Advanced Features
+
+### Custom Clustering
+```python
+strategy = CosineStrategy(
+ linkage_method='complete', # Alternative clustering method
+ max_dist=0.4, # Larger clusters
+ model_name='sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2' # Multilingual support
+)
+```
+
+### Content Filtering Pipeline
+```python
+strategy = CosineStrategy(
+ semantic_filter="pricing plans features",
+ word_count_threshold=15,
+ sim_threshold=0.5,
+ top_k=3
+)
+
+async def extract_pricing_features(url: str):
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(
+ url=url,
+ extraction_strategy=strategy
+ )
+
+ if result.success:
+ content = json.loads(result.extracted_content)
+ return {
+ 'pricing_features': content,
+ 'clusters': len(content),
+ 'similarity_scores': [item['score'] for item in content]
+ }
+```
+
+## Best Practices
+
+1. **Adjust Thresholds Iteratively**
+ - Start with default values
+ - Adjust based on results
+ - Monitor clustering quality
+
+2. **Choose Appropriate Word Count Thresholds**
+ - Higher for articles (100+)
+ - Lower for reviews/comments (20+)
+ - Medium for product descriptions (50+)
+
+3. **Optimize Performance**
+ ```python
+ strategy = CosineStrategy(
+ word_count_threshold=10, # Filter early
+ top_k=5, # Limit results
+ verbose=True # Monitor performance
+ )
+ ```
+
+4. **Handle Different Content Types**
+ ```python
+ # For mixed content pages
+ strategy = CosineStrategy(
+ semantic_filter="product features",
+ sim_threshold=0.4, # More flexible matching
+ max_dist=0.3, # Larger clusters
+ top_k=3 # Multiple relevant sections
+ )
+ ```
+
+## Error Handling
+
+```python
+try:
+ result = await crawler.arun(
+ url="https://example.com",
+ extraction_strategy=strategy
+ )
+
+ if result.success:
+ content = json.loads(result.extracted_content)
+ if not content:
+ print("No relevant content found")
+ else:
+ print(f"Extraction failed: {result.error_message}")
+
+except Exception as e:
+ print(f"Error during extraction: {str(e)}")
+```
+
+The Cosine Strategy is particularly effective when:
+- Content structure is inconsistent
+- You need semantic understanding
+- You want to find similar content blocks
+- Structure-based extraction (CSS/XPath) isn't reliable
+
+It works well with other strategies and can be used as a pre-processing step for LLM-based extraction.
\ No newline at end of file
diff --git a/docs/md_v2/extraction/css-advanced.md b/docs/md_v2/extraction/css-advanced.md
new file mode 100644
index 0000000000000000000000000000000000000000..393b79a5605097c7a7064a00eb9a713170d1ba30
--- /dev/null
+++ b/docs/md_v2/extraction/css-advanced.md
@@ -0,0 +1,282 @@
+# Advanced Usage of JsonCssExtractionStrategy
+
+While the basic usage of JsonCssExtractionStrategy is powerful for simple structures, its true potential shines when dealing with complex, nested HTML structures. This section will explore advanced usage scenarios, demonstrating how to extract nested objects, lists, and nested lists.
+
+## Hypothetical Website Example
+
+Let's consider a hypothetical e-commerce website that displays product categories, each containing multiple products. Each product has details, reviews, and related items. This complex structure will allow us to demonstrate various advanced features of JsonCssExtractionStrategy.
+
+Assume the HTML structure looks something like this:
+
+```html
+
+
Electronics
+
+
Smartphone X
+
$999
+
+ TechCorp
+ X-2000
+
+
+ 5G capable
+ 6.5" OLED screen
+ 128GB storage
+
+
+
+
John D.
+
4.5
+
Great phone, love the camera!
+
+
+
Jane S.
+
5
+
Best smartphone I've ever owned.
+
+
+
+
+
+
+
Sample Product
+
$19.99
+
This is a sample product.
+
+
Sample Product
+
+
$19.99
+
Limited time offer.
+
+
Sample Article
+
+ John Doe
+ Writer and editor
+
+
+
Product with Features
+
+ Feature 1
+ Feature 2
+ Feature 3
+
+
+
Special Product
+
+ ```
+ - **Schema**:
+ ```python
+ schema = {
+ "baseSelector": ".product",
+ "fields": [
+ {"name": "title", "selector": ".title", "type": "text", "transform": "uppercase"}
+ ]
+ }
+ ```
+ - **Explanation**: The `transform` property changes the `title` to uppercase, useful for standardized outputs.
+
+#### **8. Full JSON-CSS Extraction Example**
+ - Combining all elements in a single schema example for a comprehensive crawl:
+ - **Example HTML**:
+ ```html
+
+
Featured Product
+
+
$99.99
+
Best product of the year.
+
+ Durable
+ Eco-friendly
+
+
+
Sample Product
+
$19.99
+
This is a sample product.
+
+
Sample Product
+
+
$19.99
+
Limited time offer.
+
+
Sample Article
+
+ John Doe
+ Writer and editor
+
+
+
Product with Features
+
+ Feature 1
+ Feature 2
+ Feature 3
+
+
+
Special Product
+
+ ```
+ - **Schema**:
+ ```python
+ schema = {
+ "baseSelector": ".product",
+ "fields": [
+ {"name": "title", "selector": ".title", "type": "text", "transform": "uppercase"}
+ ]
+ }
+ ```
+ - **Explanation**: The `transform` property changes the `title` to uppercase, useful for standardized outputs.
+
+#### **8. Full JSON-CSS Extraction Example**
+ - Combining all elements in a single schema example for a comprehensive crawl:
+ - **Example HTML**:
+ ```html
+
+
Featured Product
+
+
$99.99
+
Best product of the year.
+
+ Durable
+ Eco-friendly
+
+
` in your requests. Otherwise, the service is open to anyone.
+
+---
+
+## 4. Docker Compose for Multi-Container Workflows
+
+You can also use **Docker Compose** to manage multiple services. Below is an **experimental** snippet:
+
+```yaml
+version: '3.8'
+
+services:
+ crawl4ai:
+ image: unclecode/crawl4ai:basic
+ ports:
+ - "11235:11235"
+ environment:
+ - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-}
+ - OPENAI_API_KEY=${OPENAI_API_KEY:-}
+ # Additional env variables as needed
+ volumes:
+ - /dev/shm:/dev/shm
+```
+
+To run:
+
+```bash
+docker-compose up -d
+```
+
+And to stop:
+
+```bash
+docker-compose down
+```
+
+**Troubleshooting**:
+
+- **Check logs**: `docker-compose logs -f crawl4ai`
+- **Remove orphan containers**: `docker-compose down --remove-orphans`
+- **Remove networks**: `docker network rm `
+
+---
+
+## 5. Making Requests to the Container
+
+**Base URL**: `http://localhost:11235`
+
+### Example: Basic Crawl
+
+```python
+import requests
+
+task_request = {
+ "urls": "https://example.com",
+ "priority": 10
+}
+
+response = requests.post("http://localhost:11235/crawl", json=task_request)
+task_id = response.json()["task_id"]
+
+# Poll for status
+status_url = f"http://localhost:11235/task/{task_id}"
+status = requests.get(status_url).json()
+print(status)
+```
+
+If you used an API token, do:
+
+```python
+headers = {"Authorization": "Bearer your_secret_token"}
+response = requests.post(
+ "http://localhost:11235/crawl",
+ headers=headers,
+ json=task_request
+)
+```
+
+---
+
+## 6. Docker + New Crawler Config Approach
+
+### Using `BrowserConfig` & `CrawlerRunConfig` in Requests
+
+The Docker-based solution can accept **crawler configurations** in the request JSON (legacy doc might show direct parameters, but we want to embed them in `crawler_params` or `extra` to align with the new approach). For example:
+
+```python
+import requests
+
+request_data = {
+ "urls": "https://www.nbcnews.com/business",
+ "crawler_params": {
+ "headless": True,
+ "browser_type": "chromium",
+ "verbose": True,
+ "page_timeout": 30000,
+ # ... any other BrowserConfig-like fields
+ },
+ "extra": {
+ "word_count_threshold": 50,
+ "bypass_cache": True
+ }
+}
+
+response = requests.post("http://localhost:11235/crawl", json=request_data)
+task_id = response.json()["task_id"]
+```
+
+This is the recommended style if you want to replicate `BrowserConfig` and `CrawlerRunConfig` settings in Docker mode.
+
+---
+
+## 7. Example: JSON Extraction in Docker
+
+```python
+import requests
+import json
+
+# Define a schema for CSS extraction
+schema = {
+ "name": "Coinbase Crypto Prices",
+ "baseSelector": ".cds-tableRow-t45thuk",
+ "fields": [
+ {
+ "name": "crypto",
+ "selector": "td:nth-child(1) h2",
+ "type": "text"
+ },
+ {
+ "name": "symbol",
+ "selector": "td:nth-child(1) p",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": "td:nth-child(2)",
+ "type": "text"
+ }
+ ]
+}
+
+request_data = {
+ "urls": "https://www.coinbase.com/explore",
+ "extraction_config": {
+ "type": "json_css",
+ "params": {"schema": schema}
+ },
+ "crawler_params": {
+ "headless": True,
+ "verbose": True
+ }
+}
+
+resp = requests.post("http://localhost:11235/crawl", json=request_data)
+task_id = resp.json()["task_id"]
+
+# Poll for status
+status = requests.get(f"http://localhost:11235/task/{task_id}").json()
+if status["status"] == "completed":
+ extracted_content = status["result"]["extracted_content"]
+ data = json.loads(extracted_content)
+ print("Extracted:", len(data), "entries")
+else:
+ print("Task still in progress or failed.")
+```
+
+---
+
+## 8. Why This Docker Is Temporary
+
+**We are building a new, stable approach**:
+
+- The current Docker container is **experimental** and might break with future releases.
+- We plan a stable release in **2025** with a more robust API, versioning, and orchestration.
+- If you use this Docker in production, do so at your own risk and be prepared for **breaking changes**.
+
+**Community**: Because Crawl4AI is open-source, you can track progress or contribute to the new Docker approach. Check the [GitHub repository](https://github.com/unclecode/crawl4ai) for roadmaps and updates.
+
+---
+
+## 9. Known Limitations & Next Steps
+
+1. **Not Production-Ready**: This Docker approach lacks extensive security, logging, or advanced config for large-scale usage.
+2. **Ongoing Changes**: Expect API changes. The official stable version is targeted for **2025**.
+3. **LLM Integrations**: Docker images are big if you want GPU or multiple model providers. We might unify these in a future build.
+4. **Performance**: For concurrency or large crawls, you may need to tune resources (memory, CPU) and watch out for ephemeral storage.
+5. **Version Pinning**: If you must deploy, pin your Docker tag to a specific version (e.g., `:basic-0.3.7`) to avoid surprise updates.
+
+### Next Steps
+
+- **Watch the Repository**: For announcements on the new Docker architecture.
+- **Experiment**: Use this Docker for test or dev environments, but keep an eye out for breakage.
+- **Contribute**: If you have ideas or improvements, open a PR or discussion.
+- **Check Roadmaps**: See our [GitHub issues](https://github.com/unclecode/crawl4ai/issues) or [Roadmap doc](https://github.com/unclecode/crawl4ai/blob/main/ROADMAP.md) to find upcoming releases.
+
+---
+
+## 10. Summary
+
+**Deploying with Docker** can simplify running Crawl4AI as a service. However:
+
+- **This Docker** approach is **legacy** and subject to removal/overhaul.
+- For production, please weigh the risks carefully.
+- Detailed “new Docker approach” is coming in **2025**.
+
+We hope this guide helps you do a quick spin-up of Crawl4AI in Docker for **experimental** usage. Stay tuned for the fully-supported version!
\ No newline at end of file
diff --git a/docs/md_v3/tutorials/getting-started.md b/docs/md_v3/tutorials/getting-started.md
new file mode 100644
index 0000000000000000000000000000000000000000..b148e6e1e53ce399087da5b32644216fe674315b
--- /dev/null
+++ b/docs/md_v3/tutorials/getting-started.md
@@ -0,0 +1,272 @@
+# Getting Started with Crawl4AI
+
+Welcome to **Crawl4AI**, an open-source LLM friendly Web Crawler & Scraper. In this tutorial, you’ll:
+
+1. **Install** Crawl4AI (both via pip and Docker, with notes on platform challenges).
+2. Run your **first crawl** using minimal configuration.
+3. Generate **Markdown** output (and learn how it’s influenced by content filters).
+4. Experiment with a simple **CSS-based extraction** strategy.
+5. See a glimpse of **LLM-based extraction** (including open-source and closed-source model options).
+
+---
+
+## 1. Introduction
+
+Crawl4AI provides:
+- An asynchronous crawler, **`AsyncWebCrawler`**.
+- Configurable browser and run settings via **`BrowserConfig`** and **`CrawlerRunConfig`**.
+- Automatic HTML-to-Markdown conversion via **`DefaultMarkdownGenerator`** (supports additional filters).
+- Multiple extraction strategies (LLM-based or “traditional” CSS/XPath-based).
+
+By the end of this guide, you’ll have installed Crawl4AI, performed a basic crawl, generated Markdown, and tried out two extraction strategies.
+
+---
+
+## 2. Installation
+
+### 2.1 Python + Playwright
+
+#### Basic Pip Installation
+
+```bash
+pip install crawl4ai
+crawl4ai-setup
+
+# Verify your installation
+crawl4ai-doctor
+```
+
+If you encounter any browser-related issues, you can install them manually:
+```bash
+python -m playwright install --with-deps chrome chromium
+```
+
+- **`crawl4ai-setup`** installs and configures Playwright (Chromium by default).
+
+We cover advanced installation and Docker in the [Installation](#installation) section.
+
+---
+
+## 3. Your First Crawl
+
+Here’s a minimal Python script that creates an **`AsyncWebCrawler`**, fetches a webpage, and prints the first 300 characters of its Markdown output:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def main():
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun("https://example.com")
+ print(result.markdown[:300]) # Print first 300 chars
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**What’s happening?**
+- **`AsyncWebCrawler`** launches a headless browser (Chromium by default).
+- It fetches `https://example.com`.
+- Crawl4AI automatically converts the HTML into Markdown.
+
+You now have a simple, working crawl!
+
+---
+
+## 4. Basic Configuration (Light Introduction)
+
+Crawl4AI’s crawler can be heavily customized using two main classes:
+
+1. **`BrowserConfig`**: Controls browser behavior (headless or full UI, user agent, JavaScript toggles, etc.).
+2. **`CrawlerRunConfig`**: Controls how each crawl runs (caching, extraction, timeouts, hooking, etc.).
+
+Below is an example with minimal usage:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def main():
+ browser_conf = BrowserConfig(headless=True) # or False to see the browser
+ run_conf = CrawlerRunConfig(cache_mode="BYPASS")
+
+ async with AsyncWebCrawler(config=browser_conf) as crawler:
+ result = await crawler.arun(
+ url="https://example.com",
+ config=run_conf
+ )
+ print(result.markdown)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+We’ll explore more advanced config in later tutorials (like enabling proxies, PDF output, multi-tab sessions, etc.). For now, just note how you pass these objects to manage crawling.
+
+---
+
+## 5. Generating Markdown Output
+
+By default, Crawl4AI automatically generates Markdown from each crawled page. However, the exact output depends on whether you specify a **markdown generator** or **content filter**.
+
+- **`result.markdown`**:
+ The direct HTML-to-Markdown conversion.
+- **`result.markdown.fit_markdown`**:
+ The same content after applying any configured **content filter** (e.g., `PruningContentFilter`).
+
+### Example: Using a Filter with `DefaultMarkdownGenerator`
+
+```python
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.content_filter_strategy import PruningContentFilter
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+md_generator = DefaultMarkdownGenerator(
+ content_filter=PruningContentFilter(threshold=0.4, threshold_type="fixed")
+)
+
+config = CrawlerRunConfig(markdown_generator=md_generator)
+
+async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun("https://news.ycombinator.com", config=config)
+ print("Raw Markdown length:", len(result.markdown.raw_markdown))
+ print("Fit Markdown length:", len(result.markdown.fit_markdown))
+```
+
+**Note**: If you do **not** specify a content filter or markdown generator, you’ll typically see only the raw Markdown. We’ll dive deeper into these strategies in a dedicated **Markdown Generation** tutorial.
+
+---
+
+## 6. Simple Data Extraction (CSS-based)
+
+Crawl4AI can also extract structured data (JSON) using CSS or XPath selectors. Below is a minimal CSS-based example:
+
+```python
+import asyncio
+import json
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+
+async def main():
+ schema = {
+ "name": "Example Items",
+ "baseSelector": "div.item",
+ "fields": [
+ {"name": "title", "selector": "h2", "type": "text"},
+ {"name": "link", "selector": "a", "type": "attribute", "attribute": "href"}
+ ]
+ }
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(
+ url="https://example.com/items",
+ config=CrawlerRunConfig(
+ extraction_strategy=JsonCssExtractionStrategy(schema)
+ )
+ )
+ # The JSON output is stored in 'extracted_content'
+ data = json.loads(result.extracted_content)
+ print(data)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**Why is this helpful?**
+- Great for repetitive page structures (e.g., item listings, articles).
+- No AI usage or costs.
+- The crawler returns a JSON string you can parse or store.
+
+---
+
+## 7. Simple Data Extraction (LLM-based)
+
+For more complex or irregular pages, a language model can parse text intelligently into a structure you define. Crawl4AI supports **open-source** or **closed-source** providers:
+
+- **Open-Source Models** (e.g., `ollama/llama3.3`, `no_token`)
+- **OpenAI Models** (e.g., `openai/gpt-4`, requires `api_token`)
+- Or any provider supported by the underlying library
+
+Below is an example using **open-source** style (no token) and closed-source:
+
+```python
+import os
+import json
+import asyncio
+from pydantic import BaseModel, Field
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+class PricingInfo(BaseModel):
+ model_name: str = Field(..., description="Name of the AI model")
+ input_fee: str = Field(..., description="Fee for input tokens")
+ output_fee: str = Field(..., description="Fee for output tokens")
+
+async def main():
+ # 1) Open-Source usage: no token required
+ llm_strategy_open_source = LLMExtractionStrategy(
+ provider="ollama/llama3.3", # or "any-other-local-model"
+ api_token="no_token", # for local models, no API key is typically required
+ schema=PricingInfo.schema(),
+ extraction_type="schema",
+ instruction="""
+ From this page, extract all AI model pricing details in JSON format.
+ Each entry should have 'model_name', 'input_fee', and 'output_fee'.
+ """,
+ temperature=0
+ )
+
+ # 2) Closed-Source usage: API key for OpenAI, for example
+ openai_token = os.getenv("OPENAI_API_KEY", "sk-YOUR_API_KEY")
+ llm_strategy_openai = LLMExtractionStrategy(
+ provider="openai/gpt-4",
+ api_token=openai_token,
+ schema=PricingInfo.schema(),
+ extraction_type="schema",
+ instruction="""
+ From this page, extract all AI model pricing details in JSON format.
+ Each entry should have 'model_name', 'input_fee', and 'output_fee'.
+ """,
+ temperature=0
+ )
+
+ # We'll demo the open-source approach here
+ config = CrawlerRunConfig(extraction_strategy=llm_strategy_open_source)
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(
+ url="https://example.com/pricing",
+ config=config
+ )
+ print("LLM-based extraction JSON:", result.extracted_content)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**What’s happening?**
+- We define a Pydantic schema (`PricingInfo`) describing the fields we want.
+- The LLM extraction strategy uses that schema and your instructions to transform raw text into structured JSON.
+- Depending on the **provider** and **api_token**, you can use local models or a remote API.
+
+---
+
+## 8. Next Steps
+
+Congratulations! You have:
+1. Installed Crawl4AI (via pip, with Docker as an option).
+2. Performed a simple crawl and printed Markdown.
+3. Seen how adding a **markdown generator** + **content filter** can produce “fit” Markdown.
+4. Experimented with **CSS-based** extraction for repetitive data.
+5. Learned the basics of **LLM-based** extraction (open-source and closed-source).
+
+If you are ready for more, check out:
+
+- **Installation**: Learn more on how to install Crawl4AI and set up Playwright.
+- **Focus on Configuration**: Learn to customize browser settings, caching modes, advanced timeouts, etc.
+- **Markdown Generation Basics**: Dive deeper into content filtering and “fit markdown” usage.
+- **Dynamic Pages & Hooks**: Tackle sites with “Load More” buttons, login forms, or JavaScript complexities.
+- **Deployment**: Run Crawl4AI in Docker containers and scale across multiple nodes.
+- **Explanations & How-To Guides**: Explore browser contexts, identity-based crawling, hooking, performance, and more.
+
+Crawl4AI is a powerful tool for extracting data and generating Markdown from virtually any website. Enjoy exploring, and we hope you build amazing AI-powered applications with it!
diff --git a/docs/md_v3/tutorials/getting-warmer.md b/docs/md_v3/tutorials/getting-warmer.md
new file mode 100644
index 0000000000000000000000000000000000000000..b2deb414328d934c87a2cc8e1d9eb054731d5291
--- /dev/null
+++ b/docs/md_v3/tutorials/getting-warmer.md
@@ -0,0 +1,527 @@
+# Crawl4AI Quick Start Guide: Your All-in-One AI-Ready Web Crawling & AI Integration Solution
+
+Crawl4AI, the **#1 trending GitHub repository**, streamlines web content extraction into AI-ready formats. Perfect for AI assistants, semantic search engines, or data pipelines, Crawl4AI transforms raw HTML into structured Markdown or JSON effortlessly. Integrate with LLMs, open-source models, or your own retrieval-augmented generation workflows.
+
+**What Crawl4AI is not:**
+
+Crawl4AI is not a replacement for traditional web scraping libraries, Selenium, or Playwright. It's not designed as a general-purpose web automation tool. Instead, Crawl4AI has a specific, focused goal:
+
+- To generate perfect, AI-friendly data (particularly for LLMs) from web content
+- To maximize speed and efficiency in data extraction and processing
+- To operate at scale, from Raspberry Pi to cloud infrastructures
+
+Crawl4AI is engineered with a "scale-first" mindset, aiming to handle millions of links while maintaining exceptional performance. It's super efficient and fast, optimized to:
+
+1. Transform raw web content into structured, LLM-ready formats (Markdown/JSON)
+2. Implement intelligent extraction strategies to reduce reliance on costly API calls
+3. Provide a streamlined pipeline for AI data preparation and ingestion
+
+In essence, Crawl4AI bridges the gap between web content and AI systems, focusing on delivering high-quality, processed data rather than offering broad web automation capabilities.
+
+**Key Links:**
+
+- **Website:** [https://crawl4ai.com](https://crawl4ai.com)
+- **GitHub:** [https://github.com/unclecode/crawl4ai](https://github.com/unclecode/crawl4ai)
+- **Colab Notebook:** [Try on Google Colab](https://colab.research.google.com/drive/1SgRPrByQLzjRfwoRNq1wSGE9nYY_EE8C?usp=sharing)
+- **Quickstart Code Example:** [quickstart_async.config.py](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/quickstart_async.config.py)
+- **Examples Folder:** [Crawl4AI Examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples)
+
+---
+
+## Table of Contents
+
+- [Crawl4AI Quick Start Guide: Your All-in-One AI-Ready Web Crawling \& AI Integration Solution](#crawl4ai-quick-start-guide-your-all-in-one-ai-ready-web-crawling--ai-integration-solution)
+ - [Table of Contents](#table-of-contents)
+ - [1. Introduction \& Key Concepts](#1-introduction--key-concepts)
+ - [2. Installation \& Environment Setup](#2-installation--environment-setup)
+ - [Test Your Installation](#test-your-installation)
+ - [3. Core Concepts \& Configuration](#3-core-concepts--configuration)
+ - [4. Basic Crawling \& Simple Extraction](#4-basic-crawling--simple-extraction)
+ - [5. Markdown Generation \& AI-Optimized Output](#5-markdown-generation--ai-optimized-output)
+ - [6. Structured Data Extraction (CSS, XPath, LLM)](#6-structured-data-extraction-css-xpath-llm)
+ - [7. Advanced Extraction: LLM \& Open-Source Models](#7-advanced-extraction-llm--open-source-models)
+ - [8. Page Interactions, JS Execution, \& Dynamic Content](#8-page-interactions-js-execution--dynamic-content)
+ - [9. Media, Links, \& Metadata Handling](#9-media-links--metadata-handling)
+ - [10. Authentication \& Identity Preservation](#10-authentication--identity-preservation)
+ - [Manual Setup via User Data Directory](#manual-setup-via-user-data-directory)
+ - [Using `storage_state`](#using-storage_state)
+ - [11. Proxy \& Security Enhancements](#11-proxy--security-enhancements)
+ - [12. Screenshots, PDFs \& File Downloads](#12-screenshots-pdfs--file-downloads)
+ - [13. Caching \& Performance Optimization](#13-caching--performance-optimization)
+ - [14. Hooks for Custom Logic](#14-hooks-for-custom-logic)
+ - [15. Dockerization \& Scaling](#15-dockerization--scaling)
+ - [16. Troubleshooting \& Common Pitfalls](#16-troubleshooting--common-pitfalls)
+ - [17. Comprehensive End-to-End Example](#17-comprehensive-end-to-end-example)
+ - [18. Further Resources \& Community](#18-further-resources--community)
+
+---
+
+## 1. Introduction & Key Concepts
+
+Crawl4AI transforms websites into structured, AI-friendly data. It efficiently handles large-scale crawling, integrates with both proprietary and open-source LLMs, and optimizes content for semantic search or RAG pipelines.
+
+**Quick Test:**
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler
+
+async def test_run():
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun("https://example.com")
+ print(result.markdown)
+
+asyncio.run(test_run())
+```
+
+If you see Markdown output, everything is working!
+
+**More info:** [See /docs/introduction](#) or [1_introduction.ex.md](https://github.com/unclecode/crawl4ai/blob/main/introduction.ex.md)
+
+---
+
+## 2. Installation & Environment Setup
+
+```bash
+# Install the package
+pip install crawl4ai
+crawl4ai-setup
+
+# Install Playwright with system dependencies (recommended)
+playwright install --with-deps # Installs all browsers
+
+# Or install specific browsers:
+playwright install --with-deps chrome # Recommended for Colab/Linux
+playwright install --with-deps firefox
+playwright install --with-deps webkit
+playwright install --with-deps chromium
+
+# Keep Playwright updated periodically
+playwright install
+```
+
+> **Note**: For Google Colab and some Linux environments, use `chrome` instead of `chromium` - it tends to work more reliably.
+
+### Test Your Installation
+Try these one-liners:
+
+```python
+# Visible browser test
+python -c "from playwright.sync_api import sync_playwright; p = sync_playwright().start(); browser = p.chromium.launch(headless=False); page = browser.new_page(); page.goto('https://example.com'); input('Press Enter to close...')"
+
+# Headless test (for servers/CI)
+python -c "from playwright.sync_api import sync_playwright; p = sync_playwright().start(); browser = p.chromium.launch(headless=True); page = browser.new_page(); page.goto('https://example.com'); print(f'Title: {page.title()}'); browser.close()"
+```
+
+You should see a browser window (in visible test) loading example.com. If you get errors, try with Firefox using `playwright install --with-deps firefox`.
+
+
+**Try in Colab:**
+[Open Colab Notebook](https://colab.research.google.com/drive/1SgRPrByQLzjRfwoRNq1wSGE9nYY_EE8C?usp=sharing)
+
+**More info:** [See /docs/configuration](#) or [2_configuration.md](https://github.com/unclecode/crawl4ai/blob/main/configuration.md)
+
+---
+
+## 3. Core Concepts & Configuration
+
+Use `AsyncWebCrawler`, `CrawlerRunConfig`, and `BrowserConfig` to control crawling.
+
+**Example config:**
+
+```python
+from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
+
+browser_config = BrowserConfig(
+ headless=True,
+ verbose=True,
+ viewport_width=1080,
+ viewport_height=600,
+ text_mode=False,
+ ignore_https_errors=True,
+ java_script_enabled=True
+)
+
+run_config = CrawlerRunConfig(
+ css_selector="article.main",
+ word_count_threshold=50,
+ excluded_tags=['nav','footer'],
+ exclude_external_links=True,
+ wait_for="css:.article-loaded",
+ page_timeout=60000,
+ delay_before_return_html=1.0,
+ mean_delay=0.1,
+ max_range=0.3,
+ process_iframes=True,
+ remove_overlay_elements=True,
+ js_code="""
+ (async () => {
+ window.scrollTo(0, document.body.scrollHeight);
+ await new Promise(r => setTimeout(r, 2000));
+ document.querySelector('.load-more')?.click();
+ })();
+ """
+)
+
+# Use: ENABLED, DISABLED, BYPASS, READ_ONLY, WRITE_ONLY
+# run_config.cache_mode = CacheMode.ENABLED
+```
+
+**Prefixes:**
+
+- `http://` or `https://` for live pages
+- `file://local.html` for local
+- `raw:` for raw HTML strings
+
+**More info:** [See /docs/async_webcrawler](#) or [3_async_webcrawler.ex.md](https://github.com/unclecode/crawl4ai/blob/main/async_webcrawler.ex.md)
+
+---
+
+## 4. Basic Crawling & Simple Extraction
+
+```python
+async with AsyncWebCrawler(config=browser_config) as crawler:
+ result = await crawler.arun("https://news.example.com/article", config=run_config)
+ print(result.markdown) # Basic markdown content
+```
+
+**More info:** [See /docs/browser_context_page](#) or [4_browser_context_page.ex.md](https://github.com/unclecode/crawl4ai/blob/main/browser_context_page.ex.md)
+
+---
+
+## 5. Markdown Generation & AI-Optimized Output
+
+After crawling, `result.markdown_v2` provides:
+
+- `raw_markdown`: Unfiltered markdown
+- `markdown_with_citations`: Links as references at the bottom
+- `references_markdown`: A separate list of reference links
+- `fit_markdown`: Filtered, relevant markdown (e.g., after BM25)
+- `fit_html`: The HTML used to produce `fit_markdown`
+
+**Example:**
+
+```python
+print("RAW:", result.markdown_v2.raw_markdown[:200])
+print("CITED:", result.markdown_v2.markdown_with_citations[:200])
+print("REFERENCES:", result.markdown_v2.references_markdown)
+print("FIT MARKDOWN:", result.markdown_v2.fit_markdown)
+```
+
+For AI training, `fit_markdown` focuses on the most relevant content.
+
+**More info:** [See /docs/markdown_generation](#) or [5_markdown_generation.ex.md](https://github.com/unclecode/crawl4ai/blob/main/markdown_generation.ex.md)
+
+---
+
+## 6. Structured Data Extraction (CSS, XPath, LLM)
+
+Extract JSON data without LLMs:
+
+**CSS:**
+
+```python
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+
+schema = {
+ "name": "Products",
+ "baseSelector": ".product",
+ "fields": [
+ {"name": "title", "selector": "h2", "type": "text"},
+ {"name": "price", "selector": ".price", "type": "text"}
+ ]
+}
+run_config.extraction_strategy = JsonCssExtractionStrategy(schema)
+```
+
+**XPath:**
+
+```python
+from crawl4ai.extraction_strategy import JsonXPathExtractionStrategy
+
+xpath_schema = {
+ "name": "Articles",
+ "baseSelector": "//div[@class='article']",
+ "fields": [
+ {"name":"headline","selector":".//h1","type":"text"},
+ {"name":"summary","selector":".//p[@class='summary']","type":"text"}
+ ]
+}
+run_config.extraction_strategy = JsonXPathExtractionStrategy(xpath_schema)
+```
+
+**More info:** [See /docs/extraction_strategies](#) or [7_extraction_strategies.ex.md](https://github.com/unclecode/crawl4ai/blob/main/extraction_strategies.ex.md)
+
+---
+
+## 7. Advanced Extraction: LLM & Open-Source Models
+
+Use LLMExtractionStrategy for complex tasks. Works with OpenAI or open-source models (e.g., Ollama).
+
+```python
+from pydantic import BaseModel
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+class TravelData(BaseModel):
+ destination: str
+ attractions: list
+
+run_config.extraction_strategy = LLMExtractionStrategy(
+ provider="ollama/nemotron",
+ schema=TravelData.schema(),
+ instruction="Extract destination and top attractions."
+)
+```
+
+**More info:** [See /docs/extraction_strategies](#) or [7_extraction_strategies.ex.md](https://github.com/unclecode/crawl4ai/blob/main/extraction_strategies.ex.md)
+
+---
+
+## 8. Page Interactions, JS Execution, & Dynamic Content
+
+Insert `js_code` and use `wait_for` to ensure content loads. Example:
+
+```python
+run_config.js_code = """
+(async () => {
+ document.querySelector('.load-more')?.click();
+ await new Promise(r => setTimeout(r, 2000));
+})();
+"""
+run_config.wait_for = "css:.item-loaded"
+```
+
+**More info:** [See /docs/page_interaction](#) or [11_page_interaction.md](https://github.com/unclecode/crawl4ai/blob/main/page_interaction.md)
+
+---
+
+## 9. Media, Links, & Metadata Handling
+
+`result.media["images"]`: List of images with `src`, `score`, `alt`. Score indicates relevance.
+
+`result.media["videos"]`, `result.media["audios"]` similarly hold media info.
+
+`result.links["internal"]`, `result.links["external"]`, `result.links["social"]`: Categorized links. Each link has `href`, `text`, `context`, `type`.
+
+`result.metadata`: Title, description, keywords, author.
+
+**Example:**
+
+```python
+# Images
+for img in result.media["images"]:
+ print("Image:", img["src"], "Score:", img["score"], "Alt:", img.get("alt","N/A"))
+
+# Links
+for link in result.links["external"]:
+ print("External Link:", link["href"], "Text:", link["text"])
+
+# Metadata
+print("Page Title:", result.metadata["title"])
+print("Description:", result.metadata["description"])
+```
+
+**More info:** [See /docs/content_selection](#) or [8_content_selection.ex.md](https://github.com/unclecode/crawl4ai/blob/main/content_selection.ex.md)
+
+---
+
+## 10. Authentication & Identity Preservation
+
+### Manual Setup via User Data Directory
+
+1. **Open Chrome with a custom user data dir:**
+
+ ```bash
+ "C:\Program Files\Google\Chrome\Application\chrome.exe" --user-data-dir="C:\MyChromeProfile"
+ ```
+
+ On macOS:
+
+ ```bash
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --user-data-dir="/Users/username/ChromeProfiles/MyProfile"
+ ```
+
+2. **Log in to sites, solve CAPTCHAs, adjust settings manually.**
+ The browser saves cookies/localStorage in that directory.
+
+3. **Use `user_data_dir` in `BrowserConfig`:**
+
+ ```python
+ browser_config = BrowserConfig(
+ headless=True,
+ user_data_dir="/Users/username/ChromeProfiles/MyProfile"
+ )
+ ```
+
+ Now the crawler starts with those cookies, sessions, etc.
+
+### Using `storage_state`
+
+Alternatively, export and reuse storage states:
+
+```python
+browser_config = BrowserConfig(
+ headless=True,
+ storage_state="mystate.json" # Pre-saved state
+)
+```
+
+No repeated logins needed.
+
+**More info:** [See /docs/storage_state](#) or [16_storage_state.md](https://github.com/unclecode/crawl4ai/blob/main/storage_state.md)
+
+---
+
+## 11. Proxy & Security Enhancements
+
+Use `proxy_config` for authenticated proxies:
+
+```python
+browser_config.proxy_config = {
+ "server": "http://proxy.example.com:8080",
+ "username": "proxyuser",
+ "password": "proxypass"
+}
+```
+
+Combine with `headers` or `ignore_https_errors` as needed.
+
+**More info:** [See /docs/proxy_security](#) or [14_proxy_security.md](https://github.com/unclecode/crawl4ai/blob/main/proxy_security.md)
+
+---
+
+## 12. Screenshots, PDFs & File Downloads
+
+Enable `screenshot=True` or `pdf=True` in `CrawlerRunConfig`:
+
+```python
+run_config.screenshot = True
+run_config.pdf = True
+```
+
+After crawling:
+
+```python
+if result.screenshot:
+ with open("page.png", "wb") as f:
+ f.write(result.screenshot)
+
+if result.pdf:
+ with open("page.pdf", "wb") as f:
+ f.write(result.pdf)
+```
+
+**File Downloads:**
+
+```python
+browser_config.accept_downloads = True
+browser_config.downloads_path = "./downloads"
+run_config.js_code = """document.querySelector('a.download')?.click();"""
+
+# After crawl:
+print("Downloaded files:", result.downloaded_files)
+```
+
+**More info:** [See /docs/screenshot_and_pdf_export](#) or [15_screenshot_and_pdf_export.md](https://github.com/unclecode/crawl4ai/blob/main/screenshot_and_pdf_export.md)
+Also [10_file_download.md](https://github.com/unclecode/crawl4ai/blob/main/file_download.md)
+
+---
+
+## 13. Caching & Performance Optimization
+
+Set `cache_mode` to reuse fetch results:
+
+```python
+from crawl4ai import CacheMode
+run_config.cache_mode = CacheMode.ENABLED
+```
+
+Adjust delays, increase concurrency, or use `text_mode=True` for faster extraction.
+
+**More info:** [See /docs/cache_modes](#) or [9_cache_modes.md](https://github.com/unclecode/crawl4ai/blob/main/cache_modes.md)
+
+---
+
+## 14. Hooks for Custom Logic
+
+Hooks let you run code at specific lifecycle events without creating pages manually in `on_browser_created`.
+
+Use `on_page_context_created` to apply routing or modify page contexts before crawling the URL:
+
+**Example Hook:**
+
+```python
+async def on_page_context_created_hook(context, page, **kwargs):
+ # Block all images to speed up load
+ await context.route("**/*.{png,jpg,jpeg}", lambda route: route.abort())
+ print("[HOOK] Image requests blocked")
+
+async with AsyncWebCrawler(config=browser_config) as crawler:
+ crawler.crawler_strategy.set_hook("on_page_context_created", on_page_context_created_hook)
+ result = await crawler.arun("https://imageheavy.example.com", config=run_config)
+ print("Crawl finished with images blocked.")
+```
+
+This hook is clean and doesn’t create a separate page itself—it just modifies the current context/page setup.
+
+**More info:** [See /docs/hooks_auth](#) or [13_hooks_auth.md](https://github.com/unclecode/crawl4ai/blob/main/hooks_auth.md)
+
+---
+
+## 15. Dockerization & Scaling
+
+Use Docker images:
+
+- AMD64 basic:
+
+```bash
+docker pull unclecode/crawl4ai:basic-amd64
+docker run -p 11235:11235 unclecode/crawl4ai:basic-amd64
+```
+
+- ARM64 for M1/M2:
+
+```bash
+docker pull unclecode/crawl4ai:basic-arm64
+docker run -p 11235:11235 unclecode/crawl4ai:basic-arm64
+```
+
+- GPU support:
+
+```bash
+docker pull unclecode/crawl4ai:gpu-amd64
+docker run --gpus all -p 11235:11235 unclecode/crawl4ai:gpu-amd64
+```
+
+Scale with load balancers or Kubernetes.
+
+**More info:** [See /docs/proxy_security (for proxy) or relevant Docker instructions in README](#)
+
+---
+
+## 16. Troubleshooting & Common Pitfalls
+
+- Empty results? Relax filters, check selectors.
+- Timeouts? Increase `page_timeout` or refine `wait_for`.
+- CAPTCHAs? Use `user_data_dir` or `storage_state` after manual solving.
+- JS errors? Try headful mode for debugging.
+
+Check [examples](https://github.com/unclecode/crawl4ai/tree/main/docs/examples) & [quickstart_async.config.py](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/quickstart_async.config.py) for more code.
+
+---
+
+## 17. Comprehensive End-to-End Example
+
+Combine hooks, JS execution, PDF saving, LLM extraction—see [quickstart_async.config.py](https://github.com/unclecode/crawl4ai/blob/main/docs/examples/quickstart_async.config.py) for a full example.
+
+---
+
+## 18. Further Resources & Community
+
+- **Docs:** [https://crawl4ai.com](https://crawl4ai.com)
+- **Issues & PRs:** [https://github.com/unclecode/crawl4ai/issues](https://github.com/unclecode/crawl4ai/issues)
+
+Follow [@unclecode](https://x.com/unclecode) for news & community updates.
+
+**Happy Crawling!**
+Leverage Crawl4AI to feed your AI models with clean, structured web data today.
diff --git a/docs/md_v3/tutorials/hooks-custom.md b/docs/md_v3/tutorials/hooks-custom.md
new file mode 100644
index 0000000000000000000000000000000000000000..2f1440652156987473ffedb48f1139f17475e643
--- /dev/null
+++ b/docs/md_v3/tutorials/hooks-custom.md
@@ -0,0 +1,335 @@
+# Hooks & Custom Code
+
+Crawl4AI supports a **hook** system that lets you run your own Python code at specific points in the crawling pipeline. By injecting logic into these hooks, you can automate tasks like:
+
+- **Authentication** (log in before navigating)
+- **Content manipulation** (modify HTML, inject scripts, etc.)
+- **Session or browser configuration** (e.g., adjusting user agents, local storage)
+- **Custom data collection** (scrape extra details or track state at each stage)
+
+In this tutorial, you’ll learn about:
+
+1. What hooks are available
+2. How to attach code to each hook
+3. Practical examples (auth flows, user agent changes, content manipulation, etc.)
+
+> **Prerequisites**
+> - Familiar with [AsyncWebCrawler Basics](./async-webcrawler-basics.md).
+> - Comfortable with Python async/await.
+
+---
+
+## 1. Overview of Available Hooks
+
+| Hook Name | Called When / Purpose | Context / Objects Provided |
+|--------------------------|-----------------------------------------------------------------|-----------------------------------------------------|
+| **`on_browser_created`** | Immediately after the browser is launched, but **before** any page or context is created. | **Browser** object only (no `page` yet). Use it for broad browser-level config. |
+| **`on_page_context_created`** | Right after a new page context is created. Perfect for setting default timeouts, injecting scripts, etc. | Typically provides `page` and `context`. |
+| **`on_user_agent_updated`** | Whenever the user agent changes. For advanced user agent logic or additional header updates. | Typically provides `page` and updated user agent string. |
+| **`on_execution_started`** | Right before your main crawling logic runs (before rendering the page). Good for one-time setup or variable initialization. | Typically provides `page`, possibly `context`. |
+| **`before_goto`** | Right before navigating to the URL (i.e., `page.goto(...)`). Great for setting cookies, altering the URL, or hooking in authentication steps. | Typically provides `page`, `context`, and `goto_params`. |
+| **`after_goto`** | Immediately after navigation completes, but before scraping. For post-login checks or initial content adjustments. | Typically provides `page`, `context`, `response`. |
+| **`before_retrieve_html`** | Right before retrieving or finalizing the page’s HTML content. Good for in-page manipulation (e.g., removing ads or disclaimers). | Typically provides `page` or final HTML reference. |
+| **`before_return_html`** | Just before the HTML is returned to the crawler pipeline. Last chance to alter or sanitize content. | Typically provides final HTML or a `page`. |
+
+### A Note on `on_browser_created` (the “unbrowser” hook)
+- **No `page`** object is available because no page context exists yet. You can, however, set up browser-wide properties.
+- For example, you might control [CDP sessions][cdp] or advanced browser flags here.
+
+---
+
+## 2. Registering Hooks
+
+You can attach hooks by calling:
+
+```python
+crawler.crawler_strategy.set_hook("hook_name", your_hook_function)
+```
+
+or by passing a `hooks` dictionary to `AsyncWebCrawler` or your strategy constructor:
+
+```python
+hooks = {
+ "before_goto": my_before_goto_hook,
+ "after_goto": my_after_goto_hook,
+ # ... etc.
+}
+async with AsyncWebCrawler(hooks=hooks) as crawler:
+ ...
+```
+
+### Hook Signature
+
+Each hook is a function (async or sync, depending on your usage) that receives **certain parameters**—most often `page`, `context`, or custom arguments relevant to that stage. The library then awaits or calls your hook before continuing.
+
+---
+
+## 3. Real-Life Examples
+
+Below are concrete scenarios where hooks come in handy.
+
+---
+
+### 3.1 Authentication Before Navigation
+
+One of the most frequent tasks is logging in or applying authentication **before** the crawler navigates to a URL (so that the user is recognized immediately).
+
+#### Using `before_goto`
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def before_goto_auth_hook(page, context, goto_params, **kwargs):
+ """
+ Example: Set cookies or localStorage to simulate login.
+ This hook runs right before page.goto() is called.
+ """
+ # Example: Insert cookie-based auth or local storage data
+ # (You could also do more complex actions, like fill forms if you already have a 'page' open.)
+ print("[HOOK] Setting auth data before goto.")
+ await context.add_cookies([
+ {
+ "name": "session",
+ "value": "abcd1234",
+ "domain": "example.com",
+ "path": "/"
+ }
+ ])
+ # Optionally manipulate goto_params if needed:
+ # goto_params["url"] = goto_params["url"] + "?debug=1"
+
+async def main():
+ hooks = {
+ "before_goto": before_goto_auth_hook
+ }
+
+ browser_cfg = BrowserConfig(headless=True)
+ crawler_cfg = CrawlerRunConfig()
+
+ async with AsyncWebCrawler(config=browser_cfg, hooks=hooks) as crawler:
+ result = await crawler.arun(url="https://example.com/protected", config=crawler_cfg)
+ if result.success:
+ print("[OK] Logged in and fetched protected page.")
+ else:
+ print("[ERROR]", result.error_message)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**Key Points**
+- `before_goto` receives `page`, `context`, `goto_params` so you can add cookies, localStorage, or even change the URL itself.
+- If you need to run a real login flow (submitting forms), consider `on_browser_created` or `on_page_context_created` if you want to do it once at the start.
+
+---
+
+### 3.2 Setting Up the Browser in `on_browser_created`
+
+If you need to do advanced browser-level configuration (e.g., hooking into the Chrome DevTools Protocol, adjusting command-line flags, etc.), you’ll use `on_browser_created`. No `page` is available yet, but you can set up the **browser** instance itself.
+
+```python
+async def on_browser_created_hook(browser, **kwargs):
+ """
+ Runs immediately after the browser is created, before any pages.
+ 'browser' here is a Playwright Browser object.
+ """
+ print("[HOOK] Browser created. Setting up custom stuff.")
+ # Possibly connect to DevTools or create an incognito context
+ # Example (pseudo-code):
+ # devtools_url = await browser.new_context(devtools=True)
+
+# Usage:
+async with AsyncWebCrawler(hooks={"on_browser_created": on_browser_created_hook}) as crawler:
+ ...
+```
+
+---
+
+### 3.3 Adjusting Page or Context in `on_page_context_created`
+
+If you’d like to set default timeouts or inject scripts right after a page context is spun up:
+
+```python
+async def on_page_context_created_hook(page, context, **kwargs):
+ print("[HOOK] Page context created. Setting default timeouts or scripts.")
+ await page.set_default_timeout(20000) # 20 seconds
+ # Possibly inject a script or set user locale
+
+# Usage:
+hooks = {
+ "on_page_context_created": on_page_context_created_hook
+}
+```
+
+---
+
+### 3.4 Dynamically Updating User Agents
+
+`on_user_agent_updated` is fired whenever the strategy updates the user agent. For instance, you might want to set certain cookies or console-log changes for debugging:
+
+```python
+async def on_user_agent_updated_hook(page, context, new_ua, **kwargs):
+ print(f"[HOOK] User agent updated to {new_ua}")
+ # Maybe add a custom header based on new UA
+ await context.set_extra_http_headers({"X-UA-Source": new_ua})
+
+hooks = {
+ "on_user_agent_updated": on_user_agent_updated_hook
+}
+```
+
+---
+
+### 3.5 Initializing Stuff with `on_execution_started`
+
+`on_execution_started` runs before your main crawling logic. It’s a good place for short, one-time setup tasks (like clearing old caches, or storing a timestamp).
+
+```python
+async def on_execution_started_hook(page, context, **kwargs):
+ print("[HOOK] Execution started. Setting a start timestamp or logging.")
+ context.set_default_navigation_timeout(45000) # 45s if your site is slow
+
+hooks = {
+ "on_execution_started": on_execution_started_hook
+}
+```
+
+---
+
+### 3.6 Post-Processing with `after_goto`
+
+After the crawler finishes navigating (i.e., the page has presumably loaded), you can do additional checks or manipulations—like verifying you’re on the right page, or removing interstitials:
+
+```python
+async def after_goto_hook(page, context, response, **kwargs):
+ """
+ Called right after page.goto() finishes, but before the crawler extracts HTML.
+ """
+ if response and response.ok:
+ print("[HOOK] After goto. Status:", response.status)
+ # Maybe remove popups or check if we landed on a login failure page.
+ await page.evaluate("""() => {
+ const popup = document.querySelector(".annoying-popup");
+ if (popup) popup.remove();
+ }""")
+ else:
+ print("[HOOK] Navigation might have failed, status not ok or no response.")
+
+hooks = {
+ "after_goto": after_goto_hook
+}
+```
+
+---
+
+### 3.7 Last-Minute Modifications in `before_retrieve_html` or `before_return_html`
+
+Sometimes you need to tweak the page or raw HTML right before it’s captured.
+
+```python
+async def before_retrieve_html_hook(page, context, **kwargs):
+ """
+ Modify the DOM just before the crawler finalizes the HTML.
+ """
+ print("[HOOK] Removing adverts before capturing HTML.")
+ await page.evaluate("""() => {
+ const ads = document.querySelectorAll(".ad-banner");
+ ads.forEach(ad => ad.remove());
+ }""")
+
+async def before_return_html_hook(page, context, html, **kwargs):
+ """
+ 'html' is the near-finished HTML string. Return an updated string if you like.
+ """
+ # For example, remove personal data or certain tags from the final text
+ print("[HOOK] Sanitizing final HTML.")
+ sanitized_html = html.replace("PersonalInfo:", "[REDACTED]")
+ return sanitized_html
+
+hooks = {
+ "before_retrieve_html": before_retrieve_html_hook,
+ "before_return_html": before_return_html_hook
+}
+```
+
+**Note**: If you want to make last-second changes in `before_return_html`, you can manipulate the `html` string directly. Return a new string if you want to override.
+
+---
+
+## 4. Putting It All Together
+
+You can combine multiple hooks in a single run. For instance:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def on_browser_created_hook(browser, **kwargs):
+ print("[HOOK] Browser is up, no page yet. Good for broad config.")
+
+async def before_goto_auth_hook(page, context, goto_params, **kwargs):
+ print("[HOOK] Adding cookies for auth.")
+ await context.add_cookies([{"name": "session", "value": "abcd1234", "domain": "example.com"}])
+
+async def after_goto_log_hook(page, context, response, **kwargs):
+ if response:
+ print("[HOOK] after_goto: Status code:", response.status)
+
+async def main():
+ hooks = {
+ "on_browser_created": on_browser_created_hook,
+ "before_goto": before_goto_auth_hook,
+ "after_goto": after_goto_log_hook
+ }
+
+ browser_cfg = BrowserConfig(headless=True)
+ crawler_cfg = CrawlerRunConfig(verbose=True)
+
+ async with AsyncWebCrawler(config=browser_cfg, hooks=hooks) as crawler:
+ result = await crawler.arun("https://example.com/protected", config=crawler_cfg)
+ if result.success:
+ print("[OK] Protected page length:", len(result.html))
+ else:
+ print("[ERROR]", result.error_message)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+This example:
+
+1. **`on_browser_created`** sets up the brand-new browser instance.
+2. **`before_goto`** ensures you inject an auth cookie before accessing the page.
+3. **`after_goto`** logs the resulting HTTP status code.
+
+---
+
+## 5. Common Pitfalls & Best Practices
+
+1. **Hook Order**: If multiple hooks do overlapping tasks (e.g., two `before_goto` hooks), be mindful of conflicts or repeated logic.
+2. **Async vs Sync**: Some hooks might be used in a synchronous or asynchronous style. Confirm your function signature. If the crawler expects `async`, define `async def`.
+3. **Mutating goto_params**: `goto_params` is a dict that eventually goes to Playwright’s `page.goto()`. Changing the `url` or adding extra fields can be powerful but can also lead to confusion. Document your changes carefully.
+4. **Browser vs Page vs Context**: Not all hooks have both `page` and `context`. For example, `on_browser_created` only has access to **`browser`**.
+5. **Avoid Overdoing It**: Hooks are powerful but can lead to complexity. If you find yourself writing massive code inside a hook, consider if a separate “how-to” function with a simpler approach might suffice.
+
+---
+
+## Conclusion & Next Steps
+
+**Hooks** let you bend Crawl4AI to your will:
+
+- **Authentication** (cookies, localStorage) with `before_goto`
+- **Browser-level config** with `on_browser_created`
+- **Page or context config** with `on_page_context_created`
+- **Content modifications** before capturing HTML (`before_retrieve_html` or `before_return_html`)
+
+**Where to go next**:
+
+- **[Identity-Based Crawling & Anti-Bot](./identity-anti-bot.md)**: Combine hooks with advanced user simulation to avoid bot detection.
+- **[Reference → AsyncPlaywrightCrawlerStrategy](../../reference/browser-strategies.md)**: Learn more about how hooks are implemented under the hood.
+- **[How-To Guides](../../how-to/)**: Check short, specific recipes for tasks like scraping multiple pages with repeated “Load More” clicks.
+
+With the hook system, you have near-complete control over the browser’s lifecycle—whether it’s setting up environment variables, customizing user agents, or manipulating the HTML. Enjoy the freedom to create sophisticated, fully customized crawling pipelines!
+
+**Last Updated**: 2024-XX-XX
diff --git a/docs/md_v3/tutorials/json-extraction-basic.md b/docs/md_v3/tutorials/json-extraction-basic.md
new file mode 100644
index 0000000000000000000000000000000000000000..1a9b79e608737a675d5f09844eb254130fadb81c
--- /dev/null
+++ b/docs/md_v3/tutorials/json-extraction-basic.md
@@ -0,0 +1,395 @@
+# Extracting JSON (No LLM)
+
+One of Crawl4AI’s **most powerful** features is extracting **structured JSON** from websites **without** relying on large language models. By defining a **schema** with CSS or XPath selectors, you can extract data instantly—even from complex or nested HTML structures—without the cost, latency, or environmental impact of an LLM.
+
+**Why avoid LLM for basic extractions?**
+
+1. **Faster & Cheaper**: No API calls or GPU overhead.
+2. **Lower Carbon Footprint**: LLM inference can be energy-intensive. A well-defined schema is practically carbon-free.
+3. **Precise & Repeatable**: CSS/XPath selectors do exactly what you specify. LLM outputs can vary or hallucinate.
+4. **Scales Readily**: For thousands of pages, schema-based extraction runs quickly and in parallel.
+
+Below, we’ll explore how to craft these schemas and use them with **JsonCssExtractionStrategy** (or **JsonXPathExtractionStrategy** if you prefer XPath). We’ll also highlight advanced features like **nested fields** and **base element attributes**.
+
+---
+
+## 1. Intro to Schema-Based Extraction
+
+A schema defines:
+
+1. A **base selector** that identifies each “container” element on the page (e.g., a product row, a blog post card).
+2. **Fields** describing which CSS/XPath selectors to use for each piece of data you want to capture (text, attribute, HTML block, etc.).
+3. **Nested** or **list** types for repeated or hierarchical structures.
+
+For example, if you have a list of products, each one might have a name, price, reviews, and “related products.” This approach is faster and more reliable than an LLM for consistent, structured pages.
+
+---
+
+## 2. Simple Example: Crypto Prices
+
+Let’s begin with a **simple** schema-based extraction using the `JsonCssExtractionStrategy`. Below is a snippet that extracts cryptocurrency prices from a site (similar to the legacy Coinbase example). Notice we **don’t** call any LLM:
+
+```python
+import json
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+
+async def extract_crypto_prices():
+ # 1. Define a simple extraction schema
+ schema = {
+ "name": "Crypto Prices",
+ "baseSelector": "div.crypto-row", # Repeated elements
+ "fields": [
+ {
+ "name": "coin_name",
+ "selector": "h2.coin-name",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": "span.coin-price",
+ "type": "text"
+ }
+ ]
+ }
+
+ # 2. Create the extraction strategy
+ extraction_strategy = JsonCssExtractionStrategy(schema, verbose=True)
+
+ # 3. Set up your crawler config (if needed)
+ config = CrawlerRunConfig(
+ # e.g., pass js_code or wait_for if the page is dynamic
+ # wait_for="css:.crypto-row:nth-child(20)"
+ cache_mode = CacheMode.BYPASS,
+ extraction_strategy=extraction_strategy,
+ )
+
+ async with AsyncWebCrawler(verbose=True) as crawler:
+ # 4. Run the crawl and extraction
+ result = await crawler.arun(
+ url="https://example.com/crypto-prices",
+
+ config=config
+ )
+
+ if not result.success:
+ print("Crawl failed:", result.error_message)
+ return
+
+ # 5. Parse the extracted JSON
+ data = json.loads(result.extracted_content)
+ print(f"Extracted {len(data)} coin entries")
+ print(json.dumps(data[0], indent=2) if data else "No data found")
+
+asyncio.run(extract_crypto_prices())
+```
+
+**Highlights**:
+
+- **`baseSelector`**: Tells us where each “item” (crypto row) is.
+- **`fields`**: Two fields (`coin_name`, `price`) using simple CSS selectors.
+- Each field defines a **`type`** (e.g., `text`, `attribute`, `html`, `regex`, etc.).
+
+No LLM is needed, and the performance is **near-instant** for hundreds or thousands of items.
+
+---
+
+### **XPath Example with `raw://` HTML**
+
+Below is a short example demonstrating **XPath** extraction plus the **`raw://`** scheme. We’ll pass a **dummy HTML** directly (no network request) and define the extraction strategy in `CrawlerRunConfig`.
+
+```python
+import json
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.extraction_strategy import JsonXPathExtractionStrategy
+
+async def extract_crypto_prices_xpath():
+ # 1. Minimal dummy HTML with some repeating rows
+ dummy_html = """
+
+
+
+
Bitcoin
+ $28,000
+
+
+
Ethereum
+ $1,800
+
+
+
+ """
+
+ # 2. Define the JSON schema (XPath version)
+ schema = {
+ "name": "Crypto Prices via XPath",
+ "baseSelector": "//div[@class='crypto-row']",
+ "fields": [
+ {
+ "name": "coin_name",
+ "selector": ".//h2[@class='coin-name']",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": ".//span[@class='coin-price']",
+ "type": "text"
+ }
+ ]
+ }
+
+ # 3. Place the strategy in the CrawlerRunConfig
+ config = CrawlerRunConfig(
+ extraction_strategy=JsonXPathExtractionStrategy(schema, verbose=True)
+ )
+
+ # 4. Use raw:// scheme to pass dummy_html directly
+ raw_url = f"raw://{dummy_html}"
+
+ async with AsyncWebCrawler(verbose=True) as crawler:
+ result = await crawler.arun(
+ url=raw_url,
+ config=config
+ )
+
+ if not result.success:
+ print("Crawl failed:", result.error_message)
+ return
+
+ data = json.loads(result.extracted_content)
+ print(f"Extracted {len(data)} coin rows")
+ if data:
+ print("First item:", data[0])
+
+asyncio.run(extract_crypto_prices_xpath())
+```
+
+**Key Points**:
+
+1. **`JsonXPathExtractionStrategy`** is used instead of `JsonCssExtractionStrategy`.
+2. **`baseSelector`** and each field’s `"selector"` use **XPath** instead of CSS.
+3. **`raw://`** lets us pass `dummy_html` with no real network request—handy for local testing.
+4. Everything (including the extraction strategy) is in **`CrawlerRunConfig`**.
+
+That’s how you keep the config self-contained, illustrate **XPath** usage, and demonstrate the **raw** scheme for direct HTML input—all while avoiding the old approach of passing `extraction_strategy` directly to `arun()`.
+
+---
+
+## 3. Advanced Schema & Nested Structures
+
+Real sites often have **nested** or repeated data—like categories containing products, which themselves have a list of reviews or features. For that, we can define **nested** or **list** (and even **nested_list**) fields.
+
+### Sample E-Commerce HTML
+
+We have a **sample e-commerce** HTML file on GitHub (example):
+```
+https://gist.githubusercontent.com/githubusercontent/2d7b8ba3cd8ab6cf3c8da771ddb36878/raw/1ae2f90c6861ce7dd84cc50d3df9920dee5e1fd2/sample_ecommerce.html
+```
+This snippet includes categories, products, features, reviews, and related items. Let’s see how to define a schema that fully captures that structure **without LLM**.
+
+```python
+schema = {
+ "name": "E-commerce Product Catalog",
+ "baseSelector": "div.category",
+ # (1) We can define optional baseFields if we want to extract attributes from the category container
+ "baseFields": [
+ {"name": "data_cat_id", "type": "attribute", "attribute": "data-cat-id"},
+ ],
+ "fields": [
+ {
+ "name": "category_name",
+ "selector": "h2.category-name",
+ "type": "text"
+ },
+ {
+ "name": "products",
+ "selector": "div.product",
+ "type": "nested_list", # repeated sub-objects
+ "fields": [
+ {
+ "name": "name",
+ "selector": "h3.product-name",
+ "type": "text"
+ },
+ {
+ "name": "price",
+ "selector": "p.product-price",
+ "type": "text"
+ },
+ {
+ "name": "details",
+ "selector": "div.product-details",
+ "type": "nested", # single sub-object
+ "fields": [
+ {"name": "brand", "selector": "span.brand", "type": "text"},
+ {"name": "model", "selector": "span.model", "type": "text"}
+ ]
+ },
+ {
+ "name": "features",
+ "selector": "ul.product-features li",
+ "type": "list",
+ "fields": [
+ {"name": "feature", "type": "text"}
+ ]
+ },
+ {
+ "name": "reviews",
+ "selector": "div.review",
+ "type": "nested_list",
+ "fields": [
+ {"name": "reviewer", "selector": "span.reviewer", "type": "text"},
+ {"name": "rating", "selector": "span.rating", "type": "text"},
+ {"name": "comment", "selector": "p.review-text", "type": "text"}
+ ]
+ },
+ {
+ "name": "related_products",
+ "selector": "ul.related-products li",
+ "type": "list",
+ "fields": [
+ {"name": "name", "selector": "span.related-name", "type": "text"},
+ {"name": "price", "selector": "span.related-price", "type": "text"}
+ ]
+ }
+ ]
+ }
+ ]
+}
+```
+
+Key Takeaways:
+
+- **Nested vs. List**:
+ - **`type: "nested"`** means a **single** sub-object (like `details`).
+ - **`type: "list"`** means multiple items that are **simple** dictionaries or single text fields.
+ - **`type: "nested_list"`** means repeated **complex** objects (like `products` or `reviews`).
+- **Base Fields**: We can extract **attributes** from the container element via `"baseFields"`. For instance, `"data_cat_id"` might be `data-cat-id="elect123"`.
+- **Transforms**: We can also define a `transform` if we want to lower/upper case, strip whitespace, or even run a custom function.
+
+### Running the Extraction
+
+```python
+import json
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
+
+ecommerce_schema = {
+ # ... the advanced schema from above ...
+}
+
+async def extract_ecommerce_data():
+ strategy = JsonCssExtractionStrategy(ecommerce_schema, verbose=True)
+
+ config = CrawlerRunConfig()
+
+ async with AsyncWebCrawler(verbose=True) as crawler:
+ result = await crawler.arun(
+ url="https://gist.githubusercontent.com/githubusercontent/2d7b8ba3cd8ab6cf3c8da771ddb36878/raw/1ae2f90c6861ce7dd84cc50d3df9920dee5e1fd2/sample_ecommerce.html",
+ extraction_strategy=strategy,
+ config=config
+ )
+
+ if not result.success:
+ print("Crawl failed:", result.error_message)
+ return
+
+ # Parse the JSON output
+ data = json.loads(result.extracted_content)
+ print(json.dumps(data, indent=2) if data else "No data found.")
+
+asyncio.run(extract_ecommerce_data())
+```
+
+If all goes well, you get a **structured** JSON array with each “category,” containing an array of `products`. Each product includes `details`, `features`, `reviews`, etc. All of that **without** an LLM.
+
+---
+
+## 4. Why “No LLM” Is Often Better
+
+1. **Zero Hallucination**: Schema-based extraction doesn’t guess text. It either finds it or not.
+2. **Guaranteed Structure**: The same schema yields consistent JSON across many pages, so your downstream pipeline can rely on stable keys.
+3. **Speed**: LLM-based extraction can be 10–1000x slower for large-scale crawling.
+4. **Scalable**: Adding or updating a field is a matter of adjusting the schema, not re-tuning a model.
+
+**When might you consider an LLM?** Possibly if the site is extremely unstructured or you want AI summarization. But always try a schema approach first for repeated or consistent data patterns.
+
+---
+
+## 5. Base Element Attributes & Additional Fields
+
+It’s easy to **extract attributes** (like `href`, `src`, or `data-xxx`) from your base or nested elements using:
+
+```json
+{
+ "name": "href",
+ "type": "attribute",
+ "attribute": "href",
+ "default": null
+}
+```
+
+You can define them in **`baseFields`** (extracted from the main container element) or in each field’s sub-lists. This is especially helpful if you need an item’s link or ID stored in the parent ``.
+
+---
+
+## 6. Putting It All Together: Larger Example
+
+Consider a blog site. We have a schema that extracts the **URL** from each post card (via `baseFields` with an `"attribute": "href"`), plus the title, date, summary, and author:
+
+```python
+schema = {
+ "name": "Blog Posts",
+ "baseSelector": "a.blog-post-card",
+ "baseFields": [
+ {"name": "post_url", "type": "attribute", "attribute": "href"}
+ ],
+ "fields": [
+ {"name": "title", "selector": "h2.post-title", "type": "text", "default": "No Title"},
+ {"name": "date", "selector": "time.post-date", "type": "text", "default": ""},
+ {"name": "summary", "selector": "p.post-summary", "type": "text", "default": ""},
+ {"name": "author", "selector": "span.post-author", "type": "text", "default": ""}
+ ]
+}
+```
+
+Then run with `JsonCssExtractionStrategy(schema)` to get an array of blog post objects, each with `"post_url"`, `"title"`, `"date"`, `"summary"`, `"author"`.
+
+---
+
+## 7. Tips & Best Practices
+
+1. **Inspect the DOM** in Chrome DevTools or Firefox’s Inspector to find stable selectors.
+2. **Start Simple**: Verify you can extract a single field. Then add complexity like nested objects or lists.
+3. **Test** your schema on partial HTML or a test page before a big crawl.
+4. **Combine with JS Execution** if the site loads content dynamically. You can pass `js_code` or `wait_for` in `CrawlerRunConfig`.
+5. **Look at Logs** when `verbose=True`: if your selectors are off or your schema is malformed, it’ll often show warnings.
+6. **Use baseFields** if you need attributes from the container element (e.g., `href`, `data-id`), especially for the “parent” item.
+7. **Performance**: For large pages, make sure your selectors are as narrow as possible.
+
+---
+
+## 8. Conclusion
+
+With **JsonCssExtractionStrategy** (or **JsonXPathExtractionStrategy**), you can build powerful, **LLM-free** pipelines that:
+
+- Scrape any consistent site for structured data.
+- Support nested objects, repeating lists, or advanced transformations.
+- Scale to thousands of pages quickly and reliably.
+
+**Next Steps**:
+
+- Explore the [Advanced Usage of JSON Extraction](../../explanations/extraction-chunking.md) for deeper details on schema nesting, transformations, or hooking.
+- Combine your extracted JSON with advanced filtering or summarization in a second pass if needed.
+- For dynamic pages, combine strategies with `js_code` or infinite scroll hooking to ensure all content is loaded.
+
+**Remember**: For repeated, structured data, you don’t need to pay for or wait on an LLM. A well-crafted schema plus CSS or XPath gets you the data faster, cleaner, and cheaper—**the real power** of Crawl4AI.
+
+**Last Updated**: 2024-XX-XX
+
+---
+
+That’s it for **Extracting JSON (No LLM)**! You’ve seen how schema-based approaches (either CSS or XPath) can handle everything from simple lists to deeply nested product catalogs—instantly, with minimal overhead. Enjoy building robust scrapers that produce consistent, structured JSON for your data pipelines!
\ No newline at end of file
diff --git a/docs/md_v3/tutorials/json-extraction-llm.md b/docs/md_v3/tutorials/json-extraction-llm.md
new file mode 100644
index 0000000000000000000000000000000000000000..5b9369d9f007f3f66e9a18cffdf76b32ecc2d46c
--- /dev/null
+++ b/docs/md_v3/tutorials/json-extraction-llm.md
@@ -0,0 +1,334 @@
+Below is a **draft** of the **Extracting JSON (LLM)** tutorial, illustrating how to use large language models for structured data extraction in Crawl4AI. It highlights key parameters (like chunking, overlap, instruction, schema) and explains how the system remains **provider-agnostic** via LightLLM. Adjust field names or code snippets to match your repository’s specifics.
+
+---
+
+# Extracting JSON (LLM)
+
+In some cases, you need to extract **complex or unstructured** information from a webpage that a simple CSS/XPath schema cannot easily parse. Or you want **AI**-driven insights, classification, or summarization. For these scenarios, Crawl4AI provides an **LLM-based extraction strategy** that:
+
+1. Works with **any** large language model supported by [LightLLM](https://github.com/LightLLM) (Ollama, OpenAI, Claude, and more).
+2. Automatically splits content into chunks (if desired) to handle token limits, then combines results.
+3. Lets you define a **schema** (like a Pydantic model) or a simpler “block” extraction approach.
+
+**Important**: LLM-based extraction can be slower and costlier than schema-based approaches. If your page data is highly structured, consider using [`JsonCssExtractionStrategy`](./json-extraction-basic.md) or [`JsonXPathExtractionStrategy`](./json-extraction-basic.md) first. But if you need AI to interpret or reorganize content, read on!
+
+---
+
+## 1. Why Use an LLM?
+
+- **Complex Reasoning**: If the site’s data is unstructured, scattered, or full of natural language context.
+- **Semantic Extraction**: Summaries, knowledge graphs, or relational data that require comprehension.
+- **Flexible**: You can pass instructions to the model to do more advanced transformations or classification.
+
+---
+
+## 2. Provider-Agnostic via LightLLM
+
+Crawl4AI uses a “provider string” (e.g., `"openai/gpt-4o"`, `"ollama/llama2.0"`, `"aws/titan"`) to identify your LLM. **Any** model that LightLLM supports is fair game. You just provide:
+
+- **`provider`**: The `
/` identifier (e.g., `"openai/gpt-4"`, `"ollama/llama2"`, `"huggingface/google-flan"`, etc.).
+- **`api_token`**: If needed (for OpenAI, HuggingFace, etc.); local models or Ollama might not require it.
+- **`api_base`** (optional): If your provider has a custom endpoint.
+
+This means you **aren’t locked** into a single LLM vendor. Switch or experiment easily.
+
+---
+
+## 3. How LLM Extraction Works
+
+### 3.1 Flow
+
+1. **Chunking** (optional): The HTML or markdown is split into smaller segments if it’s very long (based on `chunk_token_threshold`, overlap, etc.).
+2. **Prompt Construction**: For each chunk, the library forms a prompt that includes your **`instruction`** (and possibly schema or examples).
+3. **LLM Inference**: Each chunk is sent to the model in parallel or sequentially (depending on your concurrency).
+4. **Combining**: The results from each chunk are merged and parsed into JSON.
+
+### 3.2 `extraction_type`
+
+- **`"schema"`**: The model tries to return JSON conforming to your Pydantic-based schema.
+- **`"block"`**: The model returns freeform text, or smaller JSON structures, which the library collects.
+
+For structured data, `"schema"` is recommended. You provide `schema=YourPydanticModel.model_json_schema()`.
+
+---
+
+## 4. Key Parameters
+
+Below is an overview of important LLM extraction parameters. All are typically set inside `LLMExtractionStrategy(...)`. You then put that strategy in your `CrawlerRunConfig(..., extraction_strategy=...)`.
+
+1. **`provider`** (str): e.g., `"openai/gpt-4"`, `"ollama/llama2"`.
+2. **`api_token`** (str): The API key or token for that model. May not be needed for local models.
+3. **`schema`** (dict): A JSON schema describing the fields you want. Usually generated by `YourModel.model_json_schema()`.
+4. **`extraction_type`** (str): `"schema"` or `"block"`.
+5. **`instruction`** (str): Prompt text telling the LLM what you want extracted. E.g., “Extract these fields as a JSON array.”
+6. **`chunk_token_threshold`** (int): Maximum tokens per chunk. If your content is huge, you can break it up for the LLM.
+7. **`overlap_rate`** (float): Overlap ratio between adjacent chunks. E.g., `0.1` means 10% of each chunk is repeated to preserve context continuity.
+8. **`apply_chunking`** (bool): Set `True` to chunk automatically. If you want a single pass, set `False`.
+9. **`input_format`** (str): Determines **which** crawler result is passed to the LLM. Options include:
+ - `"markdown"`: The raw markdown (default).
+ - `"fit_markdown"`: The filtered “fit” markdown if you used a content filter.
+ - `"html"`: The cleaned or raw HTML.
+10. **`extra_args`** (dict): Additional LLM parameters like `temperature`, `max_tokens`, `top_p`, etc.
+11. **`show_usage()`**: A method you can call to print out usage info (token usage per chunk, total cost if known).
+
+**Example**:
+
+```python
+extraction_strategy = LLMExtractionStrategy(
+ provider="openai/gpt-4",
+ api_token="YOUR_OPENAI_KEY",
+ schema=MyModel.model_json_schema(),
+ extraction_type="schema",
+ instruction="Extract a list of items from the text with 'name' and 'price' fields.",
+ chunk_token_threshold=1200,
+ overlap_rate=0.1,
+ apply_chunking=True,
+ input_format="html",
+ extra_args={"temperature": 0.1, "max_tokens": 1000},
+ verbose=True
+)
+```
+
+---
+
+## 5. Putting It in `CrawlerRunConfig`
+
+**Important**: In Crawl4AI, all strategy definitions should go inside the `CrawlerRunConfig`, not directly as a param in `arun()`. Here’s a full example:
+
+```python
+import os
+import asyncio
+import json
+from pydantic import BaseModel, Field
+from typing import List
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+class Product(BaseModel):
+ name: str
+ price: str
+
+async def main():
+ # 1. Define the LLM extraction strategy
+ llm_strategy = LLMExtractionStrategy(
+ provider="openai/gpt-4o-mini", # e.g. "ollama/llama2"
+ api_token=os.getenv('OPENAI_API_KEY'),
+ schema=Product.schema_json(), # Or use model_json_schema()
+ extraction_type="schema",
+ instruction="Extract all product objects with 'name' and 'price' from the content.",
+ chunk_token_threshold=1000,
+ overlap_rate=0.0,
+ apply_chunking=True,
+ input_format="markdown", # or "html", "fit_markdown"
+ extra_args={"temperature": 0.0, "max_tokens": 800}
+ )
+
+ # 2. Build the crawler config
+ crawl_config = CrawlerRunConfig(
+ extraction_strategy=llm_strategy,
+ cache_mode=CacheMode.BYPASS
+ )
+
+ # 3. Create a browser config if needed
+ browser_cfg = BrowserConfig(headless=True)
+
+ async with AsyncWebCrawler(config=browser_cfg) as crawler:
+ # 4. Let's say we want to crawl a single page
+ result = await crawler.arun(
+ url="https://example.com/products",
+ config=crawl_config
+ )
+
+ if result.success:
+ # 5. The extracted content is presumably JSON
+ data = json.loads(result.extracted_content)
+ print("Extracted items:", data)
+
+ # 6. Show usage stats
+ llm_strategy.show_usage() # prints token usage
+ else:
+ print("Error:", result.error_message)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+## 6. Chunking Details
+
+### 6.1 `chunk_token_threshold`
+
+If your page is large, you might exceed your LLM’s context window. **`chunk_token_threshold`** sets the approximate max tokens per chunk. The library calculates word→token ratio using `word_token_rate` (often ~0.75 by default). If chunking is enabled (`apply_chunking=True`), the text is split into segments.
+
+### 6.2 `overlap_rate`
+
+To keep context continuous across chunks, we can overlap them. E.g., `overlap_rate=0.1` means each subsequent chunk includes 10% of the previous chunk’s text. This is helpful if your needed info might straddle chunk boundaries.
+
+### 6.3 Performance & Parallelism
+
+By chunking, you can potentially process multiple chunks in parallel (depending on your concurrency settings and the LLM provider). This reduces total time if the site is huge or has many sections.
+
+---
+
+## 7. Input Format
+
+By default, **LLMExtractionStrategy** uses `input_format="markdown"`, meaning the **crawler’s final markdown** is fed to the LLM. You can change to:
+
+- **`html`**: The cleaned HTML or raw HTML (depending on your crawler config) goes into the LLM.
+- **`fit_markdown`**: If you used, for instance, `PruningContentFilter`, the “fit” version of the markdown is used. This can drastically reduce tokens if you trust the filter.
+- **`markdown`**: Standard markdown output from the crawler’s `markdown_generator`.
+
+This setting is crucial: if the LLM instructions rely on HTML tags, pick `"html"`. If you prefer a text-based approach, pick `"markdown"`.
+
+```python
+LLMExtractionStrategy(
+ # ...
+ input_format="html", # Instead of "markdown" or "fit_markdown"
+)
+```
+
+---
+
+## 8. Token Usage & Show Usage
+
+To keep track of tokens and cost, each chunk is processed with an LLM call. We record usage in:
+
+- **`usages`** (list): token usage per chunk or call.
+- **`total_usage`**: sum of all chunk calls.
+- **`show_usage()`**: prints a usage report (if the provider returns usage data).
+
+```python
+llm_strategy = LLMExtractionStrategy(...)
+# ...
+llm_strategy.show_usage()
+# e.g. “Total usage: 1241 tokens across 2 chunk calls”
+```
+
+If your model provider doesn’t return usage info, these fields might be partial or empty.
+
+---
+
+## 9. Example: Building a Knowledge Graph
+
+Below is a snippet combining **`LLMExtractionStrategy`** with a Pydantic schema for a knowledge graph. Notice how we pass an **`instruction`** telling the model what to parse.
+
+```python
+import os
+import json
+import asyncio
+from typing import List
+from pydantic import BaseModel, Field
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
+from crawl4ai.extraction_strategy import LLMExtractionStrategy
+
+class Entity(BaseModel):
+ name: str
+ description: str
+
+class Relationship(BaseModel):
+ entity1: Entity
+ entity2: Entity
+ description: str
+ relation_type: str
+
+class KnowledgeGraph(BaseModel):
+ entities: List[Entity]
+ relationships: List[Relationship]
+
+async def main():
+ # LLM extraction strategy
+ llm_strat = LLMExtractionStrategy(
+ provider="openai/gpt-4",
+ api_token=os.getenv('OPENAI_API_KEY'),
+ schema=KnowledgeGraph.schema_json(),
+ extraction_type="schema",
+ instruction="Extract entities and relationships from the content. Return valid JSON.",
+ chunk_token_threshold=1400,
+ apply_chunking=True,
+ input_format="html",
+ extra_args={"temperature": 0.1, "max_tokens": 1500}
+ )
+
+ crawl_config = CrawlerRunConfig(
+ extraction_strategy=llm_strat,
+ cache_mode=CacheMode.BYPASS
+ )
+
+ async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
+ # Example page
+ url = "https://www.nbcnews.com/business"
+ result = await crawler.arun(url=url, config=crawl_config)
+
+ if result.success:
+ with open("kb_result.json", "w", encoding="utf-8") as f:
+ f.write(result.extracted_content)
+ llm_strat.show_usage()
+ else:
+ print("Crawl failed:", result.error_message)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**Key Observations**:
+
+- **`extraction_type="schema"`** ensures we get JSON fitting our `KnowledgeGraph`.
+- **`input_format="html"`** means we feed HTML to the model.
+- **`instruction`** guides the model to output a structured knowledge graph.
+
+---
+
+## 10. Best Practices & Caveats
+
+1. **Cost & Latency**: LLM calls can be slow or expensive. Consider chunking or smaller coverage if you only need partial data.
+2. **Model Token Limits**: If your page + instruction exceed the context window, chunking is essential.
+3. **Instruction Engineering**: Well-crafted instructions can drastically improve output reliability.
+4. **Schema Strictness**: `"schema"` extraction tries to parse the model output as JSON. If the model returns invalid JSON, partial extraction might happen, or you might get an error.
+5. **Parallel vs. Serial**: The library can process multiple chunks in parallel, but you must watch out for rate limits on certain providers.
+6. **Check Output**: Sometimes, an LLM might omit fields or produce extraneous text. You may want to post-validate with Pydantic or do additional cleanup.
+
+---
+
+## 11. Conclusion
+
+**LLM-based extraction** in Crawl4AI is **provider-agnostic**, letting you choose from hundreds of models via LightLLM. It’s perfect for **semantically complex** tasks or generating advanced structures like knowledge graphs. However, it’s **slower** and potentially costlier than schema-based approaches. Keep these tips in mind:
+
+- Put your LLM strategy **in `CrawlerRunConfig`**.
+- Use **`input_format`** to pick which form (markdown, HTML, fit_markdown) the LLM sees.
+- Tweak **`chunk_token_threshold`**, **`overlap_rate`**, and **`apply_chunking`** to handle large content efficiently.
+- Monitor token usage with `show_usage()`.
+
+If your site’s data is consistent or repetitive, consider [`JsonCssExtractionStrategy`](./json-extraction-basic.md) first for speed and simplicity. But if you need an **AI-driven** approach, `LLMExtractionStrategy` offers a flexible, multi-provider solution for extracting structured JSON from any website.
+
+**Next Steps**:
+
+1. **Experiment with Different Providers**
+ - Try switching the `provider` (e.g., `"ollama/llama2"`, `"openai/gpt-4o"`, etc.) to see differences in speed, accuracy, or cost.
+ - Pass different `extra_args` like `temperature`, `top_p`, and `max_tokens` to fine-tune your results.
+
+2. **Combine With Other Strategies**
+ - Use [content filters](../../how-to/content-filters.md) like BM25 or Pruning prior to LLM extraction to remove noise and reduce token usage.
+ - Apply a [CSS or XPath extraction strategy](./json-extraction-basic.md) first for obvious, structured data, then send only the tricky parts to the LLM.
+
+3. **Performance Tuning**
+ - If pages are large, tweak `chunk_token_threshold`, `overlap_rate`, or `apply_chunking` to optimize throughput.
+ - Check the usage logs with `show_usage()` to keep an eye on token consumption and identify potential bottlenecks.
+
+4. **Validate Outputs**
+ - If using `extraction_type="schema"`, parse the LLM’s JSON with a Pydantic model for a final validation step.
+ - Log or handle any parse errors gracefully, especially if the model occasionally returns malformed JSON.
+
+5. **Explore Hooks & Automation**
+ - Integrate LLM extraction with [hooks](./hooks-custom.md) for complex pre/post-processing.
+ - Use a multi-step pipeline: crawl, filter, LLM-extract, then store or index results for further analysis.
+
+6. **Scale and Deploy**
+ - Combine your LLM extraction setup with [Docker or other deployment solutions](./docker-quickstart.md) to run at scale.
+ - Monitor memory usage and concurrency if you call LLMs frequently.
+
+**Last Updated**: 2024-XX-XX
+
+---
+
+That’s it for **Extracting JSON (LLM)**—now you can harness AI to parse, classify, or reorganize data on the web. Happy crawling!
\ No newline at end of file
diff --git a/docs/md_v3/tutorials/link-media-analysis.md b/docs/md_v3/tutorials/link-media-analysis.md
new file mode 100644
index 0000000000000000000000000000000000000000..229fad8d8ade2ad2148cb75c3c9892abce3d4aaf
--- /dev/null
+++ b/docs/md_v3/tutorials/link-media-analysis.md
@@ -0,0 +1,295 @@
+Below is a **draft** of the **“Link & Media Analysis”** tutorial. It demonstrates how to access and filter links, handle domain restrictions, and manage media (especially images) using Crawl4AI’s configuration options. Feel free to adjust examples and text to match your exact workflow or preferences.
+
+---
+
+# Link & Media Analysis
+
+In this tutorial, you’ll learn how to:
+
+1. Extract links (internal, external) from crawled pages
+2. Filter or exclude specific domains (e.g., social media or custom domains)
+3. Access and manage media data (especially images) in the crawl result
+4. Configure your crawler to exclude or prioritize certain images
+
+> **Prerequisites**
+> - You have completed or are familiar with the [AsyncWebCrawler Basics](./async-webcrawler-basics.md) tutorial.
+> - You can run Crawl4AI in your environment (Playwright, Python, etc.).
+
+---
+
+Below is a revised version of the **Link Extraction** and **Media Extraction** sections that includes example data structures showing how links and media items are stored in `CrawlResult`. Feel free to adjust any field names or descriptions to match your actual output.
+
+---
+
+## 1. Link Extraction
+
+### 1.1 `result.links`
+
+When you call `arun()` or `arun_many()` on a URL, Crawl4AI automatically extracts links and stores them in the `links` field of `CrawlResult`. By default, the crawler tries to distinguish **internal** links (same domain) from **external** links (different domains).
+
+**Basic Example**:
+
+```python
+from crawl4ai import AsyncWebCrawler
+
+async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun("https://www.example.com")
+ if result.success:
+ internal_links = result.links.get("internal", [])
+ external_links = result.links.get("external", [])
+ print(f"Found {len(internal_links)} internal links, {len(external_links)} external links.")
+
+ # Each link is typically a dictionary with fields like:
+ # { "href": "...", "text": "...", "title": "...", "base_domain": "..." }
+ if internal_links:
+ print("Sample Internal Link:", internal_links[0])
+ else:
+ print("Crawl failed:", result.error_message)
+```
+
+**Structure Example**:
+
+```python
+result.links = {
+ "internal": [
+ {
+ "href": "https://kidocode.com/",
+ "text": "",
+ "title": "",
+ "base_domain": "kidocode.com"
+ },
+ {
+ "href": "https://kidocode.com/degrees/technology",
+ "text": "Technology Degree",
+ "title": "KidoCode Tech Program",
+ "base_domain": "kidocode.com"
+ },
+ # ...
+ ],
+ "external": [
+ # possibly other links leading to third-party sites
+ ]
+}
+```
+
+- **`href`**: The raw hyperlink URL.
+- **`text`**: The link text (if any) within the `` tag.
+- **`title`**: The `title` attribute of the link (if present).
+- **`base_domain`**: The domain extracted from `href`. Helpful for filtering or grouping by domain.
+
+---
+
+## 2. Domain Filtering
+
+Some websites contain hundreds of third-party or affiliate links. You can filter out certain domains at **crawl time** by configuring the crawler. The most relevant parameters in `CrawlerRunConfig` are:
+
+- **`exclude_external_links`**: If `True`, discard any link pointing outside the root domain.
+- **`exclude_social_media_domains`**: Provide a list of social media platforms (e.g., `["facebook.com", "twitter.com"]`) to exclude from your crawl.
+- **`exclude_social_media_links`**: If `True`, automatically skip known social platforms.
+- **`exclude_domains`**: Provide a list of custom domains you want to exclude (e.g., `["spammyads.com", "tracker.net"]`).
+
+### 2.1 Example: Excluding External & Social Media Links
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def main():
+ crawler_cfg = CrawlerRunConfig(
+ exclude_external_links=True, # No links outside primary domain
+ exclude_social_media_links=True # Skip recognized social media domains
+ )
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun(
+ "https://www.example.com",
+ config=crawler_cfg
+ )
+ if result.success:
+ print("[OK] Crawled:", result.url)
+ print("Internal links count:", len(result.links.get("internal", [])))
+ print("External links count:", len(result.links.get("external", [])))
+ # Likely zero external links in this scenario
+ else:
+ print("[ERROR]", result.error_message)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+### 2.2 Example: Excluding Specific Domains
+
+If you want to let external links in, but specifically exclude a domain (e.g., `suspiciousads.com`), do this:
+
+```python
+crawler_cfg = CrawlerRunConfig(
+ exclude_domains=["suspiciousads.com"]
+)
+```
+
+This approach is handy when you still want external links but need to block certain sites you consider spammy.
+
+---
+
+## 3. Media Extraction
+
+### 3.1 Accessing `result.media`
+
+By default, Crawl4AI collects images, audio, and video URLs it finds on the page. These are stored in `result.media`, a dictionary keyed by media type (e.g., `images`, `videos`, `audio`).
+
+**Basic Example**:
+
+```python
+if result.success:
+ images_info = result.media.get("images", [])
+ print(f"Found {len(images_info)} images in total.")
+ for i, img in enumerate(images_info[:5]): # Inspect just the first 5
+ print(f"[Image {i}] URL: {img['src']}")
+ print(f" Alt text: {img.get('alt', '')}")
+ print(f" Score: {img.get('score')}")
+ print(f" Description: {img.get('desc', '')}\n")
+```
+
+**Structure Example**:
+
+```python
+result.media = {
+ "images": [
+ {
+ "src": "https://cdn.prod.website-files.com/.../Group%2089.svg",
+ "alt": "coding school for kids",
+ "desc": "Trial Class Degrees degrees All Degrees AI Degree Technology ...",
+ "score": 3,
+ "type": "image",
+ "group_id": 0,
+ "format": None,
+ "width": None,
+ "height": None
+ },
+ # ...
+ ],
+ "videos": [
+ # Similar structure but with video-specific fields
+ ],
+ "audio": [
+ # Similar structure but with audio-specific fields
+ ]
+}
+```
+
+Depending on your Crawl4AI version or scraping strategy, these dictionaries can include fields like:
+
+- **`src`**: The media URL (e.g., image source)
+- **`alt`**: The alt text for images (if present)
+- **`desc`**: A snippet of nearby text or a short description (optional)
+- **`score`**: A heuristic relevance score if you’re using content-scoring features
+- **`width`**, **`height`**: If the crawler detects dimensions for the image/video
+- **`type`**: Usually `"image"`, `"video"`, or `"audio"`
+- **`group_id`**: If you’re grouping related media items, the crawler might assign an ID
+
+With these details, you can easily filter out or focus on certain images (for instance, ignoring images with very low scores or a different domain), or gather metadata for analytics.
+
+### 3.2 Excluding External Images
+
+If you’re dealing with heavy pages or want to skip third-party images (advertisements, for example), you can turn on:
+
+```python
+crawler_cfg = CrawlerRunConfig(
+ exclude_external_images=True
+)
+```
+
+This setting attempts to discard images from outside the primary domain, keeping only those from the site you’re crawling.
+
+### 3.3 Additional Media Config
+
+- **`screenshot`**: Set to `True` if you want a full-page screenshot stored as `base64` in `result.screenshot`.
+- **`pdf`**: Set to `True` if you want a PDF version of the page in `result.pdf`.
+- **`wait_for_images`**: If `True`, attempts to wait until images are fully loaded before final extraction.
+
+---
+
+## 4. Putting It All Together: Link & Media Filtering
+
+Here’s a combined example demonstrating how to filter out external links, skip certain domains, and exclude external images:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def main():
+ # Suppose we want to keep only internal links, remove certain domains,
+ # and discard external images from the final crawl data.
+ crawler_cfg = CrawlerRunConfig(
+ exclude_external_links=True,
+ exclude_domains=["spammyads.com"],
+ exclude_social_media_links=True, # skip Twitter, Facebook, etc.
+ exclude_external_images=True, # keep only images from main domain
+ wait_for_images=True, # ensure images are loaded
+ verbose=True
+ )
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun("https://www.example.com", config=crawler_cfg)
+
+ if result.success:
+ print("[OK] Crawled:", result.url)
+
+ # 1. Links
+ in_links = result.links.get("internal", [])
+ ext_links = result.links.get("external", [])
+ print("Internal link count:", len(in_links))
+ print("External link count:", len(ext_links)) # should be zero with exclude_external_links=True
+
+ # 2. Images
+ images = result.media.get("images", [])
+ print("Images found:", len(images))
+
+ # Let's see a snippet of these images
+ for i, img in enumerate(images[:3]):
+ print(f" - {img['src']} (alt={img.get('alt','')}, score={img.get('score','N/A')})")
+ else:
+ print("[ERROR] Failed to crawl. Reason:", result.error_message)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+## 5. Common Pitfalls & Tips
+
+1. **Conflicting Flags**:
+ - `exclude_external_links=True` but then also specifying `exclude_social_media_links=True` is typically fine, but understand that the first setting already discards *all* external links. The second becomes somewhat redundant.
+ - `exclude_external_images=True` but want to keep some external images? Currently no partial domain-based setting for images, so you might need a custom approach or hook logic.
+
+2. **Relevancy Scores**:
+ - If your version of Crawl4AI or your scraping strategy includes an `img["score"]`, it’s typically a heuristic based on size, position, or content analysis. Evaluate carefully if you rely on it.
+
+3. **Performance**:
+ - Excluding certain domains or external images can speed up your crawl, especially for large, media-heavy pages.
+ - If you want a “full” link map, do *not* exclude them. Instead, you can post-filter in your own code.
+
+4. **Social Media Lists**:
+ - `exclude_social_media_links=True` typically references an internal list of known social domains like Facebook, Twitter, LinkedIn, etc. If you need to add or remove from that list, look for library settings or a local config file (depending on your version).
+
+---
+
+## 6. Next Steps
+
+Now that you understand how to manage **Link & Media Analysis**, you can:
+
+- Fine-tune which links are stored or discarded in your final results
+- Control which images (or other media) appear in `result.media`
+- Filter out entire domains or social media platforms to keep your dataset relevant
+
+**Recommended Follow-Ups**:
+- **[Advanced Features (Proxy, PDF, Screenshots)](./advanced-features.md)**: If you want to capture screenshots or save the page as a PDF for archival or debugging.
+- **[Hooks & Custom Code](./hooks-custom.md)**: For more specialized logic, such as automated “infinite scroll” or repeated “Load More” button clicks.
+- **Reference**: Check out [CrawlerRunConfig Reference](../../reference/configuration.md) for a comprehensive parameter list.
+
+**Last updated**: 2024-XX-XX
+
+---
+
+**That’s it for Link & Media Analysis!** You’re now equipped to filter out unwanted sites and zero in on the images and videos that matter for your project.
\ No newline at end of file
diff --git a/docs/md_v3/tutorials/markdown-basics.md b/docs/md_v3/tutorials/markdown-basics.md
new file mode 100644
index 0000000000000000000000000000000000000000..48498709e38a2edeb345f9e4da5818d94e18eebf
--- /dev/null
+++ b/docs/md_v3/tutorials/markdown-basics.md
@@ -0,0 +1,382 @@
+Below is a **draft** of the **Markdown Generation Basics** tutorial that incorporates your current Crawl4AI design and terminology. It introduces the default markdown generator, explains the concept of content filters (BM25 and Pruning), and covers the `MarkdownGenerationResult` object in a coherent, step-by-step manner. Adjust parameters or naming as needed to align with your actual codebase.
+
+---
+
+# Markdown Generation Basics
+
+One of Crawl4AI’s core features is generating **clean, structured markdown** from web pages. Originally built to solve the problem of extracting only the “actual” content and discarding boilerplate or noise, Crawl4AI’s markdown system remains one of its biggest draws for AI workflows.
+
+In this tutorial, you’ll learn:
+
+1. How to configure the **Default Markdown Generator**
+2. How **content filters** (BM25 or Pruning) help you refine markdown and discard junk
+3. The difference between raw markdown (`result.markdown`) and filtered markdown (`fit_markdown`)
+
+> **Prerequisites**
+> - You’ve completed or read [AsyncWebCrawler Basics](./async-webcrawler-basics.md) to understand how to run a simple crawl.
+> - You know how to configure `CrawlerRunConfig`.
+
+---
+
+## 1. Quick Example
+
+Here’s a minimal code snippet that uses the **DefaultMarkdownGenerator** with no additional filtering:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+async def main():
+ config = CrawlerRunConfig(
+ markdown_generator=DefaultMarkdownGenerator()
+ )
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun("https://example.com", config=config)
+
+ if result.success:
+ print("Raw Markdown Output:\n")
+ print(result.markdown) # The unfiltered markdown from the page
+ else:
+ print("Crawl failed:", result.error_message)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**What’s happening?**
+- `CrawlerRunConfig(markdown_generator=DefaultMarkdownGenerator())` instructs Crawl4AI to convert the final HTML into markdown at the end of each crawl.
+- The resulting markdown is accessible via `result.markdown`.
+
+---
+
+## 2. How Markdown Generation Works
+
+### 2.1 HTML-to-Text Conversion (Forked & Modified)
+
+Under the hood, **DefaultMarkdownGenerator** uses a specialized HTML-to-text approach that:
+
+- Preserves headings, code blocks, bullet points, etc.
+- Removes extraneous tags (scripts, styles) that don’t add meaningful content.
+- Can optionally generate references for links or skip them altogether.
+
+A set of **options** (passed as a dict) allows you to customize precisely how HTML converts to markdown. These map to standard html2text-like configuration plus your own enhancements (e.g., ignoring internal links, preserving certain tags verbatim, or adjusting line widths).
+
+### 2.2 Link Citations & References
+
+By default, the generator can convert ` ` elements into `[text][1]` citations, then place the actual links at the bottom of the document. This is handy for research workflows that demand references in a structured manner.
+
+### 2.3 Optional Content Filters
+
+Before or after the HTML-to-Markdown step, you can apply a **content filter** (like BM25 or Pruning) to reduce noise and produce a “fit_markdown”—a heavily pruned version focusing on the page’s main text. We’ll cover these filters shortly.
+
+---
+
+## 3. Configuring the Default Markdown Generator
+
+You can tweak the output by passing an `options` dict to `DefaultMarkdownGenerator`. For example:
+
+```python
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+
+async def main():
+ # Example: ignore all links, don't escape HTML, and wrap text at 80 characters
+ md_generator = DefaultMarkdownGenerator(
+ options={
+ "ignore_links": True,
+ "escape_html": False,
+ "body_width": 80
+ }
+ )
+
+ config = CrawlerRunConfig(
+ markdown_generator=md_generator
+ )
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun("https://example.com/docs", config=config)
+ if result.success:
+ print("Markdown:\n", result.markdown[:500]) # Just a snippet
+ else:
+ print("Crawl failed:", result.error_message)
+
+if __name__ == "__main__":
+ import asyncio
+ asyncio.run(main())
+```
+
+Some commonly used `options`:
+
+- **`ignore_links`** (bool): Whether to remove all hyperlinks in the final markdown.
+- **`ignore_images`** (bool): Remove all `![image]()` references.
+- **`escape_html`** (bool): Turn HTML entities into text (default is often `True`).
+- **`body_width`** (int): Wrap text at N characters. `0` or `None` means no wrapping.
+- **`skip_internal_links`** (bool): If `True`, omit `#localAnchors` or internal links referencing the same page.
+- **`include_sup_sub`** (bool): Attempt to handle `` / `` in a more readable way.
+
+---
+
+## 4. Content Filters
+
+**Content filters** selectively remove or rank sections of text before turning them into Markdown. This is especially helpful if your page has ads, nav bars, or other clutter you don’t want.
+
+### 4.1 BM25ContentFilter
+
+If you have a **search query**, BM25 is a good choice:
+
+```python
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import BM25ContentFilter
+from crawl4ai import CrawlerRunConfig
+
+bm25_filter = BM25ContentFilter(
+ user_query="machine learning",
+ bm25_threshold=1.2,
+ use_stemming=True
+)
+
+md_generator = DefaultMarkdownGenerator(
+ content_filter=bm25_filter,
+ options={"ignore_links": True}
+)
+
+config = CrawlerRunConfig(markdown_generator=md_generator)
+```
+
+- **`user_query`**: The term you want to focus on. BM25 tries to keep only content blocks relevant to that query.
+- **`bm25_threshold`**: Raise it to keep fewer blocks; lower it to keep more.
+- **`use_stemming`**: If `True`, variations of words match (e.g., “learn,” “learning,” “learnt”).
+
+**No query provided?** BM25 tries to glean a context from page metadata, or you can simply treat it as a scorched-earth approach that discards text with low generic score. Realistically, you want to supply a query for best results.
+
+### 4.2 PruningContentFilter
+
+If you **don’t** have a specific query, or if you just want a robust “junk remover,” use `PruningContentFilter`. It analyzes text density, link density, HTML structure, and known patterns (like “nav,” “footer”) to systematically prune extraneous or repetitive sections.
+
+```python
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+prune_filter = PruningContentFilter(
+ threshold=0.5,
+ threshold_type="fixed", # or "dynamic"
+ min_word_threshold=50
+)
+```
+
+- **`threshold`**: Score boundary. Blocks below this score get removed.
+- **`threshold_type`**:
+ - `"fixed"`: Straight comparison (`score >= threshold` keeps the block).
+ - `"dynamic"`: The filter adjusts threshold in a data-driven manner.
+- **`min_word_threshold`**: Discard blocks under N words as likely too short or unhelpful.
+
+**When to Use PruningContentFilter**
+- You want a broad cleanup without a user query.
+- The page has lots of repeated sidebars, footers, or disclaimers that hamper text extraction.
+
+---
+
+## 5. Using Fit Markdown
+
+When a content filter is active, the library produces two forms of markdown inside `result.markdown_v2` or (if using the simplified field) `result.markdown`:
+
+1. **`raw_markdown`**: The full unfiltered markdown.
+2. **`fit_markdown`**: A “fit” version where the filter has removed or trimmed noisy segments.
+
+**Note**:
+- In earlier examples, you may see references to `result.markdown_v2`. Depending on your library version, you might access `result.markdown`, `result.markdown_v2`, or an object named `MarkdownGenerationResult`. The idea is the same: you’ll have a raw version and a filtered (“fit”) version if a filter is used.
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+async def main():
+ config = CrawlerRunConfig(
+ markdown_generator=DefaultMarkdownGenerator(
+ content_filter=PruningContentFilter(threshold=0.6),
+ options={"ignore_links": True}
+ )
+ )
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun("https://news.example.com/tech", config=config)
+ if result.success:
+ print("Raw markdown:\n", result.markdown)
+
+ # If a filter is used, we also have .fit_markdown:
+ md_object = result.markdown_v2 # or your equivalent
+ print("Filtered markdown:\n", md_object.fit_markdown)
+ else:
+ print("Crawl failed:", result.error_message)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+---
+
+## 6. The `MarkdownGenerationResult` Object
+
+If your library stores detailed markdown output in an object like `MarkdownGenerationResult`, you’ll see fields such as:
+
+- **`raw_markdown`**: The direct HTML-to-markdown transformation (no filtering).
+- **`markdown_with_citations`**: A version that moves links to reference-style footnotes.
+- **`references_markdown`**: A separate string or section containing the gathered references.
+- **`fit_markdown`**: The filtered markdown if you used a content filter.
+- **`fit_html`**: The corresponding HTML snippet used to generate `fit_markdown` (helpful for debugging or advanced usage).
+
+**Example**:
+
+```python
+md_obj = result.markdown_v2 # your library’s naming may vary
+print("RAW:\n", md_obj.raw_markdown)
+print("CITED:\n", md_obj.markdown_with_citations)
+print("REFERENCES:\n", md_obj.references_markdown)
+print("FIT:\n", md_obj.fit_markdown)
+```
+
+**Why Does This Matter?**
+- You can supply `raw_markdown` to an LLM if you want the entire text.
+- Or feed `fit_markdown` into a vector database to reduce token usage.
+- `references_markdown` can help you keep track of link provenance.
+
+---
+
+Below is a **revised section** under “Combining Filters (BM25 + Pruning)” that demonstrates how you can run **two** passes of content filtering without re-crawling, by taking the HTML (or text) from a first pass and feeding it into the second filter. It uses real code patterns from the snippet you provided for **BM25ContentFilter**, which directly accepts **HTML** strings (and can also handle plain text with minimal adaptation).
+
+---
+
+## 7. Combining Filters (BM25 + Pruning) in Two Passes
+
+You might want to **prune out** noisy boilerplate first (with `PruningContentFilter`), and then **rank what’s left** against a user query (with `BM25ContentFilter`). You don’t have to crawl the page twice. Instead:
+
+1. **First pass**: Apply `PruningContentFilter` directly to the raw HTML from `result.html` (the crawler’s downloaded HTML).
+2. **Second pass**: Take the pruned HTML (or text) from step 1, and feed it into `BM25ContentFilter`, focusing on a user query.
+
+### Two-Pass Example
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
+from crawl4ai.content_filter_strategy import PruningContentFilter, BM25ContentFilter
+from bs4 import BeautifulSoup
+
+async def main():
+ # 1. Crawl with minimal or no markdown generator, just get raw HTML
+ config = CrawlerRunConfig(
+ # If you only want raw HTML, you can skip passing a markdown_generator
+ # or provide one but focus on .html in this example
+ )
+
+ async with AsyncWebCrawler() as crawler:
+ result = await crawler.arun("https://example.com/tech-article", config=config)
+
+ if not result.success or not result.html:
+ print("Crawl failed or no HTML content.")
+ return
+
+ raw_html = result.html
+
+ # 2. First pass: PruningContentFilter on raw HTML
+ pruning_filter = PruningContentFilter(threshold=0.5, min_word_threshold=50)
+
+ # filter_content returns a list of "text chunks" or cleaned HTML sections
+ pruned_chunks = pruning_filter.filter_content(raw_html)
+ # This list is basically pruned content blocks, presumably in HTML or text form
+
+ # For demonstration, let's combine these chunks back into a single HTML-like string
+ # or you could do further processing. It's up to your pipeline design.
+ pruned_html = "\n".join(pruned_chunks)
+
+ # 3. Second pass: BM25ContentFilter with a user query
+ bm25_filter = BM25ContentFilter(
+ user_query="machine learning",
+ bm25_threshold=1.2,
+ language="english"
+ )
+
+ bm25_chunks = bm25_filter.filter_content(pruned_html) # returns a list of text chunks
+
+ if not bm25_chunks:
+ print("Nothing matched the BM25 query after pruning.")
+ return
+
+ # 4. Combine or display final results
+ final_text = "\n---\n".join(bm25_chunks)
+
+ print("==== PRUNED OUTPUT (first pass) ====")
+ print(pruned_html[:500], "... (truncated)") # preview
+
+ print("\n==== BM25 OUTPUT (second pass) ====")
+ print(final_text[:500], "... (truncated)")
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+### What’s Happening?
+
+1. **Raw HTML**: We crawl once and store the raw HTML in `result.html`.
+2. **PruningContentFilter**: Takes HTML + optional parameters. It extracts blocks of text or partial HTML, removing headings/sections deemed “noise.” It returns a **list of text chunks**.
+3. **Combine or Transform**: We join these pruned chunks back into a single HTML-like string. (Alternatively, you could store them in a list for further logic—whatever suits your pipeline.)
+4. **BM25ContentFilter**: We feed the pruned string into `BM25ContentFilter` with a user query. This second pass further narrows the content to chunks relevant to “machine learning.”
+
+**No Re-Crawling**: We used `raw_html` from the first pass, so there’s no need to run `arun()` again—**no second network request**.
+
+### Tips & Variations
+
+- **Plain Text vs. HTML**: If your pruned output is mostly text, BM25 can still handle it; just keep in mind it expects a valid string input. If you supply partial HTML (like `"some text
"`), it will parse it as HTML.
+- **Chaining in a Single Pipeline**: If your code supports it, you can chain multiple filters automatically. Otherwise, manual two-pass filtering (as shown) is straightforward.
+- **Adjust Thresholds**: If you see too much or too little text in step one, tweak `threshold=0.5` or `min_word_threshold=50`. Similarly, `bm25_threshold=1.2` can be raised/lowered for more or fewer chunks in step two.
+
+### One-Pass Combination?
+
+If your codebase or pipeline design allows applying multiple filters in one pass, you could do so. But often it’s simpler—and more transparent—to run them sequentially, analyzing each step’s result.
+
+**Bottom Line**: By **manually chaining** your filtering logic in two passes, you get powerful incremental control over the final content. First, remove “global” clutter with Pruning, then refine further with BM25-based query relevance—without incurring a second network crawl.
+
+---
+
+## 8. Common Pitfalls & Tips
+
+1. **No Markdown Output?**
+ - Make sure the crawler actually retrieved HTML. If the site is heavily JS-based, you may need to enable dynamic rendering or wait for elements.
+ - Check if your content filter is too aggressive. Lower thresholds or disable the filter to see if content reappears.
+
+2. **Performance Considerations**
+ - Very large pages with multiple filters can be slower. Consider `cache_mode` to avoid re-downloading.
+ - If your final use case is LLM ingestion, consider summarizing further or chunking big texts.
+
+3. **Take Advantage of `fit_markdown`**
+ - Great for RAG pipelines, semantic search, or any scenario where extraneous boilerplate is unwanted.
+ - Still verify the textual quality—some sites have crucial data in footers or sidebars.
+
+4. **Adjusting `html2text` Options**
+ - If you see lots of raw HTML slipping into the text, turn on `escape_html`.
+ - If code blocks look messy, experiment with `mark_code` or `handle_code_in_pre`.
+
+---
+
+## 9. Summary & Next Steps
+
+In this **Markdown Generation Basics** tutorial, you learned to:
+
+- Configure the **DefaultMarkdownGenerator** with HTML-to-text options.
+- Use **BM25ContentFilter** for query-specific extraction or **PruningContentFilter** for general noise removal.
+- Distinguish between raw and filtered markdown (`fit_markdown`).
+- Leverage the `MarkdownGenerationResult` object to handle different forms of output (citations, references, etc.).
+
+**Where to go from here**:
+
+- **[Extracting JSON (No LLM)](./json-extraction-basic.md)**: If you need structured data instead of markdown, check out the library’s JSON extraction strategies.
+- **[Advanced Features](./advanced-features.md)**: Combine markdown generation with proxies, PDF exports, and more.
+- **[Explanations → Content Filters vs. Extraction Strategies](../../explanations/extraction-chunking.md)**: Dive deeper into how filters differ from chunking or semantic extraction.
+
+Now you can produce high-quality Markdown from any website, focusing on exactly the content you need—an essential step for powering AI models, summarization pipelines, or knowledge-base queries.
+
+**Last Updated**: 2024-XX-XX
+
+---
+
+That’s it for **Markdown Generation Basics**! Enjoy generating clean, noise-free markdown for your LLM workflows, content archives, or research.
\ No newline at end of file
diff --git a/docs/md_v3/tutorials/targeted-crawling.md b/docs/md_v3/tutorials/targeted-crawling.md
new file mode 100644
index 0000000000000000000000000000000000000000..f5fe2b77c8dd22281a05e66a47c4b23e19cdbc82
--- /dev/null
+++ b/docs/md_v3/tutorials/targeted-crawling.md
@@ -0,0 +1,227 @@
+Below is a **draft** of a follow-up tutorial, **“Smart Crawling Techniques,”** building on the **“AsyncWebCrawler Basics”** tutorial. This tutorial focuses on three main points:
+
+1. **Advanced usage of CSS selectors** (e.g., partial extraction, exclusions)
+2. **Handling iframes** (if relevant for your workflow)
+3. **Waiting for dynamic content** using `wait_for`, including the new `css:` and `js:` prefixes
+
+Feel free to adjust code snippets, wording, or emphasis to match your library updates or user feedback.
+
+---
+
+# Smart Crawling Techniques
+
+In the previous tutorial ([AsyncWebCrawler Basics](./async-webcrawler-basics.md)), you learned how to create an `AsyncWebCrawler` instance, run a basic crawl, and inspect the `CrawlResult`. Now it’s time to explore some of the **targeted crawling** features that let you:
+
+1. Select specific parts of a webpage using CSS selectors
+2. Exclude or ignore certain page elements
+3. Wait for dynamic content to load using `wait_for` (with `css:` or `js:` rules)
+4. (Optionally) Handle iframes if your target site embeds additional content
+
+> **Prerequisites**
+> - You’ve read or completed [AsyncWebCrawler Basics](./async-webcrawler-basics.md).
+> - You have a working environment for Crawl4AI (Playwright installed, etc.).
+
+---
+
+## 1. Targeting Specific Elements with CSS Selectors
+
+### 1.1 Simple CSS Selector Usage
+
+Let’s say you only need to crawl the main article content of a news page. By setting `css_selector` in `CrawlerRunConfig`, your final HTML or Markdown output focuses on that region. For example:
+
+```python
+import asyncio
+from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
+
+async def main():
+ browser_cfg = BrowserConfig(headless=True)
+ crawler_cfg = CrawlerRunConfig(
+ css_selector=".article-body", # Only capture .article-body content
+ excluded_tags=["nav", "footer"] # Optional: skip big nav & footer sections
+ )
+
+ async with AsyncWebCrawler(config=browser_cfg) as crawler:
+ result = await crawler.arun(
+ url="https://news.example.com/story/12345",
+ config=crawler_cfg
+ )
+ if result.success:
+ print("[OK] Extracted content length:", len(result.html))
+ else:
+ print("[ERROR]", result.error_message)
+
+if __name__ == "__main__":
+ asyncio.run(main())
+```
+
+**Key Parameters**:
+- **`css_selector`**: Tells the crawler to focus on `.article-body`.
+- **`excluded_tags`**: Tells the crawler to skip specific HTML tags altogether (e.g., `nav` or `footer`).
+
+**Tip**: For extremely noisy pages, you can further refine how you exclude certain elements by using `excluded_selector`, which takes a CSS selector you want removed from the final output.
+
+### 1.2 Excluding Content with `excluded_selector`
+
+If you want to remove certain sections within `.article-body` (like “related stories” sidebars), set:
+
+```python
+CrawlerRunConfig(
+ css_selector=".article-body",
+ excluded_selector=".related-stories, .ads-banner"
+)
+```
+
+This combination grabs the main article content while filtering out sidebars or ads.
+
+---
+
+## 2. Handling Iframes
+
+Some sites embed extra content via `