#feat added simplest scholar mode

app.py

@@ -21,9 +21,9 @@ from transformers import AutoTokenizer, AutoModelForSequenceClassification
 from google.cloud import storage
 
 if gr.NO_RELOAD:
-    from humanize import humanize_text, device
-    # humanize_text = None
-    # device = None
+    # from humanize import humanize_text, device
+    humanize_text = None
+    device = None
     from utils import remove_special_characters, split_text_allow_complete_sentences_nltk
     from google_search import google_search, months, domain_list, build_date
     from ai_generate import generate, citations_to_html, remove_citations, display_cited_text, llm_wrapper
@@ -796,6 +796,24 @@ def save_humanizer_feedback_to_cloud_storage(data, humanizer_feedback):
         gr.Warning("Nothing humanized to save yet!")
 
 
+scholar_urls = [
+    "arxiv.org",
+    "aclanthology.org",
+    "ieeexplore.ieee.org",
+    "researchgate.net",
+    # "scholar.google.com",
+    "springer.com",
+    # "sciencedirect.com", # 400
+    # "onlinelibrary.wiley.com", # 400
+    "jstor.org", # 400
+    "semanticscholar.org",
+    "biorxiv.org",
+    "medrxiv.org",
+    "ssrn.com",
+    "pubmed.ncbi.nlm.nih.gov",
+    "cochranelibrary.com",
+]
+
 def generate_and_format(
     input_role,
     topic,
@@ -812,6 +830,7 @@ def generate_and_format(
     num_examples,
     conclusion_type,
     google_search_check,
+    scholar_mode_check,
     year_from,
     month_from,
     day_from,
@@ -838,14 +857,18 @@ def generate_and_format(
         final_query = llm_wrapper(
             input_role, topic, context, model="OpenAI GPT 4o", task_type="internet", temperature=0.7
         )
-        if include_sites:
-            site_queries = [f"site:{site.strip()}" for site in include_sites.split(",")]
-            final_query += " " + " OR ".join(site_queries)
-        if exclude_sites:
-            exclude_queries = [f"-site:{site.strip()}" for site in exclude_sites.split(",")]
-            final_query += " " + " ".join(exclude_queries)
+        if scholar_mode_check:
+            scholar_site_queries = [f"site:{site.strip()}" for site in scholar_urls]
+            final_query += " " + " OR ".join(scholar_site_queries)
+        else: 
+            if include_sites:
+                site_queries = [f"site:{site.strip()}" for site in include_sites.split(",")]
+                final_query += " " + " OR ".join(site_queries)
+            if exclude_sites:
+                exclude_queries = [f"-site:{site.strip()}" for site in exclude_sites.split(",")]
+                final_query += " " + " ".join(exclude_queries)
         print(f"Google Search Query: {final_query}")
-        url_content = google_search(final_query, sorted_date, domains_to_include)
+        url_content = google_search(final_query, sorted_date, domains_to_include, scholar_mode_check)
 
     yt_content = {}
     if yt_url:
@@ -1077,24 +1100,27 @@ with gr.Blocks(
                     )
                 with gr.Group(visible=google_default) as search_options:
                     with gr.Row():
-                        include_sites = gr.Textbox(
-                            label="Include Specific Websites",
-                            placeholder="Enter comma-separated keywords",
-                            elem_classes="input-highlight-yellow",
-                        )
-                    with gr.Row():
-                        exclude_sites = gr.Textbox(
-                            label="Exclude Specific Websites",
-                            placeholder="Enter comma-separated keywords",
-                            elem_classes="input-highlight-yellow",
-                        )
-                    with gr.Row():
-                        domains_to_include = gr.Dropdown(
-                            domain_list,
-                            value=domain_list,
-                            multiselect=True,
-                            label="Domains To Include",
-                        )
+                        scholar_mode_check = gr.Checkbox(label="Enable Scholar Mode", value=False)
+                    with gr.Group(visible=True) as site_options:
+                        with gr.Row():
+                            include_sites = gr.Textbox(
+                                label="Include Specific Websites",
+                                placeholder="Enter comma-separated keywords",
+                                elem_classes="input-highlight-yellow",
+                            )
+                        with gr.Row():
+                            exclude_sites = gr.Textbox(
+                                label="Exclude Specific Websites",
+                                placeholder="Enter comma-separated keywords",
+                                elem_classes="input-highlight-yellow",
+                            )
+                        with gr.Row():
+                            domains_to_include = gr.Dropdown(
+                                domain_list,
+                                value=domain_list,
+                                multiselect=True,
+                                label="Domains To Include",
+                            )
                     with gr.Row():
                         month_from = gr.Dropdown(
                             choices=months,
@@ -1224,7 +1250,9 @@ with gr.Blocks(
         else:
             return gr.update(visible=False)
 
-    google_search_check.change(search_visible, inputs=google_search_check, outputs=search_options)
+    google_search_check.change(
+        lambda toggle: gr.update(visible=toggle), inputs=google_search_check, outputs=search_options
+    )
     # ai_detector_dropdown.change(highlight_visible, inputs=ai_detector_dropdown, outputs=highlighted_text)
     # output_article.change(regenerate_visible, inputs=output_article, outputs=ai_comments)
     # ai_comments.change(regenerate_visible, inputs=output_article, outputs=regenerate_btn)
@@ -1257,6 +1285,7 @@ with gr.Blocks(
             # ai_generator,
             # input_api,
             google_search_check,
+            scholar_mode_check,
             year_from,
             month_from,
             day_from,
@@ -1293,6 +1322,7 @@ with gr.Blocks(
             # ai_generator,
             # input_api,
             google_search_check,
+            scholar_mode_check,
             year_from,
             month_from,
             day_from,

google_search.py

@@ -193,14 +193,12 @@ def google_search_urls(
     return url_list
 
 
-def google_search(
-    topic,
-    sorted_date,
-    domains_to_include,
-):
+def google_search(topic, sorted_date, domains_to_include, scholar_mode_check):
     api_key = os.environ.get("GOOGLE_SEARCH_API_KEY")
     cse_id = os.environ.get("GOOGLE_SEARCH_CSE_ID")
     start_time = time.perf_counter()
+    if scholar_mode_check:
+        topic += " -filetype:pdf"
     url_list = google_search_urls(
         topic,
         sorted_date,