concurrent pool for process_url

web_search.py

@@ -50,14 +50,11 @@ def search(msg, query_phrase):
 
 # Define a function to make a single URL request and process the response
 def process_url(url):
-    processed_page = []
+    processed_page = {}
     start_time = time.time()
-    site = ut.extract_site(url)
-    result = ""
     try:
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
-            result = ""
             try:
                 client = ZenRowsClient(os.getenv('zenrows_api_key'))
                 response = client.get(url)
@@ -65,22 +62,19 @@ def process_url(url):
                 result = response.text
                 if len(result) > 0:
                     if "an error has occurred" not in result.lower() and "permission to view this page" not in result.lower() and "403 ERROR" not in result.lower() and "have been blocked" not in result.lower() and "too many requests" not in result.lower():
-                        processed_page.append(
-                            {
+                        processed_page = {
                                 "source": ut.extract_domain(url),
                                 "url": url,
                                 "text": result,
                             }
-                        )
+                    print(f"Processed {url}: {len(result)} {int((time.time() - start_time) * 1000)} ms")
+                    return processed_page
             except Exception:
                 traceback.print_exc()
                 return processed_page
     except Exception:
         traceback.print_exc()
-        print(f"{site} err")
-        pass
-    print(f"Processed {site}: {len(result)} {int((time.time() - start_time) * 1000)} ms")
-    return processed_page
+        return processed_page
 
 
 def process_urls(urls):
@@ -90,7 +84,9 @@ def process_urls(urls):
 
     try:
         with ThreadPoolExecutor(max_workers=len(urls)) as pool:
-            results = pool.map(process_url, urls)
+            for result in pool.map(process_url, urls):
+                print(f'returned {result}')
+                results.append(result)
     except:
         traceback.print_exc()