arabellastrange commited on
Commit
911812d
·
1 Parent(s): bf0597d

expception handling

Browse files
Files changed (1) hide show
  1. web_search.py +7 -5
web_search.py CHANGED
@@ -77,17 +77,19 @@ def process_url(url, timeout):
77
  options.add_argument("--disable-gpu")
78
  options.add_argument("--disable-dev-shm-usage")
79
  result = ""
80
-
81
- driver = webdriver.Chrome(options=options)
82
- logger.info(f"*****setting page load timeout {timeout}")
83
- driver.set_page_load_timeout(timeout)
84
-
85
  try:
 
 
 
86
  driver.get(url)
87
  response = driver.page_source
88
  result = response_text_extract(url=url, response=response)
89
  except selenium.common.exceptions.TimeoutException:
90
  return "", url
 
 
 
 
91
  except Exception:
92
  traceback.print_exc()
93
  logger.info(f"{site} err")
 
77
  options.add_argument("--disable-gpu")
78
  options.add_argument("--disable-dev-shm-usage")
79
  result = ""
 
 
 
 
 
80
  try:
81
+ driver = webdriver.Chrome(options=options)
82
+ logger.info(f"*****setting page load timeout {timeout}")
83
+ driver.set_page_load_timeout(timeout)
84
  driver.get(url)
85
  response = driver.page_source
86
  result = response_text_extract(url=url, response=response)
87
  except selenium.common.exceptions.TimeoutException:
88
  return "", url
89
+ except selenium.common.exceptions.WebDriverException:
90
+ traceback.print_exc()
91
+ logger.info(f"webdriver failed to load")
92
+ return "", url
93
  except Exception:
94
  traceback.print_exc()
95
  logger.info(f"{site} err")