adding_log_visualizer (#1)

- Adding logging visualizer (a130fc382412b1c3f664e294da1aeb43ed30fa00)
- Keeping default of checking every 10 min (ccb39b1f08fe0f04412d9e0401a1dc556b149fb0)
- Increasing log lines to visualize (ef89b19123267a7694493f28cd25a8ac8ed4942b)
- Adding better console handling (d800d714b0f2ae12ffaa23c053413f1951123238)


Co-authored-by: Derek Thomas <derek-thomas@users.noreply.huggingface.co>

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 .ipynb_checkpoints
 *ipynb
 .vscode/
+.idea/
 
 eval-queue/
 eval-results/

app.py

@@ -1,27 +1,26 @@
-import sys
 import logging
-import subprocess
-import gradio as gr
-from apscheduler.schedulers.background import BackgroundScheduler
+import sys
 
-logging.basicConfig(level=logging.ERROR)
+import gradio as gr
+from main_backend_lighteval import run_auto_eval
+from src.display.log_visualizer import log_file_to_html_string
+from src.display.css_html_js import dark_mode_gradio_js
+from src.envs import REFRESH_RATE
 
-from src.logging import LOGGER, read_logs
+logging.basicConfig(level=logging.INFO)
 
-sys.stdout = LOGGER
-sys.stderr = LOGGER
 
-#subprocess.run(["python", "scripts/fix_harness_import.py"])
+intro_md = f"""
+# Intro
+This is just a visual for the auto evaluator. Note that the lines of the log visual are reversed.
+# Logs
+"""
 
-def launch_backend():
-    _ = subprocess.run(["python", "main_backend_lighteval.py"])
+with gr.Blocks(js=dark_mode_gradio_js) as demo:
+    with gr.Tab("Application"):
+        gr.Markdown(intro_md)
+        dummy = gr.Markdown(run_auto_eval, every=REFRESH_RATE, visible=False)
+        output = gr.HTML(log_file_to_html_string, every=10)
 
-demo = gr.Blocks()
-with demo:
-    logs = gr.Code(interactive=False)
-    demo.load(read_logs, None, logs, every=1)
-    
-scheduler = BackgroundScheduler()
-scheduler.add_job(launch_backend, "interval", seconds=60) # will only allow one job to be run at the same time
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+if __name__ == '__main__':
+    demo.queue(default_concurrency_limit=40).launch(server_name="0.0.0.0", show_error=True, server_port=7860)

main_backend_harness.py

@@ -11,9 +11,11 @@ from src.backend.sort_queue import sort_models_by_priority
 
 from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, DEVICE, API, LIMIT, TOKEN
 from src.about import Tasks, NUM_FEWSHOT
+from src.logging import setup_logger
 TASKS_HARNESS = [task.value.benchmark for task in Tasks]
 
-logging.basicConfig(level=logging.ERROR)
+# logging.basicConfig(level=logging.ERROR)
+logger = setup_logger(__name__)
 pp = pprint.PrettyPrinter(width=80)
 
 PENDING_STATUS = "PENDING"
@@ -51,7 +53,7 @@ def run_auto_eval():
         return
 
     eval_request = eval_requests[0]
-    pp.pprint(eval_request)
+    logger.info(pp.pformat(eval_request))
 
     set_eval_request(
         api=API,

main_backend_lighteval.py

@@ -11,8 +11,11 @@ from src.backend.sort_queue import sort_models_by_priority
 
 from src.envs import QUEUE_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO, EVAL_RESULTS_PATH_BACKEND, API, LIMIT, TOKEN, ACCELERATOR, VENDOR, REGION
 from src.about import TASKS_LIGHTEVAL
+from src.logging import setup_logger
 
-logging.basicConfig(level=logging.ERROR)
+logger = setup_logger(__name__)
+
+# logging.basicConfig(level=logging.ERROR)
 pp = pprint.PrettyPrinter(width=80)
 
 PENDING_STATUS = "PENDING"
@@ -44,13 +47,14 @@ def run_auto_eval():
     # Sort the evals by priority (first submitted first run)
     eval_requests = sort_models_by_priority(api=API, models=eval_requests)
 
-    print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
+    logger.info(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
 
     if len(eval_requests) == 0:
         return
 
     eval_request = eval_requests[0]
-    pp.pprint(eval_request)
+    logger.info(pp.pformat(eval_request))
+
 
     set_eval_request(
         api=API,

requirements.txt

@@ -16,4 +16,9 @@ tokenizers>=0.15.0
 git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
 git+https://github.com/huggingface/lighteval.git#egg=lighteval
 accelerate==0.24.1
-sentencepiece
+sentencepiece
+
+# Log Visualizer
+beautifulsoup4==4.12.2
+lxml==4.9.3
+rich==13.3.4

src/backend/manage_requests.py

@@ -5,6 +5,9 @@ from typing import Optional
 
 from huggingface_hub import HfApi, snapshot_download
 from src.envs import TOKEN
+from src.logging import setup_logger
+
+logger = setup_logger(__name__)
 
 @dataclass
 class EvalRequest:
@@ -103,20 +106,20 @@ def check_completed_evals(
 
     for eval_request in running_evals:
         model = eval_request.model
-        print("====================================")
-        print(f"Checking {model}")
+        logger.info("====================================")
+        logger.info(f"Checking {model}")
 
         output_path = model
         output_file = f"{local_dir_results}/{output_path}/results*.json"
         output_file_exists = len(glob.glob(output_file)) > 0
 
         if output_file_exists:
-            print(
+            logger.info(
                 f"EXISTS output file exists for {model} setting it to {completed_status}"
             )
             set_eval_request(api, eval_request, completed_status, hf_repo, local_dir)
         else:
-            print(
+            logger.info(
                 f"No result file found for {model} setting it to {failed_status}"
             )
             set_eval_request(api, eval_request, failed_status, hf_repo, local_dir)

src/backend/run_eval_suite_harness.py

@@ -7,18 +7,20 @@ from lm_eval import tasks, evaluator, utils
 
 from src.envs import RESULTS_REPO, API
 from src.backend.manage_requests import EvalRequest
+from src.logging import setup_logger
 
 logging.getLogger("openai").setLevel(logging.WARNING)
+logger = setup_logger(__name__)
 
 def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, local_dir: str, results_repo: str, no_cache=True, limit=None):
     if limit:
-        print(
+        logger.info(
             "WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
         )
 
     task_names = utils.pattern_match(task_names, tasks.ALL_TASKS)
 
-    print(f"Selected Tasks: {task_names}")
+    logger.info(f"Selected Tasks: {task_names}")
 
     results = evaluator.simple_evaluate(
         model="hf-causal-experimental", # "hf-causal"
@@ -38,14 +40,14 @@ def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_siz
     results["config"]["model_sha"] = eval_request.revision
 
     dumped = json.dumps(results, indent=2)
-    print(dumped)
+    logger.info(dumped)
 
     output_path = os.path.join(local_dir, *eval_request.model.split("/"), f"results_{datetime.now()}.json")
     os.makedirs(os.path.dirname(output_path), exist_ok=True)
     with open(output_path, "w") as f:
         f.write(dumped)
 
-    print(evaluator.make_table(results))
+    logger.info(evaluator.make_table(results))
 
     API.upload_file(
         path_or_fileobj=output_path,

src/backend/run_eval_suite_lighteval.py

@@ -7,12 +7,14 @@ from lighteval.main_accelerate import main, EnvConfig, create_model_config, load
 
 from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN
 from src.backend.manage_requests import EvalRequest
+from src.logging import setup_logger
 
 logging.getLogger("openai").setLevel(logging.WARNING)
+logger = setup_logger(__name__)
 
 def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int, local_dir: str, accelerator: str, region: str, vendor: str, instance_size: str, instance_type: str, limit=None):
     if limit:
-        print("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
+        logger.info("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")
 
     args = {
             "endpoint_model_name":f"{eval_request.model}_{eval_request.precision}".lower(),
@@ -43,7 +45,7 @@ def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int,
         results["config"]["model_sha"] = eval_request.revision
 
         dumped = json.dumps(results, indent=2)
-        print(dumped)
+        logger.info(dumped)
     except Exception: # if eval failed, we force a cleanup
         env_config = EnvConfig(token=TOKEN, cache_dir=args['cache_dir'])
 

src/display/css_html_js.py

@@ -0,0 +1,20 @@
+style_content = """
+pre, code {
+    background-color: #272822;
+}
+    .scrollable {
+        font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
+        height: 500px;
+        overflow: auto;
+    }
+    """
+dark_mode_gradio_js = """
+function refresh() {
+    const url = new URL(window.location);
+
+    if (url.searchParams.get('__theme') !== 'dark') {
+        url.searchParams.set('__theme', 'dark');
+        window.location.href = url.href;
+    }
+}
+"""

src/display/log_visualizer.py

@@ -0,0 +1,42 @@
+from io import StringIO
+from pathlib import Path
+
+from bs4 import BeautifulSoup
+from rich.console import Console
+from rich.syntax import Syntax
+
+from src.display.css_html_js import style_content
+from src.envs import NUM_LINES_VISUALIZE
+from src.logging import log_file
+
+proj_dir = Path(__name__).parent
+
+
+def log_file_to_html_string():
+    with open(log_file, "rt") as f:
+        # Seek to the end of the file minus 300 lines
+        # Read the last 300 lines of the file
+        lines = f.readlines()
+        lines = lines[-NUM_LINES_VISUALIZE:]
+
+        # Syntax-highlight the last 300 lines of the file using the Python lexer and Monokai style
+        output = "".join(reversed(lines))
+        syntax = Syntax(output, "python", theme="monokai", word_wrap=True)
+
+    console = Console(record=True, width=150, style="#272822", file=StringIO())
+    console.print(syntax)
+    html_content = console.export_html(inline_styles=True)
+
+    # Parse the HTML content using BeautifulSoup
+    soup = BeautifulSoup(html_content, 'lxml')
+
+    # Modify the <pre> tag
+    pre_tag = soup.pre
+    pre_tag['class'] = 'scrollable'
+    del pre_tag['style']
+
+    # Add your custom styles and the .scrollable CSS to the <style> tag
+    style_tag = soup.style
+    style_tag.append(style_content)
+
+    return soup.prettify()

src/envs.py

@@ -31,5 +31,8 @@ EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
 EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
+REFRESH_RATE = 10 * 60  # 10 min
+NUM_LINES_VISUALIZE = 300
+
 API = HfApi(token=TOKEN)
 

src/logging.py

@@ -1,32 +1,54 @@
 import sys
-from src.envs import API
-
-class Logger:
-    def __init__(self, filename):
-        self.terminal = sys.stdout
-        self.log = open(filename, "a+")
-
-    def write(self, message):
-        self.terminal.write(message)
-        self.log.write(message)
-        
-    def flush(self):
-        self.terminal.flush()
-        self.log.flush()
-        
-    def isatty(self):
-        return False    
-
-def read_logs():
-    sys.stdout.flush()
-    #API.upload_file(
-    #    path_or_fileobj="output.log",
-    #    path_in_repo="demo-backend.log",
-    #    repo_id="demo-leaderboard-backend/logs",
-    #    repo_type="dataset",
-    #)
-
-    with open("output.log", "r") as f:
-        return f.read()
-
-LOGGER =  Logger("output.log")
+from pathlib import Path
+
+proj_dir = Path(__file__).parents[1]
+
+log_file = proj_dir/"output.log"
+
+
+import logging
+
+
+def setup_logger(name: str):
+    logger = logging.getLogger(name)
+    logger.setLevel(logging.INFO)
+
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+    # Create a file handler to write logs to a file
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    return logger
+
+# class Logger:
+#     def __init__(self):
+#         self.terminal = sys.stdout
+#         self.log = open(log_file, "a+")
+#
+#     def write(self, message):
+#         self.terminal.write(message)
+#         self.log.write(message)
+#
+#     def flush(self):
+#         self.terminal.flush()
+#         self.log.flush()
+#
+#     def isatty(self):
+#         return False
+#
+# def read_logs():
+#     sys.stdout.flush()
+#     #API.upload_file(
+#     #    path_or_fileobj="output.log",
+#     #    path_in_repo="demo-backend.log",
+#     #    repo_id="demo-leaderboard-backend/logs",
+#     #    repo_type="dataset",
+#     #)
+#
+#     with open(log_file, "r") as f:
+#         return f.read()
+#
+# LOGGER =  Logger()