File size: 1,432 Bytes
2a5f9fb
df66f6e
2a5f9fb
 
9b14fa5
1ffc326
 
9b14fa5
f982b8e
9b14fa5
08ae6c5
 
9b14fa5
 
 
3e6770c
08ae6c5
 
6902167
18abd06
 
9b14fa5
3e6770c
2a5f9fb
3e6770c
df9a594
9833cdb
 
2a5f9fb
1ffc326
9b14fa5
395eff6
9833cdb
395eff6
 
1ffc326
 
2a5f9fb
8b88d2c
 
 
efeee6d
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
import os

from huggingface_hub import HfApi


# Info to change for your repository
# ----------------------------------
TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org

OWNER = "demo-leaderboard-backend"  # Change to your org - don't forget to create a results and request dataset

# For harness evaluations
DEVICE = "cpu"  # "cuda:0" if you add compute, for harness evaluations
LIMIT = 20  # !!!! For testing, should be None for actual evaluations!!!
NUM_FEWSHOT = 0  # Change with your few shot for the Harness evaluations
TASKS_HARNESS = ["anli_r1", "logiqa"]

# For lighteval evaluations
ACCELERATOR = "cpu"
REGION = "us-east-1"
VENDOR = "aws"
TASKS_LIGHTEVAL = "lighteval|anli:r1|0|0,lighteval|logiqa|0|0"
# To add your own tasks, edit the custom file and launch it with `custom|myothertask|0|0``

# ---------------------------------------------------
REPO_ID = f"{OWNER}/backend"
QUEUE_REPO = f"{OWNER}/requests"
RESULTS_REPO = f"{OWNER}/results"

# If you setup a cache later, just change HF_HOME
CACHE_PATH = os.getenv("HF_HOME", ".")

# Local caches
EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")

REFRESH_RATE = 10 * 60  # 10 min
NUM_LINES_VISUALIZE = 300

API = HfApi(token=TOKEN)