Add

.gitattributes

@@ -3116,3 +3116,51 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 4b284b84bc4v2/evaluation/generation/examples.4b284b84bc4v2_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text
 4b284b84bc4v2/evaluation/generation/examples.4b284b84bc4v2_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text
 4b284b84bc4v2/evaluation/generation/examples.4b284b84bc4v2_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-wiki_lingua_en_tldr_en_1.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-web_nlg_en_PALM_prompt_3.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-web_nlg_en_PALM_prompt_5.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-wiki_lingua_en_tldr_en_2.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-wiki_lingua_en_tldr_en_5.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_gem_xsum_article_DOC_summary_0.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-web_nlg_en_PALM_prompt_1.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-web_nlg_en_PALM_prompt_4.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-wiki_lingua_en_tldr_en_3.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_gem_xsum_article_DOC_summary_2.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_gem_xsum_article_DOC_summary_5.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-web_nlg_en_PALM_prompt_0.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-web_nlg_en_PALM_prompt_2.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_e2e_nlg_cleaned_generate_text_restaurant_3.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_gem_xsum_article_DOC_summary_4.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_GEM-wiki_lingua_en_tldr_en_4.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_GEM-wiki_lingua_en_tldr_en_0.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_e2e_nlg_cleaned_generate_text_restaurant_0.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_e2e_nlg_cleaned_generate_text_restaurant_1.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_e2e_nlg_cleaned_generate_text_restaurant_4.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_gem_xsum_article_DOC_summary_3.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_e2e_nlg_cleaned_generate_text_restaurant_2.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b1b9c4/evaluation/generation/examples.4b284b1b9c4_gem_xsum_article_DOC_summary_1.jsonl filter=lfs diff=lfs merge=lfs -text
+4b284b6bc4/evaluation/generation/examples.4b284b6bc4_e2e_nlg_cleaned_generate_text_restaurant_5.jsonl filter=lfs diff=lfs merge=lfs -text

4b284b12bc4/latest

@@ -0,0 +1 @@
+global_step80108

4b284b12bc4/sbatch_4b284b12bval.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b12bc4val
+VARIANT_CKPT=lm1-4b2-84b-c4-repetitions/4b284b12bc4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=2
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 25611230000
+# -> Samples: 12505484
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

4b284b12bc4/tensorboard_4b284b12bc4val/events.out.tfevents.1682762126.nid006361.45223.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f919c1294214557e109fa48f1ef5c77c27e44762991f870399ece229ffdc44bb
+size 980

4b284b17bc4/latest

@@ -0,0 +1 @@
+global_step80108

4b284b17bc4/sbatch_4b284b17bval.sh

@@ -0,0 +1,168 @@
+#!/bin/bash
+#SBATCH --exclude=nid007571,nid007112,nid006774,nid007502,nid007506,nid007507,nid005145,nid006692,nid007218,nid007123,nid006124,nid006123,nid007496,nid007237,nid006852,nid007206,nid006947,nid007212,nid006977,nid007222,nid005444,nid007219,nid007493,nid007221,nid005300,nid005619,nid006118,nid005203,nid006113,nid006481,nid007077,nid005208,nid005207,nid005879,nid005901
+#SBATCH --nodes=32
+#SBATCH --ntasks-per-node=1
+#SBATCH --cpus-per-task=40
+#SBATCH --mem=256G
+#SBATCH -p standard-g
+#SBATCH -t 48:00:00
+#SBATCH --gpus-per-node=mi250:8
+#SBATCH --exclusive=user
+#SBATCH --hint=nomultithread
+#SBATCH --account=project_462000119
+#SBATCH -o logs/%j.out
+#SBATCH -e logs/%j.err
+
+VARIANT=4b284b17bc4val
+VARIANT_CKPT=lm1-4b2-84b-c4-repetitions/4b284b17bc4
+
+# if run without sbatch, invoke here
+if [ -z $SLURM_JOB_ID ]; then
+    mkdir -p logs
+    sbatch "$0"
+    exit
+fi
+
+set -euo pipefail
+
+# symlink logs/latest.out and logs/latest.err
+ln -f -s $SLURM_JOB_ID.out logs/latest.out
+ln -f -s $SLURM_JOB_ID.err logs/latest.err
+
+KILL_SWITCH_PATH=kill-switch-$VARIANT
+CHECKPOINT_PATH=$VARIANT_CKPT
+TENSORBOARD_PATH=tensorboard_$VARIANT
+
+# Data
+VOCAB_FILE="gpt2/vocab.json"
+MERGE_FILE="gpt2/merges.txt"
+#DATA_PATH="/scratch/project_462000119/data/pile/megatron_data/meg-gpt2_pile_text_document"
+TRAIN_DATA_PATH=train1b5.txt
+# "train: 1.0 0:1 /scratch/project_462000119/data/c4_subsampled/gpt2tok_c4_en_12B_text_document"
+VALID_DATA_PATH=val.txt
+# "validation: 1.0 0:1 /scratch/project_462000119/data/c4_validation/gpt2tok_c4validation_rerun_text_document"
+
+PP_SIZE=1
+TP_SIZE=2
+
+MICRO_BATCH_SIZE=2
+GRADIENT_ACCUMULATION_STEPS=2
+WORLD_SIZE=$((SLURM_GPUS_ON_NODE*SLURM_JOB_NUM_NODES))
+GLOBAL_BATCH_SIZE=$((MICRO_BATCH_SIZE*WORLD_SIZE*GRADIENT_ACCUMULATION_STEPS))
+
+# Model parameters
+source model_params.sh
+MODEL_PARAM=("${PARAM_4516M[@]}")
+NHIDDEN=${MODEL_PARAM[0]}
+FFN_HIDDEN_SIZE=${MODEL_PARAM[1]}
+KV_SIZE=${MODEL_PARAM[2]}
+NHEADS=${MODEL_PARAM[3]}
+NLAYERS=${MODEL_PARAM[4]}
+SEQ_LEN=2048
+
+echo "Model parameters: d_model $NHIDDEN ffw_size $FFN_HIDDEN_SIZE kv_size $KV_SIZE n_heads $NHEADS n_layers $NLAYERS"
+
+SAVE_INTERVAL=1000
+
+# Tokens: 25611230000
+# -> Samples: 12505484
+TRAIN_SAMPLES=1
+
+OPTIMIZER_ARGS=" \
+    --optimizer adam \
+    --adam-beta1 0.9 \
+    --adam-beta2 0.999 \
+    --adam-eps 1e-8 \
+    --lr 2e-4 \
+    --min-lr 2e-5 \
+    --lr-decay-style cosine \
+    --lr-decay-samples $TRAIN_SAMPLES \
+    --lr-warmup-samples 0 \
+    --clip-grad 1.0 \
+    --weight-decay 1e-1 \
+    --override-lr-scheduler \
+    --reset-progress \
+    --no-load-optim \
+    "
+
+GPT_ARGS=" \
+    --num-layers $NLAYERS \
+    --hidden-size $NHIDDEN \
+    --num-attention-heads $NHEADS \
+    --kv-channels $KV_SIZE \
+    --ffn-hidden-size $FFN_HIDDEN_SIZE \
+    --seq-length $SEQ_LEN \
+    --max-position-embeddings $SEQ_LEN \
+    --micro-batch-size $MICRO_BATCH_SIZE \
+    --global-batch-size $GLOBAL_BATCH_SIZE \
+    --train-samples $TRAIN_SAMPLES \
+    --vocab-file $VOCAB_FILE \
+    --merge-file $MERGE_FILE \
+    --clip-grad 1.0 \
+    --kill-switch-path $KILL_SWITCH_PATH \
+    --bf16 \
+    $OPTIMIZER_ARGS \
+    "
+
+OUTPUT_ARGS=" \
+    --log-interval 10 \
+    --save-interval $SAVE_INTERVAL \
+    --eval-interval 1 \
+    --eval-iters 100 \
+    --eval-only true \
+    --tensorboard-dir $TENSORBOARD_PATH \
+    --tensorboard-queue-size 5 \
+    --log-timers-to-tensorboard \
+    --log-batch-size-to-tensorboard \
+    --log-validation-ppl-to-tensorboard \
+    "
+
+ZERO_STAGE=0
+
+mkdir -p ds_configs
+DS_CONFIG_PATH="ds_configs/$SLURM_JOB_ID.json"
+
+cat <<EOF > $DS_CONFIG_PATH
+{
+    "train_micro_batch_size_per_gpu": $MICRO_BATCH_SIZE,
+    "train_batch_size": $GLOBAL_BATCH_SIZE,
+    "gradient_clipping": 1.0,
+    "zero_optimization": {
+        "stage": $ZERO_STAGE
+    },
+    "bf16": {
+        "enabled": true
+    },
+    "steps_per_print": 2000,
+    "wall_clock_breakdown": false
+}
+EOF
+
+DEEPSPEED_ARGS=" \
+    --deepspeed \
+    --deepspeed_config $DS_CONFIG_PATH \
+    --zero-stage $ZERO_STAGE \
+    "
+
+CMD=" \
+    Megatron-DeepSpeed/pretrain_gpt.py \
+    --tensor-model-parallel-size $TP_SIZE \
+    --pipeline-model-parallel-size $PP_SIZE \
+    $GPT_ARGS \
+    $OUTPUT_ARGS \
+    --save $CHECKPOINT_PATH \
+    --load $CHECKPOINT_PATH \
+    --train-weighted-split-paths-path $TRAIN_DATA_PATH \
+    --valid-weighted-split-paths-path $VALID_DATA_PATH \
+    --data-impl mmap \
+     $DEEPSPEED_ARGS \
+    "
+
+echo $CMD
+
+echo "START $SLURM_JOBID: $(date)"
+
+# bash launch_srun_32.sh $CMD
+srun --label launch.sh $CMD
+
+echo "END $SLURM_JOBID: $(date)"

4b284b17bc4/tensorboard_4b284b17bc4val/events.out.tfevents.1682762116.nid006180.66890.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:711b6f20e54a3aae0a6a3ecd72666bad45ea2eef635b9b819efa891222c43896
+size 980

4b284b1b9c4/evaluation/generation/merged.csv

@@ -0,0 +1 @@
+dataset,fewshots,prompt,metric,value

4b284b1b9c4/evaluation/generation/merged.json

@@ -0,0 +1 @@
+{}

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_0.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.318,0.014734079309311901,0
+anli_r2,acc,0.354,0.015129868238451773,0
+anli_r3,acc,0.3458333333333333,0.013736245342311012,0
+arc_challenge,acc,0.23890784982935154,0.012461071376316617,0
+arc_challenge,acc_norm,0.2593856655290102,0.012808273573927104,0
+arc_easy,acc,0.49284511784511786,0.010258733022446362,0
+arc_easy,acc_norm,0.45791245791245794,0.010223371342195902,0
+boolq,acc,0.5229357798165137,0.008735849459018513,1
+cb,acc,0.42857142857142855,0.06672848092813058,1
+cb,f1,0.22644574398960363,,1
+copa,acc,0.72,0.04512608598542127,0
+hellaswag,acc,0.40141406094403503,0.004891826692722825,0
+hellaswag,acc_norm,0.5024895439155547,0.004989719559439899,0
+piqa,acc,0.704570184983678,0.010644731559342462,0
+piqa,acc_norm,0.7083786724700761,0.010604441527428793,0
+rte,acc,0.5523465703971119,0.02993107036293953,0
+sciq,acc,0.708,0.014385511563477345,0
+sciq,acc_norm,0.647,0.015120172605483692,0
+storycloze_2016,acc,0.6707642971672902,0.010867199207548979,0
+winogrande,acc,0.5090765588003157,0.014050170094497707,0

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_0.json

@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.318,
+            "acc_stderr": 0.014734079309311901
+        },
+        "anli_r2": {
+            "acc": 0.354,
+            "acc_stderr": 0.015129868238451773
+        },
+        "anli_r3": {
+            "acc": 0.3458333333333333,
+            "acc_stderr": 0.013736245342311012
+        },
+        "cb": {
+            "acc": 0.42857142857142855,
+            "acc_stderr": 0.06672848092813058,
+            "f1": 0.22644574398960363
+        },
+        "copa": {
+            "acc": 0.72,
+            "acc_stderr": 0.04512608598542127
+        },
+        "hellaswag": {
+            "acc": 0.40141406094403503,
+            "acc_stderr": 0.004891826692722825,
+            "acc_norm": 0.5024895439155547,
+            "acc_norm_stderr": 0.004989719559439899
+        },
+        "rte": {
+            "acc": 0.5523465703971119,
+            "acc_stderr": 0.02993107036293953
+        },
+        "winogrande": {
+            "acc": 0.5090765588003157,
+            "acc_stderr": 0.014050170094497707
+        },
+        "storycloze_2016": {
+            "acc": 0.6707642971672902,
+            "acc_stderr": 0.010867199207548979
+        },
+        "boolq": {
+            "acc": 0.5229357798165137,
+            "acc_stderr": 0.008735849459018513
+        },
+        "arc_easy": {
+            "acc": 0.49284511784511786,
+            "acc_stderr": 0.010258733022446362,
+            "acc_norm": 0.45791245791245794,
+            "acc_norm_stderr": 0.010223371342195902
+        },
+        "arc_challenge": {
+            "acc": 0.23890784982935154,
+            "acc_stderr": 0.012461071376316617,
+            "acc_norm": 0.2593856655290102,
+            "acc_norm_stderr": 0.012808273573927104
+        },
+        "sciq": {
+            "acc": 0.708,
+            "acc_stderr": 0.014385511563477345,
+            "acc_norm": 0.647,
+            "acc_norm_stderr": 0.015120172605483692
+        },
+        "piqa": {
+            "acc": 0.704570184983678,
+            "acc_stderr": 0.010644731559342462,
+            "acc_norm": 0.7083786724700761,
+            "acc_norm_stderr": 0.010604441527428793
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_1.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.343,0.015019206922356951,0
+anli_r2,acc,0.356,0.015149042659306621,0
+anli_r3,acc,0.3433333333333333,0.01371263383046586,0
+arc_challenge,acc,0.23890784982935154,0.012461071376316614,0
+arc_challenge,acc_norm,0.26535836177474403,0.012902554762313967,0
+arc_easy,acc,0.49284511784511786,0.010258733022446367,0
+arc_easy,acc_norm,0.4696969696969697,0.01024092360872654,0
+boolq,acc,0.5333333333333333,0.008725599880049204,1
+cb,acc,0.4107142857142857,0.06633634150359541,1
+cb,f1,0.19658119658119658,,1
+copa,acc,0.73,0.0446196043338474,0
+hellaswag,acc,0.4047002589125672,0.004898308167211846,0
+hellaswag,acc_norm,0.4994025094602669,0.004989777848791008,0
+piqa,acc,0.7110990206746464,0.010575111841364905,0
+piqa,acc_norm,0.7072905331882481,0.010616044462393094,0
+rte,acc,0.5018050541516246,0.030096267148976633,0
+sciq,acc,0.724,0.014142984975740668,0
+sciq,acc_norm,0.688,0.014658474370509001,0
+storycloze_2016,acc,0.6563335114911811,0.010982724236255945,0
+winogrande,acc,0.5193370165745856,0.014041972733712974,0

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_1.json

@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.343,
+            "acc_stderr": 0.015019206922356951
+        },
+        "anli_r2": {
+            "acc": 0.356,
+            "acc_stderr": 0.015149042659306621
+        },
+        "anli_r3": {
+            "acc": 0.3433333333333333,
+            "acc_stderr": 0.01371263383046586
+        },
+        "cb": {
+            "acc": 0.4107142857142857,
+            "acc_stderr": 0.06633634150359541,
+            "f1": 0.19658119658119658
+        },
+        "copa": {
+            "acc": 0.73,
+            "acc_stderr": 0.0446196043338474
+        },
+        "hellaswag": {
+            "acc": 0.4047002589125672,
+            "acc_stderr": 0.004898308167211846,
+            "acc_norm": 0.4994025094602669,
+            "acc_norm_stderr": 0.004989777848791008
+        },
+        "rte": {
+            "acc": 0.5018050541516246,
+            "acc_stderr": 0.030096267148976633
+        },
+        "winogrande": {
+            "acc": 0.5193370165745856,
+            "acc_stderr": 0.014041972733712974
+        },
+        "storycloze_2016": {
+            "acc": 0.6563335114911811,
+            "acc_stderr": 0.010982724236255945
+        },
+        "boolq": {
+            "acc": 0.5333333333333333,
+            "acc_stderr": 0.008725599880049204
+        },
+        "arc_easy": {
+            "acc": 0.49284511784511786,
+            "acc_stderr": 0.010258733022446367,
+            "acc_norm": 0.4696969696969697,
+            "acc_norm_stderr": 0.01024092360872654
+        },
+        "arc_challenge": {
+            "acc": 0.23890784982935154,
+            "acc_stderr": 0.012461071376316614,
+            "acc_norm": 0.26535836177474403,
+            "acc_norm_stderr": 0.012902554762313967
+        },
+        "sciq": {
+            "acc": 0.724,
+            "acc_stderr": 0.014142984975740668,
+            "acc_norm": 0.688,
+            "acc_norm_stderr": 0.014658474370509001
+        },
+        "piqa": {
+            "acc": 0.7110990206746464,
+            "acc_stderr": 0.010575111841364905,
+            "acc_norm": 0.7072905331882481,
+            "acc_norm_stderr": 0.010616044462393094
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_2.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.322,0.014782913600996652,0
+anli_r2,acc,0.361,0.015195720118175113,0
+anli_r3,acc,0.33916666666666667,0.013672343491681815,0
+arc_challenge,acc,0.24829351535836178,0.012624912868089758,0
+arc_challenge,acc_norm,0.2815699658703072,0.013143376735009026,0
+arc_easy,acc,0.494949494949495,0.01025926010256586,0
+arc_easy,acc_norm,0.4730639730639731,0.010244884740620094,0
+boolq,acc,0.5281345565749236,0.008731199646681927,1
+cb,acc,0.4642857142857143,0.06724777654937658,1
+cb,f1,0.21666666666666667,,1
+copa,acc,0.72,0.04512608598542127,0
+hellaswag,acc,0.4039036048595897,0.00489675785702255,0
+hellaswag,acc_norm,0.5044811790479984,0.004989581008163205,0
+piqa,acc,0.7067464635473341,0.010621818421101926,0
+piqa,acc_norm,0.7165397170837867,0.010515057791152041,0
+rte,acc,0.5054151624548736,0.030094698123239966,0
+sciq,acc,0.753,0.013644675781314133,0
+sciq,acc_norm,0.715,0.014282120955200471,0
+storycloze_2016,acc,0.6547300908605024,0.010994860223187675,0
+winogrande,acc,0.5303867403314917,0.014026510839428734,0

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_2.json

@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.322,
+            "acc_stderr": 0.014782913600996652
+        },
+        "anli_r2": {
+            "acc": 0.361,
+            "acc_stderr": 0.015195720118175113
+        },
+        "anli_r3": {
+            "acc": 0.33916666666666667,
+            "acc_stderr": 0.013672343491681815
+        },
+        "cb": {
+            "acc": 0.4642857142857143,
+            "acc_stderr": 0.06724777654937658,
+            "f1": 0.21666666666666667
+        },
+        "copa": {
+            "acc": 0.72,
+            "acc_stderr": 0.04512608598542127
+        },
+        "hellaswag": {
+            "acc": 0.4039036048595897,
+            "acc_stderr": 0.00489675785702255,
+            "acc_norm": 0.5044811790479984,
+            "acc_norm_stderr": 0.004989581008163205
+        },
+        "rte": {
+            "acc": 0.5054151624548736,
+            "acc_stderr": 0.030094698123239966
+        },
+        "winogrande": {
+            "acc": 0.5303867403314917,
+            "acc_stderr": 0.014026510839428734
+        },
+        "storycloze_2016": {
+            "acc": 0.6547300908605024,
+            "acc_stderr": 0.010994860223187675
+        },
+        "boolq": {
+            "acc": 0.5281345565749236,
+            "acc_stderr": 0.008731199646681927
+        },
+        "arc_easy": {
+            "acc": 0.494949494949495,
+            "acc_stderr": 0.01025926010256586,
+            "acc_norm": 0.4730639730639731,
+            "acc_norm_stderr": 0.010244884740620094
+        },
+        "arc_challenge": {
+            "acc": 0.24829351535836178,
+            "acc_stderr": 0.012624912868089758,
+            "acc_norm": 0.2815699658703072,
+            "acc_norm_stderr": 0.013143376735009026
+        },
+        "sciq": {
+            "acc": 0.753,
+            "acc_stderr": 0.013644675781314133,
+            "acc_norm": 0.715,
+            "acc_norm_stderr": 0.014282120955200471
+        },
+        "piqa": {
+            "acc": 0.7067464635473341,
+            "acc_stderr": 0.010621818421101926,
+            "acc_norm": 0.7165397170837867,
+            "acc_norm_stderr": 0.010515057791152041
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_3.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.323,0.014794927843348639,0
+anli_r2,acc,0.36,0.015186527932040122,0
+anli_r3,acc,0.3425,0.013704669762934725,0
+arc_challenge,acc,0.24914675767918087,0.012639407111926437,0
+arc_challenge,acc_norm,0.26109215017064846,0.012835523909473855,0
+arc_easy,acc,0.4936868686868687,0.01025896566804444,0
+arc_easy,acc_norm,0.47558922558922556,0.010247548905242267,0
+boolq,acc,0.5357798165137615,0.008722635482201088,1
+cb,acc,0.48214285714285715,0.0673769750864465,1
+cb,f1,0.2195121951219512,,1
+copa,acc,0.69,0.04648231987117316,0
+hellaswag,acc,0.4055964947221669,0.004900036261309041,0
+hellaswag,acc_norm,0.5056761601274646,0.0049894598716091814,0
+piqa,acc,0.7132752992383025,0.010551314503108056,0
+piqa,acc_norm,0.7165397170837867,0.010515057791152051,0
+rte,acc,0.5054151624548736,0.030094698123239966,0
+sciq,acc,0.752,0.013663187134877658,0
+sciq,acc_norm,0.713,0.014312087053809961,0
+storycloze_2016,acc,0.6568679850347408,0.010978648097499872,0
+winogrande,acc,0.5240726124704025,0.014036189665395132,0

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_3.json

@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.323,
+            "acc_stderr": 0.014794927843348639
+        },
+        "anli_r2": {
+            "acc": 0.36,
+            "acc_stderr": 0.015186527932040122
+        },
+        "anli_r3": {
+            "acc": 0.3425,
+            "acc_stderr": 0.013704669762934725
+        },
+        "cb": {
+            "acc": 0.48214285714285715,
+            "acc_stderr": 0.0673769750864465,
+            "f1": 0.2195121951219512
+        },
+        "copa": {
+            "acc": 0.69,
+            "acc_stderr": 0.04648231987117316
+        },
+        "hellaswag": {
+            "acc": 0.4055964947221669,
+            "acc_stderr": 0.004900036261309041,
+            "acc_norm": 0.5056761601274646,
+            "acc_norm_stderr": 0.0049894598716091814
+        },
+        "rte": {
+            "acc": 0.5054151624548736,
+            "acc_stderr": 0.030094698123239966
+        },
+        "winogrande": {
+            "acc": 0.5240726124704025,
+            "acc_stderr": 0.014036189665395132
+        },
+        "storycloze_2016": {
+            "acc": 0.6568679850347408,
+            "acc_stderr": 0.010978648097499872
+        },
+        "boolq": {
+            "acc": 0.5357798165137615,
+            "acc_stderr": 0.008722635482201088
+        },
+        "arc_easy": {
+            "acc": 0.4936868686868687,
+            "acc_stderr": 0.01025896566804444,
+            "acc_norm": 0.47558922558922556,
+            "acc_norm_stderr": 0.010247548905242267
+        },
+        "arc_challenge": {
+            "acc": 0.24914675767918087,
+            "acc_stderr": 0.012639407111926437,
+            "acc_norm": 0.26109215017064846,
+            "acc_norm_stderr": 0.012835523909473855
+        },
+        "sciq": {
+            "acc": 0.752,
+            "acc_stderr": 0.013663187134877658,
+            "acc_norm": 0.713,
+            "acc_norm_stderr": 0.014312087053809961
+        },
+        "piqa": {
+            "acc": 0.7132752992383025,
+            "acc_stderr": 0.010551314503108056,
+            "acc_norm": 0.7165397170837867,
+            "acc_norm_stderr": 0.010515057791152051
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_4.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.337,0.014955087918653607,0
+anli_r2,acc,0.362,0.015204840912919498,0
+anli_r3,acc,0.36583333333333334,0.013910212062701167,0
+arc_challenge,acc,0.24146757679180889,0.012506564839739434,0
+arc_challenge,acc_norm,0.26621160409556316,0.012915774781523214,0
+arc_easy,acc,0.4903198653198653,0.010257860554461127,0
+arc_easy,acc_norm,0.47474747474747475,0.010246690042583842,0
+boolq,acc,0.5409785932721712,0.008715635308774413,1
+cb,acc,0.5178571428571429,0.06737697508644647,1
+cb,f1,0.27628205128205124,,1
+copa,acc,0.67,0.04725815626252606,0
+hellaswag,acc,0.40619398526190004,0.004901178917900842,0
+hellaswag,acc_norm,0.5022903804023103,0.004989729059957431,0
+piqa,acc,0.7072905331882481,0.010616044462393092,0
+piqa,acc_norm,0.7072905331882481,0.010616044462393094,0
+rte,acc,0.5090252707581228,0.030091559826331334,0
+sciq,acc,0.761,0.013493000446937591,0
+sciq,acc_norm,0.726,0.014111099288259588,0
+storycloze_2016,acc,0.6606092998396579,0.010949682016358629,0
+winogrande,acc,0.5280189423835833,0.01403040421340578,0

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_4.json

@@ -0,0 +1,87 @@
+{
+    "results": {
+        "anli_r1": {
+            "acc": 0.337,
+            "acc_stderr": 0.014955087918653607
+        },
+        "anli_r2": {
+            "acc": 0.362,
+            "acc_stderr": 0.015204840912919498
+        },
+        "anli_r3": {
+            "acc": 0.36583333333333334,
+            "acc_stderr": 0.013910212062701167
+        },
+        "cb": {
+            "acc": 0.5178571428571429,
+            "acc_stderr": 0.06737697508644647,
+            "f1": 0.27628205128205124
+        },
+        "copa": {
+            "acc": 0.67,
+            "acc_stderr": 0.04725815626252606
+        },
+        "hellaswag": {
+            "acc": 0.40619398526190004,
+            "acc_stderr": 0.004901178917900842,
+            "acc_norm": 0.5022903804023103,
+            "acc_norm_stderr": 0.004989729059957431
+        },
+        "rte": {
+            "acc": 0.5090252707581228,
+            "acc_stderr": 0.030091559826331334
+        },
+        "winogrande": {
+            "acc": 0.5280189423835833,
+            "acc_stderr": 0.01403040421340578
+        },
+        "storycloze_2016": {
+            "acc": 0.6606092998396579,
+            "acc_stderr": 0.010949682016358629
+        },
+        "boolq": {
+            "acc": 0.5409785932721712,
+            "acc_stderr": 0.008715635308774413
+        },
+        "arc_easy": {
+            "acc": 0.4903198653198653,
+            "acc_stderr": 0.010257860554461127,
+            "acc_norm": 0.47474747474747475,
+            "acc_norm_stderr": 0.010246690042583842
+        },
+        "arc_challenge": {
+            "acc": 0.24146757679180889,
+            "acc_stderr": 0.012506564839739434,
+            "acc_norm": 0.26621160409556316,
+            "acc_norm_stderr": 0.012915774781523214
+        },
+        "sciq": {
+            "acc": 0.761,
+            "acc_stderr": 0.013493000446937591,
+            "acc_norm": 0.726,
+            "acc_norm_stderr": 0.014111099288259588
+        },
+        "piqa": {
+            "acc": 0.7072905331882481,
+            "acc_stderr": 0.010616044462393092,
+            "acc_norm": 0.7072905331882481,
+            "acc_norm_stderr": 0.010616044462393094
+        }
+    },
+    "versions": {
+        "anli_r1": 0,
+        "anli_r2": 0,
+        "anli_r3": 0,
+        "cb": 1,
+        "copa": 0,
+        "hellaswag": 0,
+        "rte": 0,
+        "winogrande": 0,
+        "storycloze_2016": 0,
+        "boolq": 1,
+        "arc_easy": 0,
+        "arc_challenge": 0,
+        "sciq": 0,
+        "piqa": 0
+    }
+}

4b284b1b9c4/evaluation/rankeval/4b284b1b9c4_5.csv

@@ -0,0 +1,21 @@
+task,metric,value,err,version
+anli_r1,acc,0.36,0.015186527932040126,0
+anli_r2,acc,0.354,0.015129868238451773,0
+anli_r3,acc,0.3625,0.013883037874225516,0
+arc_challenge,acc,0.25853242320819114,0.012794553754288682,0
+arc_challenge,acc_norm,0.2696245733788396,0.012968040686869147,0
+arc_easy,acc,0.4877946127946128,0.010256726235129018,0
+arc_easy,acc_norm,0.4713804713804714,0.010242962617927192,0
+boolq,acc,0.5324159021406728,0.008726657178723137,1
+cb,acc,0.5178571428571429,0.06737697508644647,1
+cb,f1,0.25430894308943086,,1
+copa,acc,0.71,0.04560480215720683,0
+hellaswag,acc,0.40400318661621193,0.004896952378506924,0
+hellaswag,acc_norm,0.5022903804023103,0.004989729059957428,0
+piqa,acc,0.7067464635473341,0.010621818421101928,0
+piqa,acc_norm,0.70620239390642,0.01062757408051481,0
+rte,acc,0.48014440433212996,0.0300727231673172,0
+sciq,acc,0.766,0.01339490288966001,0
+sciq,acc_norm,0.734,0.01397996564514515,0
+storycloze_2016,acc,0.6563335114911811,0.010982724236255948,0
+winogrande,acc,0.5224940805051302,0.014038257824059874,0