Upload files manually

README.md

@@ -0,0 +1,54 @@
+---
+base_model: asas-ai/bloom_3B_8bit
+tags:
+- generated_from_trainer
+model-index:
+- name: bloom_3B_8bit_qlora_arc
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# bloom_3B_8bit_qlora_arc
+
+This model is a fine-tuned version of [asas-ai/bloom_3B_8bit](https://huggingface.co/asas-ai/bloom_3B_8bit) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: constant
+- lr_scheduler_warmup_ratio: 0.03
+- training_steps: 2200
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.33.1
+- Pytorch 2.0.1+cu117
+- Datasets 2.4.0
+- Tokenizers 0.12.1

adapter_config.json

@@ -0,0 +1,23 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "asas-ai/bloom_3B_8bit",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "query_key_value",
+    "dense",
+    "dense_h_to_4h",
+    "dense_4h_to_h"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd270e71bdd4835a8e216ae2c7d6abc5b8e1009fd69afc2032198ae648aaf9c4
+size 314660941

checkpoint-1760/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.6.0.dev0
+
+- PEFT 0.6.0.dev0

checkpoint-1760/adapter_config.json

@@ -0,0 +1,23 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "asas-ai/bloom_3B_8bit",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "query_key_value",
+    "dense",
+    "dense_h_to_4h",
+    "dense_4h_to_h"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-1760/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f3c2c8fe9422fbb3c301aa7cfacd13f00a49a4db6b864d52686f971dc4077e8
+size 314660941

checkpoint-1760/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84aca5fb7574ab47b2cd4fd8bae90f7124a7bbbfbb5a6d7eae0907d06b721865
+size 629293957

checkpoint-1760/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abeab98ed02d29fff4ca87823f8764decec2518c3125fc99d1cbdbbfe3829c98
+size 14575

checkpoint-1760/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85c5ff61dc870ad27d377b69c3d781f714f8b44a93915b84c28547ea26cc839f
+size 627

checkpoint-1760/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-1760/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c1abd73425d69d27b6933af4fa2a004568434169689b37d1314c6ca3a1d2a7f
+size 14500541

checkpoint-1760/tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-1760/trainer_state.json

@@ -0,0 +1,67 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 12.59391771019678,
+  "eval_steps": 500,
+  "global_step": 1760,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.57,
+      "learning_rate": 0.0002,
+      "loss": 5.6303,
+      "step": 220
+    },
+    {
+      "epoch": 3.15,
+      "learning_rate": 0.0002,
+      "loss": 4.293,
+      "step": 440
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 0.0002,
+      "loss": 3.5603,
+      "step": 660
+    },
+    {
+      "epoch": 6.3,
+      "learning_rate": 0.0002,
+      "loss": 2.9014,
+      "step": 880
+    },
+    {
+      "epoch": 7.87,
+      "learning_rate": 0.0002,
+      "loss": 2.189,
+      "step": 1100
+    },
+    {
+      "epoch": 9.45,
+      "learning_rate": 0.0002,
+      "loss": 1.5293,
+      "step": 1320
+    },
+    {
+      "epoch": 11.02,
+      "learning_rate": 0.0002,
+      "loss": 1.0569,
+      "step": 1540
+    },
+    {
+      "epoch": 12.59,
+      "learning_rate": 0.0002,
+      "loss": 0.6748,
+      "step": 1760
+    }
+  ],
+  "logging_steps": 220,
+  "max_steps": 2200,
+  "num_train_epochs": 16,
+  "save_steps": 220,
+  "total_flos": 3.250832601612288e+16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1760/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddc777fb4d61b1fe91af2901bb4f81a16ee62e977481f37fb79f3b098b82dc49
+size 4091

checkpoint-1980/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.6.0.dev0
+
+- PEFT 0.6.0.dev0

checkpoint-1980/adapter_config.json

@@ -0,0 +1,23 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "asas-ai/bloom_3B_8bit",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "query_key_value",
+    "dense",
+    "dense_h_to_4h",
+    "dense_4h_to_h"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-1980/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53fb17b5f34deeba26786b8b3433fde5985ad57c7c1cf9a8f324c35b99c2ac35
+size 314660941

checkpoint-1980/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:549d20c50b4a306d68af37fc902f5d27b7ef9cb147b80472ca8760ff1f0d70c3
+size 629293957

checkpoint-1980/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3850d65902fee21423ea9db2d0ad32f1d6442a1bdf55bd5b48a9b74fc7b2db7a
+size 14575

checkpoint-1980/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b7b3cad452dc4595e700cca73b74b2842931f81f5ad69a77dcc8aa064e6bd19
+size 627

checkpoint-1980/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-1980/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c1abd73425d69d27b6933af4fa2a004568434169689b37d1314c6ca3a1d2a7f
+size 14500541

checkpoint-1980/tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-1980/trainer_state.json

@@ -0,0 +1,73 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 14.168157423971378,
+  "eval_steps": 500,
+  "global_step": 1980,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.57,
+      "learning_rate": 0.0002,
+      "loss": 5.6303,
+      "step": 220
+    },
+    {
+      "epoch": 3.15,
+      "learning_rate": 0.0002,
+      "loss": 4.293,
+      "step": 440
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 0.0002,
+      "loss": 3.5603,
+      "step": 660
+    },
+    {
+      "epoch": 6.3,
+      "learning_rate": 0.0002,
+      "loss": 2.9014,
+      "step": 880
+    },
+    {
+      "epoch": 7.87,
+      "learning_rate": 0.0002,
+      "loss": 2.189,
+      "step": 1100
+    },
+    {
+      "epoch": 9.45,
+      "learning_rate": 0.0002,
+      "loss": 1.5293,
+      "step": 1320
+    },
+    {
+      "epoch": 11.02,
+      "learning_rate": 0.0002,
+      "loss": 1.0569,
+      "step": 1540
+    },
+    {
+      "epoch": 12.59,
+      "learning_rate": 0.0002,
+      "loss": 0.6748,
+      "step": 1760
+    },
+    {
+      "epoch": 14.17,
+      "learning_rate": 0.0002,
+      "loss": 0.4278,
+      "step": 1980
+    }
+  ],
+  "logging_steps": 220,
+  "max_steps": 2200,
+  "num_train_epochs": 16,
+  "save_steps": 220,
+  "total_flos": 3.653981899352064e+16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1980/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddc777fb4d61b1fe91af2901bb4f81a16ee62e977481f37fb79f3b098b82dc49
+size 4091

checkpoint-2200/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.6.0.dev0
+
+- PEFT 0.6.0.dev0

checkpoint-2200/adapter_config.json

@@ -0,0 +1,23 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "asas-ai/bloom_3B_8bit",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "query_key_value",
+    "dense",
+    "dense_h_to_4h",
+    "dense_4h_to_h"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-2200/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd270e71bdd4835a8e216ae2c7d6abc5b8e1009fd69afc2032198ae648aaf9c4
+size 314660941

checkpoint-2200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d778c94e9e50fe16b6b6a08cb72e283c124568cc8ba6daa79d8e916e28d89aa
+size 629293957

checkpoint-2200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:195ec7d46360a0bb51a8f66bf32e760c632dc4fd549a899f042e9abf86177ca8
+size 14575

checkpoint-2200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f10a983aa914555fea6e5c0db8d7ddbaebbe7e28546c78ee0e93ac76cbc28436
+size 627

checkpoint-2200/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-2200/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c1abd73425d69d27b6933af4fa2a004568434169689b37d1314c6ca3a1d2a7f
+size 14500541

checkpoint-2200/tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-2200/trainer_state.json

@@ -0,0 +1,79 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 15.742397137745975,
+  "eval_steps": 500,
+  "global_step": 2200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.57,
+      "learning_rate": 0.0002,
+      "loss": 5.6303,
+      "step": 220
+    },
+    {
+      "epoch": 3.15,
+      "learning_rate": 0.0002,
+      "loss": 4.293,
+      "step": 440
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 0.0002,
+      "loss": 3.5603,
+      "step": 660
+    },
+    {
+      "epoch": 6.3,
+      "learning_rate": 0.0002,
+      "loss": 2.9014,
+      "step": 880
+    },
+    {
+      "epoch": 7.87,
+      "learning_rate": 0.0002,
+      "loss": 2.189,
+      "step": 1100
+    },
+    {
+      "epoch": 9.45,
+      "learning_rate": 0.0002,
+      "loss": 1.5293,
+      "step": 1320
+    },
+    {
+      "epoch": 11.02,
+      "learning_rate": 0.0002,
+      "loss": 1.0569,
+      "step": 1540
+    },
+    {
+      "epoch": 12.59,
+      "learning_rate": 0.0002,
+      "loss": 0.6748,
+      "step": 1760
+    },
+    {
+      "epoch": 14.17,
+      "learning_rate": 0.0002,
+      "loss": 0.4278,
+      "step": 1980
+    },
+    {
+      "epoch": 15.74,
+      "learning_rate": 0.0002,
+      "loss": 0.3039,
+      "step": 2200
+    }
+  ],
+  "logging_steps": 220,
+  "max_steps": 2200,
+  "num_train_epochs": 16,
+  "save_steps": 220,
+  "total_flos": 4.05707266234368e+16,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddc777fb4d61b1fe91af2901bb4f81a16ee62e977481f37fb79f3b098b82dc49
+size 4091

runs/Sep06_17-05-26_nsvpph1i44/events.out.tfevents.1694019942.nsvpph1i44.91.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:398d5ade4a0f5f125438e987e7597db8b6dfe8ec296b2843afe29b75ead31565
+size 6582

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddc777fb4d61b1fe91af2901bb4f81a16ee62e977481f37fb79f3b098b82dc49
+size 4091