Upload folder using huggingface_hub

README.md

@@ -0,0 +1,117 @@
+---
+language:
+- en
+license: apache-2.0
+datasets:
+- teknium/OpenHermes-2.5
+base_model: mistral-community/Mixtral-8x22B-v0.1
+tags:
+- exl2
+---
+
+# mixtral-8x22b-instruct-oh - EXL2 3.5bpw
+
+This is a 3.5bpw EXL2 quant of [fireworks-ai/mixtral-8x22b-instruct-oh](https://huggingface.co/fireworks-ai/mixtral-8x22b-instruct-oh)
+
+Details about the model can be found at the above model page.
+
+## EXL2 Version
+
+These quants were made with exllamav2 version 0.0.18. Quants made on this version of EXL2 may not work on older versions of the exllamav2 library.
+
+If you have problems loading these models, please update Text Generation WebUI to the latest version.
+
+## Perplexity Scoring
+
+Below are the perplexity scores for the EXL2 models. A lower score is better. 
+
+_TODO_
+
+### Perplexity Script
+
+This was the script used for perplexity testing.
+
+```bash
+#!/bin/bash
+
+# Activate the conda environment
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate exllamav2
+
+# Set the model name and bit size
+MODEL_NAME="mixtral-8x22b-instruct-oh"
+BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)
+
+# Print the markdown table header
+echo "| Quant Level | Perplexity Score |"
+echo "|-------------|------------------|"
+
+for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
+do
+  MODEL_DIR="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
+  if [ -d "$MODEL_DIR" ]; then
+    output=$(python test_inference.py -m "$MODEL_DIR" -gs 44,48 -ed data/wikitext/wikitext-2-v1.parquet)
+    score=$(echo "$output" | grep -oP 'Evaluation perplexity: \K[\d.]+')
+    echo "| $BIT_PRECISION | $score |"
+  fi
+done
+```
+
+
+## Quant Details
+
+This is the script used for quantization.
+
+```bash
+#!/bin/bash
+
+# Activate the conda environment
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate exllamav2
+
+# Set the model name and bit size
+MODEL_NAME="mixtral-8x22b-instruct-oh"
+
+# Define variables
+MODEL_DIR="models/$MODEL_NAME"
+OUTPUT_DIR="exl2_$MODEL_NAME"
+MEASUREMENT_FILE="measurements/$MODEL_NAME.json"
+
+# Create the measurement file if needed
+if [ ! -f "$MEASUREMENT_FILE" ]; then
+    echo "Creating $MEASUREMENT_FILE"
+    # Create directories
+    if [ -d "$OUTPUT_DIR" ]; then
+        rm -r "$OUTPUT_DIR"
+    fi
+    mkdir "$OUTPUT_DIR"
+
+    python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -om $MEASUREMENT_FILE
+fi
+
+# Choose one of the below. Either create a single quant for testing or a batch of them.
+# BIT_PRECISIONS=(2.25)
+BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)
+
+for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
+do
+    CONVERTED_FOLDER="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
+
+    # If it doesn't already exist, make the quant
+    if [ ! -d "$CONVERTED_FOLDER" ]; then
+
+        echo "Creating $CONVERTED_FOLDER"
+
+        # Create directories
+        if [ -d "$OUTPUT_DIR" ]; then
+            rm -r "$OUTPUT_DIR"
+        fi
+        mkdir "$OUTPUT_DIR"
+        mkdir "$CONVERTED_FOLDER"
+        
+        # Run conversion commands  
+        python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -m $MEASUREMENT_FILE -b $BIT_PRECISION -cf $CONVERTED_FOLDER
+
+    fi
+done
+```

config.json

@@ -0,0 +1,42 @@
+{
+    "_name_or_path": "mistral-community/Mixtral-8x22B-v0.1",
+    "architectures": [
+        "MixtralForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 6144,
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "max_position_embeddings": 65536,
+    "model_type": "mixtral",
+    "num_attention_heads": 48,
+    "num_experts_per_tok": 2,
+    "num_hidden_layers": 56,
+    "num_key_value_heads": 8,
+    "num_local_experts": 8,
+    "output_router_logits": false,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000,
+    "router_aux_loss_coef": 0.001,
+    "router_jitter_noise": 0.0,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "float16",
+    "transformers_version": "4.39.3",
+    "use_cache": true,
+    "vocab_size": 32000,
+    "quantization_config": {
+        "quant_method": "exl2",
+        "version": "0.0.18",
+        "bits": 3.5,
+        "head_bits": 6,
+        "calibration": {
+            "rows": 100,
+            "length": 2048,
+            "dataset": "(default)"
+        }
+    }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}

output-00001-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f1a7639d2b8e4c42479552d0aa27e55cbce7a26ab848fdb093bfcd0f96d3ded
+size 8590100792

output-00002-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e16708996d1d6e39610b2e298aec8b59ccedf0b2fb98f9191126bcce116764be
+size 8574820312

output-00003-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbfc51975c895cf8b6bb90a7f3ce468f6949cc8b277afd23b52e0acce85dc905
+size 8588408864

output-00004-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb2dcb188b4f316c5c78684b8b348d3856c7fefd1e958c0da118664cef27c472
+size 8587680448

output-00005-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0e026d0d2e4eeca568d2e9c61a58603710e3454edc45e1d07fad4b1039bc3e4
+size 8560884456

output-00006-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b05f3d58555fae86e8889e8403a565b8c3a00847d71f3911d19b63d60a358247
+size 8562558264

output-00007-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:412f4f2b076f125a668fd5c0e319d4446c293fcec0c594d3c2d404203651fc3f
+size 8555457888

output-00008-of-00008.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:baf749979a385f39813883c8a6c56be2459054ff2ca27010e398ce92ab4eff1a
+size 1884737688

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- set message_roles = ['USER', 'ASSISTANT'] -%}\n{%- set ns = namespace(initial_system_message_handled=false, messages=messages) -%}\n{{ bos_token }}\n{%- for message in ns.messages -%}\n  {%- if message['role'] | upper == 'SYSTEM' and not ns.initial_system_message_handled -%}\n    {%- set ns.initial_system_message_handled = true -%}\n    {{'SYSTEM: ' + message['content'] }}\n  {%- elif message['role'] | upper != 'SYSTEM' -%}\n    {%- if (message['role'] | upper == 'USER') != ((loop.index0 - (1 if ns.initial_system_message_handled else 0)) % 2 == 0) -%}\n      {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif -%}\n    {%- if loop.index0 > 0 or ns.initial_system_message_handled -%}\n      {{ '\\n\\n' }}\n    {%- endif -%}\n    {%- if message['role'] | upper == 'USER' -%}\n      {{ 'USER: ' + message['content'] }}\n    {%- elif message['role'] | upper == 'ASSISTANT' -%}\n      {{ 'ASSISTANT: ' + message['content'] + eos_token}}\n    {%- endif -%}\n  {%- endif -%}\n{%- endfor -%}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}