Upload folder using huggingface_hub

README.md

@@ -0,0 +1,117 @@
+---
+language:
+- en
+license: apache-2.0
+datasets:
+- teknium/OpenHermes-2.5
+base_model: mistral-community/Mixtral-8x22B-v0.1
+tags:
+- exl2
+---
+
+# mixtral-8x22b-instruct-oh - EXL2 5.5bpw
+
+This is a 5.5bpw EXL2 quant of [fireworks-ai/mixtral-8x22b-instruct-oh](https://huggingface.co/fireworks-ai/mixtral-8x22b-instruct-oh)
+
+Details about the model can be found at the above model page.
+
+## EXL2 Version
+
+These quants were made with exllamav2 version 0.0.18. Quants made on this version of EXL2 may not work on older versions of the exllamav2 library.
+
+If you have problems loading these models, please update Text Generation WebUI to the latest version.
+
+## Perplexity Scoring
+
+Below are the perplexity scores for the EXL2 models. A lower score is better. 
+
+_TODO_
+
+### Perplexity Script
+
+This was the script used for perplexity testing.
+
+```bash
+#!/bin/bash
+
+# Activate the conda environment
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate exllamav2
+
+# Set the model name and bit size
+MODEL_NAME="mixtral-8x22b-instruct-oh"
+BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)
+
+# Print the markdown table header
+echo "| Quant Level | Perplexity Score |"
+echo "|-------------|------------------|"
+
+for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
+do
+  MODEL_DIR="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
+  if [ -d "$MODEL_DIR" ]; then
+    output=$(python test_inference.py -m "$MODEL_DIR" -gs 44,48 -ed data/wikitext/wikitext-2-v1.parquet)
+    score=$(echo "$output" | grep -oP 'Evaluation perplexity: \K[\d.]+')
+    echo "| $BIT_PRECISION | $score |"
+  fi
+done
+```
+
+
+## Quant Details
+
+This is the script used for quantization.
+
+```bash
+#!/bin/bash
+
+# Activate the conda environment
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate exllamav2
+
+# Set the model name and bit size
+MODEL_NAME="mixtral-8x22b-instruct-oh"
+
+# Define variables
+MODEL_DIR="models/$MODEL_NAME"
+OUTPUT_DIR="exl2_$MODEL_NAME"
+MEASUREMENT_FILE="measurements/$MODEL_NAME.json"
+
+# Create the measurement file if needed
+if [ ! -f "$MEASUREMENT_FILE" ]; then
+    echo "Creating $MEASUREMENT_FILE"
+    # Create directories
+    if [ -d "$OUTPUT_DIR" ]; then
+        rm -r "$OUTPUT_DIR"
+    fi
+    mkdir "$OUTPUT_DIR"
+
+    python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -om $MEASUREMENT_FILE
+fi
+
+# Choose one of the below. Either create a single quant for testing or a batch of them.
+# BIT_PRECISIONS=(2.25)
+BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)
+
+for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
+do
+    CONVERTED_FOLDER="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
+
+    # If it doesn't already exist, make the quant
+    if [ ! -d "$CONVERTED_FOLDER" ]; then
+
+        echo "Creating $CONVERTED_FOLDER"
+
+        # Create directories
+        if [ -d "$OUTPUT_DIR" ]; then
+            rm -r "$OUTPUT_DIR"
+        fi
+        mkdir "$OUTPUT_DIR"
+        mkdir "$CONVERTED_FOLDER"
+        
+        # Run conversion commands  
+        python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -m $MEASUREMENT_FILE -b $BIT_PRECISION -cf $CONVERTED_FOLDER
+
+    fi
+done
+```

config.json

@@ -0,0 +1,42 @@
+{
+    "_name_or_path": "mistral-community/Mixtral-8x22B-v0.1",
+    "architectures": [
+        "MixtralForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 6144,
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "max_position_embeddings": 65536,
+    "model_type": "mixtral",
+    "num_attention_heads": 48,
+    "num_experts_per_tok": 2,
+    "num_hidden_layers": 56,
+    "num_key_value_heads": 8,
+    "num_local_experts": 8,
+    "output_router_logits": false,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000,
+    "router_aux_loss_coef": 0.001,
+    "router_jitter_noise": 0.0,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "float16",
+    "transformers_version": "4.39.3",
+    "use_cache": true,
+    "vocab_size": 32000,
+    "quantization_config": {
+        "quant_method": "exl2",
+        "version": "0.0.18",
+        "bits": 5.5,
+        "head_bits": 6,
+        "calibration": {
+            "rows": 100,
+            "length": 2048,
+            "dataset": "(default)"
+        }
+    }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}

output-00001-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:405e6d0ea391f4d86af889a3ba54eee11445cc56ff8d9be73277819ea634cdad
+size 8590074272

output-00002-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9eb35b3db7a65213baa44a513af6f55fb1a229b36f0c970b289420d87b936818
+size 8589661744

output-00003-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66a11d02597f71a1c8aed385de2b14a9cfdc6443644f4a2d4385494bee91d96d
+size 8590025576

output-00004-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd21041f485aa3ad1a985295036c39925abbc65ecadb47d5e5679e3cf0bb3b81
+size 8566465896

output-00005-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3654e0837a793236e37423c7fc2064578c25cfa76dea96fb0e792449f46547b9
+size 8573119224

output-00006-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c57f87fe9f317a15e4e07b11de70230b09d46ca94e95dc45603f59898453cb21
+size 8545072456

output-00007-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25efeac38d979df49b7828e2c564021202471c20e013c02fc633e97932c3c322
+size 8522328432

output-00008-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1eed7bc528c486e1251473a3356fc68eb78cfd142ff62be67f2ab84767aeca71
+size 8545072472

output-00009-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b1ec9d04088454a60a3901df818a02f48c1716724e40d53d6fd8ea72446bd8d
+size 8522328432

output-00010-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97111da7a7f5a66ca9fdb76bfdf8308315bf13d1704f76599e2dd3ab57c497fa
+size 8531766616

output-00011-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d045951be8a9b1e779bfa18c3c5ff328f50a61088bb3346e1348fbb338d344b
+size 8534706272

output-00012-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a2186fe78839bfc40b1e28f06d0bad4771a60926dd4dd1a618a5ef9fd50068a
+size 2845467632

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- set message_roles = ['USER', 'ASSISTANT'] -%}\n{%- set ns = namespace(initial_system_message_handled=false, messages=messages) -%}\n{{ bos_token }}\n{%- for message in ns.messages -%}\n  {%- if message['role'] | upper == 'SYSTEM' and not ns.initial_system_message_handled -%}\n    {%- set ns.initial_system_message_handled = true -%}\n    {{'SYSTEM: ' + message['content'] }}\n  {%- elif message['role'] | upper != 'SYSTEM' -%}\n    {%- if (message['role'] | upper == 'USER') != ((loop.index0 - (1 if ns.initial_system_message_handled else 0)) % 2 == 0) -%}\n      {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif -%}\n    {%- if loop.index0 > 0 or ns.initial_system_message_handled -%}\n      {{ '\\n\\n' }}\n    {%- endif -%}\n    {%- if message['role'] | upper == 'USER' -%}\n      {{ 'USER: ' + message['content'] }}\n    {%- elif message['role'] | upper == 'ASSISTANT' -%}\n      {{ 'ASSISTANT: ' + message['content'] + eos_token}}\n    {%- endif -%}\n  {%- endif -%}\n{%- endfor -%}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}