Upload folder using huggingface_hub

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "Fredithefish/MysticRP4",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.3",
+  "use_cache": true,
+  "vocab_size": 32000
+}

mergekit_config.yml

@@ -0,0 +1,17 @@
+slices:
+  - sources:
+      - model: Fredithefish/MysticRP4
+        layer_range: [0, 32]
+      - model: NeverSleep/NoromaidxOpenGPT4-2
+        layer_range: [0, 32]
+
+merge_method: slerp
+base_model: Fredithefish/MysticRP4
+parameters:
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5 # fallback for rest of tensors
+dtype: bfloat16

model-00001-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e317fafbc2a03d5ee861c443bd001d9a1c09550ec7ebc6bab51d0b050fc50227
+size 1963199472

model-00002-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e49fb6d1f87ab9d56a4c411205f3a391ab9186fc47249db67faf528a16bee00
+size 1996556608

model-00003-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6dfe18f6d463baaf1f26a8a54eb2382e719930e350853b12252daf5063f5f2ca
+size 1956793496

model-00004-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50c08893c48f7947c4ec9ccbec768069ca60f26088dcad297e5be01f9a80f256
+size 1996490952

model-00005-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72a000c280d93cdc52c5345b5a6feb454d2311fe8c00eb19d5fbbd9045efe207
+size 1996524632

model-00006-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67994c0f6f7e99161166233e9005e652ba44b2bf133b70f3a5d326c980137b6d
+size 1979853920

model-00007-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c06b141d84b7dbdd761671ba1ffd7e0722ef50b88c75d778486eda6421c8892d
+size 1912613160

model-00008-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f56861120939a733a22203128d170145f09c252a20580dd4277fbe54ab5e62b
+size 1996507568

model-00009-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08c490fdddbc0d154adffa4af02e6f83355c0a3a2bd0cebf3dc103238c19548c
+size 1963019112

model-00010-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cd00ccef65b98789b51f0d246be570fda394ca5b9da366c4761277fce6b606b
+size 1996490952

model-00011-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f49af5d90bcc77c97694c7834ed58cd90caca47cbcbf72aed1aa520e24e3c32c
+size 1963068152

model-00012-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ebd5045976ae6afa2733ef2cef6eda93fd341f559bb7f257361a29178f38391a
+size 1996490952

model-00013-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a612f0ef0c6833cfd32cf92f534c7e162b16bee0faf9f1d947a23def5ad5133d
+size 1929489960

model-00014-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4111873aec39a62bc9e7e21f38ad8e1ed63dd43b7de0866e5338df0674d7a0e
+size 1996556592

model-00015-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93a3b362bb3e79f5aed7fd73e3ebbbef95429c4e602ba8db9bd4c96c2b47204d
+size 1962945160

model-00016-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:039ac84c74253629e88fd60f6e89ceac3cc811db09186ea640b07902aa776637
+size 1996573240

model-00017-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ae9141aa1d065554e5a5e378502d959ec1478ef8aca935db4e3bce55de4ede7
+size 1962936872

model-00018-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f29f0dfccc1c3925eeee909b904882717cea9f22a50f5f5d2679f1f3cf9ac43
+size 1996507584

model-00019-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15019fae393ace7eb703aeb438487d816eda9805653effb88ef0c375b8721b48
+size 1996631048

model-00020-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b088a1d18910242a444889b590f9f2efa2dd812e1f827c1b0e1cb1f5f7addfcb
+size 1979722632

model-00021-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6495157e7721baa6669192021ca8123ca8ab067808e92dde31772651b82ae0b3
+size 1996655968

model-00022-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aec51ed86a70132105906c5d2fae032dfd536c6b3cbf9630e9d55b0b62d1baff
+size 1996490968

model-00023-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6524b68d0600a1e49fb56f6f8cbf52c22a12acf3ed01ffdd98ae23272ea4197c
+size 1962953488

model-00024-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e389d21bb5b4db29f750e315aec967d3e545ff933cdb291e4457504d2bcc0bf8
+size 1996556624

model-00025-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4477ba071b66e140899b2b4729a63cbc8127ca16c6c12b2aa586e79006bd0ec9
+size 1996490968

model-00026-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9407f59b63694655b97a58f07e76be09853972862efd9f93ad6400d986f2b82
+size 1963084800

model-00027-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9a4d1cbad9b203ee4ab5b375b7807c59670c9c4786319e16ac9ccb6d5e4c728
+size 1996490968

model-00028-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5925c8c27a89e0fb0942416a870bab1cf874b9a64f284c9645a2b2f2d4b91f2
+size 1929416016

model-00029-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4c5eb1369d2687e31268577dbc4cc10ac8d91f12c374051c66206fde5131fd2
+size 1996639352

model-00030-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00ccdc6b5131c81a5ddfd0a04129c33ece090cd2f8ea08ddcd1cc73eeb81be8e
+size 1929382328

model-00031-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6237eeaef8fdc5290ea3de5600b7d9a42a096d9b2c9ddeb139b6883af96c5224
+size 1963101416

model-00032-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3183dc280c418ae818e39a5d4ba06a06d2052d970e3147c6617793d8b006f1c8
+size 1996490968

model-00033-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c76fc309d9af757bec5a516b5a1bc8c83287d35f69ef7f4d15984dbf9c5b5396
+size 1962953488

model-00034-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:502253e5f3a914dd057da129b0af101ed47a4f9f40bac4d938f895af2dffc4f1
+size 1996556624

model-00035-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2b7630c381fecd781ef58b30899c11d1e21bde24261e81a85d9e4b6ad1d05c9
+size 1962936872

model-00036-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b6d0c308b6b673b30a9351d8f5827991bd981f19f4de80691d1f20508fc8234
+size 1996638888

model-00037-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72da9b6f382b64decd8f4d5eb84008916847505c1ff2f6917f3cf39303239657
+size 1962936872

model-00038-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:878e0ae2e80ff4a174aad471149e5571edd661391ca832b5a7f570e6dae2c147
+size 1996598624

model-00039-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:138a1307662b50294e90d6df09cf367d42e9bb2a05270053db1669bf3c40899e
+size 1929390632

model-00040-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa0c6c0d0bf7f3f01dbea5171fe180a1222848628972823b35058c8323f35ee2
+size 1996490968

model-00041-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e3d64e6631ca4f2943e30ba2813d6fa186135503a79ef7218514daa8fa3a79f
+size 1962953488

model-00042-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d78f35fae9799eae9abdefe3dbe5133fc9e9f3f7dbb2ccde3fe09a9fb6f13f7
+size 1996490968

model-00043-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:696486dd62f3b31da9ef45c12328ebd5fab9df386d2b61e9802f28d83949cbde
+size 1962953488

model-00044-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73d88bcb0f4d7ab1f7b12221093bae855d190f68b1d6d9a73b6b4a325cbbe964
+size 1996556624

model-00045-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51caa2812124cdb34af6c53a24ed26914609e11bad94a286f8ce0f1178bbfa5d
+size 1879050320

model-00046-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bdf32d1b72dbd9688706c2fb1c667d11e46ea9a23c29783c8164a1a7c06050bd
+size 1990315152

model-00047-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:753a0f8bd0a0a144a374c7eca21357e58c44ce0d35e5a78ce48225c536e3e433
+size 1962961792

model-00048-of-00048.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a68780731751cb3d23b8e57539589445e729190cef59ebb11e31f308f11d49cb
+size 587203248