Trained with Unsloth

config.json

@@ -13,14 +13,17 @@
   "hidden_size": 3072,
   "initializer_range": 0.02,
   "intermediate_size": 24576,
-  "max_position_embeddings": 8192,
+  "max_position_embeddings": 16384,
   "model_type": "gemma",
   "num_attention_heads": 16,
   "num_hidden_layers": 28,
   "num_key_value_heads": 16,
   "pad_token_id": 0,
   "rms_norm_eps": 1e-06,
-  "rope_scaling": null,
+  "rope_scaling": {
+    "factor": 2.0,
+    "type": "linear"
+  },
   "rope_theta": 10000.0,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.40.1",

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16e7f0ca7782e62a941aa3340ed9b0a60a3e14494ae155cacf74d4da99a08866
+oid sha256:d5df9566a7b47803175f8632bcd14a607f635f73062c4d9dff1eed8429bac9e6
 size 4995496656

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e64f985d1231d6a8775bee4be3a358eb8afb71ebb851b2a6716ae10f6420c75b
+oid sha256:08603609ec765e38788126340fae2c4c414b956c2e878ef8ee38a6201ef0d1d3
 size 4982953168

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be2dfebb1f542b70a29ebb4e5e7672bee0e80a1fa5c5be71bfe3996ff6a14279
+oid sha256:2881faa29f5c5fde536e729f19aabdaf7e69f7a1042c4a2c3da4dacc19aaad7e
 size 4982953200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d45c2c7316995780fdbe5d14d7fa1cbef8168e17d142f16328adcee4c6a8ac4
+oid sha256:809dcdd309927e6b6ad160ed1afe14dc6112ffc3d07de06e010da2ab11e6473f
 size 2113988336