include retnet modeling

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "RetNet"]
+	path = RetNet
+	url = https://github.com/syncdoth/RetNet.git

RetNet

@@ -0,0 +1 @@
+Subproject commit a4253a7bc16519459320c140a7e3d14b5f017b32

config.json

@@ -1,9 +1,11 @@
 {
   "activation_dropout": 0.0,
   "activation_fn": "swish",
-  "architectures": [
-    "RetNetForCausalLM"
-  ],
+  "architectures": ["LightNovelIntroRetNetModelConfig"],
+  "auto_map": {
+    "AutoConfig": "modeling_retnet.LightNovelIntroRetNetModelConfig",
+    "AutoModelForCausalLM": "modeling_retnet.LightNovelIntroRetNetModel"
+  },
   "decoder_embed_dim": 1280,
   "decoder_ffn_embed_dim": 2560,
   "decoder_layers": 12,
@@ -18,7 +20,7 @@
   "initializer_range": 0.02,
   "is_decoder": true,
   "layernorm_embedding": true,
-  "layernorm_eps": 1e-06,
+  "layernorm_eps": 1e-6,
   "model_type": "retnet",
   "no_scale_embedding": false,
   "output_retentions": false,

generation_config.json

@@ -2,5 +2,12 @@
   "_from_model_config": true,
   "eos_token_id": 50256,
   "pad_token_id": 50256,
+  "do_sample": true,
+  "temperature": 1.0,
+  "top_p": 0.95,
+  "top_k": 20,
+  "no_repeat_ngram_size": 3,
+  "repetition_penalty": 1.2,
+  "num_beams": 1,
   "transformers_version": "4.34.0"
 }

modeling_retnet.py

@@ -0,0 +1,8 @@
+from RetNet.retnet.modeling_retnet import RetNetForCausalLM, RetNetConfig
+
+
+LightNovelIntroRetNetModelConfig = RetNetConfig
+
+
+class LightNovelIntroRetNetModel(RetNetForCausalLM):
+    config_class = LightNovelIntroRetNetModelConfig