Add model and tokenizer files

Browse files

Files changed (8) hide show

added_tokens.json +118 -0
config.json +98 -0
merges.txt +0 -0
pytorch_model.bin +3 -0
special_tokens_map.json +56 -0
tokenizer.json +0 -0
tokenizer_config.json +65 -0
vocab.json +0 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,118 @@

+{
+  "<add>": 32101,
+  "<c>": 32213,
+  "<c_plus_plus>": 32215,
+  "<c_sharp>": 32214,
+  "<del>": 32102,
+  "<e0>": 32204,
+  "<e10>": 32194,
+  "<e11>": 32193,
+  "<e12>": 32192,
+  "<e13>": 32191,
+  "<e14>": 32190,
+  "<e15>": 32189,
+  "<e16>": 32188,
+  "<e17>": 32187,
+  "<e18>": 32186,
+  "<e19>": 32185,
+  "<e1>": 32203,
+  "<e20>": 32184,
+  "<e21>": 32183,
+  "<e22>": 32182,
+  "<e23>": 32181,
+  "<e24>": 32180,
+  "<e25>": 32179,
+  "<e26>": 32178,
+  "<e27>": 32177,
+  "<e28>": 32176,
+  "<e29>": 32175,
+  "<e2>": 32202,
+  "<e30>": 32174,
+  "<e31>": 32173,
+  "<e32>": 32172,
+  "<e33>": 32171,
+  "<e34>": 32170,
+  "<e35>": 32169,
+  "<e36>": 32168,
+  "<e37>": 32167,
+  "<e38>": 32166,
+  "<e39>": 32165,
+  "<e3>": 32201,
+  "<e40>": 32164,
+  "<e41>": 32163,
+  "<e42>": 32162,
+  "<e43>": 32161,
+  "<e44>": 32160,
+  "<e45>": 32159,
+  "<e46>": 32158,
+  "<e47>": 32157,
+  "<e48>": 32156,
+  "<e49>": 32155,
+  "<e4>": 32200,
+  "<e50>": 32154,
+  "<e51>": 32153,
+  "<e52>": 32152,
+  "<e53>": 32151,
+  "<e54>": 32150,
+  "<e55>": 32149,
+  "<e56>": 32148,
+  "<e57>": 32147,
+  "<e58>": 32146,
+  "<e59>": 32145,
+  "<e5>": 32199,
+  "<e60>": 32144,
+  "<e61>": 32143,
+  "<e62>": 32142,
+  "<e63>": 32141,
+  "<e64>": 32140,
+  "<e65>": 32139,
+  "<e66>": 32138,
+  "<e67>": 32137,
+  "<e68>": 32136,
+  "<e69>": 32135,
+  "<e6>": 32198,
+  "<e70>": 32134,
+  "<e71>": 32133,
+  "<e72>": 32132,
+  "<e73>": 32131,
+  "<e74>": 32130,
+  "<e75>": 32129,
+  "<e76>": 32128,
+  "<e77>": 32127,
+  "<e78>": 32126,
+  "<e79>": 32125,
+  "<e7>": 32197,
+  "<e80>": 32124,
+  "<e81>": 32123,
+  "<e82>": 32122,
+  "<e83>": 32121,
+  "<e84>": 32120,
+  "<e85>": 32119,
+  "<e86>": 32118,
+  "<e87>": 32117,
+  "<e88>": 32116,
+  "<e89>": 32115,
+  "<e8>": 32196,
+  "<e90>": 32114,
+  "<e91>": 32113,
+  "<e92>": 32112,
+  "<e93>": 32111,
+  "<e94>": 32110,
+  "<e95>": 32109,
+  "<e96>": 32108,
+  "<e97>": 32107,
+  "<e98>": 32106,
+  "<e99>": 32105,
+  "<e9>": 32195,
+  "<en>": 32206,
+  "<end>": 32104,
+  "<go>": 32212,
+  "<java>": 32208,
+  "<javascript>": 32209,
+  "<keep>": 32100,
+  "<msg>": 32205,
+  "<php>": 32211,
+  "<python>": 32207,
+  "<ruby>": 32210,
+  "<start>": 32103
+}

config.json ADDED Viewed

	@@ -0,0 +1,98 @@

+{
+  "_name_or_path": "microsoft/codereviewer",
+  "add_token_id": 32101,
+  "architectures": [
+    "T5ForConditionalGeneration"
+  ],
+  "bos_token_id": 1,
+  "d_ff": 3072,
+  "d_kv": 64,
+  "d_model": 768,
+  "decoder_start_token_id": 0,
+  "del_token_id": 32102,
+  "dense_act_fn": "relu",
+  "dropout_rate": 0.1,
+  "end_token_id": 32104,
+  "eos_token_id": 2,
+  "feed_forward_proj": "relu",
+  "gradient_checkpointing": false,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": false,
+  "keep_token_id": 32100,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "lang_id": {
+    "<c>": 32213,
+    "<c_plus_plus>": 32215,
+    "<c_sharp>": 32214,
+    "<en>": 32206,
+    "<go>": 32212,
+    "<java>": 32208,
+    "<javascript>": 32209,
+    "<php>": 32211,
+    "<python>": 32207,
+    "<ruby>": 32210
+  },
+  "lang_tokens": [
+    "<en>",
+    "<python>",
+    "<java>",
+    "<javascript>",
+    "<ruby>",
+    "<php>",
+    "<go>",
+    "<c>",
+    "<c_sharp>",
+    "<c_plus_plus>"
+  ],
+  "layer_norm_epsilon": 1e-06,
+  "mask_token_id": 4,
+  "model_type": "t5",
+  "n_positions": 512,
+  "num_decoder_layers": 12,
+  "num_heads": 12,
+  "num_layers": 12,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "start_token_id": 32103,
+  "task_specific_params": {
+    "summarization": {
+      "early_stopping": true,
+      "length_penalty": 2.0,
+      "max_length": 200,
+      "min_length": 30,
+      "no_repeat_ngram_size": 3,
+      "num_beams": 4,
+      "prefix": "summarize: "
+    },
+    "translation_en_to_de": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to German: "
+    },
+    "translation_en_to_fr": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to French: "
+    },
+    "translation_en_to_ro": {
+      "early_stopping": true,
+      "max_length": 300,
+      "num_beams": 4,
+      "prefix": "translate English to Romanian: "
+    }
+  },
+  "torch_dtype": "float32",
+  "transformers_version": "4.21.3",
+  "use_cache": true,
+  "vocab_size": 32216
+}

merges.txt ADDED Viewed

The diff for this file is too large to render. See raw diff

pytorch_model.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:c37503607ff56790e49418536b1d9e59b08ecfd0f27aa5e037bfc48d4d8a2da1
+size 891967487

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,56 @@

+{
+  "additional_special_tokens": [
+    "<add>",
+    "<del>",
+    "<keep>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,65 @@

+{
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "__type": "AddedToken",
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 512,
+  "name_or_path": "microsoft/codereviewer",
+  "pad_token": {
+    "__type": "AddedToken",
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "__type": "AddedToken",
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "special_tokens_map_file": "/home/shuailu/lushuai/code_review/PreViewer/pretrained_models/codet5/special_tokens_map.json",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

vocab.json ADDED Viewed

The diff for this file is too large to render. See raw diff