config.json

{
 "vocab_size": 32064,
 "hidden_size": 3072,
 "intermediate_size": 8192,
 "num_hidden_layers": 32,
 "num_attention_heads": 32,
 "num_key_value_heads": 32,
 "resid_pdrop": 0.0,
 "embd_pdrop": 0.0,
 "attention_dropout": 0.0,
 "hidden_act": "silu",
 "max_position_embeddings": 131072,
 "original_max_position_embeddings": 4096,
 "initializer_range": 0.02,
 "rms_norm_eps": 1e-05,
 "use_cache": true,
 "rope_theta": 10000.0,
 "rope_scaling": {
  "long_factor": [
   1.0299999713897705,
   1.0499999523162842,
   1.0499999523162842,
   1.0799999237060547,
   1.2299998998641968,
   1.2299998998641968,
   1.2999999523162842,
   1.4499999284744263,
   1.5999999046325684,
   1.6499998569488525,
   1.8999998569488525,
   2.859999895095825,
   3.68999981880188,
   5.419999599456787,
   5.489999771118164,
   5.489999771118164,
   9.09000015258789,
   11.579999923706055,
   15.65999984741211,
   15.769999504089355,
   15.789999961853027,
   18.360000610351562,
   21.989999771118164,
   23.079999923706055,
   30.009998321533203,
   32.35000228881836,
   32.590003967285156,
   35.56000518798828,
   39.95000457763672,
   53.840003967285156,
   56.20000457763672,
   57.95000457763672,
   59.29000473022461,
   59.77000427246094,
   59.920005798339844,
   61.190006256103516,
   61.96000671386719,
   62.50000762939453,
   63.3700065612793,
   63.48000717163086,
   63.48000717163086,
   63.66000747680664,
   63.850006103515625,
   64.08000946044922,
   64.760009765625,
   64.80001068115234,
   64.81001281738281,
   64.81001281738281
  ],
  "short_factor": [
   1.05,
   1.05,
   1.05,
   1.1,
   1.1,
   1.1500000000000001,
   1.2000000000000002,
   1.2500000000000002,
   1.3000000000000003,
   1.3500000000000003,
   1.5000000000000004,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.000000000000001,
   2.0500000000000007,
   2.0500000000000007,
   2.0500000000000007,
   2.1000000000000005,
   2.1000000000000005,
   2.1000000000000005,
   2.1500000000000004,
   2.1500000000000004,
   2.3499999999999996,
   2.549999999999999,
   2.5999999999999988,
   2.5999999999999988,
   2.7499999999999982,
   2.849999999999998,
   2.849999999999998,
   2.9499999999999975
  ],
  "type": "su"
 },
 "sliding_window": 262144,
 "return_dict": true,
 "output_hidden_states": false,
 "output_attentions": false,
 "torchscript": false,
 "torch_dtype": "float16",
 "use_bfloat16": false,
 "tf_legacy_loss": false,
 "pruned_heads": {},
 "tie_word_embeddings": false,
 "chunk_size_feed_forward": 0,
 "is_encoder_decoder": false,
 "is_decoder": false,
 "cross_attention_hidden_size": null,
 "add_cross_attention": false,
 "tie_encoder_decoder": false,
 "max_length": 20,
 "min_length": 0,
 "do_sample": false,
 "early_stopping": false,
 "num_beams": 1,
 "num_beam_groups": 1,
 "diversity_penalty": 0.0,
 "temperature": 1.0,
 "top_k": 50,
 "top_p": 1.0,
 "typical_p": 1.0,
 "repetition_penalty": 1.0,
 "length_penalty": 1.0,
 "no_repeat_ngram_size": 0,
 "encoder_no_repeat_ngram_size": 0,
 "bad_words_ids": null,
 "num_return_sequences": 1,
 "output_scores": false,
 "return_dict_in_generate": false,
 "forced_bos_token_id": null,
 "forced_eos_token_id": null,
 "remove_invalid_values": false,
 "exponential_decay_length_penalty": null,
 "suppress_tokens": null,
 "begin_suppress_tokens": null,
 "architectures": [
  "Phi3ForCausalLM"
 ],
 "finetuning_task": null,
 "id2label": {
  "0": "LABEL_0",
  "1": "LABEL_1"
 },
 "label2id": {
  "LABEL_0": 0,
  "LABEL_1": 1
 },
 "tokenizer_class": null,
 "prefix": null,
 "bos_token_id": 1,
 "pad_token_id": 32000,
 "eos_token_id": 32000,
 "sep_token_id": null,
 "decoder_start_token_id": null,
 "task_specific_params": null,
 "problem_type": null,
 "_name_or_path": "microsoft/Phi-3-mini-128k-instruct",
 "transformers_version": "4.41.2",
 "auto_map": {
  "AutoConfig": "microsoft/Phi-3-mini-128k-instruct--configuration_phi3.Phi3Config",
  "AutoModelForCausalLM": "microsoft/Phi-3-mini-128k-instruct--modeling_phi3.Phi3ForCausalLM",
  "AutoModelForSequenceClassification": "microsoft/Phi-3-mini-128k-instruct--modeling_phi3.Phi3ForSequenceClassification",
  "AutoModelForTokenClassification": "microsoft/Phi-3-mini-128k-instruct--modeling_phi3.Phi3ForTokenClassification"
 },
 "model_type": "phi3",
 "attention_bias": false,
 "num_experts": 2,
 "num_experts_per_tok": 2,
 "router_layers": [
  "gate_up_proj",
  "down_proj"
 ],
 "router_layers_index": [
  0,
  1,
  2,
  3,
  4,
  5,
  6,
  7,
  8,
  9,
  10,
  11,
  12,
  13,
  14,
  15,
  16,
  17,
  18,
  19,
  20,
  21,
  22,
  23,
  24,
  25,
  26,
  27,
  28,
  29,
  30,
  31
 ]
}