mlc-chat-config.json

{
  "model_type": "llama",
  "quantization": "q4f16_1",
  "model_config": {
    "hidden_size": 4096,
    "intermediate_size": 14336,
    "num_attention_heads": 32,
    "num_hidden_layers": 32,
    "rms_norm_eps": 1e-05,
    "vocab_size": 128256,
    "position_embedding_base": 500000.0,
    "context_window_size": 8192,
    "prefill_chunk_size": 8192,
    "num_key_value_heads": 8,
    "head_dim": 128,
    "tensor_parallel_shards": 1,
    "max_batch_size": 80
  },
  "vocab_size": 128256,
  "context_window_size": 8192,
  "sliding_window_size": -1,
  "prefill_chunk_size": 8192,
  "attention_sink_size": -1,
  "tensor_parallel_shards": 1,
  "mean_gen_len": 256,
  "max_gen_len": 1024,
  "shift_fill_factor": 0.3,
  "temperature": 0.7,
  "presence_penalty": 0.0,
  "frequency_penalty": 0.0,
  "repetition_penalty": 1.0,
  "top_p": 0.95,
  "conv_template": {
    "name": "llama-3",
    "system_template": "{system_message}",
    "system_message": "You are an AI tutor. You have to help a student learning programming. The program uses Python. You have to strictly follow the format for the final output as instructed below. While coming up with any explanations or codes, think step-by-step and justify your steps.",
    "system_prefix_token_ids": [
      128000
    ],
    "add_role_after_system_message": true,
    "roles": {
      "user": "user",
      "assistant": "assistant"
    },
    "role_templates": {
      "user": "{user_message}",
      "assistant": "{assistant_message}",
      "tool": "{tool_message}"
    },
    "messages": [],
    "seps": [
      "<|eot_id|><|start_header_id|>"
    ],
    "role_content_sep": "<|end_header_id|>\n\n",
    "role_empty_sep": "<|end_header_id|>\n\n",
    "stop_str": [
      "<|end_of_text|>",
      "<|eot_id|>"
    ],
    "stop_token_ids": [
      128001,
      128009
    ],
    "function_string": "",
    "use_function_calling": false
  },
  "pad_token_id": 0,
  "bos_token_id": 128000,
  "eos_token_id": [
    128001,
    128009
  ],
  "tokenizer_files": [
    "tokenizer.json",
    "tokenizer_config.json"
  ],
  "version": "0.1.0"
}