|
main: build = 3003 (d298382a) |
|
main: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu |
|
main: seed = 1716745611 |
|
llama_model_loader: loaded meta data with 25 key-value pairs and 291 tensors from Mistral-7B-Instruct-v0.3-IMat-GGUF/Mistral-7B-Instruct-v0.3.gguf (version GGUF V3 (latest)) |
|
llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. |
|
llama_model_loader: - kv 0: general.architecture str = llama |
|
llama_model_loader: - kv 1: general.name str = Mistral-7B-Instruct-v0.3 |
|
llama_model_loader: - kv 2: llama.block_count u32 = 32 |
|
llama_model_loader: - kv 3: llama.context_length u32 = 32768 |
|
llama_model_loader: - kv 4: llama.embedding_length u32 = 4096 |
|
llama_model_loader: - kv 5: llama.feed_forward_length u32 = 14336 |
|
llama_model_loader: - kv 6: llama.attention.head_count u32 = 32 |
|
llama_model_loader: - kv 7: llama.attention.head_count_kv u32 = 8 |
|
llama_model_loader: - kv 8: llama.rope.freq_base f32 = 1000000.000000 |
|
llama_model_loader: - kv 9: llama.attention.layer_norm_rms_epsilon f32 = 0.000010 |
|
llama_model_loader: - kv 10: general.file_type u32 = 0 |
|
llama_model_loader: - kv 11: llama.vocab_size u32 = 32768 |
|
llama_model_loader: - kv 12: llama.rope.dimension_count u32 = 128 |
|
llama_model_loader: - kv 13: tokenizer.ggml.model str = llama |
|
llama_model_loader: - kv 14: tokenizer.ggml.pre str = default |
|
llama_model_loader: - kv 15: tokenizer.ggml.tokens arr[str,32768] = ["<unk>", "<s>", "</s>", "[INST]", "[... |
|
llama_model_loader: - kv 16: tokenizer.ggml.scores arr[f32,32768] = [0.000000, 0.000000, 0.000000, 0.0000... |
|
llama_model_loader: - kv 17: tokenizer.ggml.token_type arr[i32,32768] = [2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ... |
|
llama_model_loader: - kv 18: tokenizer.ggml.bos_token_id u32 = 1 |
|
llama_model_loader: - kv 19: tokenizer.ggml.eos_token_id u32 = 2 |
|
llama_model_loader: - kv 20: tokenizer.ggml.unknown_token_id u32 = 0 |
|
llama_model_loader: - kv 21: tokenizer.ggml.add_bos_token bool = true |
|
llama_model_loader: - kv 22: tokenizer.ggml.add_eos_token bool = false |
|
llama_model_loader: - kv 23: tokenizer.chat_template str = {{ bos_token }}{% for message in mess... |
|
llama_model_loader: - kv 24: general.quantization_version u32 = 2 |
|
llama_model_loader: - type f32: 291 tensors |
|
llm_load_vocab: special tokens definition check successful ( 1027/32768 ). |
|
llm_load_print_meta: format = GGUF V3 (latest) |
|
llm_load_print_meta: arch = llama |
|
llm_load_print_meta: vocab type = SPM |
|
llm_load_print_meta: n_vocab = 32768 |
|
llm_load_print_meta: n_merges = 0 |
|
llm_load_print_meta: n_ctx_train = 32768 |
|
llm_load_print_meta: n_embd = 4096 |
|
llm_load_print_meta: n_head = 32 |
|
llm_load_print_meta: n_head_kv = 8 |
|
llm_load_print_meta: n_layer = 32 |
|
llm_load_print_meta: n_rot = 128 |
|
llm_load_print_meta: n_embd_head_k = 128 |
|
llm_load_print_meta: n_embd_head_v = 128 |
|
llm_load_print_meta: n_gqa = 4 |
|
llm_load_print_meta: n_embd_k_gqa = 1024 |
|
llm_load_print_meta: n_embd_v_gqa = 1024 |
|
llm_load_print_meta: f_norm_eps = 0.0e+00 |
|
llm_load_print_meta: f_norm_rms_eps = 1.0e-05 |
|
llm_load_print_meta: f_clamp_kqv = 0.0e+00 |
|
llm_load_print_meta: f_max_alibi_bias = 0.0e+00 |
|
llm_load_print_meta: f_logit_scale = 0.0e+00 |
|
llm_load_print_meta: n_ff = 14336 |
|
llm_load_print_meta: n_expert = 0 |
|
llm_load_print_meta: n_expert_used = 0 |
|
llm_load_print_meta: causal attn = 1 |
|
llm_load_print_meta: pooling type = 0 |
|
llm_load_print_meta: rope type = 0 |
|
llm_load_print_meta: rope scaling = linear |
|
llm_load_print_meta: freq_base_train = 1000000.0 |
|
llm_load_print_meta: freq_scale_train = 1 |
|
llm_load_print_meta: n_yarn_orig_ctx = 32768 |
|
llm_load_print_meta: rope_finetuned = unknown |
|
llm_load_print_meta: ssm_d_conv = 0 |
|
llm_load_print_meta: ssm_d_inner = 0 |
|
llm_load_print_meta: ssm_d_state = 0 |
|
llm_load_print_meta: ssm_dt_rank = 0 |
|
llm_load_print_meta: model type = 7B |
|
llm_load_print_meta: model ftype = all F32 |
|
llm_load_print_meta: model params = 7.25 B |
|
llm_load_print_meta: model size = 27.00 GiB (32.00 BPW) |
|
llm_load_print_meta: general.name = Mistral-7B-Instruct-v0.3 |
|
llm_load_print_meta: BOS token = 1 '<s>' |
|
llm_load_print_meta: EOS token = 2 '</s>' |
|
llm_load_print_meta: UNK token = 0 '<unk>' |
|
llm_load_print_meta: LF token = 781 '<0x0A>' |
|
ggml_cuda_init: GGML_CUDA_FORCE_MMQ: no |
|
ggml_cuda_init: CUDA_USE_TENSOR_CORES: yes |
|
ggml_cuda_init: found 1 CUDA devices: |
|
Device 0: NVIDIA GeForce RTX 4090, compute capability 8.9, VMM: yes |
|
llm_load_tensors: ggml ctx size = 0.30 MiB |
|
llm_load_tensors: offloading 25 repeating layers to GPU |
|
llm_load_tensors: offloaded 25/33 layers to GPU |
|
llm_load_tensors: CPU buffer size = 27649.02 MiB |
|
llm_load_tensors: CUDA0 buffer size = 20800.78 MiB |
|
................................................................................................... |
|
llama_new_context_with_model: n_ctx = 512 |
|
llama_new_context_with_model: n_batch = 512 |
|
llama_new_context_with_model: n_ubatch = 512 |
|
llama_new_context_with_model: flash_attn = 0 |
|
llama_new_context_with_model: freq_base = 1000000.0 |
|
llama_new_context_with_model: freq_scale = 1 |
|
llama_kv_cache_init: CUDA_Host KV buffer size = 14.00 MiB |
|
llama_kv_cache_init: CUDA0 KV buffer size = 50.00 MiB |
|
llama_new_context_with_model: KV self size = 64.00 MiB, K (f16): 32.00 MiB, V (f16): 32.00 MiB |
|
llama_new_context_with_model: CUDA_Host output buffer size = 0.12 MiB |
|
llama_new_context_with_model: CUDA0 compute buffer size = 584.00 MiB |
|
llama_new_context_with_model: CUDA_Host compute buffer size = 9.01 MiB |
|
llama_new_context_with_model: graph nodes = 1030 |
|
llama_new_context_with_model: graph splits = 81 |
|
|
|
system_info: n_threads = 25 / 32 | AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | FMA = 1 | NEON = 0 | SVE = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | |
|
compute_imatrix: tokenizing the input .. |
|
compute_imatrix: tokenization took 135.736 ms |
|
compute_imatrix: computing over 228 chunks with batch_size 512 |
|
compute_imatrix: 0.89 seconds per pass - ETA 3.35 minutes |
|
[1]3.7636,[2]2.8000,[3]2.8426,[4]2.9852,[5]3.3598,[6]3.2903,[7]3.0106,[8]3.4845,[9]3.6286, |
|
save_imatrix: stored collected data after 10 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[10]4.0039,[11]4.1684,[12]4.1105,[13]4.3626,[14]4.1543,[15]4.4947,[16]4.6395,[17]4.8555,[18]4.9841,[19]5.1167, |
|
save_imatrix: stored collected data after 20 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[20]5.2947,[21]5.4169,[22]5.3040,[23]5.1116,[24]5.1945,[25]4.9353,[26]4.7608,[27]4.6577,[28]4.6066,[29]4.6324, |
|
save_imatrix: stored collected data after 30 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[30]4.7295,[31]4.8700,[32]4.9863,[33]5.0148,[34]5.0800,[35]4.9080,[36]4.8060,[37]4.7535,[38]4.7688,[39]4.7671, |
|
save_imatrix: stored collected data after 40 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[40]4.7312,[41]4.7783,[42]4.7807,[43]4.8577,[44]4.9476,[45]4.9572,[46]5.0449,[47]5.1809,[48]5.2907,[49]5.4296, |
|
save_imatrix: stored collected data after 50 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[50]5.5129,[51]5.5297,[52]5.4888,[53]5.4517,[54]5.3621,[55]5.4166,[56]5.4671,[57]5.5175,[58]5.5679,[59]5.5963, |
|
save_imatrix: stored collected data after 60 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[60]5.6691,[61]5.7043,[62]5.7493,[63]5.7634,[64]5.7868,[65]5.8185,[66]5.8538,[67]5.8974,[68]5.9436,[69]5.9592, |
|
save_imatrix: stored collected data after 70 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[70]5.9939,[71]5.9563,[72]5.9138,[73]5.8805,[74]5.8533,[75]5.8415,[76]5.8319,[77]5.8048,[78]5.7549,[79]5.7344, |
|
save_imatrix: stored collected data after 80 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[80]5.7353,[81]5.7068,[82]5.7606,[83]5.7953,[84]5.8060,[85]5.7514,[86]5.7651,[87]5.7297,[88]5.6835,[89]5.6742, |
|
save_imatrix: stored collected data after 90 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[90]5.6662,[91]5.6798,[92]5.6800,[93]5.6960,[94]5.6911,[95]5.6394,[96]5.6009,[97]5.5938,[98]5.6197,[99]5.6320, |
|
save_imatrix: stored collected data after 100 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[100]5.6231,[101]5.5923,[102]5.5684,[103]5.5735,[104]5.5669,[105]5.5525,[106]5.5375,[107]5.5369,[108]5.5448,[109]5.5596, |
|
save_imatrix: stored collected data after 110 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[110]5.5428,[111]5.5494,[112]5.5424,[113]5.5360,[114]5.5276,[115]5.5334,[116]5.5280,[117]5.5200,[118]5.4913,[119]5.4975, |
|
save_imatrix: stored collected data after 120 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[120]5.5215,[121]5.5294,[122]5.5230,[123]5.5310,[124]5.5376,[125]5.5595,[126]5.5121,[127]5.5065,[128]5.4851,[129]5.4567, |
|
save_imatrix: stored collected data after 130 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[130]5.4815,[131]5.4541,[132]5.4251,[133]5.3966,[134]5.3706,[135]5.3428,[136]5.3176,[137]5.2943,[138]5.2737,[139]5.2594, |
|
save_imatrix: stored collected data after 140 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[140]5.2534,[141]5.2456,[142]5.2224,[143]5.2169,[144]5.2027,[145]5.1923,[146]5.1832,[147]5.1723,[148]5.1677,[149]5.1549, |
|
save_imatrix: stored collected data after 150 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[150]5.1456,[151]5.1634,[152]5.1416,[153]5.1476,[154]5.1775,[155]5.2033,[156]5.2186,[157]5.2349,[158]5.2531,[159]5.2859, |
|
save_imatrix: stored collected data after 160 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[160]5.3100,[161]5.3264,[162]5.3332,[163]5.3395,[164]5.3552,[165]5.3545,[166]5.3633,[167]5.3803,[168]5.3897,[169]5.4034, |
|
save_imatrix: stored collected data after 170 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[170]5.4033,[171]5.4220,[172]5.4360,[173]5.4390,[174]5.4497,[175]5.4365,[176]5.4510,[177]5.4551,[178]5.4716,[179]5.4665, |
|
save_imatrix: stored collected data after 180 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[180]5.4838,[181]5.4842,[182]5.4795,[183]5.4779,[184]5.4745,[185]5.4854,[186]5.4930,[187]5.5137,[188]5.5144,[189]5.4943, |
|
save_imatrix: stored collected data after 190 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[190]5.5264,[191]5.5579,[192]5.5876,[193]5.6364,[194]5.6699,[195]5.6796,[196]5.6899,[197]5.6743,[198]5.6784,[199]5.6951, |
|
save_imatrix: stored collected data after 200 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[200]5.7213,[201]5.7168,[202]5.7140,[203]5.7213,[204]5.7329,[205]5.7356,[206]5.7411,[207]5.7491,[208]5.7586,[209]5.7729, |
|
save_imatrix: stored collected data after 210 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[210]5.7936,[211]5.7835,[212]5.7839,[213]5.7774,[214]5.7734,[215]5.7671,[216]5.7610,[217]5.7611,[218]5.7819,[219]5.7653, |
|
save_imatrix: stored collected data after 220 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
[220]5.7753,[221]5.8063,[222]5.8261,[223]5.8534,[224]5.8680,[225]5.8672,[226]5.8428,[227]5.8242,[228]5.8091, |
|
save_imatrix: stored collected data after 228 chunks in Mistral-7B-Instruct-v0.3-IMat-GGUF/imatrix.dat |
|
|
|
llama_print_timings: load time = 2897.33 ms |
|
llama_print_timings: sample time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) |
|
llama_print_timings: prompt eval time = 183295.25 ms / 116736 tokens ( 1.57 ms per token, 636.87 tokens per second) |
|
llama_print_timings: eval time = 0.00 ms / 1 runs ( 0.00 ms per token, inf tokens per second) |
|
llama_print_timings: total time = 185956.92 ms / 116737 tokens |
|
|
|
Final estimate: PPL = 5.8091 +/- 0.05771 |
|
|
