{ "_name_or_path": "tiiuae/falcon-40b", "alibi": false, "apply_residual_connection_post_layernorm": false, "architectures": [ "RWForCausalLM" ], "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_RW.RWConfig", "AutoModel": "modelling_RW.RWModel", "AutoModelForCausalLM": "modelling_RW.RWForCausalLM", "AutoModelForQuestionAnswering": "modelling_RW.RWForQuestionAnswering", "AutoModelForSequenceClassification": "modelling_RW.RWForSequenceClassification", "AutoModelForTokenClassification": "modelling_RW.RWForTokenClassification" }, "bias": false, "bos_token_id": null, "eos_token_id": 11, "hidden_dropout": 0.0, "hidden_size": 8192, "initializer_range": 0.02, "layer_norm_epsilon": 1e-05, "model_type": "RefinedWeb", "n_head": 128, "n_head_kv": 8, "n_layer": 60, "pad_token_id": 11, "parallel_attn": true, "torch_dtype": "bfloat16", "transformers_version": "4.30.2", "use_cache": true, "vocab_size": 65040 }