Upload params_train.yaml with huggingface_hub

params_train.yaml

@@ -0,0 +1,194 @@
+train_input:
+  batch_size: 256
+  data_processor: GptHDF5MapDataProcessor
+  data_dir: /cra-406/datasets/jais_instruction_datasets/v12p2/tokenized_mlv2_2k/
+  num_workers: 1
+  persistent_workers: true
+  prefetch_factor: 10
+  repeat: true
+  shuffle: false
+  shuffle_seed: 1
+  use_worker_cache: false
+  vocab_size: 84992
+eval_input:
+  batch_size: 32
+  data_dir: /cb/customers/g42/datasets/multilingual_v2/pile_val_correct_packed
+  data_processor: GptHDF5MapDataProcessor
+  num_workers: 1
+  repeat: false
+  shuffle: false
+  use_worker_cache: false
+  vocab_size: 84992
+model:
+  mixed_precision: true
+  fp16_type: cbfloat16
+  boundary_casting: false
+  lora_params: null
+  vocab_size: 84992
+  embedding_layer_norm: false
+  embedding_dropout_rate: 0.0
+  share_embedding_weights: true
+  position_embedding_type: alibi
+  max_position_embeddings: 2048
+  position_embedding_offset: 0
+  num_relative_attention_buckets: 32
+  rotary_dim: null
+  rope_theta: 10000
+  pad_rope: false
+  alibi_trainable_slopes: false
+  pos_scaling_factor: 1.0
+  hidden_size: 1088
+  num_hidden_layers: 14
+  dropout_rate: 0.0
+  norm_type: layernorm
+  layer_norm_epsilon: 1.0e-05
+  num_heads: 17
+  attention_module: aiayn_attention
+  extra_attention_params: {}
+  attention_type: scaled_dot_product
+  attention_dropout_rate: 0.0
+  use_projection_bias_in_attention: true
+  use_ffn_bias_in_attention: true
+  attention_softmax_fp32: false
+  attention_kernel: optimized_beta
+  attention_sliding_window_length: null
+  scale_qk_dot_by_layer_idx: false
+  fixed_sparse_attention: null
+  filter_size: 2912
+  nonlinearity: swiglu
+  use_ffn_bias: true
+  use_bias_in_output: false
+  loss_scaling: num_tokens
+  loss_weight: 1.0
+  embeddings_scale: 9.1705785388303
+  scale_qk_dot_by_d: true
+  output_logits_scale: 0.2576902348606329
+  initializer:
+    name: truncated_normal
+    mean: 0.0
+    std: 0.04203434605680388
+    a: -0.08406869211360776
+    b: 0.08406869211360776
+    nonlinearity: null
+    mode: null
+    scale: null
+    distribution: null
+  initializer_range: 0.02
+  embedding_initializer:
+    name: truncated_normal
+    mean: 0.0
+    std: 0.0866560243479838
+    a: -0.1733120486959676
+    b: 0.1733120486959676
+    nonlinearity: null
+    mode: null
+    scale: null
+    distribution: null
+  output_layer_initializer:
+    name: truncated_normal
+    mean: 0.0
+    std: 0.007943744727823684
+    a: -0.015887489455647368
+    b: 0.015887489455647368
+    nonlinearity: null
+    mode: null
+    scale: null
+    distribution: null
+  compute_eval_metrics: true
+sparsity: null
+optimizer:
+  optimizer_type: AdamW
+  weight_decay: 0.1
+  log_summaries: true
+  loss_scaling_factor: dynamic
+  learning_rate:
+  - end_learning_rate: 0.0016
+    initial_learning_rate: 0.0
+    scheduler: Linear
+    total_iters: 695
+  - end_learning_rate: 0.00016
+    initial_learning_rate: 0.0016
+    scheduler: Linear
+    total_iters: 23995
+  max_gradient_norm: 1.0
+  adjust_learning_rate:
+    decoder_kernel: 0.23529411764705882
+  betas:
+  - 0.9
+  - 0.95
+  correct_bias: true
+  eps: 1.0e-08
+runconfig:
+  steps_per_epoch: null
+  max_steps: 24690
+  mgmt_address: null
+  mount_dirs:
+  - /cra-406
+  num_epochs: null
+  python_paths:
+  - /cra-406/workdirs/modelzoos/rel-2.2.1/modelzoo/src
+  compile_dir: null
+  checkpoint_path: /cra-406/workdirs/240209_Jais_series_v3/artifacts/model_dir_256M/checkpoint_240320.mdl
+  credentials_path: null
+  debug_args_path: null
+  retrace_every_iteration: null
+  eval_steps: 5219
+  init_method: env://
+  job_time_sec: null
+  job_labels:
+  - Name=Neha_Sengupta
+  - Organization=Core42
+  - Model=Jais_256M
+  - Mode=Train
+  - Num_CSX=4
+  - Language=Bilingual
+  - Type=Train
+  - Dataset=v12p2
+  job_priority: p3
+  seed: 1
+  mgmt_namespace: null
+  load_checkpoint_states: model
+  target_device: CSX
+  mode: train
+  wsc_log_level: null
+  autoload_last_checkpoint: true
+  check_loss_values: true
+  disable_strict_checkpoint_loading: null
+  dist_addr: localhost:8888
+  dist_backend: nccl
+  checkpoint_steps: 8231
+  disable_version_check: true
+  drop_data: false
+  enable_distributed: false
+  model_dir: artifacts/jais_256M_v12p2_gbs256
+  save_initial_checkpoint: false
+  precision_opt_level: 1
+  num_workers_per_csx: 0
+  validate_only: null
+  logging: null
+  sync_batchnorm: false
+  compile_only: null
+  log_steps: 1
+  num_steps: null
+  transfer_processes: null
+  num_wgt_servers: null
+  num_csx: 4
+  num_act_servers: null
+  eval_frequency: null
+  execute_crd_memory_gi: null
+  compile_crd_memory_gi: null
+  op_profiler_config: null
+  dump_activations: false
+  log_input_summaries: false
+  main_process_id: 0
+  max_checkpoints: 100000
+  summary_dir: null
+  lazy_initialization: true
+  use_cstorch_optimizer_step: false
+  wrk_memory_gi: null
+  act_memory_gi: null
+  cmd_memory_gi: null
+  wgt_memory_gi: null
+  experimental: {}
+  ini: null
+  debug_args: null