Spaces:
Runtime error
Runtime error
| name: megatron_gpt_1.3b | |
| restore_from_path: null # used when starting from a .nemo file | |
| trainer: | |
| devices: 8 | |
| num_nodes: 1 | |
| accelerator: gpu | |
| precision: bf16 | |
| logger: False # logger provided by exp_manager | |
| enable_checkpointing: False | |
| replace_sampler_ddp: False | |
| max_epochs: -1 # PTL default. In practice, max_steps will be reached first. | |
| max_steps: 200 # consumed_samples = global_step * micro_batch_size * data_parallel_size * accumulate_grad_batches | |
| log_every_n_steps: 1 | |
| val_check_interval: 20 | |
| # check_val_every_n_epoch: null | |
| limit_val_batches: 2 | |
| limit_test_batches: 0 | |
| accumulate_grad_batches: 1 # do not modify, grad acc is automatic for training megatron models | |
| gradient_clip_val: 1.0 | |
| benchmark: False | |
| exp_manager: | |
| # set this to save checkpoints | |
| explicit_log_dir: ppo_sentiments_logs | |
| exp_dir: null | |
| name: megatron_gpt_1.3b_ppo_sentiments | |
| create_tensorboard_logger: False | |
| create_wandb_logger: False | |
| wandb_logger_kwargs: | |
| project: trlxnemo | |
| name: megatron_gpt_1.3b_ppo_sentiments | |
| resume_if_exists: False | |
| resume_ignore_no_checkpoint: True | |
| # set this to save checkpoints | |
| create_checkpoint_callback: False | |
| checkpoint_callback_params: | |
| monitor: reduced_train_loss | |
| save_top_k: 1 | |
| mode: min | |
| always_save_nemo: False # saves nemo file during validation, not implemented for model parallel | |
| save_nemo_on_train_end: True # not recommended when training large models on clusters with short time limits | |
| filename: 'megatron_gpt-{reduced_train_loss:.2f}-{step}-{consumed_samples}' | |
| model_parallel_size: ${multiply:${model.tensor_model_parallel_size}, ${model.pipeline_model_parallel_size}} | |
| log_step_timing: True | |
| step_timing_kwargs: | |
| sync_cuda: True | |
| buffer_size: 5 | |
| model: | |
| global_batch_size: 256 | |
| micro_batch_size: 32 | |
| tensor_model_parallel_size: 1 | |
| pipeline_model_parallel_size: 1 | |
| encoder_seq_length: 2048 | |
| max_position_embeddings: 2048 | |
| num_layers: 24 | |
| hidden_size: 2048 | |
| ffn_hidden_size: 3072 | |
| num_attention_heads: 16 | |
| init_method_std: 0.015 | |
| hidden_dropout: 0.0 | |
| attention_dropout: 0.0 | |
| ffn_dropout: 0.0 | |
| kv_channels: null | |
| apply_query_key_layer_scaling: true | |
| layernorm_epsilon: 1.0e-05 | |
| make_vocab_size_divisible_by: 128 | |
| pre_process: true | |
| post_process: true | |
| tokenizer: | |
| library: megatron | |
| type: GPT2BPETokenizer | |
| model: null | |
| vocab_file: /artifacts/vocab.json | |
| merge_file: /artifacts/merges.txt | |
| native_amp_init_scale: 4294967296 | |
| native_amp_growth_interval: 1000 | |
| fp32_residual_connection: false | |
| fp16_lm_cross_entropy: false | |
| megatron_amp_O2: True | |
| sync_batch_comm: False | |
| seed: 1234 | |
| use_cpu_initialization: false | |
| onnx_safe: false | |
| activations_checkpoint_method: null | |
| activations_checkpoint_granularity: null | |
| activations_checkpoint_num_layers: null | |
| gradient_as_bucket_view: True | |
| resume_from_checkpoint: null | |
| sequence_parallel: True | |
| data: | |
| data_prefix: | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_00_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_01_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_02_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_03_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_04_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_05_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_06_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_07_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_08_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_09_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_10_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_11_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_12_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_13_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_14_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_15_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_16_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_17_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_18_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_19_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_20_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_21_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_22_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_23_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_24_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_25_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_26_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_27_text_document | |
| - 0.0333 | |
| - /preproc_data/my-gpt3_28_text_document | |
| - 0.0334 | |
| - /preproc_data/my-gpt3_29_text_document | |
| data_impl: mmap | |
| splits_string: 99990,8,2 | |
| seq_length: 2048 | |
| skip_warmup: true | |
| num_workers: 0 | |
| dataloader_type: single | |
| reset_position_ids: false | |
| reset_attention_mask: false | |
| eod_mask_loss: True | |
| optim: | |
| name: distributed_fused_adam | |
| lr: 6e-05 | |
| weight_decay: 1e-06 | |
| betas: | |
| - 0.9 | |
| - 0.95 | |
| sched: | |
| name: CosineAnnealing | |
| warmup_steps: 0 | |
| constant_steps: 100000000 | |
| min_lr: 5e-05 | |
| precision: bf16 | |
| vocab_file: nemo:c4aec99015da48ba8cbcba41b48feb2c_vocab.json | |
| merges_file: nemo:50284f68eefe440e850c4fb42c4d13e7_merges.txt | |