File size: 3,424 Bytes
7934b29 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 |
# BERT Pretraining from Text
name: &name PretrainingBERTFromText
trainer:
devices: 1 # the number of gpus, 0 for CPU, or list with gpu indices
num_nodes: 1
max_epochs: 2 # the number of training epochs
max_steps: -1 # precedence over max_epochs
accumulate_grad_batches: 1 # accumulates grads every k batches
precision: 16 # 16 to use AMP
accelerator: gpu
gradient_clip_val: 0.0
log_every_n_steps: 1
val_check_interval: 1.0 # check once per epoch .25 for 4 times per epoch
enable_checkpointing: False # provided by exp_manager
logger: false # provided by exp_manager
model:
nemo_path: null # exported .nemo path
only_mlm_loss: false # only use masked language model without next sentence prediction
num_tok_classification_layers: 1 # number of token classification head output layers
num_seq_classification_layers: 2 # number of sequence classification head output layers
max_seq_length: 128
# The maximum total input sequence length after tokenization. Sequences longer than this
# will be truncated, and sequences shorter than this will be padded.
mask_prob: 0.15
# Probability of masking a token in the input text during data processing.
short_seq_prob: 0.1
# Probability of having a sequence shorter than the maximum sequence length `max_seq_length` in data processing.",
language_model:
pretrained_model_name: bert-base-uncased
lm_checkpoint: null
config:
attention_probs_dropout_prob: 0.1
hidden_act: gelu
hidden_dropout_prob: 0.1
hidden_size: 768
initializer_range: 0.02
intermediate_size: 3072
max_position_embeddings: 512
num_attention_heads: 12
num_hidden_layers: 12
type_vocab_size: 2
vocab_size: 30522
config_file: null # json file, precedence over config
tokenizer:
tokenizer_name: ${model.language_model.pretrained_model_name} # tokenizer that inherits from TokenizerSpec
vocab_file: null # path to vocab file
tokenizer_model: null # tokenizer model for sentencepiece
special_tokens: # only necessary for adding transformer/bert-specific special tokens to tokenizer if the tokenizer does not already have these inherently.
unk_token: '[UNK]'
sep_token: '[SEP]'
pad_token: '[PAD]'
bos_token: '[CLS]'
mask_token: '[MASK]'
eos_token: '[SEP]'
cls_token: '[CLS]'
train_ds:
data_file: ??? # path to data file
max_seq_length: ${model.max_seq_length}
mask_prob: ${model.mask_prob}
short_seq_prob: ${model.short_seq_prob}
batch_size: 16 # per GPU
shuffle: true
num_samples: -1
num_workers: 2
drop_last: false
pin_memory: false
validation_ds:
data_file: ??? # path to data file
max_seq_length: ${model.max_seq_length}
mask_prob: ${model.mask_prob}
short_seq_prob: ${model.short_seq_prob}
batch_size: 16 # per GPU
shuffle: false
num_samples: -1
num_workers: 2
drop_last: false
pin_memory: false
optim:
name: adamw
lr: 3e-5
weight_decay: 0.0
sched:
name: CosineAnnealing
warmup_steps: null
warmup_ratio: 0.1
min_lr: 0.0
last_epoch: -1
exp_manager:
exp_dir: null # where to store logs and checkpoints
name: *name # name of experiment
create_tensorboard_logger: True
create_checkpoint_callback: True
hydra:
run:
dir: .
job_logging:
root:
handlers: null
|