pretrain_wikibio_lstm.cfg

# Model/Embeddings
word_vec_size: 300  # Word embedding size for src and tgt
share_embeddings: True  # Share embeddings from src and tgt

# Model/Embedding Features
feat_vec_size: -1  # Attribute embedding size. -1 means <nb_features>**0.7
feat_merge: mlp  # Merge action for incorporating feature embeddings [concat|sum|mlp]


# Model Structure
model_type: text  # Type of source model to use [text|img|audio]
model_dtype: fp32
encoder_type: brnn  # Type of encoder [rnn|brnn|transformer|cnn]
decoder_type: rnn  # Type of decoder [rnn|transformer|cnn]
bidirectional_encoder: true
param_init: 0.1  # Uniform distribution with support (-param_init, +param_init)

layers: 2
rnn_size: 300
input_feed: 1
bridge: True
rnn_type: LSTM


# Model/Attention
global_attention: general  # Type of attn to use [dot|general|mlp|none]
global_attention_function: softmax  # [softmax|sparsemax]
generator_function: softmax

# Model/Copy
copy_attn: True
reuse_copy_attn: True  # Reuse standard attention for copy
copy_attn_force: True  # When available, train to copy


#  Files and logs
data: experiments/wikibio/pretraining-lstm/data/data  # path to datafile from preprocess.py
save_model: experiments/wikibio/pretraining-lstm/models/model  # path to store checkpoints
log_file: experiments/wikibio/pretraining-lstm/train-log.txt

report_every: 250  # log current loss every X steps
save_checkpoint_steps: 1000  # save a cp every X steps


# Gpu related:
gpu_ranks: [0]  # ids of gpus to use
world_size: 1  # total number of distributed processes
gpu_backend: nccl  # type of torch distributed backend
gpu_verbose_level: 0
master_ip: localhost
master_port: 10000
seed: 123


# Optimization & training
batch_size: 32
batch_type: sents
normalization: sents
accum_count: [4]  # Update weights every X batches
accum_steps: [0]  # steps at which accum counts value changes
valid_steps: 100000  # run models on validation set every X steps
train_steps: 100000
optim: adam
max_grad_norm: 5
dropout: .3
adam_beta1: 0.9
adam_beta2: 0.999
label_smoothing: 0.0
average_decay: 0
average_every: 1

# Learning rate
learning_rate: 0.001
learning_rate_decay: 0.5  # lr *= lr_decay
start_decay_step: 5000
decay_steps: 10000