How to fix optimizer in SGD to Adam?

[sinyoung-park]

i just want to change optimizer type='SGD' to 'Adam'. And than i change the optimizer option

%%writefile configs/resnet/resnet34_8xb32_in1k_custom.py

base = [
'../base/models/resnet34.py', '../base/datasets/imagenet_bs32.py',
'../base/schedules/imagenet_bs256.py', '../base/default_runtime.py'
]

model settings

model = dict(
type='ImageClassifier',
backbone=dict(
type='ResNet',
depth=34,
num_stages=4,
out_indices=(3, ),
style='pytorch'),
neck=dict(type='GlobalAveragePooling'),
head=dict(
type='LinearClsHead',
num_classes=5,
in_channels=512,
loss=dict(type='CrossEntropyLoss', loss_weight=0.8),
topk=(1, 5),
))

dataset settings

dataset_type = 'CustomDataset'
img_norm_cfg = dict(
mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
train_pipeline = [
dict(type='LoadImageFromFile'),
dict(type='RandomResizedCrop', size=224),
dict(type='RandomFlip', flip_prob=0.5, direction='horizontal'),
dict(type='Normalize', **img_norm_cfg),
dict(type='ImageToTensor', keys=['img']),
dict(type='ToTensor', keys=['gt_label']),
dict(type='Collect', keys=['img', 'gt_label'])
]
test_pipeline = [
dict(type='LoadImageFromFile'),
dict(type='Resize', size=(256, -1)),
dict(type='CenterCrop', crop_size=224),
dict(type='Normalize', **img_norm_cfg),
dict(type='ImageToTensor', keys=['img']),
dict(type='Collect', keys=['img'])
]
data = dict(
samples_per_gpu=8,
workers_per_gpu=2,
train=dict(
type=dataset_type,
data_prefix='/content/mmclassification/pcldata/train_set',
ann_file='/content/mmclassification/pcldata/train.txt',
classes = ['8slm', '12slm', '16slm', '20slm', '24slm'],
pipeline=train_pipeline),
val=dict(
type=dataset_type,
data_prefix='/content/mmclassification/pcldata/val_set',
ann_file='/content/mmclassification/pcldata/val.txt',
classes=['8slm', '12slm', '16slm', '20slm', '24slm'],
pipeline=test_pipeline),
test=dict(
# replace data/val with data/test for standard test
type=dataset_type,
data_prefix='/content/mmclassification/pcldata/test_set',
ann_file='/content/mmclassification/pcldata/test.txt',
classes=['8slm', '12slm', '16slm', '20slm', '24slm'],
pipeline=test_pipeline))
evaluation = dict(interval=1, metric='accuracy')

optimizer

optimizer = optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)
optimizer_config = dict(grad_clip=None)

learning policy

lr_config = dict(policy='step', step=[30, 60, 90])
runner = dict(type='EpochBasedRunner', max_epochs=12)

checkpoint saving

checkpoint_config = dict(interval=1)

yapf:disable

log_config = dict(
interval=100,
hooks=[
dict(type='TextLoggerHook'),
# dict(type='TensorboardLoggerHook')
])

yapf:enable

dist_params = dict(backend='nccl')
log_level = 'INFO'
load_from = None
resume_from = None
workflow = [('train', 1)]
it is configfile before.

%%writefile configs/resnet/resnet34_8xb32_in1k_custom.py

base = [
'../base/models/resnet34.py', '../base/datasets/imagenet_bs32.py',
'../base/schedules/imagenet_bs256.py', '../base/default_runtime.py'
]

model settings

model = dict(
type='ImageClassifier',
backbone=dict(
type='ResNet',
depth=34,
num_stages=4,
out_indices=(3, ),
style='pytorch'),
neck=dict(type='GlobalAveragePooling'),
head=dict(
type='LinearClsHead',
num_classes=5,
in_channels=512,
loss=dict(type='CrossEntropyLoss', loss_weight=0.8),
topk=(1, 5),
))

dataset settings

dataset_type = 'CustomDataset'
img_norm_cfg = dict(
mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
train_pipeline = [
dict(type='LoadImageFromFile'),
dict(type='RandomResizedCrop', size=224),
dict(type='RandomFlip', flip_prob=0.5, direction='horizontal'),
dict(type='Normalize', **img_norm_cfg),
dict(type='ImageToTensor', keys=['img']),
dict(type='ToTensor', keys=['gt_label']),
dict(type='Collect', keys=['img', 'gt_label'])
]
test_pipeline = [
dict(type='LoadImageFromFile'),
dict(type='Resize', size=(256, -1)),
dict(type='CenterCrop', crop_size=224),
dict(type='Normalize', **img_norm_cfg),
dict(type='ImageToTensor', keys=['img']),
dict(type='Collect', keys=['img'])
]
data = dict(
samples_per_gpu=8,
workers_per_gpu=2,
train=dict(
type=dataset_type,
data_prefix='/content/mmclassification/pcldata/train_set',
ann_file='/content/mmclassification/pcldata/train.txt',
classes = ['8slm', '12slm', '16slm', '20slm', '24slm'],
pipeline=train_pipeline),
val=dict(
type=dataset_type,
data_prefix='/content/mmclassification/pcldata/val_set',
ann_file='/content/mmclassification/pcldata/val.txt',
classes=['8slm', '12slm', '16slm', '20slm', '24slm'],
pipeline=test_pipeline),
test=dict(
# replace data/val with data/test for standard test
type=dataset_type,
data_prefix='/content/mmclassification/pcldata/test_set',
ann_file='/content/mmclassification/pcldata/test.txt',
classes=['8slm', '12slm', '16slm', '20slm', '24slm'],
pipeline=test_pipeline))
evaluation = dict(interval=1, metric='accuracy')

optimizer

optimizer = optimizer = dict(type='Adam', lr=0.01,Beta_1=0.9, Beta_2=0.999,epsilon=1e-07 ,weight_decay=None)
optimizer_config = dict(grad_clip=None)

learning policy

lr_config = dict(policy='step', step=[30, 60, 90])
runner = dict(type='EpochBasedRunner', max_epochs=12)

checkpoint saving

checkpoint_config = dict(interval=1)

yapf:disable

log_config = dict(
interval=100,
hooks=[
dict(type='TextLoggerHook'),
# dict(type='TensorboardLoggerHook')
])

yapf:enable

dist_params = dict(backend='nccl')
log_level = 'INFO'
load_from = None
resume_from = None
workflow = [('train', 1)]
and it is after config file code.
when i run this code, Even though I didn't put momentum, the code recognizes momentum and says this is an argument that doesn't need momentum. how can i fix it?

[Ezra-Yu]

We are also experimenting with the AdamW, Lion optimizer on ImageNet1-k, but we are not currently getting comparable results to SGD on ResNet either. It is currently found that using a smaller lr, a larger wd and the addition of optim_wrapper setting would improve the results.

optim_wrapper = dict(
    optimizer=dict(type='AdamW',xxxxxxx.....)),
    clip_grad=dict(max_norm=1.0),
    paramwise_cfg=dict(
        norm_decay_mult=0.0,
        bias_decay_mult=0.0,),
)