uploading local files

.gitignore

@@ -134,4 +134,5 @@ checkpoint/
 checkpoints/
 
 *.zip
-aurocs/
+aurocs/
+pacscheck/

afterbreak.sh

@@ -0,0 +1,2 @@
+export CUDA_VISIBLE_DEVICES='0'
+python train.py --dataset cifar100 --model resnet56 --epochs 160 --schedule-steps 80 120 --loss LogitNorm

analyze.py

@@ -30,14 +30,16 @@ def get_basic_curves(logfilename):
     auroc = []
     for line in lines:
         if 'INFO:  End of epoch' in line:
-            train_loss.append(float(line.split('train_loss : ')[1].split(" ")[0]))
-            val_loss.append(float(line.split('val_loss : ')[1].split(" ")[0]))
-            top1_train.append(float(line.split('top1_train : ')[1].split(" ")[0]))
-            top1.append(float(line.split('top1 : ')[1].split(" ")[0]))
-            sce.append(float(line.split('SCE : ')[1].split(" ")[0]))
-            ece.append(float(line.split('ECE : ')[1].split(" ")[0]))
+            # print(line)
+            # print(line.split('train_loss : '))
+            train_loss.append(float(line.split('train_loss: ')[1].split(" ")[0]))
+            val_loss.append(float(line.split('val_loss: ')[1].split(" ")[0]))
+            top1_train.append(float(line.split('top1_train: ')[1].split(" ")[0]))
+            top1.append(float(line.split('top1: ')[1].split(" ")[0]))
+            sce.append(float(line.split('SCE: ')[1].split(" ")[0]))
+            ece.append(float(line.split('ECE: ')[1].split(" ")[0]))
             try:
-                auroc.append(float(line.split('AUROC : ')[1].split(" ")[0]))
+                auroc.append(float(line.split('AUROC: ')[1].split(" ")[0]))
             except:
                 pass
     return train_loss, val_loss, top1_train, top1, sce, ece, auroc

calibration_library/calibrators.py

@@ -109,7 +109,7 @@ def fit(self, train_loader, lr=0.001, epochs=25, patience=10):
         scheduler = EarlyStopping(patience=patience)
 
         # send model to gpu
-        self.cuda(1)
+        self.cuda()
 
         last_loss = 0.0
 

datasets/pacs.py

@@ -24,6 +24,7 @@
 }
 
 def get_train_valid_test_loader(args, target_type="art"):
+    target_type=args.target_type
     # Prepare Pytorch train/test Datasets
     train_set = datasets.ImageFolder(path_dict["photo"], transform=transf)
     test_set = datasets.ImageFolder(path_dict[target_type], transform=transf)
@@ -34,6 +35,7 @@ def get_train_valid_test_loader(args, target_type="art"):
     return train_loader, test_loader, test_loader
 
 def get_datasets(args, target_type="art"):
+    target_type=args.target_type
     train_set = datasets.ImageFolder(path_dict["photo"])
     test_set = datasets.ImageFolder(path_dict[target_type])
     return train_set, test_set

pacscheck.py

@@ -0,0 +1,36 @@
+# iterate on random seeds and check if the PACS is working
+
+import os
+import json
+base=" python train.py --dataset pacs --model resnet_pacs --train-batch-size 256 --test-batch-size 256 --epochs 30 --schedule-steps 20 --lr 0.01 --seed "
+losses=["cross_entropy"]
+import sys
+args=sys.argv
+# main function
+for seed in range(int(args[1]),int(args[2])):
+    for loss in losses:
+        sces=[]
+        # for a seed execute command and store the console output
+        os.system(base+str(seed)+" --loss "+loss+" --target_type=art"+" 2> pacscheck/"+str(seed)+".txt")
+        with open('train_results_anirudha.json') as f :
+            data = json.load(f)
+            sces.append(data[-1]['SCE'])
+        # collect result from json object
+        os.system(base+str(seed)+" --loss "+loss+" --target_type=cartoon"+" 2> pacscheck/"+str(seed)+".txt")
+        with open('train_results_anirudha.json') as f :
+            data = json.load(f)
+            sces.append(data[-1]['SCE'])
+        os.system(base+str(seed)+" --loss "+loss+" --target_type=sketch"+" 2> pacscheck/"+str(seed)+".txt")
+        with open('train_results_anirudha.json') as f :
+            data = json.load(f)
+            sces.append(data[-1]['SCE'])
+        # store the results in csv
+        with open('pacscheck.csv', 'a') as f:
+            f.write(str(seed)+","+str(sces[0])+","+str(sces[1])+","+str(sces[2])+","+str((sces[0]+sces[1]+sces[2])/3)+"\n")
+
+
+
+
+
+
+

posthoc_calibrate.py

@@ -11,9 +11,11 @@
 from calibration_library.calibrators import TemperatureScaling, DirichletScaling
 
 import logging
+import torch.nn as nn 
 
 if __name__ == "__main__":
-
+    t={"a":"aa","b":"bb","c":"cc"}
+    print(t)
     args = parse_args()
     logging.basicConfig(level=logging.INFO, 
                         format="%(levelname)s:  %(message)s",
@@ -33,9 +35,13 @@
 
     checkpoint_folder = os.path.dirname(args.checkpoint)
     saved_model_dict = torch.load(args.checkpoint)
-
-    model = model_dict[args.model](num_classes=num_classes, alpha=args.alpha)
-    model.load_state_dict(saved_model_dict['state_dict'])
+    model = model_dict[args.model](num_classes=num_classes, alpha=args.alpha, args=args)
+    # model.load_state_dict(saved_model_dict['state_dict'])
+    try:
+        model.load_state_dict(saved_model_dict['state_dict'])
+    except:
+        model=nn.DataParallel(model)
+        model.load_state_dict(saved_model_dict['state_dict'])
     model.cuda()
 
     # set up dataset
@@ -49,10 +55,10 @@
     metric_log_path = os.path.join(checkpoint_folder, 'temperature.txt')
     logger = Logger(metric_log_path, resume=os.path.exists(metric_log_path))
 
-    logger.set_names(['temprature', 'SCE', 'ECE'])
+    logger.set_names(['temprature', 'top1','SCE', 'ECE', 'ALL'])
 
-    test_loss, top1, top3, top5, cce_score, ece_score = test(testloader, model, criterion)
-    logger.append(["1.0", cce_score, ece_score])
+    test_loss, top1, top3, top5, cce_score, ece_score, all_metrics = test(testloader, model, criterion)
+    logger.append(["1.0", top1,cce_score, ece_score,str(all_metrics)])
 
     # Set up temperature scaling
     temperature_model = TemperatureScaling(base_model=model)
@@ -61,8 +67,10 @@
     logging.info("Running temp scaling:")
     temperature_model.calibrate(valloader)
 
-    test_loss, top1, top3, top5, cce_score, ece_score = test(testloader, temperature_model, criterion)
-    logger.append(["{:.2f}".format(temperature_model.T), cce_score, ece_score])
+    test_loss, top1, top3, top5, cce_score, ece_score,all_metrics = test(testloader, temperature_model, criterion)
+    print(["{:.2f}".format(temperature_model.T), cce_score, ece_score,str(all_metrics)])
+    print(len(["{:.2f}".format(temperature_model.T), cce_score, ece_score,str(all_metrics)]))
+    logger.append(["{:.2f}".format(temperature_model.T), top1, cce_score, ece_score,str(all_metrics)])
     logger.close()
 
 
@@ -74,7 +82,7 @@
     # set up loggers
     metric_log_path = os.path.join(checkpoint_folder, 'dirichlet.txt')
     logger = Logger(metric_log_path, resume=os.path.exists(metric_log_path))
-    logger.set_names(['method', 'test_nll', 'top1', 'top3', 'top5', 'SCE', 'ECE'])
+    logger.set_names(['method', 'test_nll', 'top1', 'top3', 'top5', 'SCE', 'ECE','ALL'])
 
     min_stats = {}
     min_error = float('inf')
@@ -87,8 +95,8 @@
 
             # calibrate
             dir_model.calibrate(valloader, lr=args.lr, epochs=args.epochs, patience=args.patience)
-            val_nll, _, _, _, _, _ = test(valloader, dir_model, criterion)
-            test_loss, top1, top3, top5, sce_score, ece_score = test(testloader, dir_model, criterion)
+            val_nll, _, _, _, _, _,_ = test(valloader, dir_model, criterion)
+            test_loss, top1, top3, top5, sce_score, ece_score,all_metrics = test(testloader, dir_model, criterion)
 
             if val_nll < min_error:
                 min_error = val_nll
@@ -99,16 +107,18 @@
                     "top5" : top5,
                     "ece_score" : ece_score,
                     "sce_score" : sce_score,
-                    "pair" : (l, m)
+                    "pair" : (l, m),
+                    "all": all_metrics
                 }
 
-            logger.append(["Dir=({:.2f},{:.2f})".format(l, m), test_loss, top1, top3, top5, sce_score, ece_score])
+            logger.append(["Dir=({:.2f},{:.2f})".format(l, m), test_loss, top1, top3, top5, sce_score, ece_score,str(all_metrics)])
 
     logger.append(["Best_Dir={}".format(min_stats["pair"]), 
                                             min_stats["test_loss"], 
                                             min_stats["top1"], 
                                             min_stats["top3"], 
                                             min_stats["top5"], 
                                             min_stats["sce_score"], 
-                                            min_stats["ece_score"]])
+                                            min_stats["ece_score"],
+                                            min_stats["all"]])
 

solvers/loss.py

@@ -290,6 +290,11 @@ def __init__(self):
         super(MDCA,self).__init__()
 
     def forward(self , output, target):
+
+        # TODO:  DELETE THIS LINE
+        norms = torch.norm(output, p=2, dim=-1, keepdim=True) + 1e-7
+        output = torch.div(output, norms) / 1.0
+
         output = torch.softmax(output, dim=1)
         # [batch, classes]
         loss = torch.tensor(0.0).cuda()

solvers/runners.py

@@ -4,8 +4,10 @@
 import numpy as np
 from calibration_library.metrics import ECELoss, SCELoss
 from sklearn.metrics import roc_auc_score
-torch.manual_seed(0)
-np.random.seed(0)
+from utils import parse_args
+args=parse_args()
+torch.manual_seed(args.seed)
+np.random.seed(args.seed)
 
 def train(trainloader, model, optimizer, criterion):
     # switch to train mode

temp-2.sh

@@ -105,18 +105,123 @@ python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80
 
 export CUDA_VISIBLE_DEVICES='1'
 screen bash -c \
-"python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
-python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
-python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
-python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+"python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
 "
 
 
 
-export CUDA_VISIBLE_DEVICES='0,7'
+export CUDA_VISIBLE_DEVICES='2,3,7'
 screen bash -c \
-"python3 train.py --dataset imagenet --model resnet50_imagenet --epochs 160 --schedule-steps 80 120 --loss LogitNorm+MDCA --beta 5.0
-python3 train.py --dataset imagenet --model resnet50_imagenet --epochs 160 --schedule-steps 80 120 --loss LogitNorm+MDCA --beta 5.0
-python3 train.py --dataset imagenet --model resnet50_imagenet --epochs 160 --schedule-steps 80 120 --loss LogitNorm+MDCA --beta 5.0
-python3 train.py --dataset imagenet --model resnet50_imagenet --epochs 160 --schedule-steps 80 120 --loss LogitNorm+MDCA --beta 5.0
+"python3 train.py --dataset imagenet --model resnet50_imagenet --epochs 160 --schedule-steps 80 120 --loss LogitNorm+MDCA --beta 20.0
+python3 train.py --dataset imagenet --model resnet50_imagenet --epochs 160 --schedule-steps 80 120 --loss LogitNorm+MDCA --beta 20.0
+python3 train.py --dataset imagenet --model resnet50_imagenet --epochs 160 --schedule-steps 80 120 --loss LogitNorm+MDCA --beta 20.0
+python3 train.py --dataset imagenet --model resnet50_imagenet --epochs 160 --schedule-steps 80 120 --loss LogitNorm+MDCA --beta 20.0
+"
+
+
+export CUDA_VISIBLE_DEVICES='1'
+screen bash -c \
+"python3 train.py --dataset cifar100 --model resnet32 --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+python3 train.py --dataset cifar100 --model resnet32 --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+python3 train.py --dataset cifar100 --model resnet32 --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+python3 train.py --dataset cifar100 --model resnet32 --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+"
+
+
+export CUDA_VISIBLE_DEVICES='0'
+screen bash -c \
+"python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm+MDCA --beta 5.0
+"
+export CUDA_VISIBLE_DEVICES='1'
+screen bash -c \
+"python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm
+python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm
+python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm
+python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm
+"
+export CUDA_VISIBLE_DEVICES='1'
+screen bash -c \
+"python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss cross_entropy
+python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss cross_entropy
+python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss cross_entropy
+python3 train.py --dataset svhn --model wrn --epochs 200 --schedule-steps 80 140 --loss cross_entropy
+"
+
+python posthoc_calibrate.py --checkpoint checkpoint/imagenet/04-Jul_resnet50_imagenet_LogitNorm_adjacent_anirudha/model_best.pth --model resnet50_imagenet --dataset imagenet
+python posthoc_calibrate.py --checkpoint checkpoint/imagenet/05-Jul_resnet50_imagenet_LogitNorm+MDCA_adjacent_beta=5.0_anirudha/model_best.pth --model resnet50_imagenet --dataset imagenet
+python posthoc_calibrate.py --checkpoint checkpoint/cifar10/04-Jul_resnet50_imagenet_LogitNorm_adjacent_anirudha/model_best.pth --model resnet32 --dataset cifar10
+python posthoc_calibrate.py --checkpoint checkpoint/svhn/05-Jul_resnet50_imagenet_LogitNorm+MDCA_adjacent_anirudha/model_best.pth --model wrn --dataset svhn
+
+export CUDA_VISIBLE_DEVICES='4,5,6'
+screen bash -c \
+"python3 train.py --dataset imagenet --model resnet50_imagenet --epochs 160 --schedule-steps 80 120 --loss cross_entropy
+"
+
+export CUDA_VISIBLE_DEVICES='1'
+screen bash -c \
+"python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm
+python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm
+python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm
+python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss LogitNorm
+"
+
+export CUDA_VISIBLE_DEVICES='2'
+screen bash -c \
+"python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA --beta 10.0 --gamma 3.0
+python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA --beta 10.0 --gamma 3.0
+python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA --beta 10.0 --gamma 3.0
+python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA --beta 10.0 --gamma 3.0
+"
+
+export CUDA_VISIBLE_DEVICES='3'
+screen bash -c \
+"python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+MDCA --beta 10.0
+python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+MDCA --beta 10.0 
+python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+MDCA --beta 10.0 
+python3 train.py --dataset cifar10 --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+MDCA --beta 10.0 
+"
+export CUDA_VISIBLE_DEVICES='4'
+screen bash -c \
+"python3 train.py --dataset cifar10_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA+CRL --beta 10.0 --gamma 3.0
+python3 train.py --dataset cifar10_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA+CRL --beta 10.0 --gamma 3.0
+python3 train.py --dataset cifar10_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA+CRL --beta 10.0 --gamma 3.0
+python3 train.py --dataset cifar10_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA+CRL --beta 10.0 --gamma 3.0
+"
+
+export CUDA_VISIBLE_DEVICES='6'
+screen bash -c \
+"python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss cross_entropy
+python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss cross_entropy
+python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss cross_entropy
+python3 train.py --dataset cifar100 --model wrn --epochs 200 --schedule-steps 80 140 --loss cross_entropy
+"
+
+
+export CUDA_VISIBLE_DEVICES='4'
+screen bash -c \
+"python3 train.py --dataset svhn_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA+CRL --beta 10.0 --gamma 3.0
+python3 train.py --dataset svhn_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA+CRL --beta 10.0 --gamma 3.0
+python3 train.py --dataset svhn_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA+CRL --beta 10.0 --gamma 3.0
+python3 train.py --dataset svhn_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss FL+MDCA+CRL --beta 10.0 --gamma 3.0
+"
+
+export CUDA_VISIBLE_DEVICES='3'
+screen bash -c \
+"python train.py --dataset cifar100_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+CRL --theta 1.0 && \
+python train.py --dataset cifar100_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+CRL --theta 1.0 && \
+python train.py --dataset cifar100_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+CRL --theta 1.0 && \
+python train.py --dataset cifar100_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+CRL --theta 1.0
+"
+export CUDA_VISIBLE_DEVICES='3'
+screen bash -c \
+"python train.py --dataset cifar100_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+CRL --theta 1.0 && \
+python train.py --dataset cifar100_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+CRL --theta 1.0 && \
+python train.py --dataset cifar100_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+CRL --theta 1.0 && \
+python train.py --dataset cifar100_CRL --model wrn --epochs 200 --schedule-steps 80 140 --loss NLL+CRL --theta 1.0
 "

temp-5.sh

@@ -37,3 +37,20 @@ python train_imbalanced.py --dataset im_cifar10 --model resnet32 --schedule-step
 python train_imbalanced.py --dataset im_cifar10 --model resnet32 --schedule-steps 80 120 --epochs 160 --imbalance 0.01 --loss LS+MDCA --beta 15 --alpha 0.01
 python train_imbalanced.py --dataset im_cifar10 --model resnet32 --schedule-steps 80 120 --epochs 160 --imbalance 0.01 --loss LS+MDCA --beta 20 --alpha 0.01
 
+export CUDA_VISIBLE_DEVICES=0,1
+screen bash -c \
+"python3 aucm.py >> aucm.txt
+"
+
+export CUDA_VISIBLE_DEVICES=6,7
+screen bash -c \
+"python3 aucm_mdca.py >> aucm_mdca.txt
+"
+export CUDA_VISIBLE_DEVICES=6
+screen bash -c \
+"python pacscheck.py 960 1200
+"
+export CUDA_VISIBLE_DEVICES=7
+screen bash -c \
+"python pacscheck.py 1200 1440
+"

test.py

@@ -14,11 +14,11 @@
 import logging
 import json
 
-np.random.seed(0)
+args = parse_args()
+np.random.seed(args.seed)
 
 if __name__ == "__main__":
 
-    args = parse_args()
     logging.basicConfig(level=logging.INFO, 
                         format="%(levelname)s:  %(message)s",
                         handlers=[