diff --git a/exp/attack.py b/exp/attack.py index e6dab43..9ce2f43 100644 --- a/exp/attack.py +++ b/exp/attack.py @@ -1,4 +1,5 @@ import sys +import time import numpy as np # noinspection PyPackageRequirements @@ -47,6 +48,7 @@ def __init__(self, kind: str, constr: bool, conf): self.adv_y = None self.score = None self.conf = conf or {} + self.start = self.end = 0 def reset(self, cls): self.cls = cls @@ -55,6 +57,7 @@ def reset(self, cls): self.ori_x = cls.test_x.copy() self.adv_x = None self.adv_y = None + self.start = self.end = 0 return self @property @@ -63,6 +66,7 @@ def can_validate(self): def run(self, v_model: Validation): """Generate adversarial examples and score.""" + self.start = time.time_ns() if issubclass(self.attack, CPGD): self.adv_x, self.adv_y = cpgd_apply_and_predict( self.cls.model, self.ori_x, self.ori_y, **self.conf) @@ -73,13 +77,18 @@ def run(self, v_model: Validation): self.adv_x = aml_attack.generate(x=self.ori_x) self.adv_y = np.array(self.cls.predict( self.adv_x, self.ori_y).flatten()) + self.end = time.time_ns() sys.stdout.write('\x1b[1A') sys.stdout.write('\x1b[2K') self.score.calculate( - self, v_model.constraints, v_model.scalars) + self, v_model.constraints, v_model.scalars, + dur=self.end - self.start) return self def to_dict(self): - return {'name': self.name, 'config': self.conf, - 'can_validate': self.can_validate} + return { + 'name': self.name, + 'config': self.conf, + 'can_validate': self.can_validate + } diff --git a/exp/scoring.py b/exp/scoring.py index b3bc98b..837421d 100644 --- a/exp/scoring.py +++ b/exp/scoring.py @@ -2,7 +2,7 @@ import sklearn.metrics as skm from exp import CONSTR_DICT as CD, Validation -from exp.utility import sdiv, log, logr, logd, attr_of +from exp.utility import sdiv, log, logr, logd, attr_of, dur_sec def score_valid(ori: np.ndarray, adv: np.ndarray, cd: CD, scalars): @@ -41,7 +41,8 @@ def calculate(self, true_labels, predictions): def log(self): for a in attr_of(self, (int, float)): - log(a.capitalize(), f'{getattr(self, a) * 100:.2f} %') + log(a.capitalize().replace('_', '-'), + f'{getattr(self, a) * 100:.2f} %') class AttackScore: @@ -53,8 +54,9 @@ def __init__(self): self.n_records = 0 self.n_valid = 0 self.n_valid_evades = 0 + self.dur = 0 - def calculate(self, attack, constraints, attr_range): + def calculate(self, attack, constraints, attr_range, dur): ori_x, ori_y = attack.ori_x, attack.ori_y adv_x, adv_y = attack.adv_x, attack.adv_y original = attack.cls.predict(ori_x, ori_y) @@ -67,11 +69,13 @@ def calculate(self, attack, constraints, attr_range): self.n_evasions = len(self.evasions) self.n_valid_evades = len(self.valid_evades) self.n_records = ori_x.shape[0] + self.dur = dur def log(self): logr('Evasions', self.n_evasions, self.n_records) logr('Valid', self.n_valid, self.n_records) logr('Valid+Evades', self.n_valid_evades, self.n_records) + log('Attack Duration', f'{dur_sec(self.dur) :.2f} s') class Result: @@ -93,6 +97,7 @@ def __init__(self): self.n_evasions = Result.AvgList() self.n_valid = Result.AvgList() self.n_valid_evades = Result.AvgList() + self.dur = Result.AvgList() def append(self, obj): for a in attr_of(obj, (int, float)): @@ -113,3 +118,4 @@ def log(self): logd('Valid', self.n_valid.avg, self.n_records.avg) logd('Valid Evasions', self.n_valid_evades.avg, self.n_records.avg) + log('Attack Duration', f'{dur_sec(self.dur.avg) :.2f} s') diff --git a/exp/utility.py b/exp/utility.py index 03b5a2b..311234c 100644 --- a/exp/utility.py +++ b/exp/utility.py @@ -78,7 +78,7 @@ def log(label: str, value): def logr(label: str, n: float, d: float): a, b, r = round(n, 0), round(d, 0), sdiv(n, d) - r_ftm = "{r:.2f} %" if str(r).isnumeric() else r + r_ftm = f"{r:.2f} %" if str(r).isnumeric() else r return log(label, f'{a} of {b} - {r_ftm}') @@ -89,3 +89,8 @@ def logd(label: str, n: float, d: float): def time_sec(start: time, end: time) -> int: """Time difference in seconds.""" return round((end - start) / 1e9, 1) + + +def dur_sec(dur: time) -> int: + """Time difference in seconds.""" + return round(dur / 1e9, 1)