diff --git a/Tune_new.py b/Tune_new.py index 14228f3..a221003 100644 --- a/Tune_new.py +++ b/Tune_new.py @@ -165,65 +165,6 @@ def log_metrics(evaluation_domains, reloaded, strategy_in_use): strategy=strategy_in_use ) -# performs MIA and ZRF attack on models and logs the results -def run_unlearning_and_attack_eval(forget_train_loader, retain_test_loader, reloaded, strategy_in_use, suite_runner, device, forget_class): - """ - Performs adversarial vulnerability stress tests (MIA and ZRF) in-memory - on the freshly unlearned model instance without saving it to disk first. - """ - if suite_runner is None: - raise ValueError("An active initialized UnlearningAttackSuite instance must be supplied.") - - print(f"\n>>> Initializing Threat Model Stress Testing Suite for: {strategy_in_use}") - - # 1. Dynamically map the white-box feature extraction hook to the active inner model - suite_runner.register_model_hook(reloaded.model) - - # 2. Fire the complete evaluation suite using the isolated data split subsets - results = suite_runner.run_complete_evaluation( - target_class=forget_class, - framework_name=strategy_in_use, - forget_loader=forget_train_loader, # Members split from the train data partition - retain_test_loader=retain_test_loader, # Clean non-members split from validation data - device=device - ) - - print(f" [Attack Complete] Logit MIA AUC: {results['logit_mia_auc']:.4f} | " - f"Internal MIA AUC: {results['internal_mia_auc']:.4f} | " - f"ZRF Score: {results['zrf_score']:.4f}") - - -# performs MIA and ZRF attack on models and logs the results -def run_shaddow_attack_eval(forget_train_loader, retain_test_loader, reloaded, strategy_in_use, suite_runner, device, forget_class): - """ - Performs adversarial vulnerability stress tests matching the localized - shadow architecture specifications laid out in thesis Section 5.5. - """ - if suite_runner is None: - raise ValueError("An active initialized UnlearningAttackSuite instance must be supplied.") - - print(f"\n>>> Initializing Threat Model Stress Testing Suite for: {strategy_in_use}") - - # Instantiate a clean copy of the baseline trained model to serve as the Shadow reference proxy - # (Since finetuning is done once, we read its parameters cleanly from disk) - base_shadow = Model.create(arch=ARCH, device=device, size=CLASS_SIZE) - base_shadow.load(arch=ARCH) - - # Execute the updated conditional attack framework - results = suite_runner.run_complete_evaluation( - framework_name=strategy_in_use, - target_class=forget_class, - forget_loader=forget_train_loader, - retain_test_loader=retain_test_loader, - unlearned_instance=reloaded, # The unlearned candidate model - base_shadow_instance=base_shadow, # The shadow proxy architecture - device=device - ) - - print(f" [Attack Complete] Adversary Binary Classification Accuracy: {results['mia_accuracy']:.4f}") - - - # Unlearning and strategy eval def run_unlearning_and_strategy_eval(env_dict, forget_class_idx, strategy, evaluate = False, suite_runner=None): """ @@ -393,8 +334,8 @@ if __name__ == "__main__": #dist_attacker.run_adversarial_evaluation() #dist_attacker.run_incremental_evaluation(current_class_step=i) - if suite_runner is not None: - suite_runner.shutdown_hook() + #if suite_runner is not None: + #suite_runner.shutdown_hook() except KeyboardInterrupt: print("\nprogram interrupted. Exit!") diff --git a/eval/UnlearningAttack.py b/eval/UnlearningAttack.py index 87bbb02..5497bf3 100644 --- a/eval/UnlearningAttack.py +++ b/eval/UnlearningAttack.py @@ -18,9 +18,9 @@ class UnlearningAttack: self.arch = arch self.class_size = class_size - self.hook = None - self.model = None - self._hook_features = [] + #self.hook = None + #self.model = None + #self._hook_features = [] self.criterion = nn.CrossEntropyLoss(reduction='none') self.collecting = False @@ -46,7 +46,7 @@ class UnlearningAttack: def calculate_a_dist(self, latent1, latent2): """Calculates formal A-Distance: 2 * (1 - 2 * epsilon).""" - combined = np.vstack([latent1, latent2]) + '''combined = np.vstack([latent1, latent2]) mean = np.mean(combined, axis=0) std = np.std(combined, axis=0) + 1e-8 l1 = (latent1 - mean) / std @@ -63,13 +63,16 @@ class UnlearningAttack: split = int(len(X) * 0.7) clf = LogisticRegression(solver='liblinear').fit(X[:split], y[:split]) - epsilon = 1.0 - accuracy_score(y[split:], clf.predict(X[split:])) + epsilon = 1.0 - accuracy_score(y[split:], clf.predict(X[split:]))''' + + accuracy_score = self._comput_adversarial_accuracy(latent1, latent2, axis=0) + epsilon = 1 - accuracy_score return 2.0 * np.abs(0.5 - epsilon) - def _hook_fn(self, module, input, output): + '''def _hook_fn(self, module, input, output): if not self.collecting: return flattened_embeddings = torch.flatten(output, 1) @@ -99,9 +102,11 @@ class UnlearningAttack: if hasattr(self, 'hook') and self.hook: self.hook.remove() self.hook = None - + ''' def _extract_attack_features(self, target_model, loader, device, target_class): - '''target_model.eval() + ''' + # cnahgng to black box. + target_model.eval() all_probs = [] all_entropies = [] all_losses = [] @@ -223,11 +228,11 @@ class UnlearningAttack: # Note: Latent distance is removed as it's not a black-box metric return mia_accuracy, 0.0 - def _comput_adversarial_accuracy(self, filtered, naive, axis=-1): + def _comput_adversarial_accuracy(self, filtered, naive, axis=0): # Z-Score Normalisation - filtered = (filtered - np.mean(filtered, axis=-1, keepdims=True)) / (np.std(filtered, axis=-1, keepdims=True) + 1e-8) - naive = (naive - np.mean(naive, axis=-1, keepdims=True)) / (np.std(naive, axis=-1, keepdims=True) + 1e-8) + filtered = (filtered - np.mean(filtered, axis = axis, keepdims=True)) / (np.std(filtered, axis = axis, keepdims = True) + 1e-8) + naive = (naive - np.mean(naive, axis = axis, keepdims = True)) / (np.std(naive, axis = axis, keepdims = True) + 1e-8) # shuffle indices num_images = len(filtered) @@ -282,7 +287,7 @@ class UnlearningAttack: naive = torch.cat(naive_logits, dim=0).cpu().numpy() # evaluate similarity of outputs - lookalike_accuracy = self._comput_adversarial_accuracy(filtered=filtered, naive=naive) + lookalike_accuracy = self._comput_adversarial_accuracy(filtered=filtered, naive=naive, axis = -1) # so that the metric is between 0 and 1. return 2.0 * np.abs(lookalike_accuracy - 0.5) @@ -296,7 +301,7 @@ class UnlearningAttack: with open(current_log_file, "w") as f: f.write("target_class, parameter_mia_accuracy, latent_distance_tell, lookalike_accuracy, A-Dist, JS-Dist\n") - self.register_model_hook(unlearned_instance.model) + #self.register_model_hook(unlearned_instance.model) # 1. Parameter-Space MIA and Latent Distance parameter_mia_acc, latent_dist = self.run_parameter_space_mia(