attck metrics

Tune_new.py

@@ -2,6 +2,7 @@ import torch
 import torch.nn as nn
 from torch.utils.data import DataLoader
 from sklearn.metrics import classification_report
+import copy
 
 # Framework and Utility Imports
 import SetUp
@@ -12,6 +13,8 @@ from architectures.Model import Model, Architecture
 from unlearning.CertifiedUnlearning import CertifiedUnlearning
 from unlearning.LinearFiltration import LinearFiltration
 from unlearning.WeightFiltration import WeightFiltration
+from eval.UnlearningAttack import UnlearningAttack
+from unlearning.Retrain import Retrain
 
 
 # Global Hyperparameters
@@ -147,8 +150,82 @@ def run_finetuning_or_baseline_eval(env_dict, run_training=False, lr_rate=0.0001
         print(f">> Skipping baseline log generation. Reason: {e}")
 
 
+# saves evaluation metrics to log files
+def log_metrics(evaluation_domains, reloaded, strategy_in_use):
+    
+    # Process and append metrics to target reporting paths
+    for domain in evaluation_domains:
+        print(domain["label"])
+        accuracy, report_dict = reloaded.evaluate(loader=domain["loader"], mode=domain["mode"])
+        Util._log_to_csv(
+            arch=ARCH.name,
+            mode=domain["mode"], 
+            accuracy=accuracy, 
+            report_dict=report_dict, 
+            strategy=strategy_in_use
+        )
+
+# performs MIA and ZRF attack on models and logs the results
+def run_unlearning_and_attack_eval(forget_train_loader, retain_test_loader, reloaded, strategy_in_use, suite_runner, device, forget_class):
+    """
+    Performs adversarial vulnerability stress tests (MIA and ZRF) in-memory 
+    on the freshly unlearned model instance without saving it to disk first.
+    """
+    if suite_runner is None:
+        raise ValueError("An active initialized UnlearningAttackSuite instance must be supplied.")
+
+    print(f"\n>>> Initializing Threat Model Stress Testing Suite for: {strategy_in_use}")
+    
+    # 1. Dynamically map the white-box feature extraction hook to the active inner model
+    suite_runner.register_model_hook(reloaded.model)
+    
+    # 2. Fire the complete evaluation suite using the isolated data split subsets
+    results = suite_runner.run_complete_evaluation(
+        target_class=forget_class,
+        framework_name=strategy_in_use,
+        forget_loader=forget_train_loader,      # Members split from the train data partition
+        retain_test_loader=retain_test_loader,  # Clean non-members split from validation data
+        device=device
+    )
+    
+    print(f"    [Attack Complete] Logit MIA AUC: {results['logit_mia_auc']:.4f} | "
+          f"Internal MIA AUC: {results['internal_mia_auc']:.4f} | "
+          f"ZRF Score: {results['zrf_score']:.4f}")
+    
+
+# performs MIA and ZRF attack on models and logs the results
+def run_shaddow_attack_eval(forget_train_loader, retain_test_loader, reloaded, strategy_in_use, suite_runner, device, forget_class):
+    """
+    Performs adversarial vulnerability stress tests matching the localized 
+    shadow architecture specifications laid out in thesis Section 5.5.
+    """
+    if suite_runner is None:
+        raise ValueError("An active initialized UnlearningAttackSuite instance must be supplied.")
+
+    print(f"\n>>> Initializing Threat Model Stress Testing Suite for: {strategy_in_use}")
+    
+    # Instantiate a clean copy of the baseline trained model to serve as the Shadow reference proxy
+    # (Since finetuning is done once, we read its parameters cleanly from disk)
+    base_shadow = Model.create(arch=ARCH, device=device, size=CLASS_SIZE)
+    base_shadow.load(arch=ARCH)
+    
+    # Execute the updated conditional attack framework
+    results = suite_runner.run_complete_evaluation(
+        framework_name=strategy_in_use,
+        target_class=forget_class,
+        forget_loader=forget_train_loader,      
+        retain_test_loader=retain_test_loader,  
+        unlearned_instance=reloaded,             # The unlearned candidate model
+        base_shadow_instance=base_shadow,        # The shadow proxy architecture
+        device=device
+    )
+    
+    print(f"    [Attack Complete] Adversary Binary Classification Accuracy: {results['mia_accuracy']:.4f}")
+    
+
+
 # Unlearning and strategy eval
-def run_unlearning_and_strategy_eval(env_dict, forget_class_idx, strategy, evaluate = False):
+def run_unlearning_and_strategy_eval(env_dict, forget_class_idx, strategy, evaluate = False, suite_runner=None):
     """
     Reloads a clean model state, applies the isolated unlearning framework,
     and runs specific target evaluation domain checks.
@@ -170,6 +247,9 @@ def run_unlearning_and_strategy_eval(env_dict, forget_class_idx, strategy, evalu
     reloaded = Model.create(arch=ARCH, device=device, size=CLASS_SIZE)
     reloaded.load(arch=ARCH)
 
+    # Clean un-manipulated snapshot to serve as the Parameter-Space shadow proxy reference
+    shadow_model = copy.deepcopy(reloaded)
+
     if evaluate:
         reloaded.evaluate(
             loader=retain_test_loader, mode="finetuned"
@@ -188,32 +268,43 @@ def run_unlearning_and_strategy_eval(env_dict, forget_class_idx, strategy, evalu
     else:
         reloaded = unlearned
 
-    
 
-    # Define validation tracking steps dynamically
-    evaluation_domains = [
-        {"loader": retain_test_loader, "mode": "retain", "label": "\n--- Performance on Retained Classes"},
-        {"loader": forget_test_loader, "mode": "forget", "label": "\n--- Performance on Forgotten Class"},
-        {"loader": forget_train_loader, "mode": "forget_train", "label": "\n--- Performance on Forgotten Class (Train Set - Verifying Unlearning)"}
-    ]
 
-    # Process and append metrics to target reporting paths
-    for domain in evaluation_domains:
-        print(domain["label"])
-        accuracy, report_dict = reloaded.evaluate(loader=domain["loader"], mode=domain["mode"])
-        Util._log_to_csv(
-            arch=ARCH.name,
-            mode=domain["mode"], 
-            accuracy=accuracy, 
-            report_dict=report_dict, 
-            strategy=strategy_in_use
+    is_retrained = isinstance(strategy, Retrain)
+
+    if is_retrained:
+        os.makedirs("trained_models", exist_ok=True)
+        reloaded.save(filename=f"class_{forget_class_idx}_retrained.pth")
+
+
+
+    # here we add a condition conditional statement
+    if suite_runner is not None:
+            
+        suite_runner.run_complete_evaluation(
+            framework_name=strategy_in_use,
+            target_class=forget_class_idx,
+            forget_loader=forget_train_loader,
+            retain_test_loader=forget_test_loader,
+            unlearned_instance=reloaded,            
+            base_shadow_instance=shadow_model,  
+            device=device
         )
+    else:
+
+        # Define validation tracking steps dynamically
+        evaluation_domains = [
+            {"loader": retain_test_loader, "mode": "retain", "label": "\n--- Performance on Retained Classes"},
+            {"loader": forget_test_loader, "mode": "forget", "label": "\n--- Performance on Forgotten Class"},
+            {"loader": forget_train_loader, "mode": "forget_train", "label": "\n--- Performance on Forgotten Class (Train Set - Verifying Unlearning)"}
+        ]
+        log_metrics(evaluation_domains, reloaded, strategy_in_use)
 
 
 # entry
 if __name__ == "__main__":
 
-    outer_loop = 10
+    outer_loop = 1
     inner_loop = CLASS_SIZE
     
     for k in range(outer_loop):
@@ -225,7 +316,7 @@ if __name__ == "__main__":
             # Baseline Evaluation 
             # switch finetuning for tests on strategies only, 
             # to avoid finetunning every time we test a strategy 
-            finetuning = True
+            finetuning = False
             run_finetuning_or_baseline_eval(runtime_environment, run_training = finetuning)
             # scale 16400.0 for ResNet
             scale = 20100
@@ -261,13 +352,24 @@ if __name__ == "__main__":
                 arch=ARCH                   
             )
 
+            retrain = Retrain(
+                target_class_index = 0,
+                arch = ARCH,
+                size = CLASS_SIZE,
+                lr = 0.0001,
+                epochs = 14
+
+            )
+
             strategies = [
+                retrain,
+                linear_filtration,
+                weight_filtration,
                 certified_unlearning,
-                #weight_filtration,
-                #linear_filtration
             ]
+            suite_runner = UnlearningAttack(arch=ARCH, class_size=CLASS_SIZE)
                 # Unlearning Iteration
-            for i in range(4, inner_loop):
+            for i in range(inner_loop):
 
                 for strategy in strategies:
 
@@ -282,9 +384,18 @@ if __name__ == "__main__":
                         strategy=strategy,
                         # if we are finetuning, no need to evaluate base model. 
                         # or may be never when not either!
-                        evaluate = not finetuning
+                        evaluate = False,
+                        suite_runner=suite_runner
                     )
+                # just a single class run before running all remaining classes.
+                #print(">> Single check run complete. Verification successful!")
+                #break
+                #dist_attacker.run_adversarial_evaluation()
+                #dist_attacker.run_incremental_evaluation(current_class_step=i)
+
+            if suite_runner is not None:
+                suite_runner.shutdown_hook()
 
         except KeyboardInterrupt:
-            print("program interrupted. Exit!")
+            print("\nprogram interrupted. Exit!")
             break