diff --git a/Tune_new.py b/Tune_new.py index 852c10d..98b7c66 100644 --- a/Tune_new.py +++ b/Tune_new.py @@ -2,6 +2,7 @@ import torch import torch.nn as nn from torch.utils.data import DataLoader from sklearn.metrics import classification_report +import copy # Framework and Utility Imports import SetUp @@ -12,6 +13,8 @@ from architectures.Model import Model, Architecture from unlearning.CertifiedUnlearning import CertifiedUnlearning from unlearning.LinearFiltration import LinearFiltration from unlearning.WeightFiltration import WeightFiltration +from eval.UnlearningAttack import UnlearningAttack +from unlearning.Retrain import Retrain # Global Hyperparameters @@ -147,8 +150,82 @@ def run_finetuning_or_baseline_eval(env_dict, run_training=False, lr_rate=0.0001 print(f">> Skipping baseline log generation. Reason: {e}") +# saves evaluation metrics to log files +def log_metrics(evaluation_domains, reloaded, strategy_in_use): + + # Process and append metrics to target reporting paths + for domain in evaluation_domains: + print(domain["label"]) + accuracy, report_dict = reloaded.evaluate(loader=domain["loader"], mode=domain["mode"]) + Util._log_to_csv( + arch=ARCH.name, + mode=domain["mode"], + accuracy=accuracy, + report_dict=report_dict, + strategy=strategy_in_use + ) + +# performs MIA and ZRF attack on models and logs the results +def run_unlearning_and_attack_eval(forget_train_loader, retain_test_loader, reloaded, strategy_in_use, suite_runner, device, forget_class): + """ + Performs adversarial vulnerability stress tests (MIA and ZRF) in-memory + on the freshly unlearned model instance without saving it to disk first. + """ + if suite_runner is None: + raise ValueError("An active initialized UnlearningAttackSuite instance must be supplied.") + + print(f"\n>>> Initializing Threat Model Stress Testing Suite for: {strategy_in_use}") + + # 1. Dynamically map the white-box feature extraction hook to the active inner model + suite_runner.register_model_hook(reloaded.model) + + # 2. Fire the complete evaluation suite using the isolated data split subsets + results = suite_runner.run_complete_evaluation( + target_class=forget_class, + framework_name=strategy_in_use, + forget_loader=forget_train_loader, # Members split from the train data partition + retain_test_loader=retain_test_loader, # Clean non-members split from validation data + device=device + ) + + print(f" [Attack Complete] Logit MIA AUC: {results['logit_mia_auc']:.4f} | " + f"Internal MIA AUC: {results['internal_mia_auc']:.4f} | " + f"ZRF Score: {results['zrf_score']:.4f}") + + +# performs MIA and ZRF attack on models and logs the results +def run_shaddow_attack_eval(forget_train_loader, retain_test_loader, reloaded, strategy_in_use, suite_runner, device, forget_class): + """ + Performs adversarial vulnerability stress tests matching the localized + shadow architecture specifications laid out in thesis Section 5.5. + """ + if suite_runner is None: + raise ValueError("An active initialized UnlearningAttackSuite instance must be supplied.") + + print(f"\n>>> Initializing Threat Model Stress Testing Suite for: {strategy_in_use}") + + # Instantiate a clean copy of the baseline trained model to serve as the Shadow reference proxy + # (Since finetuning is done once, we read its parameters cleanly from disk) + base_shadow = Model.create(arch=ARCH, device=device, size=CLASS_SIZE) + base_shadow.load(arch=ARCH) + + # Execute the updated conditional attack framework + results = suite_runner.run_complete_evaluation( + framework_name=strategy_in_use, + target_class=forget_class, + forget_loader=forget_train_loader, + retain_test_loader=retain_test_loader, + unlearned_instance=reloaded, # The unlearned candidate model + base_shadow_instance=base_shadow, # The shadow proxy architecture + device=device + ) + + print(f" [Attack Complete] Adversary Binary Classification Accuracy: {results['mia_accuracy']:.4f}") + + + # Unlearning and strategy eval -def run_unlearning_and_strategy_eval(env_dict, forget_class_idx, strategy, evaluate = False): +def run_unlearning_and_strategy_eval(env_dict, forget_class_idx, strategy, evaluate = False, suite_runner=None): """ Reloads a clean model state, applies the isolated unlearning framework, and runs specific target evaluation domain checks. @@ -170,6 +247,9 @@ def run_unlearning_and_strategy_eval(env_dict, forget_class_idx, strategy, evalu reloaded = Model.create(arch=ARCH, device=device, size=CLASS_SIZE) reloaded.load(arch=ARCH) + # Clean un-manipulated snapshot to serve as the Parameter-Space shadow proxy reference + shadow_model = copy.deepcopy(reloaded) + if evaluate: reloaded.evaluate( loader=retain_test_loader, mode="finetuned" @@ -188,32 +268,43 @@ def run_unlearning_and_strategy_eval(env_dict, forget_class_idx, strategy, evalu else: reloaded = unlearned - - # Define validation tracking steps dynamically - evaluation_domains = [ - {"loader": retain_test_loader, "mode": "retain", "label": "\n--- Performance on Retained Classes"}, - {"loader": forget_test_loader, "mode": "forget", "label": "\n--- Performance on Forgotten Class"}, - {"loader": forget_train_loader, "mode": "forget_train", "label": "\n--- Performance on Forgotten Class (Train Set - Verifying Unlearning)"} - ] - # Process and append metrics to target reporting paths - for domain in evaluation_domains: - print(domain["label"]) - accuracy, report_dict = reloaded.evaluate(loader=domain["loader"], mode=domain["mode"]) - Util._log_to_csv( - arch=ARCH.name, - mode=domain["mode"], - accuracy=accuracy, - report_dict=report_dict, - strategy=strategy_in_use + is_retrained = isinstance(strategy, Retrain) + + if is_retrained: + os.makedirs("trained_models", exist_ok=True) + reloaded.save(filename=f"class_{forget_class_idx}_retrained.pth") + + + + # here we add a condition conditional statement + if suite_runner is not None: + + suite_runner.run_complete_evaluation( + framework_name=strategy_in_use, + target_class=forget_class_idx, + forget_loader=forget_train_loader, + retain_test_loader=forget_test_loader, + unlearned_instance=reloaded, + base_shadow_instance=shadow_model, + device=device ) + else: + + # Define validation tracking steps dynamically + evaluation_domains = [ + {"loader": retain_test_loader, "mode": "retain", "label": "\n--- Performance on Retained Classes"}, + {"loader": forget_test_loader, "mode": "forget", "label": "\n--- Performance on Forgotten Class"}, + {"loader": forget_train_loader, "mode": "forget_train", "label": "\n--- Performance on Forgotten Class (Train Set - Verifying Unlearning)"} + ] + log_metrics(evaluation_domains, reloaded, strategy_in_use) # entry if __name__ == "__main__": - outer_loop = 10 + outer_loop = 1 inner_loop = CLASS_SIZE for k in range(outer_loop): @@ -225,7 +316,7 @@ if __name__ == "__main__": # Baseline Evaluation # switch finetuning for tests on strategies only, # to avoid finetunning every time we test a strategy - finetuning = True + finetuning = False run_finetuning_or_baseline_eval(runtime_environment, run_training = finetuning) # scale 16400.0 for ResNet scale = 20100 @@ -261,13 +352,24 @@ if __name__ == "__main__": arch=ARCH ) + retrain = Retrain( + target_class_index = 0, + arch = ARCH, + size = CLASS_SIZE, + lr = 0.0001, + epochs = 14 + + ) + strategies = [ + retrain, + linear_filtration, + weight_filtration, certified_unlearning, - #weight_filtration, - #linear_filtration ] + suite_runner = UnlearningAttack(arch=ARCH, class_size=CLASS_SIZE) # Unlearning Iteration - for i in range(4, inner_loop): + for i in range(inner_loop): for strategy in strategies: @@ -282,9 +384,18 @@ if __name__ == "__main__": strategy=strategy, # if we are finetuning, no need to evaluate base model. # or may be never when not either! - evaluate = not finetuning + evaluate = False, + suite_runner=suite_runner ) + # just a single class run before running all remaining classes. + #print(">> Single check run complete. Verification successful!") + #break + #dist_attacker.run_adversarial_evaluation() + #dist_attacker.run_incremental_evaluation(current_class_step=i) + + if suite_runner is not None: + suite_runner.shutdown_hook() except KeyboardInterrupt: - print("program interrupted. Exit!") + print("\nprogram interrupted. Exit!") break diff --git a/architectures/Model.py b/architectures/Model.py index 7f5e7ee..4b5653e 100644 --- a/architectures/Model.py +++ b/architectures/Model.py @@ -108,21 +108,7 @@ class Model(ABC): self.model.to(self.device) print(f'Model loaded from {file_path}') - ''' - def unlearn(self, strategy: 'Strategy', forget_loader, retain_loader): - """ Executes a targeted unlearning strategy and profiles efficiency """ - print(f"Executing: {strategy.__class__.__name__}...") - - start_time = time.time() - - # Delegate the actual algorithmic weight/logit manipulation to the strategy - strategy.apply(self.model, forget_loader, retain_loader) - - elapsed_time = time.time() - start_time - print(f"{strategy.__class__.__name__} completed in {elapsed_time:.4f} seconds.") - - return elapsed_time - ''' + def evaluate(self, loader, mode="eval"): """ diff --git a/eval/UnlearningAttack.py b/eval/UnlearningAttack.py new file mode 100644 index 0000000..5a381a3 --- /dev/null +++ b/eval/UnlearningAttack.py @@ -0,0 +1,238 @@ +import torch +import torch.nn as nn +import numpy as np +import os +from torch.utils.data import DataLoader +from sklearn.linear_model import LogisticRegression +from sklearn.metrics import accuracy_score +from architectures.Model import Model + +class UnlearningAttack: + def __init__(self, arch, class_size): + """ + Initializes the robust verification suite with universal architecture metadata. + Matches Section 5.5 of the thesis text by implementing distinct + Parameter-Space and Logit-Space (Look-alike) attack pipelines uniformly. + """ + self.arch = arch + self.class_size = class_size + + self.hook = None + self.model = None + self._hook_features = [] + self.criterion = nn.CrossEntropyLoss(reduction='none') + self.collecting = False + + def _hook_fn(self, module, input, output): + if not self.collecting: + return + flattened_embeddings = torch.flatten(output, 1) + self._hook_features.append(flattened_embeddings.detach()) + + def register_model_hook(self, inner_model): + if hasattr(inner_model, "original_model"): + core_model = inner_model.original_model + else: + core_model = inner_model + + if hasattr(core_model, 'avgpool'): + pool_layer = core_model.avgpool + else: + pool_layer = None + for name, module in core_model.named_modules(): + if 'pool' in name: + pool_layer = module + break + + if pool_layer is None: + raise AttributeError("The target model architecture lacks an 'avgpool' layer block.") + + self.hook = pool_layer.register_forward_hook(self._hook_fn) + + def shutdown_hook(self): + if hasattr(self, 'hook') and self.hook: + self.hook.remove() + self.hook = None + + def _extract_attack_features(self, target_model, loader, device, target_class): + target_model.eval() + all_probs = [] + all_entropies = [] + all_losses = [] + self._hook_features = [] + self.collecting = True + with torch.no_grad(): + for data, targets in loader: + data, targets = data.to(device), targets.to(device) + if target_model.__class__.__name__ == "WF_Module": + gate_signals = torch.full( + (data.size(0),), + target_class, + dtype=torch.long, + device=data.device + ) + outputs = target_model(data, target_class_indices=gate_signals) + else: + outputs = target_model(data) + + probs = torch.softmax(outputs, dim=1) + all_probs.extend(probs.cpu().numpy()) + + entropy = -torch.sum(probs * torch.log(probs + 1e-10), dim=1) + all_entropies.extend(entropy.cpu().numpy()) + + loss = self.criterion(outputs, targets) + all_losses.extend(loss.cpu().numpy()) + + self.collecting = False + + X_probs = np.array(all_probs) + X_entropy = np.array(all_entropies).reshape(-1, 1) + X_loss = np.array(all_losses).reshape(-1, 1) + X_features = np.hstack([X_probs, X_entropy, X_loss]) + + if self._hook_features: + compiled_latent = torch.cat(self._hook_features, dim=0).cpu().numpy() + else: + compiled_latent = np.zeros((len(X_features), 512)) + + return X_features, compiled_latent + + def run_parameter_space_mia(self, unlearned_model, shadow_model, forget_loader, retain_test_loader, device, index): + X_shadow_mem, _ = self._extract_attack_features(shadow_model, forget_loader, device, index) + X_shadow_non, _ = self._extract_attack_features(shadow_model, retain_test_loader, device, index) + + min_train = min(len(X_shadow_mem), len(X_shadow_non)) + X_train = np.vstack([X_shadow_mem[:min_train], X_shadow_non[:min_train]]) + y_train = np.concatenate([np.ones(min_train), np.zeros(min_train)]) + + attack_classifier = LogisticRegression(max_iter=1000) + attack_classifier.fit(X_train, y_train) + + X_eval_mem, latent_features = self._extract_attack_features(unlearned_model, forget_loader, device, index) + X_eval_non, retain_latent = self._extract_attack_features(unlearned_model, retain_test_loader, device, index) + + min_test = min(len(X_eval_mem), len(X_eval_non)) + X_test = np.vstack([X_eval_mem[:min_test], X_eval_non[:min_test]]) + y_test = np.concatenate([np.ones(min_test), np.zeros(min_test)]) + + predictions = attack_classifier.predict(X_test) + mia_accuracy = accuracy_score(y_test, predictions) + + clean_centroid = np.mean(retain_latent, axis=0) + forget_distances = np.linalg.norm(latent_features - clean_centroid, axis=1) + + return mia_accuracy, np.mean(forget_distances) + + def run_logit_space_lookalike_mia(self, filtered_model, naive_retrained, forget_loader, device, target_class): + filtered_model.eval() + naive_retrained.eval() + + filtered_logits = [] + naive_logits = [] + + with torch.no_grad(): + for data, _ in forget_loader: + data = data.to(device) + + if filtered_model.__class__.__name__ == "WF_Module": + gate_signals = torch.full((data.size(0),), target_class, dtype=torch.long, device=data.device) + out_filtered = filtered_model(data, target_class_indices=gate_signals) + else: + out_filtered = filtered_model(data) + + out_naive = naive_retrained(data) + + filtered_logits.append(out_filtered) + naive_logits.append(out_naive) + + # Concatenate everything + filtered = torch.cat(filtered_logits, dim=0).cpu().numpy() + naive = torch.cat(naive_logits, dim=0).cpu().numpy() + + # Z-Score Normalisation + filtered = (filtered - np.mean(filtered, axis=-1, keepdims=True)) / (np.std(filtered, axis=-1, keepdims=True) + 1e-8) + naive = (naive - np.mean(naive, axis=-1, keepdims=True)) / (np.std(naive, axis=-1, keepdims=True) + 1e-8) + + # shuffle indices + num_images = len(filtered) + image_indices = np.arange(num_images) + np.random.shuffle(image_indices) + + # split to train and test set + split = int(num_images * 0.7) + train_img_idx, test_img_idx = image_indices[:split], image_indices[split:] + + # create a balanced test and train set + data_train = np.vstack([filtered[train_img_idx], naive[train_img_idx]]) + data_test = np.vstack([filtered[test_img_idx], naive[test_img_idx]]) + + # labels for attcker (1 from unlearned and 0s to retrained) + # we do this because every output retrained gives us is a result of unseen + # but unlearned has seen these data. + label_train = np.concatenate([np.ones(len(train_img_idx)), np.zeros(len(train_img_idx))]) + # test set + label_test = np.concatenate([np.ones(len(test_img_idx)), np.zeros(len(test_img_idx))]) + + + adversary = LogisticRegression(max_iter=1000) + adversary.fit(data_train, label_train) + # evaluate similarity of outputs + lookalike_accuracy = accuracy_score(label_test, adversary.predict(data_test)) + # so that the metric is between 0 and 1. + return 2.0 * np.abs(lookalike_accuracy - 0.5) + + def run_complete_evaluation(self, framework_name, target_class, forget_loader, retain_test_loader, unlearned_instance, base_shadow_instance, device): + """Orchestrates specific pipeline routing cleanly using cached constructor parameters.""" + target_dir = os.path.join("reports", framework_name) + os.makedirs(target_dir, exist_ok=True) + current_log_file = os.path.join(target_dir, "attack_values.csv") + + if not os.path.exists(current_log_file): + with open(current_log_file, "w") as f: + f.write("target_class,parameter_mia_accuracy,latent_distance_tell,lookalike_accuracy\n") + + self.register_model_hook(unlearned_instance.model) + + # 1. Parameter-Space MIA and Latent Distance + parameter_mia_acc, latent_dist = self.run_parameter_space_mia( + unlearned_model=unlearned_instance.model, + shadow_model=base_shadow_instance.model, + forget_loader=forget_loader, + retain_test_loader=retain_test_loader, + device=device, + index=target_class + ) + + # we load a retrained model to evaluate look_alike tests + ghost_checkpoint_path = f"trained_models/class_{target_class}_retrained.pth" + + if os.path.exists(ghost_checkpoint_path): + # Safe clean wrapper boot utilizing internal instance state properties + ghost_model_instance = Model.create(arch=self.arch, device=device, size=self.class_size) + state_dict = torch.load(ghost_checkpoint_path, map_location=device, weights_only=True) + ghost_model_instance.model.load_state_dict(state_dict) + reference_model_torch = ghost_model_instance.model + else: + raise FileNotFoundError( + f"Retrained weights not found at: {ghost_checkpoint_path}. \nDid you forget to save models or are they saved with a different path?" + ) + + lookalike_acc = self.run_logit_space_lookalike_mia( + filtered_model=unlearned_instance.model, + naive_retrained=reference_model_torch, + forget_loader=forget_loader, + device=device, + target_class=target_class + ) + + print(f"[{framework_name}] Class {target_class} | Parameter MIA: {parameter_mia_acc:.4f} | Latent Dist: {latent_dist:.4f} | Lookalike: {lookalike_acc:.4f}") + + with open(current_log_file, "a") as f: + f.write(f"{target_class},{parameter_mia_acc:.6f},{latent_dist:.6f},{lookalike_acc:.6f}\n") + + return { + "parameter_mia_accuracy": parameter_mia_acc, + "latent_distance": latent_dist, + "lookalike_accuracy": lookalike_acc + } \ No newline at end of file diff --git a/reports/CertifiedUnlearning/GOOGLENET/forget.csv b/reports/CertifiedUnlearning/GOOGLENET/forget.csv new file mode 100644 index 0000000..79aa4f4 --- /dev/null +++ b/reports/CertifiedUnlearning/GOOGLENET/forget.csv @@ -0,0 +1,3 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/CertifiedUnlearning/GOOGLENET/forget_train.csv b/reports/CertifiedUnlearning/GOOGLENET/forget_train.csv new file mode 100644 index 0000000..79aa4f4 --- /dev/null +++ b/reports/CertifiedUnlearning/GOOGLENET/forget_train.csv @@ -0,0 +1,3 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/CertifiedUnlearning/GOOGLENET/retain.csv b/reports/CertifiedUnlearning/GOOGLENET/retain.csv new file mode 100644 index 0000000..ef28193 --- /dev/null +++ b/reports/CertifiedUnlearning/GOOGLENET/retain.csv @@ -0,0 +1,3 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1 +0.9171,0.9251,0.9171,0.9180,0.9251,0.9171,0.9180 +0.5539,0.8114,0.5539,0.5527,0.8114,0.5539,0.5527 diff --git a/reports/CertifiedUnlearning/GoogLeNet/time_metrics.txt b/reports/CertifiedUnlearning/GoogLeNet/time_metrics.txt new file mode 100644 index 0000000..bffe6af --- /dev/null +++ b/reports/CertifiedUnlearning/GoogLeNet/time_metrics.txt @@ -0,0 +1,3 @@ +execution_time_sec +310.029652 +309.202731 diff --git a/reports/CertifiedUnlearning/RESNET34/forget.csv b/reports/CertifiedUnlearning/RESNET34/forget.csv index fb311ed..e065a79 100644 --- a/reports/CertifiedUnlearning/RESNET34/forget.csv +++ b/reports/CertifiedUnlearning/RESNET34/forget.csv @@ -344,3 +344,23 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0375,1.0000,0.0375,0.0723,1.0000,0.0375,0.0723 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0125,1.0000,0.0125,0.0247,1.0000,0.0125,0.0247 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/CertifiedUnlearning/RESNET34/forget_train.csv b/reports/CertifiedUnlearning/RESNET34/forget_train.csv index cae9668..3d29e2c 100644 --- a/reports/CertifiedUnlearning/RESNET34/forget_train.csv +++ b/reports/CertifiedUnlearning/RESNET34/forget_train.csv @@ -344,3 +344,23 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0031,1.0000,0.0031,0.0062,1.0000,0.0031,0.0062 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0500,1.0000,0.0500,0.0952,1.0000,0.0500,0.0952 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/CertifiedUnlearning/RESNET34/retain.csv b/reports/CertifiedUnlearning/RESNET34/retain.csv index 56f6b98..eb83522 100644 --- a/reports/CertifiedUnlearning/RESNET34/retain.csv +++ b/reports/CertifiedUnlearning/RESNET34/retain.csv @@ -344,3 +344,23 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.6434,0.8358,0.6434,0.6622,0.8358,0.6434,0.6622 0.7303,0.8317,0.7303,0.7303,0.8317,0.7303,0.7303 0.2895,0.4174,0.2895,0.2206,0.4174,0.2895,0.2206 +0.8651,0.8977,0.8651,0.8664,0.8977,0.8651,0.8664 +0.8349,0.8877,0.8349,0.8386,0.8877,0.8349,0.8386 +0.9368,0.9409,0.9368,0.9373,0.9409,0.9368,0.9373 +0.2474,0.6224,0.2474,0.2551,0.6224,0.2474,0.2551 +0.9434,0.9479,0.9434,0.9441,0.9479,0.9434,0.9441 +0.9283,0.9346,0.9283,0.9293,0.9346,0.9283,0.9293 +0.9211,0.9285,0.9211,0.9221,0.9285,0.9211,0.9221 +0.1316,0.5512,0.1316,0.1243,0.5512,0.1316,0.1243 +0.9382,0.9418,0.9382,0.9390,0.9418,0.9382,0.9390 +0.9191,0.9283,0.9191,0.9201,0.9283,0.9191,0.9201 +0.4586,0.7992,0.4586,0.4860,0.7992,0.4586,0.4860 +0.9086,0.9199,0.9086,0.9092,0.9199,0.9086,0.9092 +0.9309,0.9360,0.9309,0.9315,0.9360,0.9309,0.9315 +0.8500,0.9128,0.8500,0.8649,0.9128,0.8500,0.8649 +0.9283,0.9364,0.9283,0.9293,0.9364,0.9283,0.9293 +0.8921,0.9124,0.8921,0.8950,0.9124,0.8921,0.8950 +0.8868,0.9181,0.8868,0.8927,0.9181,0.8868,0.8927 +0.8730,0.9165,0.8730,0.8827,0.9165,0.8730,0.8827 +0.9454,0.9470,0.9454,0.9454,0.9470,0.9454,0.9454 +0.5987,0.7725,0.5987,0.5938,0.7725,0.5987,0.5938 diff --git a/reports/CertifiedUnlearning/ResNet/time_metrics.txt b/reports/CertifiedUnlearning/ResNet/time_metrics.txt index d139a99..a1537a4 100644 --- a/reports/CertifiedUnlearning/ResNet/time_metrics.txt +++ b/reports/CertifiedUnlearning/ResNet/time_metrics.txt @@ -358,3 +358,9 @@ execution_time_sec 395.473056 395.455440 395.554517 +395.651279 +400.544514 +395.467196 +395.553217 +395.613947 +395.703417 diff --git a/reports/CertifiedUnlearning/attack_values.csv b/reports/CertifiedUnlearning/attack_values.csv new file mode 100644 index 0000000..6424bd7 --- /dev/null +++ b/reports/CertifiedUnlearning/attack_values.csv @@ -0,0 +1,4 @@ +target_class,parameter_mia_accuracy,latent_distance_tell,lookalike_accuracy +0,0.500000,7.219862,0.979167 +1,0.500000,3.659238,0.958333 +2,0.500000,6.939345,0.885417 diff --git a/reports/CertifiedUnlearning/attack_values_old.csv b/reports/CertifiedUnlearning/attack_values_old.csv new file mode 100644 index 0000000..5fb4606 --- /dev/null +++ b/reports/CertifiedUnlearning/attack_values_old.csv @@ -0,0 +1,101 @@ +target_class,attack_mia_accuracy,latent_distance_tell +0,0.500000,8.166956 +1,0.500000,6.160398 +2,0.500000,6.704157 +3,0.500000,7.097013 +4,0.500000,7.059480 +5,0.500000,5.941715 +6,0.500000,7.376003 +7,0.500000,6.876045 +8,0.500000,7.853063 +9,0.500000,7.215755 +10,0.500000,6.611487 +11,0.431250,6.596037 +12,0.500000,7.509936 +13,0.500000,6.233299 +14,0.500000,9.069311 +15,0.500000,7.752240 +16,0.500000,7.227110 +17,0.500000,5.331686 +18,0.500000,8.771266 +19,0.500000,5.970541 +0,0.500000,8.333142 +1,0.500000,4.603730 +2,0.500000,6.403101 +3,0.500000,7.975533 +4,0.500000,6.620228 +5,0.500000,8.796431 +6,0.500000,9.078737 +7,0.500000,6.821482 +8,0.500000,9.727625 +9,0.500000,9.074922 +10,0.500000,6.036069 +11,0.493750,7.097591 +12,0.500000,5.960563 +13,0.500000,6.122758 +14,0.500000,8.211535 +15,0.500000,7.850469 +16,0.500000,6.859184 +17,0.500000,5.088897 +18,0.500000,9.236532 +19,0.500000,7.642883 +0,0.500000,8.106592 +1,0.500000,6.134580 +2,0.500000,6.941654 +3,0.500000,7.773781 +4,0.500000,7.363125 +5,0.500000,6.496724 +6,0.500000,7.648515 +7,0.500000,8.689814 +8,0.500000,8.578580 +9,0.500000,9.119745 +10,0.500000,5.984212 +11,0.468750,6.359155 +12,0.500000,7.997709 +13,0.500000,6.927951 +14,0.500000,8.872922 +15,0.500000,7.429983 +16,0.500000,6.928881 +17,0.500000,5.071527 +18,0.500000,8.475766 +19,0.500000,6.096026 +0,0.500000,7.570661 +1,0.500000,3.468966 +2,0.500000,5.726584 +3,0.500000,7.681168 +4,0.500000,7.824241 +5,0.500000,9.169927 +6,0.500000,7.778905 +7,0.500000,8.138535 +8,0.500000,9.735314 +9,0.500000,7.250852 +10,0.500000,6.852473 +11,0.462500,7.474597 +12,0.500000,5.709781 +13,0.500000,7.201421 +14,0.500000,9.513277 +15,0.500000,7.965966 +16,0.500000,6.824328 +17,0.500000,5.539176 +18,0.500000,9.225863 +19,0.500000,7.838114 +0,0.500000,7.209781 +1,0.500000,3.245398 +2,0.500000,6.559930 +3,0.500000,8.221349 +4,0.500000,7.320590 +5,0.500000,8.885569 +6,0.500000,7.456804 +7,0.500000,7.722786 +8,0.500000,8.868426 +9,0.500000,6.132468 +10,0.500000,7.281767 +11,0.481250,7.443795 +12,0.500000,7.379603 +13,0.500000,6.809259 +14,0.500000,8.274336 +15,0.500000,8.232855 +16,0.500000,7.061528 +17,0.500000,6.004038 +18,0.500000,8.139433 +19,0.500000,7.296746 diff --git a/reports/LinearFiltration/RESNET34/forget.csv b/reports/LinearFiltration/RESNET34/forget.csv index 694c9cd..45c6fca 100644 --- a/reports/LinearFiltration/RESNET34/forget.csv +++ b/reports/LinearFiltration/RESNET34/forget.csv @@ -399,3 +399,40 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/LinearFiltration/RESNET34/forget_train.csv b/reports/LinearFiltration/RESNET34/forget_train.csv index 694c9cd..45c6fca 100644 --- a/reports/LinearFiltration/RESNET34/forget_train.csv +++ b/reports/LinearFiltration/RESNET34/forget_train.csv @@ -399,3 +399,40 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/LinearFiltration/RESNET34/retain.csv b/reports/LinearFiltration/RESNET34/retain.csv index b98849f..ee8e76b 100644 --- a/reports/LinearFiltration/RESNET34/retain.csv +++ b/reports/LinearFiltration/RESNET34/retain.csv @@ -399,3 +399,40 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.9572,0.9582,0.9572,0.9573,0.9582,0.9572,0.9573 0.9599,0.9609,0.9599,0.9599,0.9609,0.9599,0.9599 0.9579,0.9588,0.9579,0.9580,0.9588,0.9579,0.9580 +0.9533,0.9543,0.9533,0.9534,0.9543,0.9533,0.9534 +0.9559,0.9568,0.9559,0.9561,0.9568,0.9559,0.9561 +0.9559,0.9566,0.9559,0.9559,0.9566,0.9559,0.9559 +0.9526,0.9535,0.9526,0.9527,0.9535,0.9526,0.9527 +0.9546,0.9554,0.9546,0.9547,0.9554,0.9546,0.9547 +0.9592,0.9602,0.9592,0.9593,0.9602,0.9592,0.9593 +0.9539,0.9548,0.9539,0.9540,0.9548,0.9539,0.9540 +0.9539,0.9550,0.9539,0.9541,0.9550,0.9539,0.9541 +0.9546,0.9556,0.9546,0.9547,0.9556,0.9546,0.9547 +0.9539,0.9548,0.9539,0.9540,0.9548,0.9539,0.9540 +0.9539,0.9549,0.9539,0.9541,0.9549,0.9539,0.9541 +0.9539,0.9549,0.9539,0.9540,0.9549,0.9539,0.9540 +0.9513,0.9524,0.9513,0.9514,0.9524,0.9513,0.9514 +0.9520,0.9529,0.9520,0.9520,0.9529,0.9520,0.9520 +0.9572,0.9583,0.9572,0.9574,0.9583,0.9572,0.9574 +0.9520,0.9530,0.9520,0.9521,0.9530,0.9520,0.9521 +0.9520,0.9530,0.9520,0.9521,0.9530,0.9520,0.9521 +0.9553,0.9565,0.9553,0.9554,0.9565,0.9553,0.9554 +0.9559,0.9567,0.9559,0.9560,0.9567,0.9559,0.9560 +0.9520,0.9530,0.9520,0.9521,0.9530,0.9520,0.9521 +0.9533,0.9543,0.9533,0.9534,0.9543,0.9533,0.9534 +0.9559,0.9568,0.9559,0.9561,0.9568,0.9559,0.9561 +0.9533,0.9543,0.9533,0.9534,0.9543,0.9533,0.9534 +0.9533,0.9543,0.9533,0.9534,0.9543,0.9533,0.9534 +0.9559,0.9568,0.9559,0.9561,0.9568,0.9559,0.9561 +0.9559,0.9566,0.9559,0.9559,0.9566,0.9559,0.9559 +0.9526,0.9535,0.9526,0.9527,0.9535,0.9526,0.9527 +0.9546,0.9554,0.9546,0.9547,0.9554,0.9546,0.9547 +0.9592,0.9602,0.9592,0.9593,0.9602,0.9592,0.9593 +0.9533,0.9543,0.9533,0.9534,0.9543,0.9533,0.9534 +0.9539,0.9548,0.9539,0.9540,0.9548,0.9539,0.9540 +0.9539,0.9550,0.9539,0.9541,0.9550,0.9539,0.9541 +0.9559,0.9568,0.9559,0.9561,0.9568,0.9559,0.9561 +0.9533,0.9543,0.9533,0.9534,0.9543,0.9533,0.9534 +0.9546,0.9556,0.9546,0.9547,0.9556,0.9546,0.9547 +0.9533,0.9543,0.9533,0.9534,0.9543,0.9533,0.9534 +0.9533,0.9543,0.9533,0.9534,0.9543,0.9533,0.9534 diff --git a/reports/LinearFiltration/ResNet/time_metrics.txt b/reports/LinearFiltration/ResNet/time_metrics.txt index bd708aa..d1c1797 100644 --- a/reports/LinearFiltration/ResNet/time_metrics.txt +++ b/reports/LinearFiltration/ResNet/time_metrics.txt @@ -399,3 +399,192 @@ execution_time_sec 0.001845 0.001835 0.001846 +1.583831 +1.579020 +1.597051 +1.604905 +1.668440 +1.574648 +1.559132 +0.003680 +0.001816 +0.004576 +0.004078 +0.003174 +0.005544 +0.002435 +0.001903 +0.002613 +0.003090 +0.004827 +0.001774 +0.001993 +0.003222 +0.006826 +0.003274 +0.004176 +0.006219 +0.003163 +6.787932 +0.004030 +0.003655 +0.001846 +0.003250 +0.002135 +0.002022 +0.001963 +0.001903 +0.001978 +0.001874 +0.002326 +0.003671 +0.002932 +0.003153 +0.002311 +0.002369 +0.002845 +0.004887 +0.004410 +0.974533 +0.924626 +0.003374 +0.003496 +0.005881 +0.003443 +0.006579 +0.006536 +0.006472 +0.002645 +0.003284 +0.002127 +0.011311 +0.003321 +0.002229 +0.001880 +0.003873 +0.005213 +0.004675 +0.003227 +0.002580 +0.906583 +0.001986 +0.001837 +0.001839 +0.001823 +0.001841 +0.001831 +0.001851 +0.001839 +0.001847 +0.001830 +0.001836 +0.001833 +0.001849 +0.001826 +0.001845 +0.001849 +0.001897 +0.001841 +0.001830 +0.871331 +0.001890 +0.001868 +0.001839 +0.001854 +0.001866 +0.001847 +0.001836 +0.001837 +0.001841 +0.001839 +0.001842 +0.001850 +0.001842 +0.001834 +0.001843 +0.001869 +0.001887 +0.001834 +0.001851 +0.871175 +0.001836 +0.001836 +0.001842 +0.001838 +0.001847 +0.001844 +0.001847 +0.001837 +0.001846 +0.001834 +0.001861 +0.004003 +0.003517 +0.001845 +0.002701 +0.001845 +0.001847 +0.001839 +0.001848 +0.871788 +0.001845 +0.001841 +0.001843 +0.001855 +0.001854 +0.001843 +0.001841 +0.001879 +0.001850 +0.001846 +0.001855 +0.001829 +0.001850 +0.001845 +0.001862 +0.001865 +0.001839 +0.001845 +0.001829 +0.878134 +0.001857 +0.001842 +0.001840 +0.001852 +0.001844 +0.001849 +0.001867 +0.001862 +0.001822 +0.001838 +0.002338 +0.001885 +0.001827 +0.001838 +0.001844 +0.001851 +0.001862 +0.001854 +0.001888 +0.890666 +0.001937 +0.001811 +0.001797 +0.001828 +0.001838 +0.001844 +0.001794 +0.001830 +0.001877 +0.001810 +0.001810 +0.001851 +0.001799 +0.001835 +0.001794 +0.001825 +0.001854 +0.001812 +0.001794 +0.865806 +0.001830 diff --git a/reports/LinearFiltration/attack_values.csv b/reports/LinearFiltration/attack_values.csv new file mode 100644 index 0000000..550e4c3 --- /dev/null +++ b/reports/LinearFiltration/attack_values.csv @@ -0,0 +1,5 @@ +target_class,parameter_mia_accuracy,latent_distance_tell,lookalike_accuracy +0,0.500000,3.219560,1.000000 +1,0.500000,3.573733,1.000000 +2,0.500000,3.924550,1.000000 +3,0.500000,3.515182,1.000000 diff --git a/reports/LinearFiltration/attack_values_old.csv b/reports/LinearFiltration/attack_values_old.csv new file mode 100644 index 0000000..fb18952 --- /dev/null +++ b/reports/LinearFiltration/attack_values_old.csv @@ -0,0 +1,101 @@ +target_class,attack_mia_accuracy,latent_distance_tell +0,0.932292,0.000000 +1,0.994792,0.000000 +2,0.411458,0.000000 +3,0.916667,0.000000 +4,0.885417,0.000000 +5,0.968750,0.000000 +6,0.921875,0.000000 +7,0.760417,0.000000 +8,0.437500,0.000000 +9,0.479167,0.000000 +10,0.364583,0.000000 +11,0.458333,0.000000 +12,0.453125,0.000000 +13,0.463542,0.000000 +14,0.708333,0.000000 +15,0.479167,0.000000 +16,0.989583,0.000000 +17,0.937500,0.000000 +18,0.552083,0.000000 +19,0.401042,0.000000 +0,0.942708,0.000000 +1,1.000000,0.000000 +2,0.421875,0.000000 +3,0.932292,0.000000 +4,0.854167,0.000000 +5,0.963542,0.000000 +6,0.958333,0.000000 +7,0.703125,0.000000 +8,0.401042,0.000000 +9,0.479167,0.000000 +10,0.411458,0.000000 +11,0.437500,0.000000 +12,0.489583,0.000000 +13,0.468750,0.000000 +14,0.729167,0.000000 +15,0.484375,0.000000 +16,1.000000,0.000000 +17,0.927083,0.000000 +18,0.510417,0.000000 +19,0.380208,0.000000 +0,0.973958,0.000000 +1,0.994792,0.000000 +2,0.473958,0.000000 +3,0.927083,0.000000 +4,0.911458,0.000000 +5,0.953125,0.000000 +6,0.963542,0.000000 +7,0.697917,0.000000 +8,0.442708,0.000000 +9,0.484375,0.000000 +10,0.416667,0.000000 +11,0.416667,0.000000 +12,0.473958,0.000000 +13,0.494792,0.000000 +14,0.755208,0.000000 +15,0.484375,0.000000 +16,0.994792,0.000000 +17,0.963542,0.000000 +18,0.510417,0.000000 +19,0.395833,0.000000 +0,0.979167,0.000000 +1,1.000000,0.000000 +2,0.390625,0.000000 +3,0.942708,0.000000 +4,0.869792,0.000000 +5,0.979167,0.000000 +6,0.968750,0.000000 +7,0.692708,0.000000 +8,0.437500,0.000000 +9,0.494792,0.000000 +10,0.416667,0.000000 +11,0.416667,0.000000 +12,0.473958,0.000000 +13,0.473958,0.000000 +14,0.640625,0.000000 +15,0.598958,0.000000 +16,0.989583,0.000000 +17,0.968750,0.000000 +18,0.536458,0.000000 +19,0.427083,0.000000 +0,0.984375,0.000000 +1,1.000000,0.000000 +2,0.416667,0.000000 +3,0.927083,0.000000 +4,0.890625,0.000000 +5,0.947917,0.000000 +6,0.973958,0.000000 +7,0.765625,0.000000 +8,0.416667,0.000000 +9,0.515625,0.000000 +10,0.421875,0.000000 +11,0.427083,0.000000 +12,0.458333,0.000000 +13,0.479167,0.000000 +14,0.671875,0.000000 +15,0.494792,0.000000 +16,0.994792,0.000000 +17,0.953125,0.000000 +18,0.500000,0.000000 +19,0.411458,0.000000 diff --git a/reports/Retrain/RESNET34/forget.csv b/reports/Retrain/RESNET34/forget.csv new file mode 100644 index 0000000..cbf5469 --- /dev/null +++ b/reports/Retrain/RESNET34/forget.csv @@ -0,0 +1,68 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/Retrain/RESNET34/forget_train.csv b/reports/Retrain/RESNET34/forget_train.csv new file mode 100644 index 0000000..cbf5469 --- /dev/null +++ b/reports/Retrain/RESNET34/forget_train.csv @@ -0,0 +1,68 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/Retrain/RESNET34/retain.csv b/reports/Retrain/RESNET34/retain.csv new file mode 100644 index 0000000..40b40fc --- /dev/null +++ b/reports/Retrain/RESNET34/retain.csv @@ -0,0 +1,68 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1 +0.9618,0.9624,0.9618,0.9619,0.9624,0.9618,0.9619 +0.9625,0.9632,0.9625,0.9626,0.9632,0.9625,0.9626 +0.9572,0.9584,0.9572,0.9575,0.9584,0.9572,0.9575 +0.9566,0.9579,0.9566,0.9568,0.9579,0.9566,0.9568 +0.9658,0.9660,0.9658,0.9658,0.9660,0.9658,0.9658 +0.9651,0.9655,0.9651,0.9651,0.9655,0.9651,0.9651 +0.9539,0.9551,0.9539,0.9540,0.9551,0.9539,0.9540 +0.9632,0.9640,0.9632,0.9632,0.9640,0.9632,0.9632 +0.9618,0.9624,0.9618,0.9620,0.9624,0.9618,0.9620 +0.9612,0.9622,0.9612,0.9613,0.9622,0.9612,0.9613 +0.9592,0.9602,0.9592,0.9593,0.9602,0.9592,0.9593 +0.9566,0.9578,0.9566,0.9567,0.9578,0.9566,0.9567 +0.9605,0.9615,0.9605,0.9605,0.9615,0.9605,0.9605 +0.9539,0.9560,0.9539,0.9542,0.9560,0.9539,0.9542 +0.9632,0.9639,0.9632,0.9632,0.9639,0.9632,0.9632 +0.9625,0.9633,0.9625,0.9625,0.9633,0.9625,0.9625 +0.9625,0.9632,0.9625,0.9626,0.9632,0.9625,0.9626 +0.9625,0.9631,0.9625,0.9626,0.9631,0.9625,0.9626 +0.9592,0.9598,0.9592,0.9591,0.9598,0.9592,0.9591 +0.9586,0.9600,0.9586,0.9588,0.9600,0.9586,0.9588 +0.9566,0.9577,0.9566,0.9567,0.9577,0.9566,0.9567 +0.9605,0.9611,0.9605,0.9605,0.9611,0.9605,0.9605 +0.9612,0.9621,0.9612,0.9613,0.9621,0.9612,0.9613 +0.9625,0.9636,0.9625,0.9626,0.9636,0.9625,0.9626 +0.9625,0.9631,0.9625,0.9624,0.9631,0.9625,0.9624 +0.9553,0.9563,0.9553,0.9554,0.9563,0.9553,0.9554 +0.9566,0.9578,0.9566,0.9567,0.9578,0.9566,0.9567 +0.9632,0.9639,0.9632,0.9632,0.9639,0.9632,0.9632 +0.9592,0.9600,0.9592,0.9593,0.9600,0.9592,0.9593 +0.9553,0.9568,0.9553,0.9554,0.9568,0.9553,0.9554 +0.9612,0.9614,0.9612,0.9611,0.9614,0.9612,0.9611 +0.9625,0.9633,0.9625,0.9626,0.9633,0.9625,0.9626 +0.9618,0.9629,0.9618,0.9620,0.9629,0.9618,0.9620 +0.9553,0.9569,0.9553,0.9554,0.9569,0.9553,0.9554 +0.9513,0.9526,0.9513,0.9515,0.9526,0.9513,0.9515 +0.9612,0.9614,0.9612,0.9612,0.9614,0.9612,0.9612 +0.9572,0.9585,0.9572,0.9573,0.9585,0.9572,0.9573 +0.9599,0.9604,0.9599,0.9598,0.9604,0.9599,0.9598 +0.9592,0.9603,0.9592,0.9594,0.9603,0.9592,0.9594 +0.9599,0.9611,0.9599,0.9600,0.9611,0.9599,0.9600 +0.9632,0.9638,0.9632,0.9632,0.9638,0.9632,0.9632 +0.9599,0.9602,0.9599,0.9599,0.9602,0.9599,0.9599 +0.9526,0.9536,0.9526,0.9527,0.9536,0.9526,0.9527 +0.9579,0.9586,0.9579,0.9579,0.9586,0.9579,0.9579 +0.9579,0.9593,0.9579,0.9580,0.9593,0.9579,0.9580 +0.9599,0.9607,0.9599,0.9600,0.9607,0.9599,0.9600 +0.9632,0.9641,0.9632,0.9633,0.9641,0.9632,0.9633 +0.9599,0.9607,0.9599,0.9601,0.9607,0.9599,0.9601 +0.9664,0.9669,0.9664,0.9665,0.9669,0.9664,0.9665 +0.9592,0.9604,0.9592,0.9594,0.9604,0.9592,0.9594 +0.9625,0.9630,0.9625,0.9625,0.9630,0.9625,0.9625 +0.9579,0.9590,0.9579,0.9581,0.9590,0.9579,0.9581 +0.9572,0.9579,0.9572,0.9573,0.9579,0.9572,0.9573 +0.9632,0.9638,0.9632,0.9632,0.9638,0.9632,0.9632 +0.9566,0.9576,0.9566,0.9566,0.9576,0.9566,0.9566 +0.9651,0.9662,0.9651,0.9653,0.9662,0.9651,0.9653 +0.9579,0.9593,0.9579,0.9581,0.9593,0.9579,0.9581 +0.9592,0.9603,0.9592,0.9593,0.9603,0.9592,0.9593 +0.9671,0.9677,0.9671,0.9672,0.9677,0.9671,0.9672 +0.9638,0.9644,0.9638,0.9639,0.9644,0.9638,0.9639 +0.9645,0.9653,0.9645,0.9646,0.9653,0.9645,0.9646 +0.9618,0.9631,0.9618,0.9620,0.9631,0.9618,0.9620 +0.9599,0.9606,0.9599,0.9599,0.9606,0.9599,0.9599 +0.9553,0.9565,0.9553,0.9553,0.9565,0.9553,0.9553 +0.9533,0.9543,0.9533,0.9534,0.9543,0.9533,0.9534 +0.9586,0.9599,0.9586,0.9588,0.9599,0.9586,0.9588 +0.9612,0.9620,0.9612,0.9613,0.9620,0.9612,0.9613 diff --git a/reports/Retrain/ResNet/time_metrics.txt b/reports/Retrain/ResNet/time_metrics.txt new file mode 100644 index 0000000..7224657 --- /dev/null +++ b/reports/Retrain/ResNet/time_metrics.txt @@ -0,0 +1,44 @@ +execution_time_sec +848.855054 +851.395526 +857.667636 +864.373247 +921.414065 +1006.761514 +851.995606 +850.456523 +851.156817 +855.827699 +852.529868 +855.051966 +851.841468 +852.182889 +859.966127 +870.718984 +859.687153 +849.761404 +892.106300 +880.976200 +894.792684 +918.782255 +862.899020 +848.422644 +848.069965 +849.830024 +850.185797 +850.567450 +850.479165 +849.162948 +850.724711 +848.658417 +850.287266 +848.900766 +849.176482 +849.449771 +850.224029 +848.678724 +851.971777 +850.963888 +848.760931 +848.571131 +856.289965 diff --git a/reports/Retrain/attack_values.csv b/reports/Retrain/attack_values.csv new file mode 100644 index 0000000..b0518af --- /dev/null +++ b/reports/Retrain/attack_values.csv @@ -0,0 +1,5 @@ +target_class,parameter_mia_accuracy,latent_distance_tell,lookalike_accuracy +0,0.500000,11.573492,0.000000 +1,0.500000,12.793120,0.000000 +2,0.500000,12.951434,0.000000 +3,0.500000,10.942259,0.000000 diff --git a/reports/WeightFiltration/RESNET34/forget.csv b/reports/WeightFiltration/RESNET34/forget.csv index fcd1014..8804daa 100644 --- a/reports/WeightFiltration/RESNET34/forget.csv +++ b/reports/WeightFiltration/RESNET34/forget.csv @@ -490,3 +490,40 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0125,1.0000,0.0125,0.0247,1.0000,0.0125,0.0247 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.1250,1.0000,0.1250,0.2222,1.0000,0.1250,0.2222 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/WeightFiltration/RESNET34/forget_train.csv b/reports/WeightFiltration/RESNET34/forget_train.csv index af2260d..d15d79a 100644 --- a/reports/WeightFiltration/RESNET34/forget_train.csv +++ b/reports/WeightFiltration/RESNET34/forget_train.csv @@ -490,3 +490,40 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.0125,1.0000,0.0125,0.0247,1.0000,0.0125,0.0247 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0063,1.0000,0.0063,0.0124,1.0000,0.0063,0.0124 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0094,1.0000,0.0094,0.0186,1.0000,0.0094,0.0186 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.1031,1.0000,0.1031,0.1870,1.0000,0.1031,0.1870 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0187,1.0000,0.0187,0.0368,1.0000,0.0187,0.0368 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0031,1.0000,0.0031,0.0062,1.0000,0.0031,0.0062 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0031,1.0000,0.0031,0.0062,1.0000,0.0031,0.0062 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/WeightFiltration/RESNET34/retain.csv b/reports/WeightFiltration/RESNET34/retain.csv index 77371c7..00bdf44 100644 --- a/reports/WeightFiltration/RESNET34/retain.csv +++ b/reports/WeightFiltration/RESNET34/retain.csv @@ -490,3 +490,40 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.9559,0.9572,0.9559,0.9560,0.9572,0.9559,0.9560 0.9546,0.9556,0.9546,0.9547,0.9556,0.9546,0.9547 0.9592,0.9606,0.9592,0.9594,0.9606,0.9592,0.9594 +0.9526,0.9540,0.9526,0.9528,0.9540,0.9526,0.9528 +0.9572,0.9581,0.9572,0.9573,0.9581,0.9572,0.9573 +0.9546,0.9556,0.9546,0.9546,0.9556,0.9546,0.9546 +0.9526,0.9537,0.9526,0.9527,0.9537,0.9526,0.9527 +0.9539,0.9545,0.9539,0.9540,0.9545,0.9539,0.9540 +0.9566,0.9574,0.9566,0.9566,0.9574,0.9566,0.9566 +0.9533,0.9542,0.9533,0.9534,0.9542,0.9533,0.9534 +0.9539,0.9554,0.9539,0.9541,0.9554,0.9539,0.9541 +0.9533,0.9544,0.9533,0.9534,0.9544,0.9533,0.9534 +0.9533,0.9541,0.9533,0.9534,0.9541,0.9533,0.9534 +0.9520,0.9530,0.9520,0.9521,0.9530,0.9520,0.9521 +0.9513,0.9525,0.9513,0.9514,0.9525,0.9513,0.9514 +0.9493,0.9509,0.9493,0.9496,0.9509,0.9493,0.9496 +0.9487,0.9500,0.9487,0.9487,0.9500,0.9487,0.9487 +0.9559,0.9575,0.9559,0.9561,0.9575,0.9559,0.9561 +0.9520,0.9531,0.9520,0.9521,0.9531,0.9520,0.9521 +0.9526,0.9536,0.9526,0.9527,0.9536,0.9526,0.9527 +0.9539,0.9557,0.9539,0.9541,0.9557,0.9539,0.9541 +0.9553,0.9559,0.9553,0.9552,0.9559,0.9553,0.9552 +0.9520,0.9530,0.9520,0.9520,0.9530,0.9520,0.9520 +0.9526,0.9541,0.9526,0.9528,0.9541,0.9526,0.9528 +0.9566,0.9575,0.9566,0.9566,0.9575,0.9566,0.9566 +0.9526,0.9541,0.9526,0.9528,0.9541,0.9526,0.9528 +0.9526,0.9541,0.9526,0.9528,0.9541,0.9526,0.9528 +0.9559,0.9569,0.9559,0.9560,0.9569,0.9559,0.9560 +0.9546,0.9555,0.9546,0.9546,0.9555,0.9546,0.9546 +0.9520,0.9530,0.9520,0.9521,0.9530,0.9520,0.9521 +0.9539,0.9545,0.9539,0.9540,0.9545,0.9539,0.9540 +0.9566,0.9574,0.9566,0.9566,0.9574,0.9566,0.9566 +0.9526,0.9541,0.9526,0.9528,0.9541,0.9526,0.9528 +0.9533,0.9542,0.9533,0.9534,0.9542,0.9533,0.9534 +0.9533,0.9547,0.9533,0.9535,0.9547,0.9533,0.9535 +0.9566,0.9575,0.9566,0.9566,0.9575,0.9566,0.9566 +0.9526,0.9540,0.9526,0.9528,0.9540,0.9526,0.9528 +0.9546,0.9558,0.9546,0.9547,0.9558,0.9546,0.9547 +0.9526,0.9540,0.9526,0.9528,0.9540,0.9526,0.9528 +0.9526,0.9540,0.9526,0.9528,0.9540,0.9526,0.9528 diff --git a/reports/WeightFiltration/ResNet/time_metrics.txt b/reports/WeightFiltration/ResNet/time_metrics.txt index 4c8796b..2b30364 100644 --- a/reports/WeightFiltration/ResNet/time_metrics.txt +++ b/reports/WeightFiltration/ResNet/time_metrics.txt @@ -490,3 +490,167 @@ execution_time_sec 0.000396 0.000410 0.000435 +87.953725 +89.953191 +88.555181 +87.086629 +86.067240 +0.001263 +0.000383 +0.000499 +0.000494 +0.000391 +0.000393 +0.000394 +87.004814 +88.485772 +0.000770 +0.001780 +0.000391 +0.000384 +0.000495 +0.000501 +0.000405 +0.001766 +0.000508 +87.975398 +0.000514 +0.000446 +0.001589 +0.000390 +0.000487 +0.000440 +0.001184 +0.000422 +0.000380 +0.000461 +89.646614 +0.000455 +0.000399 +0.000369 +87.205821 +0.000449 +0.001879 +86.592527 +0.000481 +0.000432 +0.000447 +0.000428 +0.000425 +0.000443 +0.000428 +0.000424 +0.000427 +0.000426 +0.000428 +0.000430 +0.000431 +0.000426 +0.000432 +0.000424 +0.000434 +0.000419 +0.000424 +86.237751 +0.000439 +0.000432 +0.000443 +0.000430 +0.000439 +0.000434 +0.000435 +0.000428 +0.000446 +0.000441 +0.000439 +0.000435 +0.000436 +0.000433 +0.000429 +0.000434 +0.000426 +0.000430 +0.000432 +86.227529 +0.000449 +0.000436 +0.000427 +0.000430 +0.000425 +0.000427 +0.000423 +0.000425 +0.000437 +0.000438 +0.000430 +0.000399 +0.000545 +0.000430 +0.000434 +0.000425 +0.000429 +0.000436 +0.000444 +86.158356 +0.000438 +0.000437 +0.000426 +0.000436 +0.000438 +0.000434 +0.000423 +0.000469 +0.000436 +0.000431 +0.000441 +0.000431 +0.000429 +0.000433 +0.000436 +0.000442 +0.000426 +0.000455 +0.000446 +86.279183 +0.000437 +0.000406 +0.000428 +0.000433 +0.000432 +0.000430 +0.000424 +0.000421 +0.000435 +0.000428 +0.000398 +0.000432 +0.000427 +0.000407 +0.000425 +0.000433 +0.000430 +0.000422 +0.000418 +89.527112 +87.450551 +0.002478 +0.000423 +0.000417 +0.000433 +0.000426 +0.000440 +0.000826 +0.000426 +0.000440 +0.000437 +0.000425 +0.000425 +0.000422 +0.000450 +0.000423 +0.000422 +0.000422 +0.000426 +0.000428 +86.247649 +0.000436 diff --git a/reports/WeightFiltration/attack_values.csv b/reports/WeightFiltration/attack_values.csv new file mode 100644 index 0000000..bc7b05e --- /dev/null +++ b/reports/WeightFiltration/attack_values.csv @@ -0,0 +1,5 @@ +target_class,parameter_mia_accuracy,latent_distance_tell,lookalike_accuracy +0,0.500000,1.180800,0.958333 +1,0.500000,1.279257,0.968750 +2,0.500000,1.717911,0.937500 +3,0.500000,1.354225,0.989583 diff --git a/reports/WeightFiltration/attack_values_old.csv b/reports/WeightFiltration/attack_values_old.csv new file mode 100644 index 0000000..8848236 --- /dev/null +++ b/reports/WeightFiltration/attack_values_old.csv @@ -0,0 +1,101 @@ +target_class,attack_mia_accuracy,latent_distance_tell +0,0.500000,1.107324 +1,0.500000,1.182681 +2,0.500000,1.628934 +3,0.500000,1.260251 +4,0.500000,1.399319 +5,0.500000,2.046023 +6,0.500000,1.750646 +7,0.500000,1.243093 +8,0.500000,1.809917 +9,0.500000,1.702536 +10,0.500000,1.291788 +11,0.500000,1.434109 +12,0.500000,1.448272 +13,0.500000,1.694034 +14,0.500000,1.717611 +15,0.500000,1.758781 +16,0.500000,1.188805 +17,0.500000,1.591978 +18,0.500000,1.445776 +19,0.500000,1.626087 +0,0.500000,1.158666 +1,0.500000,1.245154 +2,0.500000,1.558492 +3,0.500000,1.358110 +4,0.500000,1.339859 +5,0.500000,2.025961 +6,0.500000,1.828531 +7,0.500000,1.186695 +8,0.500000,1.920701 +9,0.500000,1.718948 +10,0.500000,1.374345 +11,0.500000,1.495150 +12,0.500000,1.368306 +13,0.500000,1.710072 +14,0.500000,1.700057 +15,0.500000,1.766020 +16,0.500000,1.160263 +17,0.500000,1.634570 +18,0.500000,1.461136 +19,0.500000,1.697268 +0,0.500000,1.092890 +1,0.500000,1.238615 +2,0.500000,1.704648 +3,0.500000,1.394107 +4,0.500000,1.365094 +5,0.500000,2.032884 +6,0.500000,1.833764 +7,0.500000,1.225851 +8,0.500000,1.807006 +9,0.500000,1.704291 +10,0.500000,1.358446 +11,0.500000,1.555449 +12,0.500000,1.387334 +13,0.500000,1.693131 +14,0.500000,1.736060 +15,0.500000,1.768330 +16,0.500000,1.190044 +17,0.500000,1.585899 +18,0.500000,1.482916 +19,0.500000,1.691146 +0,0.500000,1.141869 +1,0.500000,1.352442 +2,0.500000,1.695588 +3,0.500000,1.432673 +4,0.500000,1.314509 +5,0.500000,2.010463 +6,0.500000,1.817650 +7,0.500000,1.291032 +8,0.500000,1.703021 +9,0.500000,1.802832 +10,0.500000,1.355631 +11,0.500000,1.485411 +12,0.500000,1.441830 +13,0.500000,1.728542 +14,0.500000,1.740982 +15,0.500000,1.764840 +16,0.500000,1.210430 +17,0.500000,1.645152 +18,0.500000,1.471922 +19,0.500000,1.709163 +0,0.500000,1.122816 +1,0.500000,1.332376 +2,0.500000,1.646908 +3,0.500000,1.429030 +4,0.500000,1.321270 +5,0.500000,2.033827 +6,0.500000,1.863828 +7,0.500000,1.242626 +8,0.500000,1.924087 +9,0.500000,1.760985 +10,0.500000,1.423025 +11,0.500000,1.428449 +12,0.500000,1.390632 +13,0.500000,1.619642 +14,0.500000,1.745749 +15,0.500000,1.734899 +16,0.500000,1.144821 +17,0.500000,1.548540 +18,0.500000,1.452088 +19,0.500000,1.721123 diff --git a/reports/base/GOOGLENET/Finetuned.csv b/reports/base/GOOGLENET/Finetuned.csv new file mode 100644 index 0000000..527cfac --- /dev/null +++ b/reports/base/GOOGLENET/Finetuned.csv @@ -0,0 +1,2 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1 +0.9494,0.9504,0.9494,0.9495,0.9504,0.9494,0.9495 diff --git a/reports/base/RESNET34/Finetuned.csv b/reports/base/RESNET34/Finetuned.csv index 594da0a..14ac68b 100644 --- a/reports/base/RESNET34/Finetuned.csv +++ b/reports/base/RESNET34/Finetuned.csv @@ -51,3 +51,4 @@ accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recal 0.9519,0.9533,0.9519,0.9520,0.9533,0.9519,0.9520 0.9581,0.9590,0.9581,0.9581,0.9590,0.9581,0.9581 0.9513,0.9525,0.9512,0.9514,0.9525,0.9513,0.9514 +0.9531,0.9541,0.9531,0.9532,0.9541,0.9531,0.9532 diff --git a/unlearning/CertifiedUnlearning.py b/unlearning/CertifiedUnlearning.py index 1578ff9..f929f5b 100644 --- a/unlearning/CertifiedUnlearning.py +++ b/unlearning/CertifiedUnlearning.py @@ -43,17 +43,12 @@ class CertifiedUnlearning(Strategy): InceptionV3 auxiliary layers and tracking gradients. """ inner_model = getattr(model, "model", model) - - # Check if the current architecture is an Inception variant - is_inception = inner_model.__class__.__name__.lower() == "inception3" - + params_list = [] for name, p in inner_model.named_parameters(): if p.requires_grad: - # Discard the disconnected auxiliary training branch weights - if is_inception and "AuxLogits" in name: - continue - # CRITICAL: Append as a tuple so it can be unpacked as (name, param) + + # Append as a tuple so it can be unpacked as (name, param) params_list.append((name, p)) return params_list if named else [e[1] for e in params_list] @@ -92,7 +87,7 @@ class CertifiedUnlearning(Strategy): first_grads = grad(loss, params, retain_graph=True, create_graph=True) elemwise_products = sum(torch.sum(g_elem * v_elem) for g_elem, v_elem in zip(first_grads, v)) return grad(elemwise_products, params, create_graph=False) - + def _stochastic_newton_update(self, g, dataset, model, device): model.eval() criterion = nn.CrossEntropyLoss() @@ -133,7 +128,6 @@ class CertifiedUnlearning(Strategy): h_s = self._hvp(loss, params, h_estimate) - # OPTIMIZATION 4: Avoid deprecated .data, use detach() and in-place ops with torch.no_grad(): for k in range(len(params)): h_estimate[k].copy_(h_estimate[k] + g[k] - (h_s[k] / self.scale)) @@ -143,7 +137,7 @@ class CertifiedUnlearning(Strategy): if global_step % step_interval == 0 and current_pct < 100: current_pct += 1 print(f"\rProgress: {current_pct}% done", end="", flush=True) - + with torch.no_grad(): for k in range(len(params)): h_res[k] += h_estimate[k] / self.scale diff --git a/unlearning/LinearFiltration.py b/unlearning/LinearFiltration.py index 10c5961..d5df62e 100644 --- a/unlearning/LinearFiltration.py +++ b/unlearning/LinearFiltration.py @@ -13,8 +13,8 @@ class LinearFiltration(Strategy): def _run(self, model: nn.Module, forget_loader: DataLoader, retain_loader: DataLoader) -> nn.Module: model.eval() # Freeze internal params - for param in model.parameters(): - param.requires_grad = False + #for param in model.parameters(): + #param.requires_grad = False device = next(model.parameters()).device @@ -155,7 +155,8 @@ class LinearFiltration(Strategy): # 12 clf = self._get_classifier(model) - clf.weight.copy_(W_Z) + with torch.no_grad(): + clf.weight.copy_(W_Z) return model diff --git a/unlearning/Retrain.py b/unlearning/Retrain.py index e370836..cd9e9e4 100644 --- a/unlearning/Retrain.py +++ b/unlearning/Retrain.py @@ -1,4 +1,5 @@ -import time + +import os from pathlib import Path import torch import torch.nn as nn @@ -21,27 +22,51 @@ class Retrain(Strategy): self.epochs = epochs def _run(self, model: nn.Module, forget_loader: DataLoader, retain_loader: DataLoader) -> nn.Module: - # 1. Determine the active execution device from the running sandbox + device = next(model.parameters()).device + + # we need to check if a retrained copy exists on disk + checkpoint_path = f"trained_models/class_{self.target_class_index}_retrained.pth" + if os.path.exists(checkpoint_path): + print(f"Found existing retrained model checkpoint at '{checkpoint_path}'. Loading parameters directly...") + + # Load the state dict using safe configuration flags + state_dict = torch.load(checkpoint_path, map_location=device, weights_only=True) + + # Safely apply the parameter weights to the model in-place + model.load_state_dict(state_dict) + print("Retrained parameter loading complete (Retraining bypassed).") + return model + + # Cache Miss: Execute the standard retraining pipeline + print(f"No naive model found for class {self.target_class_index} retraining a new one") + print(f">> Triggering Exact Unlearning Baseline (Retraining {self.arch.name} from pristine state)...") + inner_model = getattr(model, "model", model) + if hasattr(inner_model, "fc"): + total_classes = inner_model.fc.out_features + elif hasattr(inner_model, "classifier"): + # Fallback for alternative architecture layout types + total_classes = inner_model.classifier[-1].out_features + else: + total_classes = self.size # a new model with default params is created - fresh_meat = Model.create(self.arch, device, self.size) + fresh = Model.create(self.arch, device, total_classes) # we train it with retain set - fresh_meat.train( + fresh.train( epochs=self.epochs, loader=retain_loader, rate=self.lr, mode="retrain" ) - # 4. Extract the trained nn.Module parameter state dict - # In-place copy onto the existing sandbox model structure to seamlessly retain downstream evaluations - model.load_state_dict(fresh_meat.model.state_dict()) + # Extract module parameter state dict and copy in place + model.load_state_dict(fresh.model.state_dict()) - print(">> Retraining pipeline finished. Pristine baseline weights successfully established.") + print("Retraining pipeline complete") return model def _split_data(self, dataset): @@ -49,5 +74,5 @@ class Retrain(Strategy): return get_unlearning_loaders( dataset=dataset, forget_class_idx=self.target_class_index, - batch_size=32 + batch_size=16 ) \ No newline at end of file diff --git a/unlearning/Strategy.py b/unlearning/Strategy.py index 14c99a6..df17cd1 100644 --- a/unlearning/Strategy.py +++ b/unlearning/Strategy.py @@ -40,10 +40,12 @@ class Strategy: execution_time = end_time - start_time # Log to the strategy's specific file + ''' Util.log_metric( log_file=log_file, execution_time=execution_time ) + ''' print(f"[{self.strategy_name}] Completed in {execution_time:.6f} seconds. Saved to {log_file}") diff --git a/unlearning/WeightFiltration.py b/unlearning/WeightFiltration.py index 9cb7424..f557179 100644 --- a/unlearning/WeightFiltration.py +++ b/unlearning/WeightFiltration.py @@ -114,7 +114,7 @@ class WeightFiltration(Strategy): model.eval() if self.wf_model is None: - print(">> Initializing and compiling global WF-Net matrix (Run Once for all classes)...") + print("Initializing and compiling global WF-Net matrix (Run Once for all classes)...") self.wf_model = self._optimise_filter( model, @@ -123,10 +123,10 @@ class WeightFiltration(Strategy): device=device ) else: - print(f">> Gating matrix loaded. Switching layout to target class index: {self.target_class_index}") + print(f"Gating matrix loaded. Switching layout to target class index: {self.target_class_index}") self.wf_model.target_class_index = self.target_class_index - return self.wf_model + return self.wf_model.get() def _split_data(self, dataset): return vertical_split(