diff --git a/Data.py b/Data.py index e64a440..8ec35c7 100644 --- a/Data.py +++ b/Data.py @@ -1,8 +1,10 @@ from torchvision import datasets, transforms, models +from torch.utils.data import Dataset, DataLoader, Subset import torch import numpy as np + # train set transform def train_transform(res): return transforms.Compose([ @@ -101,3 +103,39 @@ def get_indices(dataset, identities, split_at, size = 30): test_indices.extend(indices[split_at:size]) return train_indices, test_indices + + + +def get_forget_retain_loaders(dataset: Dataset, forget_class_idx: int, batch_size: int = 32) -> tuple[DataLoader, DataLoader]: + """ + Splits an IdentitySubset or standard Dataset into forget and retain sets + based on a remapped target class index. + """ + # 1. Safely extract targets whether it's a standard dataset or a Subset wrapper + if hasattr(dataset, 'targets'): + targets = dataset.targets + elif hasattr(dataset, 'identity'): # Raw CelebA support + targets = dataset.identity + else: + # If it's an IdentitySubset or standard Subset, extract mapped targets sequentially + # This guarantees we get the 0 -> (n-1) remapped labels + targets = [dataset[i][1] for i in range(len(dataset))] + + if not isinstance(targets, torch.Tensor): + targets = torch.tensor(targets) + + # 2. Generate mask indices local to this subset + forget_indices = torch.where(targets == forget_class_idx)[0].tolist() + retain_indices = torch.where(targets != forget_class_idx)[0].tolist() + + # 3. Create PyTorch Subsets + forget_subset = Subset(dataset, forget_indices) + retain_subset = Subset(dataset, retain_indices) + + # 4. Wrap into clean DataLoaders + forget_loader = DataLoader(forget_subset, batch_size=batch_size, shuffle=False) + retain_loader = DataLoader(retain_subset, batch_size=batch_size, shuffle=True) + + print(f"[Data Split] Local Class {forget_class_idx}: {len(forget_subset)} samples | Remaining Classes: {len(retain_subset)} samples.") + + return forget_loader, retain_loader \ No newline at end of file diff --git a/LinearFiltration_metrics.txt b/LinearFiltration_metrics.txt new file mode 100644 index 0000000..ac054a6 --- /dev/null +++ b/LinearFiltration_metrics.txt @@ -0,0 +1,7 @@ +execution_time_sec +0.000996 +0.030071 +0.001182 +0.001176 +0.001229 +0.001257 diff --git a/Tune.py b/Tune.py index 789bc85..e975f72 100644 --- a/Tune.py +++ b/Tune.py @@ -7,12 +7,13 @@ from sklearn.metrics import classification_report import SetUp from Data import * #from datasets.Casia import * -#from IdentitySubset import IdentitySubset -from datasets.UniversalIdentitySubset import UniversalIdentitySubset as IdentitySubset +from IdentitySubset import IdentitySubset +#from datasets.UniversalIdentitySubset import UniversalIdentitySubset as IdentitySubset # models from architectures.Model import Model, Architecture -from unlearning import LinearFiltration, WeightFiltration, CertifiedRemoval +from unlearning.LinearFiltration import LinearFiltration +# WeightFiltration, CertifiedRemoval # numbre of classes CLASS_SIZE = 20 @@ -24,7 +25,7 @@ BATCH_SIZE = 32 SAMPLE_SIZE = 30 # this is then (full_sample - test_sample) -TRAINING_SMPLE = 28 +TRAINING_SMPLE = 27 # learning rate LR_RATE = 0.0001 @@ -96,68 +97,79 @@ print(f'> Constants : Classes = {CLASS_SIZE}, Batch = {BATCH_SIZE}, epochs = {EP # MODEL PREPARATION # cuda if exists (it does here) device = SetUp.get_device() + +for i in range(0,CLASS_SIZE): # Create model using Factory -model = Model.create( - arch = arch, - device = device, - size = CLASS_SIZE) + model = Model.create( + arch = arch, + device = device, + size = CLASS_SIZE) -# we may need to load existing model or finetune -model.train( - epochs = EPOCHS, - loader = train_loader, - rate = LR_RATE) + # we may need to load existing model or finetune + model.train( + epochs = EPOCHS, + loader = train_loader, + rate = LR_RATE) - # save. -model.save(filename=arch.name.lower()) + # save. + model.save(filename=arch.name.lower()) -# done tuning -print('Model saved!') + # done tuning -# EVALUATE + # EVALUATE + te_transform = test_transform(RESOLUTION) + # Testing + test_data = IdentitySubset( + dataset = dataset, + indices=test_indices, + id_mapping=id_map, + transform=te_transform) -te_transform = test_transform(RESOLUTION) -# Testing -test_data = IdentitySubset( - dataset = dataset, - indices=test_indices, - id_mapping=id_map, - transform=te_transform) + test_loader = DataLoader( + test_data, + batch_size=BATCH_SIZE, + shuffle=False) -test_loader = DataLoader( - test_data, - batch_size=BATCH_SIZE, - shuffle=False) + print(f"Total test images for these {CLASS_SIZE} classes: {len(test_data)}") -print(f"Total test images for these {CLASS_SIZE} classes: {len(test_data)}") + # Evaluate + model.evaluate( + loader = test_loader, + mode="finetunned" + ) -# Evaluate -model.evaluate( - loader = test_loader) + # test again + reloaded = Model.create( + arch=arch, + device = device, + size = CLASS_SIZE + ) + reloaded.load(arch = arch) + print("fine tunned model loaded") + # reloaded.evaluate( + # loader = test_loader + #) -# test again -reloaded = Model.create( - arch=arch, - device = device, - size = CLASS_SIZE + # Unlearning + FORGET_CLASS_IDX = i + + forget_test_loader, retain_test_loader = get_forget_retain_loaders( + dataset=test_data, + forget_class_idx=FORGET_CLASS_IDX, + batch_size=BATCH_SIZE ) -reloaded.load(arch = arch) -print("Evaluating loaded") -reloaded.evaluate( - loader = test_loader -) + + #retain_test_loader = DataLoader(retain_test_loader.dataset, batch_size=BATCH_SIZE, shuffle=False) + + # 3. Instantiate and apply the Linear Filtration rule + filtration = LinearFiltration(target_class_idx=FORGET_CLASS_IDX) + filtration.apply(reloaded.model) -strategies_to_test = [ - LinearFiltration(target_class_idx=12), - WeightFiltration(target_class_idx=12), - CertifiedRemoval(target_class_idx=12) -] + # 4. Final Performance Analysis + print("\n--- Performance on Retained Classes") + reloaded.evaluate(loader=retain_test_loader, mode="retain") -# Run the comparative benchmark seamlessly -execution_profiles = {} -for strategy in strategies_to_test: - # Each iteration clones weights back to fine-tuned state before running - runtime = my_model.unlearn(strategy, forget_loader, retain_loader) - execution_profiles[strategy.__class__.__name__] = runtime + print("\n--- Performance on Forgotten Class") + reloaded.evaluate(loader=forget_test_loader,mode="forget") \ No newline at end of file diff --git a/architectures/Model.py b/architectures/Model.py index f5292e2..2d5978e 100644 --- a/architectures/Model.py +++ b/architectures/Model.py @@ -43,10 +43,13 @@ class Model(ABC): if self.device.type == 'cuda': torch.cuda.synchronize() print(f"Training completed in: {time.time() - start_time:.2f}s") + def evaluate(self, loader): self.model.eval() all_preds, all_labels = [], [] print("\nEvaluating...") + + with torch.no_grad(): for inputs, labels in loader: @@ -56,9 +59,11 @@ class Model(ABC): all_preds.extend(predicted.cpu().numpy()) all_labels.extend(labels.cpu().numpy()) - accuracy = 100 * (np.array(all_preds) == np.array(all_labels)).sum() / len(all_labels) + classes = sorted(list(set(all_labels))) + + accuracy = 100 * (np.array(all_preds) == np.array(all_labels)).sum() / len(classes) print(f"Test Accuracy: {accuracy:.2f}%") - print(classification_report(all_labels, all_preds, zero_division=0)) + print(classification_report(all_labels, all_preds, labels=classes, zero_division=0)) def save(self, filename=None): @@ -101,13 +106,109 @@ class Model(ABC): start_time = time.time() # Delegate the actual algorithmic weight/logit manipulation to the strategy - strategy.apply(self.network, forget_loader, retain_loader) + strategy.apply(self.model, forget_loader, retain_loader) elapsed_time = time.time() - start_time print(f"{strategy.__class__.__name__} completed in {elapsed_time:.4f} seconds.") return elapsed_time + def evaluate(self, loader, mode="eval"): + """ + Evaluates the model, prints terminal reports, and routes metrics to + a file logger based on the current context mode. + """ + self.model.eval() + all_preds, all_labels = [], [] + print(f"\nEvaluating Domain: [{mode}]...") + + with torch.no_grad(): + for inputs, labels in loader: + inputs, labels = inputs.to(self.device), labels.to(self.device) + outputs = self.model(inputs) + _, predicted = torch.max(outputs, 1) + all_preds.extend(predicted.cpu().numpy()) + all_labels.extend(labels.cpu().numpy()) + + # Extract only the active classes evaluated in this loader slice + classes = sorted(list(set(all_labels))) + accuracy = 100 * (np.array(all_preds) == np.array(all_labels)).sum() / len(all_labels) + + print(f"Test Accuracy: {accuracy:.2f}%") + + # 1. Print standard text report to terminal + print(classification_report(all_labels, all_preds, labels=classes, zero_division=0)) + + # 2. Extract structured dictionary metrics + report_dict = classification_report( + all_labels, + all_preds, + labels=classes, + output_dict=True, + zero_division=0 + ) + + # 3. Delegate file tracking to isolated helper method + self._log_to_csv(mode, accuracy, classes, report_dict) + + + def _log_to_csv(self, mode, accuracy, classes, report_dict): + """Handles directory structures, file setups, and distinct CSV column formatting.""" + arch_name = self.__class__.__name__.lower() + save_dir = Path("reports") + save_dir.mkdir(parents=True, exist_ok=True) + csv_path = save_dir / f"{arch_name}-{mode}.csv" + + file_exists = csv_path.exists() + ''' + # Structure payload and headers based on evaluation slice type + if mode == "forget": + headers = ["accuracy", "precision", "recall", "f1-score"] + target_cls_str = str(classes[0]) + metrics = report_dict[target_cls_str] + row = [ + f"{accuracy / 100.0:.4f}", + f"{metrics['precision']:.4f}", + f"{metrics['recall']:.4f}", + f"{metrics['f1-score']:.4f}" + ] + else: + headers = [ + "accuracy", + "macro_precision", "macro_recall", "macro_f1", + "weighted_precision", "weighted_recall", "weighted_f1" + ] + row = [ + f"{accuracy / 100.0:.4f}", + f"{report_dict['macro avg']['precision']:.4f}", + f"{report_dict['macro avg']['recall']:.4f}", + f"{report_dict['macro avg']['f1-score']:.4f}", + f"{report_dict['weighted avg']['precision']:.4f}", + f"{report_dict['weighted avg']['recall']:.4f}", + f"{report_dict['weighted avg']['f1-score']:.4f}" + ]''' + headers = [ + "accuracy", + "macro_precision", "macro_recall", "macro_f1", + "weighted_precision", "weighted_recall", "weighted_f1" + ] + row = [ + f"{accuracy / 100.0:.4f}", + f"{report_dict['macro avg']['precision']:.4f}", + f"{report_dict['macro avg']['recall']:.4f}", + f"{report_dict['macro avg']['f1-score']:.4f}", + f"{report_dict['weighted avg']['precision']:.4f}", + f"{report_dict['weighted avg']['recall']:.4f}", + f"{report_dict['weighted avg']['f1-score']:.4f}" + ] + + with open(csv_path, "a") as f: + if not file_exists: + f.write(",".join(headers) + "\n") + f.write(",".join(row) + "\n") + + print(f">> Direct CSV metrics appended to {csv_path}") + # Using the factory patern here diff --git a/reports/LinearFiltration_metrics.txt b/reports/LinearFiltration_metrics.txt new file mode 100644 index 0000000..335dc33 --- /dev/null +++ b/reports/LinearFiltration_metrics.txt @@ -0,0 +1,35 @@ +execution_time_sec +0.001269 +0.001227 +0.001298 +0.001281 +0.001178 +0.001392 +0.001391 +0.001338 +0.001162 +0.001355 +0.001361 +0.001241 +0.001210 +0.001152 +0.001358 +0.001250 +0.001467 +0.001248 +0.001411 +0.001470 +0.001241 +0.001366 +0.001206 +0.001339 +0.001268 +0.002847 +0.001245 +0.001299 +0.001222 +0.001274 +0.001351 +0.001401 +0.001286 +0.001214 diff --git a/reports/resnet50-finetunned.csv b/reports/resnet50-finetunned.csv new file mode 100644 index 0000000..0a2eab6 --- /dev/null +++ b/reports/resnet50-finetunned.csv @@ -0,0 +1,21 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1 +0.9333,0.9500,0.9333,0.9264,0.9500,0.9333,0.9264 +0.9167,0.9425,0.9167,0.9111,0.9425,0.9167,0.9111 +0.9333,0.9500,0.9333,0.9264,0.9500,0.9333,0.9264 +0.8667,0.9050,0.8667,0.8625,0.9050,0.8667,0.8625 +0.9000,0.9208,0.9000,0.8976,0.9208,0.9000,0.8976 +0.9000,0.9208,0.9000,0.8926,0.9208,0.9000,0.8926 +0.9667,0.9750,0.9667,0.9657,0.9750,0.9667,0.9657 +0.9000,0.9308,0.9000,0.9012,0.9308,0.9000,0.9012 +0.9833,0.9875,0.9833,0.9829,0.9875,0.9833,0.9829 +0.9500,0.9625,0.9500,0.9486,0.9625,0.9500,0.9486 +0.8667,0.9008,0.8667,0.8551,0.9008,0.8667,0.8551 +0.9167,0.9375,0.9167,0.9093,0.9375,0.9167,0.9093 +0.9000,0.9250,0.9000,0.8921,0.9250,0.9000,0.8921 +0.9000,0.9333,0.9000,0.9024,0.9333,0.9000,0.9024 +0.9500,0.9625,0.9500,0.9486,0.9625,0.9500,0.9486 +0.9167,0.9333,0.9167,0.9148,0.9333,0.9167,0.9148 +0.9167,0.9375,0.9167,0.9143,0.9375,0.9167,0.9143 +0.9667,0.9750,0.9667,0.9657,0.9750,0.9667,0.9657 +0.9333,0.9500,0.9333,0.9314,0.9500,0.9333,0.9314 +0.9000,0.9350,0.9000,0.8957,0.9350,0.9000,0.8957 diff --git a/reports/resnet50-forget.csv b/reports/resnet50-forget.csv new file mode 100644 index 0000000..90ee200 --- /dev/null +++ b/reports/resnet50-forget.csv @@ -0,0 +1,21 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 +0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000 diff --git a/reports/resnet50-retain.csv b/reports/resnet50-retain.csv new file mode 100644 index 0000000..8f740fc --- /dev/null +++ b/reports/resnet50-retain.csv @@ -0,0 +1,21 @@ +accuracy,macro_precision,macro_recall,macro_f1,weighted_precision,weighted_recall,weighted_f1, +0.9474,0.9605,0.9474,0.9459,0.9605,0.9474,0.9459 +0.9123,0.9395,0.9123,0.9064,0.9395,0.9123,0.9064 +0.9298,0.9526,0.9298,0.9244,0.9526,0.9298,0.9244 +0.8596,0.9000,0.8596,0.8553,0.9000,0.8596,0.8553 +0.9123,0.9298,0.9123,0.9103,0.9298,0.9123,0.9103 +0.9123,0.9342,0.9123,0.9045,0.9342,0.9123,0.9045 +0.9649,0.9737,0.9649,0.9639,0.9737,0.9649,0.9639 +0.8947,0.9272,0.8947,0.8960,0.9272,0.8947,0.8960 +1.0000,1.0000,1.0000,1.0000,1.0000,1.0000,1.0000 +0.9649,0.9737,0.9649,0.9639,0.9737,0.9649,0.9639 +0.8772,0.9132,0.8772,0.8650,0.9132,0.8772,0.8650 +0.9123,0.9342,0.9123,0.9045,0.9342,0.9123,0.9045 +0.9123,0.9342,0.9123,0.9045,0.9342,0.9123,0.9045 +0.9298,0.9605,0.9298,0.9328,0.9605,0.9298,0.9328 +0.9649,0.9737,0.9649,0.9639,0.9737,0.9649,0.9639 +0.9298,0.9430,0.9298,0.9283,0.9430,0.9298,0.9283 +0.9298,0.9474,0.9298,0.9278,0.9474,0.9298,0.9278 +0.9649,0.9737,0.9649,0.9639,0.9737,0.9649,0.9639 +0.9298,0.9474,0.9298,0.9278,0.9474,0.9298,0.9278 +0.8947,0.9316,0.8947,0.8902,0.9316,0.8947,0.8902 diff --git a/unlearning/LinearFiltration.py b/unlearning/LinearFiltration.py index abb8306..ca3652b 100644 --- a/unlearning/LinearFiltration.py +++ b/unlearning/LinearFiltration.py @@ -1,29 +1,48 @@ import torch -from Strategy import Strategy +import torch.nn as nn +from .Strategy import Strategy -class NormalizingLinearFiltration(Strategy): - def __init__(self, target_class_idx): +class LinearFiltration(Strategy): + def __init__(self, target_class_idx: int): + super().__init__() # Automatically configures 'NormalizingLinearFiltration_metrics.txt' self.target_class_idx = target_class_idx - def apply(self, model, forget_loader, retain_loader): + def _run(self, model: nn.Module) -> nn.Module: model.eval() - # Freeze parameters structurally for param in model.parameters(): param.requires_grad = False with torch.no_grad(): - # we modify only classification head - # Shape: [num_classes, feature_dim] - W = model.fc.weight.data - - # Compute the normalization transformation projection matrix (A) - # (In your full code, calculate A here matching Baumhauer et al.'s equations) + W = model.fc.weight.data.clone() num_classes = W.shape[0] - A = torch.eye(num_classes, device=W.device) - # Mask/blend target class index distribution configurations here... - A[self.target_class_idx, :] = 0.0 - # 3. Direct weight matrix override: W_filtered = A * W + A = self._calculate_filtration_matrix(num_classes, self.target_class_idx, W.device) sanitized_W = torch.mm(A, W) - model.fc.weight.copy_(sanitized_W) \ No newline at end of file + model.fc.weight.copy_(sanitized_W) + + return model + + '''@staticmethod + def _calculate_filtration_matrix(num_classes: int, forget_class: int, device: torch.device) -> torch.Tensor: + A = torch.eye(num_classes, device=device) + num_remaining_classes = num_classes - 1 + for j in range(num_classes): + if j == forget_class: + A[forget_class, j] = 0.0 + else: + A[forget_class, j] = 1.0 / num_remaining_classes + return A''' + @staticmethod + def _calculate_filtration_matrix(num_classes: int, forget_class: int, device: torch.device) -> torch.Tensor: + A = torch.eye(num_classes, device=device) + num_remaining = num_classes - 1 + + # The row of the forgotten class should average the inputs of all other classes + for j in range(num_classes): + if j == forget_class: + A[forget_class, j] = 0.0 + else: + A[forget_class, j] = 1.0 / num_remaining + + return A \ No newline at end of file diff --git a/unlearning/Strategy.py b/unlearning/Strategy.py index e69de29..c41821d 100644 --- a/unlearning/Strategy.py +++ b/unlearning/Strategy.py @@ -0,0 +1,47 @@ + +import torch.nn as nn +import time +import os + +class Strategy: + """Abstract base class for unlearning algorithms with automated, strategy-specific logging.""" + + def __init__(self): + # Dynamically set file name based on the class name (e.g., 'NormalizingLinearFiltration.txt') + self.strategy_name = self.__class__.__name__ + self.log_file = f"reports/{self.strategy_name}_metrics.txt" + self._initialize_log_file() + + def _initialize_log_file(self): + """Creates a unique log file for this strategy with a header if it doesn't exist.""" + if not os.path.exists(self.log_file): + with open(self.log_file, "w") as f: + f.write("execution_time_sec\n") + + def log_metric(self, execution_time: float): + """Appends the execution time to this strategy's specific file.""" + with open(self.log_file, "a") as f: + f.write(f"{execution_time:.6f}\n") + + def apply(self, model: nn.Module) -> nn.Module: + """ + Wraps the unlearning execution with automated timing and strategy-specific logging. + DO NOT override this method in subclasses. Override _run instead. + """ + start_time = time.perf_counter() + + # Execute core unlearning logic + processed_model = self._run(model) + + end_time = time.perf_counter() + execution_time = end_time - start_time + + # Log to the strategy's specific file + self.log_metric(execution_time) + print(f"[{self.strategy_name}] Completed in {execution_time:.6f} seconds. Saved to {self.log_file}") + + return processed_model + + def _run(self, model: nn.Module) -> nn.Module: + """Subclasses implement their core unlearning logic here.""" + raise NotImplementedError \ No newline at end of file