strategies tested

Tune.py

@@ -5,14 +5,17 @@
 from torch.utils.data import DataLoader
 from sklearn.metrics import classification_report
 import SetUp
-from Data import *
-#from datasets.Casia import *
-from IdentitySubset import IdentitySubset
-#from datasets.UniversalIdentitySubset import UniversalIdentitySubset as IdentitySubset
+#from Data import *
+# from datasets.Casia import *
+#from IdentitySubset import IdentitySubset
+from sets.Data import *
+from sets.IdentitySubset import IdentitySubset
 # models
 from architectures.Model import Model, Architecture
 
 from unlearning.LinearFiltration import LinearFiltration
+from unlearning.CertifiedRemoval import CertifiedRemoval
+from unlearning.WeightFiltration import WeightFiltration
 
 import Util
 # WeightFiltration, CertifiedRemoval
@@ -20,7 +23,7 @@ import Util
 # numbre of classes 
 CLASS_SIZE = 20
 # batch 
-BATCH_SIZE = 32
+BATCH_SIZE = 16
 
 # size of images per class trainset + testset
 # 30 works best, more than that and we dont have enough data
@@ -50,7 +53,11 @@ arch = Architecture.RESNET50
 
 # DATA PREPARATION
 # load data set and prepare
-dataset = get_set()
+dataset_name = Set_Name.CELEBA
+set = Set_Name.CELEBA
+
+dataset = get_set(set_name=dataset_name)
+print(f"> {dataset.__class__.__name__} dataset loaded")
 # select identities for experiment
 #selected_identities = select_ids(
 #    dataset = dataset,
@@ -62,7 +69,7 @@ dataset = get_set()
 # that way repeated calls return the same classes
 selected_identities = select_top_ids(
     dataset=dataset,
-    class_size= CLASS_SIZE,
+    class_size=CLASS_SIZE
 )
 
 print(f'> Selected {CLASS_SIZE} random identity classes from CelebA dataset.')
@@ -72,7 +79,8 @@ print(f'> A class has {TRAINING_SMPLE} train and {SAMPLE_SIZE-TRAINING_SMPLE} te
 train_indices, test_indices = get_indices(
     dataset = dataset, 
     identities = selected_identities, 
-    split_at = TRAINING_SMPLE
+    split_at = TRAINING_SMPLE,
+    size= SAMPLE_SIZE
 )
 
 # helps map class id to index
@@ -80,7 +88,7 @@ id_map = {old_id: new_id for new_id, old_id in enumerate(selected_identities)}
 
 # we remap identities because crossEntropyLoss requires in indices 0 -> (n-1)
 # where n = class size.
-tr_transform = train_transform(res = RESOLUTION)
+tr_transform = train_transform(RESOLUTION)
 train_data = IdentitySubset(
     dataset=dataset, 
     indices=train_indices, 
@@ -100,8 +108,10 @@ print(f'> Constants : Classes = {CLASS_SIZE}, Batch = {BATCH_SIZE}, epochs = {EP
 # cuda if exists (it does here)
 device = SetUp.get_device()
 
+
 for i in range(0,CLASS_SIZE):
-# Create model using Factory
+    FORGET_CLASS_IDX = i
+    # Create model using Factory
     model = Model.create(
         arch = arch, 
         device = device, 
@@ -136,45 +146,101 @@ for i in range(0,CLASS_SIZE):
     print(f"Total test images for these {CLASS_SIZE} classes: {len(test_data)}")
 
     # Evaluate
-    mode, accuracy, report_dict = model.evaluate(
-                                    loader = test_loader,
-                                    mode="finetunned"
-                                    )
-    Util._log_to_csv(model=reloaded, mode = "finetuned", accuracy=accuracy, report_dict=report_dict, strategy="base")
+    current_mode = "Finetuned"
+    accuracy, report_dict = model.evaluate(
+        loader = test_loader,
+        mode=current_mode
+    )
 
-    # test again
-    reloaded = Model.create(
-        arch=arch,
-        device = device,
-        size = CLASS_SIZE
-        )
-    reloaded.load(arch = arch)
-    print("fine tunned model loaded")
-    # reloaded.evaluate(
-    #   loader = test_loader
-    #)
+    Util._log_to_csv(
+        arch=model.__class__.__name__, 
+        mode = current_mode, 
+        accuracy=accuracy, 
+        report_dict=report_dict, 
+        strategy="base"
+    )
 
-    # Unlearning
-    FORGET_CLASS_IDX = i
+    # unlearning algorithms
+    linear_filtration = LinearFiltration(target_class_idx=FORGET_CLASS_IDX)
+    #filtration.apply(reloaded.model)
 
-    forget_test_loader, retain_test_loader = get_forget_retain_loaders(
-        dataset=test_data, 
+    weight_filtration = WeightFiltration(num_classes = CLASS_SIZE,target_class_idx=FORGET_CLASS_IDX)
+    #weight_filtration.apply(reloaded.model)
+
+    certified_removal = CertifiedRemoval(removal_bound=0.05, epsilon=0.5, l2_reg=0.1)
+    #certified_removal.apply(reloaded.model)
+
+    # to be unlearned
+    forget_train_loader, retain_train_loader = get_unlearning_loaders(
+        dataset=train_data, 
         forget_class_idx=FORGET_CLASS_IDX, 
         batch_size=BATCH_SIZE
     )
 
-    #retain_test_loader = DataLoader(retain_test_loader.dataset, batch_size=BATCH_SIZE, shuffle=False)
-
-    # 3. Instantiate and apply the Linear Filtration rule
-    filtration = LinearFiltration(target_class_idx=FORGET_CLASS_IDX)
-    filtration.apply(reloaded.model)
+    # to evaluate
+    forget_test_loader, retain_test_loader = get_unlearning_loaders(
+            dataset=test_data, 
+            forget_class_idx=FORGET_CLASS_IDX, 
+            batch_size=BATCH_SIZE
+        )
 
 
-    # 4. Final Performance Analysis
-    print("\n--- Performance on Retained Classes")
-    mode, accuracy, report_dict = reloaded.evaluate(loader=retain_test_loader, mode="retain")
-    Util._log_to_csv(model=reloaded, mode = "retain", accuracy=accuracy, report_dict=report_dict, strategy="linearFiltration")
+    strategies = [linear_filtration, weight_filtration, certified_removal]
+    #strategies = [linear_filtration]
+    for strategy in strategies:
+        # test again
+        reloaded = Model.create(
+            arch=arch,
+            device = device,
+            size = CLASS_SIZE
+            )
+        reloaded.load(arch = arch)
+        print("fine tunned model loaded")
+        # reloaded.evaluate(
+        #   loader = test_loader
+        #)
 
-    print("\n--- Performance on Forgotten Class")
-    mode, accuracy, report_dict = reloaded.evaluate(loader=forget_test_loader,mode="forget")
-    Util._log_to_csv(model=reloaded, mode = "forgotten", accuracy=accuracy, report_dict=report_dict, strategy="linearFiltration")
+        # Unlearning
+        # train loaders passed here
+        strategy.apply(reloaded.model, forget_train_loader, retain_train_loader)
+        # Performance Analysis
+        strategy_in_use = strategy.__class__.__name__
+
+        # evaluation on retain Test_set
+        current_mode = "retain"
+        print("\n--- Performance on Retained Classes")
+        accuracy, report_dict = reloaded.evaluate(loader=retain_test_loader, mode=current_mode)
+
+        Util._log_to_csv(
+            arch=reloaded.__class__.__name__, 
+            mode = current_mode, 
+            accuracy=accuracy, 
+            report_dict=report_dict, 
+            strategy=strategy_in_use
+        )
+
+
+        # evaluation on forget Test_set
+        print("\n--- Performance on Forgotten Class")
+        current_mode = "forget"
+        accuracy, report_dict = reloaded.evaluate(loader=forget_test_loader,mode=current_mode)
+        Util._log_to_csv(
+            arch=reloaded.__class__.__name__, 
+            mode = current_mode, 
+            accuracy=accuracy, 
+            report_dict=report_dict, 
+            strategy=strategy_in_use
+        )
+
+        # evaluation on forget Train_set
+        # we expect this to be equal or highr than accuracy on Forget Test_set
+        current_mode = "forget_train"
+        print("\n--- Performance on Forgotten Class (Train Set - Verifying Unlearning)")
+        accuracy, report_dict = reloaded.evaluate(loader=forget_train_loader, mode=current_mode)
+        Util._log_to_csv(
+            arch= reloaded.__class__.__name__, 
+            mode = current_mode, 
+            accuracy=accuracy, 
+            report_dict=report_dict, 
+            strategy=strategy_in_use
+        )