facebook's implementation

unlearning/CertifiedRemoval.py

@@ -55,27 +55,9 @@ class CertifiedRemoval(Strategy):
         
         # Compute the Exact Hessian Matrix over the remaining (retained) features
         # Formula: H = (X^T * X) / N + lambda * I
-        # this will be done on CPU. requires more ram so we cant afford to do it on VRAM
-        # print(">> Computing exact Hessian matrix...")
         N_retain = retain_features.size(0)
-        # X_T_X = torch.matmul(retain_features.t(), retain_features)
-        # reg_matrix = self.l2_reg * torch.eye(retain_features.size(1))
         hessian = self._compute_hessian(retain_features=retain_features, retain_features_size = N_retain)
         
-        # Compute the gradient of the loss with respect to the forgotten data
-        # print(">> Calculating forget set gradients...")
-        # num_classes = w.size(0)
-        # Pass features through linear layer weights to get logits
-        # logits_forget = torch.matmul(forget_features, w.t()) 
-        # Apply softmax to get true class probabilities
-        # preds_softmax = torch.softmax(logits_forget, dim=1)
-        
-        # forget_labels_one_hot = torch.nn.functional.one_hot(forget_labels, num_classes=num_classes).float()
-        
-        #preds_forget = torch.matmul(forget_features, w.t())
-        #error = preds_forget - forget_labels_one_hot
-        # error = preds_softmax - forget_labels_one_hot
-        # grad_forget shape: [num_classes, 2048]
         grad_forget = self._compute_loss_gradient(
             forget_labels=forget_labels,
             forget_features=forget_features, 
@@ -87,14 +69,6 @@ class CertifiedRemoval(Strategy):
             tensor = hessian,
             gradient= grad_forget
         )
-        # print(">> Solving Newton step via system optimization...")
-        # try:
-        #     delta_w_t = torch.linalg.solve(Hessian, grad_forget.t())
-        #     delta_w = delta_w_t.t()
-        # except RuntimeError:
-        #     print(">> Warning: Hessian matrix is singular. Falling back to pseudo-inverse.")
-        #     delta_w = torch.matmul(grad_forget, torch.linalg.pinv(Hessian).t())
-
         # Apply the Certified Removal update rule: W_new = W + Delta_W
         new_w = w + delta_w
         # Calibrate noise based on your epsilon budget