played around with CASIA-WEB-FACE

.gitignore

@@ -7,6 +7,7 @@ include/
 
 # Data and Models
 data/
+datasets/
 trained_models/
 
 # Python cache

Data.py

@@ -2,6 +2,7 @@ from torchvision import datasets, transforms, models
 import torch
 import numpy as np
 
+
 # train set transform 
 def train_transform(res):
     return transforms.Compose([
@@ -25,7 +26,6 @@ def train_transform(res):
 # test set transform
 def test_transform(res):
     return transforms.Compose([
-        # Just standard resize to 224x224
         transforms.Resize((res, res)),
         transforms.ToTensor(),
         transforms.Normalize(
@@ -67,7 +67,7 @@ def select_ids( dataset, sample_size, class_size):
     return np.random.choice(eligible_ids, class_size, replace=False)
 
 # optional function to get max amount of samples per class
-def select_balanced_ids(dataset, class_size):
+def select_top_ids(dataset, class_size):
     ids, counts = get_ids_and_counts(dataset=dataset)
 
     # sort by number of images (descending)
@@ -79,11 +79,12 @@ def select_balanced_ids(dataset, class_size):
 
 
 # split class images to train and test set.
-def get_indices(dataset, identities, split_at):
+def get_indices(dataset, identities, split_at, size = 30):
     train_indices = []
     test_indices = []
 
     #training_sample = int(sample_size * training_ratio)
+    np.random.seed(42)
     for person_id in identities:
         # Get all indices for this specific person
         indices = torch.where(dataset.identity == person_id)[0].numpy()
@@ -93,6 +94,6 @@ def get_indices(dataset, identities, split_at):
         
         # split data to testing and training
         train_indices.extend(indices[:split_at])
-        test_indices.extend(indices[split_at:])
+        test_indices.extend(indices[split_at:size])
 
     return train_indices, test_indices

DataAnalyser.py

@@ -0,0 +1,50 @@
+
+#from Data import *
+from datasets.Casia import *
+
+'''
+Because the size of samples per class had the biggest impact
+on training outcome, I decided to check the maximum amount of data 
+I can get from a class. 
+The highest I can get is 
+Rank     | Identity ID  |Count 
+-----------------------------------
+1        | 3782         | 35          
+2        | 2820         | 35          
+3        | 3227         | 35          
+4        | 3745         | 34          
+5        | 3699         | 34          
+6        | 8968         | 32          
+7        | 9152         | 32          
+8        | 9256         | 32          
+9        | 2114         | 31          
+...      | ...          | ...            
+17       | 4126         | 31          
+18       | 3185         | 30                  
+...      | ...          | ...       
+50       | 3186         | 30   
+
+as can be seen, 3 classes have 35, 2 have 34, 3 have 32 and the rest have 30.
+'''
+def print_top_identity_stats(dataset, top_n=50):
+    # we get data
+    ids, counts = get_ids_and_counts(dataset)
+    # sort in descending order
+    sorted_counts, sorted_indices = torch.sort(counts, descending=True)
+
+    # coresponding sorted ids
+    sorted_ids = ids[sorted_indices]
+
+    # 4. Slice the first 'top_n' and print
+    print(f"{'Rank':<8} | {'Identity ID':<12} | {'Image Count':<12}")
+    print("-" * 35)
+    
+    for i in range(top_n):
+        identity_id = sorted_ids[i].item()
+        count = sorted_counts[i].item()
+        print(f"{i+1:<8} | {identity_id:<12} | {count:<12}")
+
+# Usage:
+dataset = get_set()
+print_top_identity_stats(dataset, 50)
+

Tune.py

@@ -1,16 +1,21 @@
-import torch
+# Finetuning a selected model 
+# on a selected dataset 
+# using selected parameters
+
 from torch.utils.data import DataLoader
 from sklearn.metrics import classification_report
 import SetUp
 from Data import *
-from IdentitySubset import IdentitySubset
+#from datasets.Casia import *
+#from IdentitySubset import IdentitySubset
+from datasets.UniversalIdentitySubset import UniversalIdentitySubset as IdentitySubset
 # models
 from architectures.Model import Model, Architecture
 
 # numbre of classes 
 CLASS_SIZE = 20
 # batch 
-BATCH_SIZE = 16
+BATCH_SIZE = 32
 
 # size of images per class trainset + testset
 # 30 works best, more than that and we dont have enough data
@@ -21,7 +26,7 @@ TRAINING_SMPLE = 28
 
 # learning rate
 LR_RATE = 0.0001
-EPOCHS = 16
+EPOCHS = 20
 
 # depends on model architecture
 # ResNet, DenseNet = 224
@@ -42,10 +47,17 @@ arch = Architecture.RESNET50
 # load data set and prepare
 dataset = get_set()
 # select identities for experiment
-selected_identities = select_ids(
+#selected_identities = select_ids(
+#    dataset = dataset,
+#    sample_size = SAMPLE_SIZE,
+#    class_size = CLASS_SIZE
+#    )
+
+# this selects the top 50 based on sample size
+# that way repeated calls return the same classes
+selected_identities = select_top_ids(
     dataset=dataset,
-    sample_size = SAMPLE_SIZE,
-    class_size = CLASS_SIZE
+    class_size= CLASS_SIZE,
 )
 
 print(f'> Selected {CLASS_SIZE} random identity classes from CelebA dataset.')

architectures/ResNet18.py

@@ -11,12 +11,12 @@ class ResNet18(Model):
         m = models.resnet18(weights=models.ResNet18_Weights.DEFAULT)
 
         # freez all layers
-        for param in m.parameters():
-            param.requires_grad = False
+        #for param in m.parameters():
+        #    param.requires_grad = False
 
         # unfreez the last two
-        for param in m.layer3.parameters(): param.requires_grad = True
-        for param in m.layer4.parameters(): param.requires_grad = True
+        #for param in m.layer3.parameters(): param.requires_grad = True
+        #for param in m.layer4.parameters(): param.requires_grad = True
         
         m.fc = nn.Linear(m.fc.in_features, self.size)
         return m

dependencies.txt

@@ -3,3 +3,4 @@ torchvision
 gdown
 numpy
 scikit-learn
+kagglehub