From f54885ba16675521d7186081cbe63e0bc5097b2b Mon Sep 17 00:00:00 2001 From: Cian Hughes Date: Thu, 12 Sep 2024 09:04:26 +0100 Subject: [PATCH] removed accidentally committed files --- src/symbolic_nn_tests/local/__init__.py | 93 -------------------- src/symbolic_nn_tests/local/model.py | 72 --------------- src/symbolic_nn_tests/local/semantic_loss.py | 88 ------------------ 3 files changed, 253 deletions(-) delete mode 100644 src/symbolic_nn_tests/local/__init__.py delete mode 100644 src/symbolic_nn_tests/local/model.py delete mode 100644 src/symbolic_nn_tests/local/semantic_loss.py diff --git a/src/symbolic_nn_tests/local/__init__.py b/src/symbolic_nn_tests/local/__init__.py deleted file mode 100644 index 4c9d141..0000000 --- a/src/symbolic_nn_tests/local/__init__.py +++ /dev/null @@ -1,93 +0,0 @@ -LEARNING_RATE = 10e-5 - - -def test(train_loss, val_loss, test_loss, version, tensorboard=True, wandb=True): - from .model import main as test_model - - logger = [] - - if tensorboard: - from lightning.pytorch.loggers import TensorBoardLogger - - tb_logger = TensorBoardLogger( - save_dir=".", - name="logs/comparison", - version=version, - ) - logger.append(tb_logger) - - if wandb: - import wandb as _wandb - from lightning.pytorch.loggers import WandbLogger - - wandb_logger = WandbLogger( - project="Symbolic_NN_Tests", - name=version, - dir="wandb", - ) - logger.append(wandb_logger) - - test_model( - logger=logger, - train_loss=train_loss, - val_loss=val_loss, - test_loss=test_loss, - lr=LEARNING_RATE, - ) - - if wandb: - _wandb.finish() - - -def run(tensorboard: bool = True, wandb: bool = True): - from . import semantic_loss - from .model import oh_vs_cat_cross_entropy - - test( - train_loss=oh_vs_cat_cross_entropy, - val_loss=oh_vs_cat_cross_entropy, - test_loss=oh_vs_cat_cross_entropy, - version="cross_entropy", - tensorboard=tensorboard, - wandb=wandb, - ) - test( - train_loss=semantic_loss.similarity_cross_entropy, - val_loss=oh_vs_cat_cross_entropy, - test_loss=oh_vs_cat_cross_entropy, - version="similarity_cross_entropy", - tensorboard=tensorboard, - wandb=wandb, - ) - test( - train_loss=semantic_loss.hasline_cross_entropy, - val_loss=oh_vs_cat_cross_entropy, - test_loss=oh_vs_cat_cross_entropy, - version="hasline_cross_entropy", - tensorboard=tensorboard, - wandb=wandb, - ) - test( - train_loss=semantic_loss.hasloop_cross_entropy, - val_loss=oh_vs_cat_cross_entropy, - test_loss=oh_vs_cat_cross_entropy, - version="hasloop_cross_entropy", - tensorboard=tensorboard, - wandb=wandb, - ) - test( - train_loss=semantic_loss.multisemantic_cross_entropy, - val_loss=oh_vs_cat_cross_entropy, - test_loss=oh_vs_cat_cross_entropy, - version="multisemantic_cross_entropy", - tensorboard=tensorboard, - wandb=wandb, - ) - test( - train_loss=semantic_loss.garbage_cross_entropy, - val_loss=oh_vs_cat_cross_entropy, - test_loss=oh_vs_cat_cross_entropy, - version="garbage_cross_entropy", - tensorboard=tensorboard, - wandb=wandb, - ) diff --git a/src/symbolic_nn_tests/local/model.py b/src/symbolic_nn_tests/local/model.py deleted file mode 100644 index 45e9080..0000000 --- a/src/symbolic_nn_tests/local/model.py +++ /dev/null @@ -1,72 +0,0 @@ -from functools import lru_cache -import torch -from torch import nn - - -model = nn.Sequential( - nn.Flatten(1, -1), - nn.Linear(784, 10), - nn.Softmax(dim=1), -) - - -def collate(batch): - x, y = zip(*batch) - x = [i[0] for i in x] - y = [torch.tensor(i) for i in y] - x = torch.stack(x) - y = torch.tensor(y) - return x, y - - -# This is just a quick, lazy way to ensure all models are trained on the same dataset -@lru_cache(maxsize=1) -def get_singleton_dataset(): - from torchvision.datasets import QMNIST - - from symbolic_nn_tests.dataloader import create_dataset - - return create_dataset( - dataset=QMNIST, - collate_fn=collate, - batch_size=128, - shuffle_train=True, - num_workers=11, - ) - - -def oh_vs_cat_cross_entropy(y_bin, y_cat): - return nn.functional.cross_entropy( - y_bin, - nn.functional.one_hot(y_cat, num_classes=10).float(), - ) - - -def main( - train_loss=oh_vs_cat_cross_entropy, - val_loss=oh_vs_cat_cross_entropy, - test_loss=oh_vs_cat_cross_entropy, - logger=None, - **kwargs, -): - import lightning as L - - from symbolic_nn_tests.train import TrainingWrapper - - if logger is None: - from lightning.pytorch.loggers import TensorBoardLogger - - logger = TensorBoardLogger(save_dir=".", name="logs/ffnn") - - train, val, test = get_singleton_dataset() - lmodel = TrainingWrapper( - model, train_loss=train_loss, val_loss=val_loss, test_loss=test_loss - ) - lmodel.configure_optimizers(**kwargs) - trainer = L.Trainer(max_epochs=20, logger=logger) - trainer.fit(model=lmodel, train_dataloaders=train, val_dataloaders=val) - trainer.test(dataloaders=test) - - -if __name__ == "__main__": - main() diff --git a/src/symbolic_nn_tests/local/semantic_loss.py b/src/symbolic_nn_tests/local/semantic_loss.py deleted file mode 100644 index 636b2aa..0000000 --- a/src/symbolic_nn_tests/local/semantic_loss.py +++ /dev/null @@ -1,88 +0,0 @@ -import torch - - -def create_semantic_cross_entropy(semantic_matrix): - def semantic_cross_entropy(input, target): - ce_loss = torch.nn.functional.cross_entropy(input, target) - - penalty_tensor = semantic_matrix[target.argmax(dim=1)] - abs_diff = (target - input).abs() - semantic_penalty = (abs_diff * penalty_tensor).sum() - return ce_loss * semantic_penalty - - def oh_vs_cat_semantic_cross_entropy(input_oh, target_cat): - return semantic_cross_entropy( - input_oh, torch.nn.functional.one_hot(target_cat, num_classes=10).float() - ) - - return oh_vs_cat_semantic_cross_entropy - - -# NOTE: This similarity matrix defines loss scaling factors for misclassification -# of numbers from our QMNIST dataset. Visually similar numbers (e.g: 3/8) are -# penalised less harshly than visually distinct numbers as this mistake is "less -# mistaken" given our understanding of the visual characteristics of numerals. -# By using this scaling matric we can inject human knowledge into the model via -# the loss function, making this an example of a "semantic loss function" -SIMILARITY_MATRIX = torch.tensor( - [ - [2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], - [1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.5, 1.0, 1.0], - [1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0], - [1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.5, 1.0], - [1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0], - [1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.5, 1.0, 1.0, 1.0], - [1.0, 1.0, 1.0, 1.0, 1.0, 1.5, 2.0, 1.0, 1.0, 1.0], - [1.0, 1.5, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0], - [1.0, 1.0, 1.0, 1.5, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0], - [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0], - ] -).to("cuda") -SIMILARITY_MATRIX /= SIMILARITY_MATRIX.sum() # Normalized to sum of 1 - -similarity_cross_entropy = create_semantic_cross_entropy(SIMILARITY_MATRIX) - - -# NOTE: The following matrix encodes a simpler semantic penalty for correctly/incorrectly -# identifying shapes with straight lines in their representation. This can be a bit fuzzy -# in cases like "9" though. -HASLINE_MATRIX = torch.tensor( - # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 - [False, True, False, False, True, True, False, True, False, True] -).to("cuda") -HASLINE_MATRIX = torch.stack([i ^ HASLINE_MATRIX for i in HASLINE_MATRIX]).type( - torch.float64 -) -HASLINE_MATRIX += 1 -HASLINE_MATRIX /= HASLINE_MATRIX.sum() # Normalize to sum of 1 - -hasline_cross_entropy = create_semantic_cross_entropy(HASLINE_MATRIX) - - -# NOTE: Similarly, we can do the same for closed circular loops in a numeric character -HASLOOP_MATRIX = torch.tensor( - # 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 - [True, False, False, False, False, False, True, False, True, True] -).to("cuda") -HASLOOP_MATRIX = torch.stack([i ^ HASLOOP_MATRIX for i in HASLOOP_MATRIX]).type( - torch.float64 -) -HASLOOP_MATRIX += 1 -HASLOOP_MATRIX /= HASLOOP_MATRIX.sum() # Normalize to sum of 1 - -hasloop_cross_entropy = create_semantic_cross_entropy(HASLOOP_MATRIX) - - -# NOTE: We can also combine all of these semantic matrices -MULTISEMANTIC_MATRIX = SIMILARITY_MATRIX * HASLINE_MATRIX * HASLOOP_MATRIX -MULTISEMANTIC_MATRIX /= MULTISEMANTIC_MATRIX.sum() - -multisemantic_cross_entropy = create_semantic_cross_entropy(MULTISEMANTIC_MATRIX) - -# NOTE: As a final test, lets make something similar to tehse but where there's no knowledge, -# just random data. This will create a benchmark for the effects of this process wothout the -# "knowledge" component -GARBAGE_MATRIX = torch.rand(10, 10).to("cuda") -GARBAGE_MATRIX /= GARBAGE_MATRIX.sum() - -garbage_cross_entropy = create_semantic_cross_entropy(GARBAGE_MATRIX)