Cian-H/symbolic_nn_tests
1
0
Fork 0
mirror of https://github.com/Cian-H/symbolic_nn_tests.git synced 2025-12-22 14:11:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

removed accidentally committed files

Browse Source
This commit is contained in:
Cian Hughes
2024-09-12 09:04:26 +01:00
parent 3b2ca2b1a5
commit f54885ba16
3 changed files with 0 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
src/symbolic_nn_tests/local/__init__.py
View File

@@ -1,93 +0,0 @@
LEARNING_RATE = 10e-5
def test(train_loss, val_loss, test_loss, version, tensorboard=True, wandb=True):
from .model import main as test_model
logger = []
if tensorboard:
from lightning.pytorch.loggers import TensorBoardLogger
tb_logger = TensorBoardLogger(
save_dir=".",
name="logs/comparison",
version=version,
)
logger.append(tb_logger)
if wandb:
import wandb as _wandb
from lightning.pytorch.loggers import WandbLogger
wandb_logger = WandbLogger(
project="Symbolic_NN_Tests",
name=version,
dir="wandb",
)
logger.append(wandb_logger)
test_model(
logger=logger,
train_loss=train_loss,
val_loss=val_loss,
test_loss=test_loss,
lr=LEARNING_RATE,
)
if wandb:
_wandb.finish()
def run(tensorboard: bool = True, wandb: bool = True):
from . import semantic_loss
from .model import oh_vs_cat_cross_entropy
test(
train_loss=oh_vs_cat_cross_entropy,
val_loss=oh_vs_cat_cross_entropy,
test_loss=oh_vs_cat_cross_entropy,
version="cross_entropy",
tensorboard=tensorboard,
wandb=wandb,
)
test(
train_loss=semantic_loss.similarity_cross_entropy,
val_loss=oh_vs_cat_cross_entropy,
test_loss=oh_vs_cat_cross_entropy,
version="similarity_cross_entropy",
tensorboard=tensorboard,
wandb=wandb,
)
test(
train_loss=semantic_loss.hasline_cross_entropy,
val_loss=oh_vs_cat_cross_entropy,
test_loss=oh_vs_cat_cross_entropy,
version="hasline_cross_entropy",
tensorboard=tensorboard,
wandb=wandb,
)
test(
train_loss=semantic_loss.hasloop_cross_entropy,
val_loss=oh_vs_cat_cross_entropy,
test_loss=oh_vs_cat_cross_entropy,
version="hasloop_cross_entropy",
tensorboard=tensorboard,
wandb=wandb,
)
test(
train_loss=semantic_loss.multisemantic_cross_entropy,
val_loss=oh_vs_cat_cross_entropy,
test_loss=oh_vs_cat_cross_entropy,
version="multisemantic_cross_entropy",
tensorboard=tensorboard,
wandb=wandb,
)
test(
train_loss=semantic_loss.garbage_cross_entropy,
val_loss=oh_vs_cat_cross_entropy,
test_loss=oh_vs_cat_cross_entropy,
version="garbage_cross_entropy",
tensorboard=tensorboard,
wandb=wandb,
)

72
src/symbolic_nn_tests/local/model.py
View File

@@ -1,72 +0,0 @@
from functools import lru_cache
import torch
from torch import nn
model = nn.Sequential(
nn.Flatten(1, -1),
nn.Linear(784, 10),
nn.Softmax(dim=1),
)
def collate(batch):
x, y = zip(*batch)
x = [i[0] for i in x]
y = [torch.tensor(i) for i in y]
x = torch.stack(x)
y = torch.tensor(y)
return x, y
# This is just a quick, lazy way to ensure all models are trained on the same dataset
@lru_cache(maxsize=1)
def get_singleton_dataset():
from torchvision.datasets import QMNIST
from symbolic_nn_tests.dataloader import create_dataset
return create_dataset(
dataset=QMNIST,
collate_fn=collate,
batch_size=128,
shuffle_train=True,
num_workers=11,
)
def oh_vs_cat_cross_entropy(y_bin, y_cat):
return nn.functional.cross_entropy(
y_bin,
nn.functional.one_hot(y_cat, num_classes=10).float(),
)
def main(
train_loss=oh_vs_cat_cross_entropy,
val_loss=oh_vs_cat_cross_entropy,
test_loss=oh_vs_cat_cross_entropy,
logger=None,
**kwargs,
):
import lightning as L
from symbolic_nn_tests.train import TrainingWrapper
if logger is None:
from lightning.pytorch.loggers import TensorBoardLogger
logger = TensorBoardLogger(save_dir=".", name="logs/ffnn")
train, val, test = get_singleton_dataset()
lmodel = TrainingWrapper(
model, train_loss=train_loss, val_loss=val_loss, test_loss=test_loss
)
lmodel.configure_optimizers(**kwargs)
trainer = L.Trainer(max_epochs=20, logger=logger)
trainer.fit(model=lmodel, train_dataloaders=train, val_dataloaders=val)
trainer.test(dataloaders=test)
if __name__ == "__main__":
main()

88
src/symbolic_nn_tests/local/semantic_loss.py
View File

@@ -1,88 +0,0 @@
import torch
def create_semantic_cross_entropy(semantic_matrix):
def semantic_cross_entropy(input, target):
ce_loss = torch.nn.functional.cross_entropy(input, target)
penalty_tensor = semantic_matrix[target.argmax(dim=1)]
abs_diff = (target - input).abs()
semantic_penalty = (abs_diff * penalty_tensor).sum()
return ce_loss * semantic_penalty
def oh_vs_cat_semantic_cross_entropy(input_oh, target_cat):
return semantic_cross_entropy(
input_oh, torch.nn.functional.one_hot(target_cat, num_classes=10).float()
)
return oh_vs_cat_semantic_cross_entropy
# NOTE: This similarity matrix defines loss scaling factors for misclassification
# of numbers from our QMNIST dataset. Visually similar numbers (e.g: 3/8) are
# penalised less harshly than visually distinct numbers as this mistake is "less
# mistaken" given our understanding of the visual characteristics of numerals.
# By using this scaling matric we can inject human knowledge into the model via
# the loss function, making this an example of a "semantic loss function"
SIMILARITY_MATRIX = torch.tensor(
[
[2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.5, 1.0, 1.0],
[1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.5, 1.0],
[1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.5, 1.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0, 1.5, 2.0, 1.0, 1.0, 1.0],
[1.0, 1.5, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0, 1.0],
[1.0, 1.0, 1.0, 1.5, 1.0, 1.0, 1.0, 1.0, 2.0, 1.0],
[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0],
]
).to("cuda")
SIMILARITY_MATRIX /= SIMILARITY_MATRIX.sum() # Normalized to sum of 1
similarity_cross_entropy = create_semantic_cross_entropy(SIMILARITY_MATRIX)
# NOTE: The following matrix encodes a simpler semantic penalty for correctly/incorrectly
# identifying shapes with straight lines in their representation. This can be a bit fuzzy
# in cases like "9" though.
HASLINE_MATRIX = torch.tensor(
# 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
[False, True, False, False, True, True, False, True, False, True]
).to("cuda")
HASLINE_MATRIX = torch.stack([i ^ HASLINE_MATRIX for i in HASLINE_MATRIX]).type(
torch.float64
)
HASLINE_MATRIX += 1
HASLINE_MATRIX /= HASLINE_MATRIX.sum() # Normalize to sum of 1
hasline_cross_entropy = create_semantic_cross_entropy(HASLINE_MATRIX)
# NOTE: Similarly, we can do the same for closed circular loops in a numeric character
HASLOOP_MATRIX = torch.tensor(
# 0, 1, 2, 3, 4, 5, 6, 7, 8, 9
[True, False, False, False, False, False, True, False, True, True]
).to("cuda")
HASLOOP_MATRIX = torch.stack([i ^ HASLOOP_MATRIX for i in HASLOOP_MATRIX]).type(
torch.float64
)
HASLOOP_MATRIX += 1
HASLOOP_MATRIX /= HASLOOP_MATRIX.sum() # Normalize to sum of 1
hasloop_cross_entropy = create_semantic_cross_entropy(HASLOOP_MATRIX)
# NOTE: We can also combine all of these semantic matrices
MULTISEMANTIC_MATRIX = SIMILARITY_MATRIX * HASLINE_MATRIX * HASLOOP_MATRIX
MULTISEMANTIC_MATRIX /= MULTISEMANTIC_MATRIX.sum()
multisemantic_cross_entropy = create_semantic_cross_entropy(MULTISEMANTIC_MATRIX)
# NOTE: As a final test, lets make something similar to tehse but where there's no knowledge,
# just random data. This will create a benchmark for the effects of this process wothout the
# "knowledge" component
GARBAGE_MATRIX = torch.rand(10, 10).to("cuda")
GARBAGE_MATRIX /= GARBAGE_MATRIX.sum()
garbage_cross_entropy = create_semantic_cross_entropy(GARBAGE_MATRIX)