mirror of
https://github.com/Cian-H/Aconity_ML_Expt1.git
synced 2025-12-22 20:51:58 +00:00
367 lines
33 KiB
Plaintext
367 lines
33 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"<h1>Experiment 2</h1>\n",
|
|
"<h3>Targetted hyperparameter tuning</h3>\n",
|
|
"<p>\n",
|
|
"By examining the results of expt1, a smaller range of hyperparameters for expt2 was chosen. This allowed for a more targetted search of the hyperparameter space to find an optimal configuration. The selected parameters for expt2 were as follows:\n",
|
|
"</p>\n",
|
|
"<ul>\n",
|
|
"<li>in_act = Linear, Mish, PBessel, or Tanhshrink</li>\n",
|
|
"<li>compressor_kernel_size = 128</li>\n",
|
|
"<li>compressor_act = Softshrink, SoftExp, or PReLU</li>\n",
|
|
"<li>conv_kernel_size = 128</li>\n",
|
|
"<li>conv_act = Sigmoid or PBessel</li>\n",
|
|
"<li>channel_combine_act = HardSigmoid or GELU</li>\n",
|
|
"<li>ff_width = 512</li>\n",
|
|
"<li>ff_depth = 2, 4, or 6</li>\n",
|
|
"<li>ff_act = CELU</li>\n",
|
|
"<li>out_act = Tanhshrink or Mish</li>\n",
|
|
"</ul>\n",
|
|
"<p>\n",
|
|
"Several of the parameters were able to be fixed to a specific value, and the remaining parameters (with the exception of <code>`in_act`</code>) were reduced to only 2 or 3 possible values, dramatically shrinking the parameter space. For this reason, a significantly less aggressive pruning algorithm was used, allowing for a more thorough search of the parameter space.\n",
|
|
"</p>"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Data handling imports\n",
|
|
"from dask.distributed import Client, LocalCluster\n",
|
|
"import dask\n",
|
|
"import dask.dataframe as dd\n",
|
|
"import dask.array as da\n",
|
|
"import numpy as np\n",
|
|
"import pickle\n",
|
|
"import random\n",
|
|
"from itertools import chain\n",
|
|
"from tqdm.auto import tqdm\n",
|
|
"\n",
|
|
"# Deep learning imports\n",
|
|
"import torch\n",
|
|
"from torch.utils.data import DataLoader\n",
|
|
"from torch import nn\n",
|
|
"from torch.nn import functional as F\n",
|
|
"from torch import optim\n",
|
|
"import pytorch_lightning as pl\n",
|
|
"from pytorch_lightning import Trainer\n",
|
|
"import optuna\n",
|
|
"from optuna.pruners import HyperbandPruner\n",
|
|
"from optuna.integration import PyTorchLightningPruningCallback\n",
|
|
"\n",
|
|
"# Suppress some warning messages from pytorch_lightning,\n",
|
|
"# It really doesn't like that i've forced it to handle a dask array!\n",
|
|
"import warnings\n",
|
|
"\n",
|
|
"warnings.filterwarnings(\"ignore\", category=UserWarning, module=pl.__name__)\n",
|
|
"\n",
|
|
"# Also, set up a log to record debug messages for failed trials\n",
|
|
"import logging\n",
|
|
"\n",
|
|
"logging.basicConfig(filename=\"debug_test.log\", encoding=\"utf-8\", level=logging.DEBUG)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from expt1 import (\n",
|
|
" Model,\n",
|
|
" Linear,\n",
|
|
" device,\n",
|
|
" activation_dispatcher,\n",
|
|
" X_train,\n",
|
|
" y_train,\n",
|
|
" X_val,\n",
|
|
" y_val,\n",
|
|
" create_collate_fn,\n",
|
|
")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"cluster = LocalCluster(n_workers=8, threads_per_worker=1)\n",
|
|
"client = Client(cluster)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Monkey patch to allow pytorch lightning to accept a dask array as a model input\n",
|
|
"from typing import Any, Generator, Iterable, Mapping, Optional, Union\n",
|
|
"\n",
|
|
"BType = Union[da.Array, torch.Tensor, str, Mapping[Any, \"BType\"], Iterable[\"BType\"]]\n",
|
|
"\n",
|
|
"unpatched = pl.utilities.data._extract_batch_size\n",
|
|
"\n",
|
|
"\n",
|
|
"def patch(batch: BType) -> Generator[Optional[int], None, None]:\n",
|
|
" if isinstance(batch, da.core.Array):\n",
|
|
" if len(batch.shape) == 0:\n",
|
|
" yield 1\n",
|
|
" else:\n",
|
|
" yield batch.shape[0]\n",
|
|
" else:\n",
|
|
" yield from unpatched(batch)\n",
|
|
"\n",
|
|
"\n",
|
|
"pl.utilities.data._extract_batch_size = patch"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Test parameters\n",
|
|
"n_epochs = 10\n",
|
|
"output_keys = list(next(iter(y_train.values())).keys())\n",
|
|
"activation_vals = list(activation_dispatcher.keys())\n",
|
|
"\n",
|
|
"\n",
|
|
"# Next we define the objective function for the hyperparameter optimization\n",
|
|
"def objective(trial):\n",
|
|
" torch.cuda.empty_cache()\n",
|
|
" objective_value = torch.inf\n",
|
|
" model = None\n",
|
|
" logger = None\n",
|
|
" try:\n",
|
|
" # Select hyperparameters for testing\n",
|
|
" in_act = (\n",
|
|
" activation_dispatcher[trial.suggest_categorical(\"in_act\", activation_vals)],\n",
|
|
" (),\n",
|
|
" {},\n",
|
|
" )\n",
|
|
" compressor_kernel_size = trial.suggest_int(\n",
|
|
" \"compressor_kernel_size\", 64, 257, 64\n",
|
|
" )\n",
|
|
" compressor_chunk_size = 128\n",
|
|
" compressor_act = (\n",
|
|
" activation_dispatcher[\n",
|
|
" trial.suggest_categorical(\"compressor_act\", activation_vals)\n",
|
|
" ],\n",
|
|
" (),\n",
|
|
" {},\n",
|
|
" )\n",
|
|
" conv_kernel_size = trial.suggest_int(\"conv_kernel_size\", 64, 257, 64)\n",
|
|
" conv_act = (\n",
|
|
" activation_dispatcher[\n",
|
|
" trial.suggest_categorical(\"conv_act\", activation_vals)\n",
|
|
" ],\n",
|
|
" (),\n",
|
|
" {},\n",
|
|
" )\n",
|
|
" channel_combine_act = (\n",
|
|
" activation_dispatcher[\n",
|
|
" trial.suggest_categorical(\"channel_combine_act\", activation_vals)\n",
|
|
" ],\n",
|
|
" (),\n",
|
|
" {},\n",
|
|
" )\n",
|
|
" ff_width = trial.suggest_int(\"ff_width\", 256, 1025, 256)\n",
|
|
" ff_depth = trial.suggest_int(\"ff_depth\", 2, 8, 2)\n",
|
|
" ff_act = (\n",
|
|
" activation_dispatcher[trial.suggest_categorical(\"ff_act\", activation_vals)],\n",
|
|
" (),\n",
|
|
" {},\n",
|
|
" )\n",
|
|
" out_size = len(output_keys)\n",
|
|
" out_act = (\n",
|
|
" activation_dispatcher[\n",
|
|
" trial.suggest_categorical(\"out_act\", activation_vals)\n",
|
|
" ],\n",
|
|
" (),\n",
|
|
" {},\n",
|
|
" )\n",
|
|
"\n",
|
|
" # Set up the model architecture and other necessary components\n",
|
|
" model = Model(\n",
|
|
" in_act=in_act,\n",
|
|
" compressor_kernel_size=compressor_kernel_size,\n",
|
|
" compressor_chunk_size=compressor_chunk_size,\n",
|
|
" compressor_act=compressor_act,\n",
|
|
" conv_kernel_size=conv_kernel_size,\n",
|
|
" conv_act=conv_act,\n",
|
|
" channel_combine_act=channel_combine_act,\n",
|
|
" ff_width=ff_width,\n",
|
|
" ff_depth=ff_depth,\n",
|
|
" ff_act=ff_act,\n",
|
|
" out_size=out_size,\n",
|
|
" out_act=out_act,\n",
|
|
" ).to(device)\n",
|
|
"\n",
|
|
" trainer = Trainer(\n",
|
|
" accelerator=\"gpu\",\n",
|
|
" max_epochs=n_epochs,\n",
|
|
" devices=1,\n",
|
|
" logger=logger,\n",
|
|
" num_sanity_val_steps=0, # Needs to be disabled or else we get an error because X is dask array\n",
|
|
" # precision=\"16-mixed\",\n",
|
|
" callbacks=[\n",
|
|
" PyTorchLightningPruningCallback(trial, monitor=\"val_loss\"),\n",
|
|
" ],\n",
|
|
" )\n",
|
|
" # Prepare datasets\n",
|
|
" train = DataLoader(\n",
|
|
" list(zip(X_train.values(), y_train.values())),\n",
|
|
" collate_fn=create_collate_fn(),\n",
|
|
" shuffle=True,\n",
|
|
" )\n",
|
|
" valid = DataLoader(\n",
|
|
" list(zip(X_val.values(), y_val.values())),\n",
|
|
" shuffle=True,\n",
|
|
" collate_fn=create_collate_fn(),\n",
|
|
" )\n",
|
|
" # Finally, train the model\n",
|
|
" trainer.fit(model, train, valid)\n",
|
|
" except torch.cuda.OutOfMemoryError as e:\n",
|
|
" logging.warning(f\"Ran out of memory in trial {trial.number}!\")\n",
|
|
" raise optuna.exceptions.TrialPruned()\n",
|
|
" except Exception as e:\n",
|
|
" logging.exception(f\"An exception occurred in trial {trial.number}: {e}\")\n",
|
|
" raise optuna.exceptions.TrialPruned()\n",
|
|
" finally:\n",
|
|
" if logger is not None:\n",
|
|
" logger.experiment.unwatch(model)\n",
|
|
" logger.experiment.finish()\n",
|
|
" del model\n",
|
|
" torch.cuda.empty_cache()\n",
|
|
" if objective_value == torch.inf:\n",
|
|
" raise optuna.exceptions.TrialPruned()\n",
|
|
" return objective_value"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[I 2023-07-31 23:49:15,744] Using an existing study with name 'Experiment 2' instead of creating a new one.\n",
|
|
"[I 2023-07-31 23:49:16,553] Trial 221 pruned. \n",
|
|
"[I 2023-07-31 23:49:16,928] Trial 222 pruned. \n",
|
|
"[I 2023-07-31 23:49:17,318] Trial 223 pruned. \n",
|
|
"[I 2023-07-31 23:49:17,682] Trial 224 pruned. \n",
|
|
"[W 2023-07-31 23:49:18,028] Trial 225 failed with parameters: {} because of the following error: KeyboardInterrupt().\n",
|
|
"Traceback (most recent call last):\n",
|
|
" File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py\", line 200, in _run_trial\n",
|
|
" value_or_values = func(trial)\n",
|
|
" ^^^^^^^^^^^\n",
|
|
" File \"/tmp/ipykernel_562333/3392796582.py\", line 16, in objective\n",
|
|
" activation_dispatcher[trial.suggest_categorical(\"in_act\", activation_vals)],\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/trial/_trial.py\", line 405, in suggest_categorical\n",
|
|
" return self._suggest(name, CategoricalDistribution(choices=choices))\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/trial/_trial.py\", line 630, in _suggest\n",
|
|
" param_value = self.study.sampler.sample_independent(\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/sampler.py\", line 471, in sample_independent\n",
|
|
" mpe_above = _ParzenEstimator(\n",
|
|
" ^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py\", line 75, in __init__\n",
|
|
" distributions=[\n",
|
|
" ^\n",
|
|
" File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py\", line 76, in <listcomp>\n",
|
|
" self._calculate_distributions(\n",
|
|
" File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py\", line 154, in _calculate_distributions\n",
|
|
" return self._calculate_categorical_distributions(\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py\", line 192, in _calculate_categorical_distributions\n",
|
|
" weights /= weights.sum(axis=1, keepdims=True)\n",
|
|
" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
|
|
" File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/numpy/core/_methods.py\", line 47, in _sum\n",
|
|
" def _sum(a, axis=None, dtype=None, out=None, keepdims=False,\n",
|
|
" \n",
|
|
"KeyboardInterrupt\n",
|
|
"[W 2023-07-31 23:49:18,040] Trial 225 failed with value None.\n"
|
|
]
|
|
},
|
|
{
|
|
"ename": "KeyboardInterrupt",
|
|
"evalue": "",
|
|
"output_type": "error",
|
|
"traceback": [
|
|
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
|
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
|
|
"Cell \u001b[0;32mIn[5], line 11\u001b[0m\n\u001b[1;32m 3\u001b[0m study_name \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mExperiment 2\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 4\u001b[0m study \u001b[39m=\u001b[39m optuna\u001b[39m.\u001b[39mcreate_study(\n\u001b[1;32m 5\u001b[0m study_name\u001b[39m=\u001b[39mstudy_name,\n\u001b[1;32m 6\u001b[0m storage\u001b[39m=\u001b[39mstorage_name,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 9\u001b[0m load_if_exists\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m,\n\u001b[1;32m 10\u001b[0m )\n\u001b[0;32m---> 11\u001b[0m study\u001b[39m.\u001b[39;49moptimize(\n\u001b[1;32m 12\u001b[0m objective,\n\u001b[1;32m 13\u001b[0m n_trials\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m 14\u001b[0m timeout\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m 15\u001b[0m )\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/study.py:443\u001b[0m, in \u001b[0;36mStudy.optimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m 339\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39moptimize\u001b[39m(\n\u001b[1;32m 340\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[1;32m 341\u001b[0m func: ObjectiveFuncType,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 348\u001b[0m show_progress_bar: \u001b[39mbool\u001b[39m \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m 349\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 350\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Optimize an objective function.\u001b[39;00m\n\u001b[1;32m 351\u001b[0m \n\u001b[1;32m 352\u001b[0m \u001b[39m Optimization is done by choosing a suitable set of hyperparameter values from a given\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 440\u001b[0m \u001b[39m If nested invocation of this method occurs.\u001b[39;00m\n\u001b[1;32m 441\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 443\u001b[0m _optimize(\n\u001b[1;32m 444\u001b[0m study\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m,\n\u001b[1;32m 445\u001b[0m func\u001b[39m=\u001b[39;49mfunc,\n\u001b[1;32m 446\u001b[0m n_trials\u001b[39m=\u001b[39;49mn_trials,\n\u001b[1;32m 447\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout,\n\u001b[1;32m 448\u001b[0m n_jobs\u001b[39m=\u001b[39;49mn_jobs,\n\u001b[1;32m 449\u001b[0m catch\u001b[39m=\u001b[39;49m\u001b[39mtuple\u001b[39;49m(catch) \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(catch, Iterable) \u001b[39melse\u001b[39;49;00m (catch,),\n\u001b[1;32m 450\u001b[0m callbacks\u001b[39m=\u001b[39;49mcallbacks,\n\u001b[1;32m 451\u001b[0m gc_after_trial\u001b[39m=\u001b[39;49mgc_after_trial,\n\u001b[1;32m 452\u001b[0m show_progress_bar\u001b[39m=\u001b[39;49mshow_progress_bar,\n\u001b[1;32m 453\u001b[0m )\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py:66\u001b[0m, in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 65\u001b[0m \u001b[39mif\u001b[39;00m n_jobs \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m---> 66\u001b[0m _optimize_sequential(\n\u001b[1;32m 67\u001b[0m study,\n\u001b[1;32m 68\u001b[0m func,\n\u001b[1;32m 69\u001b[0m n_trials,\n\u001b[1;32m 70\u001b[0m timeout,\n\u001b[1;32m 71\u001b[0m catch,\n\u001b[1;32m 72\u001b[0m callbacks,\n\u001b[1;32m 73\u001b[0m gc_after_trial,\n\u001b[1;32m 74\u001b[0m reseed_sampler_rng\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 75\u001b[0m time_start\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m 76\u001b[0m progress_bar\u001b[39m=\u001b[39;49mprogress_bar,\n\u001b[1;32m 77\u001b[0m )\n\u001b[1;32m 78\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 79\u001b[0m \u001b[39mif\u001b[39;00m n_jobs \u001b[39m==\u001b[39m \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m:\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py:163\u001b[0m, in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 163\u001b[0m frozen_trial \u001b[39m=\u001b[39m _run_trial(study, func, catch)\n\u001b[1;32m 164\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 165\u001b[0m \u001b[39m# The following line mitigates memory problems that can be occurred in some\u001b[39;00m\n\u001b[1;32m 166\u001b[0m \u001b[39m# environments (e.g., services that use computing containers such as GitHub Actions).\u001b[39;00m\n\u001b[1;32m 167\u001b[0m \u001b[39m# Please refer to the following PR for further details:\u001b[39;00m\n\u001b[1;32m 168\u001b[0m \u001b[39m# https://github.com/optuna/optuna/pull/325.\u001b[39;00m\n\u001b[1;32m 169\u001b[0m \u001b[39mif\u001b[39;00m gc_after_trial:\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py:251\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m 244\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mFalse\u001b[39;00m, \u001b[39m\"\u001b[39m\u001b[39mShould not reach.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 246\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m 247\u001b[0m frozen_trial\u001b[39m.\u001b[39mstate \u001b[39m==\u001b[39m TrialState\u001b[39m.\u001b[39mFAIL\n\u001b[1;32m 248\u001b[0m \u001b[39mand\u001b[39;00m func_err \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 249\u001b[0m \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(func_err, catch)\n\u001b[1;32m 250\u001b[0m ):\n\u001b[0;32m--> 251\u001b[0m \u001b[39mraise\u001b[39;00m func_err\n\u001b[1;32m 252\u001b[0m \u001b[39mreturn\u001b[39;00m frozen_trial\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py:200\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[39mwith\u001b[39;00m get_heartbeat_thread(trial\u001b[39m.\u001b[39m_trial_id, study\u001b[39m.\u001b[39m_storage):\n\u001b[1;32m 199\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 200\u001b[0m value_or_values \u001b[39m=\u001b[39m func(trial)\n\u001b[1;32m 201\u001b[0m \u001b[39mexcept\u001b[39;00m exceptions\u001b[39m.\u001b[39mTrialPruned \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 202\u001b[0m \u001b[39m# TODO(mamu): Handle multi-objective cases.\u001b[39;00m\n\u001b[1;32m 203\u001b[0m state \u001b[39m=\u001b[39m TrialState\u001b[39m.\u001b[39mPRUNED\n",
|
|
"Cell \u001b[0;32mIn[4], line 16\u001b[0m, in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m 12\u001b[0m logger \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 14\u001b[0m \u001b[39m# Select hyperparameters for testing\u001b[39;00m\n\u001b[1;32m 15\u001b[0m in_act \u001b[39m=\u001b[39m (\n\u001b[0;32m---> 16\u001b[0m activation_dispatcher[trial\u001b[39m.\u001b[39;49msuggest_categorical(\u001b[39m\"\u001b[39;49m\u001b[39min_act\u001b[39;49m\u001b[39m\"\u001b[39;49m, activation_vals)],\n\u001b[1;32m 17\u001b[0m (),\n\u001b[1;32m 18\u001b[0m {},\n\u001b[1;32m 19\u001b[0m )\n\u001b[1;32m 20\u001b[0m compressor_kernel_size \u001b[39m=\u001b[39m trial\u001b[39m.\u001b[39msuggest_int(\n\u001b[1;32m 21\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mcompressor_kernel_size\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m64\u001b[39m, \u001b[39m257\u001b[39m, \u001b[39m64\u001b[39m\n\u001b[1;32m 22\u001b[0m )\n\u001b[1;32m 23\u001b[0m compressor_chunk_size \u001b[39m=\u001b[39m \u001b[39m128\u001b[39m\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/trial/_trial.py:405\u001b[0m, in \u001b[0;36mTrial.suggest_categorical\u001b[0;34m(self, name, choices)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Suggest a value for the categorical parameter.\u001b[39;00m\n\u001b[1;32m 355\u001b[0m \n\u001b[1;32m 356\u001b[0m \u001b[39mThe value is sampled from ``choices``.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 400\u001b[0m \u001b[39m :ref:`configurations` tutorial describes more details and flexible usages.\u001b[39;00m\n\u001b[1;32m 401\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 402\u001b[0m \u001b[39m# There is no need to call self._check_distribution because\u001b[39;00m\n\u001b[1;32m 403\u001b[0m \u001b[39m# CategoricalDistribution does not support dynamic value space.\u001b[39;00m\n\u001b[0;32m--> 405\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_suggest(name, CategoricalDistribution(choices\u001b[39m=\u001b[39;49mchoices))\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/trial/_trial.py:630\u001b[0m, in \u001b[0;36mTrial._suggest\u001b[0;34m(self, name, distribution)\u001b[0m\n\u001b[1;32m 628\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 629\u001b[0m study \u001b[39m=\u001b[39m pruners\u001b[39m.\u001b[39m_filter_study(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstudy, trial)\n\u001b[0;32m--> 630\u001b[0m param_value \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mstudy\u001b[39m.\u001b[39;49msampler\u001b[39m.\u001b[39;49msample_independent(\n\u001b[1;32m 631\u001b[0m study, trial, name, distribution\n\u001b[1;32m 632\u001b[0m )\n\u001b[1;32m 634\u001b[0m \u001b[39m# `param_value` is validated here (invalid value like `np.nan` raises ValueError).\u001b[39;00m\n\u001b[1;32m 635\u001b[0m param_value_in_internal_repr \u001b[39m=\u001b[39m distribution\u001b[39m.\u001b[39mto_internal_repr(param_value)\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/sampler.py:471\u001b[0m, in \u001b[0;36mTPESampler.sample_independent\u001b[0;34m(self, study, trial, param_name, param_distribution)\u001b[0m\n\u001b[1;32m 467\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 468\u001b[0m mpe_below \u001b[39m=\u001b[39m _ParzenEstimator(\n\u001b[1;32m 469\u001b[0m below, {param_name: param_distribution}, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_parzen_estimator_parameters\n\u001b[1;32m 470\u001b[0m )\n\u001b[0;32m--> 471\u001b[0m mpe_above \u001b[39m=\u001b[39m _ParzenEstimator(\n\u001b[1;32m 472\u001b[0m above, {param_name: param_distribution}, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_parzen_estimator_parameters\n\u001b[1;32m 473\u001b[0m )\n\u001b[1;32m 474\u001b[0m samples_below \u001b[39m=\u001b[39m mpe_below\u001b[39m.\u001b[39msample(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_rng, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_n_ei_candidates)\n\u001b[1;32m 475\u001b[0m log_likelihoods_below \u001b[39m=\u001b[39m mpe_below\u001b[39m.\u001b[39mlog_pdf(samples_below)\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py:75\u001b[0m, in \u001b[0;36m_ParzenEstimator.__init__\u001b[0;34m(self, observations, search_space, parameters, predetermined_weights)\u001b[0m\n\u001b[1;32m 71\u001b[0m weights \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mappend(weights, [parameters\u001b[39m.\u001b[39mprior_weight])\n\u001b[1;32m 72\u001b[0m weights \u001b[39m/\u001b[39m\u001b[39m=\u001b[39m weights\u001b[39m.\u001b[39msum()\n\u001b[1;32m 73\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_mixture_distribution \u001b[39m=\u001b[39m _MixtureOfProductDistribution(\n\u001b[1;32m 74\u001b[0m weights\u001b[39m=\u001b[39mweights,\n\u001b[0;32m---> 75\u001b[0m distributions\u001b[39m=\u001b[39m[\n\u001b[1;32m 76\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_calculate_distributions(\n\u001b[1;32m 77\u001b[0m transformed_observations[:, i], search_space[param], parameters\n\u001b[1;32m 78\u001b[0m )\n\u001b[1;32m 79\u001b[0m \u001b[39mfor\u001b[39;49;00m i, param \u001b[39min\u001b[39;49;00m \u001b[39menumerate\u001b[39;49m(search_space)\n\u001b[1;32m 80\u001b[0m ],\n\u001b[1;32m 81\u001b[0m )\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py:76\u001b[0m, in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 71\u001b[0m weights \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mappend(weights, [parameters\u001b[39m.\u001b[39mprior_weight])\n\u001b[1;32m 72\u001b[0m weights \u001b[39m/\u001b[39m\u001b[39m=\u001b[39m weights\u001b[39m.\u001b[39msum()\n\u001b[1;32m 73\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_mixture_distribution \u001b[39m=\u001b[39m _MixtureOfProductDistribution(\n\u001b[1;32m 74\u001b[0m weights\u001b[39m=\u001b[39mweights,\n\u001b[1;32m 75\u001b[0m distributions\u001b[39m=\u001b[39m[\n\u001b[0;32m---> 76\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_calculate_distributions(\n\u001b[1;32m 77\u001b[0m transformed_observations[:, i], search_space[param], parameters\n\u001b[1;32m 78\u001b[0m )\n\u001b[1;32m 79\u001b[0m \u001b[39mfor\u001b[39;00m i, param \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(search_space)\n\u001b[1;32m 80\u001b[0m ],\n\u001b[1;32m 81\u001b[0m )\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py:154\u001b[0m, in \u001b[0;36m_ParzenEstimator._calculate_distributions\u001b[0;34m(self, transformed_observations, search_space, parameters)\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_calculate_distributions\u001b[39m(\n\u001b[1;32m 148\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[1;32m 149\u001b[0m transformed_observations: np\u001b[39m.\u001b[39mndarray,\n\u001b[1;32m 150\u001b[0m search_space: BaseDistribution,\n\u001b[1;32m 151\u001b[0m parameters: _ParzenEstimatorParameters,\n\u001b[1;32m 152\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m _BatchedDistributions:\n\u001b[1;32m 153\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(search_space, CategoricalDistribution):\n\u001b[0;32m--> 154\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_calculate_categorical_distributions(\n\u001b[1;32m 155\u001b[0m transformed_observations, search_space\u001b[39m.\u001b[39;49mchoices, parameters\n\u001b[1;32m 156\u001b[0m )\n\u001b[1;32m 157\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 158\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(search_space, (FloatDistribution, IntDistribution))\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py:192\u001b[0m, in \u001b[0;36m_ParzenEstimator._calculate_categorical_distributions\u001b[0;34m(self, observations, choices, parameters)\u001b[0m\n\u001b[1;32m 186\u001b[0m weights \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mfull(\n\u001b[1;32m 187\u001b[0m shape\u001b[39m=\u001b[39m(\u001b[39mlen\u001b[39m(observations) \u001b[39m+\u001b[39m consider_prior, \u001b[39mlen\u001b[39m(choices)),\n\u001b[1;32m 188\u001b[0m fill_value\u001b[39m=\u001b[39mparameters\u001b[39m.\u001b[39mprior_weight \u001b[39m/\u001b[39m (\u001b[39mlen\u001b[39m(observations) \u001b[39m+\u001b[39m consider_prior),\n\u001b[1;32m 189\u001b[0m )\n\u001b[1;32m 191\u001b[0m weights[np\u001b[39m.\u001b[39marange(\u001b[39mlen\u001b[39m(observations)), observations\u001b[39m.\u001b[39mastype(\u001b[39mint\u001b[39m)] \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n\u001b[0;32m--> 192\u001b[0m weights \u001b[39m/\u001b[39m\u001b[39m=\u001b[39m weights\u001b[39m.\u001b[39;49msum(axis\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m, keepdims\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n\u001b[1;32m 193\u001b[0m \u001b[39mreturn\u001b[39;00m _BatchedCategoricalDistributions(weights)\n",
|
|
"File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/numpy/core/_methods.py:47\u001b[0m, in \u001b[0;36m_sum\u001b[0;34m(a, axis, dtype, out, keepdims, initial, where)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_amin\u001b[39m(a, axis\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, out\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, keepdims\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m 44\u001b[0m initial\u001b[39m=\u001b[39m_NoValue, where\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m):\n\u001b[1;32m 45\u001b[0m \u001b[39mreturn\u001b[39;00m umr_minimum(a, axis, \u001b[39mNone\u001b[39;00m, out, keepdims, initial, where)\n\u001b[0;32m---> 47\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_sum\u001b[39m(a, axis\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, dtype\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, out\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, keepdims\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m 48\u001b[0m initial\u001b[39m=\u001b[39m_NoValue, where\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m):\n\u001b[1;32m 49\u001b[0m \u001b[39mreturn\u001b[39;00m umr_sum(a, axis, dtype, out, keepdims, initial, where)\n\u001b[1;32m 51\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_prod\u001b[39m(a, axis\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, dtype\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, out\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, keepdims\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m 52\u001b[0m initial\u001b[39m=\u001b[39m_NoValue, where\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m):\n",
|
|
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"storage_name = \"sqlite:///optuna.sql\"\n",
|
|
"storage_name = \"mysql+pymysql://root:Ch31121992@192.168.1.10:3306/optuna_db\"\n",
|
|
"study_name = \"Experiment 2\"\n",
|
|
"study = optuna.create_study(\n",
|
|
" study_name=study_name,\n",
|
|
" storage=storage_name,\n",
|
|
" direction=\"minimize\",\n",
|
|
" pruner=HyperbandPruner(),\n",
|
|
" load_if_exists=True,\n",
|
|
")\n",
|
|
"study.optimize(\n",
|
|
" objective,\n",
|
|
" n_trials=None,\n",
|
|
" timeout=None,\n",
|
|
")"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": ".venv",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.3"
|
|
},
|
|
"orig_nbformat": 4
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|