{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "

Experiment 2

\n", "

Targetted hyperparameter tuning

\n", "

\n", "By examining the results of expt1, a smaller range of hyperparameters for expt2 was chosen. This allowed for a more targetted search of the hyperparameter space to find an optimal configuration. The selected parameters for expt2 were as follows:\n", "

\n", "\n", "

\n", "Several of the parameters were able to be fixed to a specific value, and the remaining parameters (with the exception of `in_act`) were reduced to only 2 or 3 possible values, dramatically shrinking the parameter space. For this reason, a significantly less aggressive pruning algorithm was used, allowing for a more thorough search of the parameter space.\n", "

" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# Data handling imports\n", "from dask.distributed import Client, LocalCluster\n", "import dask\n", "import dask.dataframe as dd\n", "import dask.array as da\n", "import numpy as np\n", "import pickle\n", "import random\n", "from itertools import chain\n", "from tqdm.auto import tqdm\n", "\n", "# Deep learning imports\n", "import torch\n", "from torch.utils.data import DataLoader\n", "from torch import nn\n", "from torch.nn import functional as F\n", "from torch import optim\n", "import pytorch_lightning as pl\n", "from pytorch_lightning import Trainer\n", "import optuna\n", "from optuna.pruners import HyperbandPruner\n", "from optuna.integration import PyTorchLightningPruningCallback\n", "\n", "# Suppress some warning messages from pytorch_lightning,\n", "# It really doesn't like that i've forced it to handle a dask array!\n", "import warnings\n", "\n", "warnings.filterwarnings(\"ignore\", category=UserWarning, module=pl.__name__)\n", "\n", "# Also, set up a log to record debug messages for failed trials\n", "import logging\n", "\n", "logging.basicConfig(filename=\"debug_test.log\", encoding=\"utf-8\", level=logging.DEBUG)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "from expt1 import (\n", " Model,\n", " Linear,\n", " device,\n", " activation_dispatcher,\n", " X_train,\n", " y_train,\n", " X_val,\n", " y_val,\n", " create_collate_fn,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "cluster = LocalCluster(n_workers=8, threads_per_worker=1)\n", "client = Client(cluster)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# Monkey patch to allow pytorch lightning to accept a dask array as a model input\n", "from typing import Any, Generator, Iterable, Mapping, Optional, Union\n", "\n", "BType = Union[da.Array, torch.Tensor, str, Mapping[Any, \"BType\"], Iterable[\"BType\"]]\n", "\n", "unpatched = pl.utilities.data._extract_batch_size\n", "\n", "\n", "def patch(batch: BType) -> Generator[Optional[int], None, None]:\n", " if isinstance(batch, da.core.Array):\n", " if len(batch.shape) == 0:\n", " yield 1\n", " else:\n", " yield batch.shape[0]\n", " else:\n", " yield from unpatched(batch)\n", "\n", "\n", "pl.utilities.data._extract_batch_size = patch" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "# Test parameters\n", "n_epochs = 10\n", "output_keys = list(next(iter(y_train.values())).keys())\n", "activation_vals = list(activation_dispatcher.keys())\n", "\n", "\n", "# Next we define the objective function for the hyperparameter optimization\n", "def objective(trial):\n", " torch.cuda.empty_cache()\n", " objective_value = torch.inf\n", " model = None\n", " logger = None\n", " try:\n", " # Select hyperparameters for testing\n", " in_act = (\n", " activation_dispatcher[trial.suggest_categorical(\"in_act\", activation_vals)],\n", " (),\n", " {},\n", " )\n", " compressor_kernel_size = trial.suggest_int(\n", " \"compressor_kernel_size\", 64, 257, 64\n", " )\n", " compressor_chunk_size = 128\n", " compressor_act = (\n", " activation_dispatcher[\n", " trial.suggest_categorical(\"compressor_act\", activation_vals)\n", " ],\n", " (),\n", " {},\n", " )\n", " conv_kernel_size = trial.suggest_int(\"conv_kernel_size\", 64, 257, 64)\n", " conv_act = (\n", " activation_dispatcher[\n", " trial.suggest_categorical(\"conv_act\", activation_vals)\n", " ],\n", " (),\n", " {},\n", " )\n", " channel_combine_act = (\n", " activation_dispatcher[\n", " trial.suggest_categorical(\"channel_combine_act\", activation_vals)\n", " ],\n", " (),\n", " {},\n", " )\n", " ff_width = trial.suggest_int(\"ff_width\", 256, 1025, 256)\n", " ff_depth = trial.suggest_int(\"ff_depth\", 2, 8, 2)\n", " ff_act = (\n", " activation_dispatcher[trial.suggest_categorical(\"ff_act\", activation_vals)],\n", " (),\n", " {},\n", " )\n", " out_size = len(output_keys)\n", " out_act = (\n", " activation_dispatcher[\n", " trial.suggest_categorical(\"out_act\", activation_vals)\n", " ],\n", " (),\n", " {},\n", " )\n", "\n", " # Set up the model architecture and other necessary components\n", " model = Model(\n", " in_act=in_act,\n", " compressor_kernel_size=compressor_kernel_size,\n", " compressor_chunk_size=compressor_chunk_size,\n", " compressor_act=compressor_act,\n", " conv_kernel_size=conv_kernel_size,\n", " conv_act=conv_act,\n", " channel_combine_act=channel_combine_act,\n", " ff_width=ff_width,\n", " ff_depth=ff_depth,\n", " ff_act=ff_act,\n", " out_size=out_size,\n", " out_act=out_act,\n", " ).to(device)\n", "\n", " trainer = Trainer(\n", " accelerator=\"gpu\",\n", " max_epochs=n_epochs,\n", " devices=1,\n", " logger=logger,\n", " num_sanity_val_steps=0, # Needs to be disabled or else we get an error because X is dask array\n", " # precision=\"16-mixed\",\n", " callbacks=[\n", " PyTorchLightningPruningCallback(trial, monitor=\"val_loss\"),\n", " ],\n", " )\n", " # Prepare datasets\n", " train = DataLoader(\n", " list(zip(X_train.values(), y_train.values())),\n", " collate_fn=create_collate_fn(),\n", " shuffle=True,\n", " )\n", " valid = DataLoader(\n", " list(zip(X_val.values(), y_val.values())),\n", " shuffle=True,\n", " collate_fn=create_collate_fn(),\n", " )\n", " # Finally, train the model\n", " trainer.fit(model, train, valid)\n", " except torch.cuda.OutOfMemoryError as e:\n", " logging.warning(f\"Ran out of memory in trial {trial.number}!\")\n", " raise optuna.exceptions.TrialPruned()\n", " except Exception as e:\n", " logging.exception(f\"An exception occurred in trial {trial.number}: {e}\")\n", " raise optuna.exceptions.TrialPruned()\n", " finally:\n", " if logger is not None:\n", " logger.experiment.unwatch(model)\n", " logger.experiment.finish()\n", " del model\n", " torch.cuda.empty_cache()\n", " if objective_value == torch.inf:\n", " raise optuna.exceptions.TrialPruned()\n", " return objective_value" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[I 2023-07-31 23:49:15,744] Using an existing study with name 'Experiment 2' instead of creating a new one.\n", "[I 2023-07-31 23:49:16,553] Trial 221 pruned. \n", "[I 2023-07-31 23:49:16,928] Trial 222 pruned. \n", "[I 2023-07-31 23:49:17,318] Trial 223 pruned. \n", "[I 2023-07-31 23:49:17,682] Trial 224 pruned. \n", "[W 2023-07-31 23:49:18,028] Trial 225 failed with parameters: {} because of the following error: KeyboardInterrupt().\n", "Traceback (most recent call last):\n", " File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py\", line 200, in _run_trial\n", " value_or_values = func(trial)\n", " ^^^^^^^^^^^\n", " File \"/tmp/ipykernel_562333/3392796582.py\", line 16, in objective\n", " activation_dispatcher[trial.suggest_categorical(\"in_act\", activation_vals)],\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/trial/_trial.py\", line 405, in suggest_categorical\n", " return self._suggest(name, CategoricalDistribution(choices=choices))\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/trial/_trial.py\", line 630, in _suggest\n", " param_value = self.study.sampler.sample_independent(\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/sampler.py\", line 471, in sample_independent\n", " mpe_above = _ParzenEstimator(\n", " ^^^^^^^^^^^^^^^^^\n", " File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py\", line 75, in __init__\n", " distributions=[\n", " ^\n", " File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py\", line 76, in \n", " self._calculate_distributions(\n", " File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py\", line 154, in _calculate_distributions\n", " return self._calculate_categorical_distributions(\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py\", line 192, in _calculate_categorical_distributions\n", " weights /= weights.sum(axis=1, keepdims=True)\n", " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", " File \"/home/cianh/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/numpy/core/_methods.py\", line 47, in _sum\n", " def _sum(a, axis=None, dtype=None, out=None, keepdims=False,\n", " \n", "KeyboardInterrupt\n", "[W 2023-07-31 23:49:18,040] Trial 225 failed with value None.\n" ] }, { "ename": "KeyboardInterrupt", "evalue": "", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[5], line 11\u001b[0m\n\u001b[1;32m 3\u001b[0m study_name \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mExperiment 2\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 4\u001b[0m study \u001b[39m=\u001b[39m optuna\u001b[39m.\u001b[39mcreate_study(\n\u001b[1;32m 5\u001b[0m study_name\u001b[39m=\u001b[39mstudy_name,\n\u001b[1;32m 6\u001b[0m storage\u001b[39m=\u001b[39mstorage_name,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 9\u001b[0m load_if_exists\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m,\n\u001b[1;32m 10\u001b[0m )\n\u001b[0;32m---> 11\u001b[0m study\u001b[39m.\u001b[39;49moptimize(\n\u001b[1;32m 12\u001b[0m objective,\n\u001b[1;32m 13\u001b[0m n_trials\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m 14\u001b[0m timeout\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m 15\u001b[0m )\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/study.py:443\u001b[0m, in \u001b[0;36mStudy.optimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m 339\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39moptimize\u001b[39m(\n\u001b[1;32m 340\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[1;32m 341\u001b[0m func: ObjectiveFuncType,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 348\u001b[0m show_progress_bar: \u001b[39mbool\u001b[39m \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m 349\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 350\u001b[0m \u001b[39m \u001b[39m\u001b[39m\"\"\"Optimize an objective function.\u001b[39;00m\n\u001b[1;32m 351\u001b[0m \n\u001b[1;32m 352\u001b[0m \u001b[39m Optimization is done by choosing a suitable set of hyperparameter values from a given\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 440\u001b[0m \u001b[39m If nested invocation of this method occurs.\u001b[39;00m\n\u001b[1;32m 441\u001b[0m \u001b[39m \"\"\"\u001b[39;00m\n\u001b[0;32m--> 443\u001b[0m _optimize(\n\u001b[1;32m 444\u001b[0m study\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m,\n\u001b[1;32m 445\u001b[0m func\u001b[39m=\u001b[39;49mfunc,\n\u001b[1;32m 446\u001b[0m n_trials\u001b[39m=\u001b[39;49mn_trials,\n\u001b[1;32m 447\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout,\n\u001b[1;32m 448\u001b[0m n_jobs\u001b[39m=\u001b[39;49mn_jobs,\n\u001b[1;32m 449\u001b[0m catch\u001b[39m=\u001b[39;49m\u001b[39mtuple\u001b[39;49m(catch) \u001b[39mif\u001b[39;49;00m \u001b[39misinstance\u001b[39;49m(catch, Iterable) \u001b[39melse\u001b[39;49;00m (catch,),\n\u001b[1;32m 450\u001b[0m callbacks\u001b[39m=\u001b[39;49mcallbacks,\n\u001b[1;32m 451\u001b[0m gc_after_trial\u001b[39m=\u001b[39;49mgc_after_trial,\n\u001b[1;32m 452\u001b[0m show_progress_bar\u001b[39m=\u001b[39;49mshow_progress_bar,\n\u001b[1;32m 453\u001b[0m )\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py:66\u001b[0m, in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 65\u001b[0m \u001b[39mif\u001b[39;00m n_jobs \u001b[39m==\u001b[39m \u001b[39m1\u001b[39m:\n\u001b[0;32m---> 66\u001b[0m _optimize_sequential(\n\u001b[1;32m 67\u001b[0m study,\n\u001b[1;32m 68\u001b[0m func,\n\u001b[1;32m 69\u001b[0m n_trials,\n\u001b[1;32m 70\u001b[0m timeout,\n\u001b[1;32m 71\u001b[0m catch,\n\u001b[1;32m 72\u001b[0m callbacks,\n\u001b[1;32m 73\u001b[0m gc_after_trial,\n\u001b[1;32m 74\u001b[0m reseed_sampler_rng\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 75\u001b[0m time_start\u001b[39m=\u001b[39;49m\u001b[39mNone\u001b[39;49;00m,\n\u001b[1;32m 76\u001b[0m progress_bar\u001b[39m=\u001b[39;49mprogress_bar,\n\u001b[1;32m 77\u001b[0m )\n\u001b[1;32m 78\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 79\u001b[0m \u001b[39mif\u001b[39;00m n_jobs \u001b[39m==\u001b[39m \u001b[39m-\u001b[39m\u001b[39m1\u001b[39m:\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py:163\u001b[0m, in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m 160\u001b[0m \u001b[39mbreak\u001b[39;00m\n\u001b[1;32m 162\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 163\u001b[0m frozen_trial \u001b[39m=\u001b[39m _run_trial(study, func, catch)\n\u001b[1;32m 164\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m 165\u001b[0m \u001b[39m# The following line mitigates memory problems that can be occurred in some\u001b[39;00m\n\u001b[1;32m 166\u001b[0m \u001b[39m# environments (e.g., services that use computing containers such as GitHub Actions).\u001b[39;00m\n\u001b[1;32m 167\u001b[0m \u001b[39m# Please refer to the following PR for further details:\u001b[39;00m\n\u001b[1;32m 168\u001b[0m \u001b[39m# https://github.com/optuna/optuna/pull/325.\u001b[39;00m\n\u001b[1;32m 169\u001b[0m \u001b[39mif\u001b[39;00m gc_after_trial:\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py:251\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m 244\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39mFalse\u001b[39;00m, \u001b[39m\"\u001b[39m\u001b[39mShould not reach.\u001b[39m\u001b[39m\"\u001b[39m\n\u001b[1;32m 246\u001b[0m \u001b[39mif\u001b[39;00m (\n\u001b[1;32m 247\u001b[0m frozen_trial\u001b[39m.\u001b[39mstate \u001b[39m==\u001b[39m TrialState\u001b[39m.\u001b[39mFAIL\n\u001b[1;32m 248\u001b[0m \u001b[39mand\u001b[39;00m func_err \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 249\u001b[0m \u001b[39mand\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39misinstance\u001b[39m(func_err, catch)\n\u001b[1;32m 250\u001b[0m ):\n\u001b[0;32m--> 251\u001b[0m \u001b[39mraise\u001b[39;00m func_err\n\u001b[1;32m 252\u001b[0m \u001b[39mreturn\u001b[39;00m frozen_trial\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/study/_optimize.py:200\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[39mwith\u001b[39;00m get_heartbeat_thread(trial\u001b[39m.\u001b[39m_trial_id, study\u001b[39m.\u001b[39m_storage):\n\u001b[1;32m 199\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 200\u001b[0m value_or_values \u001b[39m=\u001b[39m func(trial)\n\u001b[1;32m 201\u001b[0m \u001b[39mexcept\u001b[39;00m exceptions\u001b[39m.\u001b[39mTrialPruned \u001b[39mas\u001b[39;00m e:\n\u001b[1;32m 202\u001b[0m \u001b[39m# TODO(mamu): Handle multi-objective cases.\u001b[39;00m\n\u001b[1;32m 203\u001b[0m state \u001b[39m=\u001b[39m TrialState\u001b[39m.\u001b[39mPRUNED\n", "Cell \u001b[0;32mIn[4], line 16\u001b[0m, in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m 12\u001b[0m logger \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m 13\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m 14\u001b[0m \u001b[39m# Select hyperparameters for testing\u001b[39;00m\n\u001b[1;32m 15\u001b[0m in_act \u001b[39m=\u001b[39m (\n\u001b[0;32m---> 16\u001b[0m activation_dispatcher[trial\u001b[39m.\u001b[39;49msuggest_categorical(\u001b[39m\"\u001b[39;49m\u001b[39min_act\u001b[39;49m\u001b[39m\"\u001b[39;49m, activation_vals)],\n\u001b[1;32m 17\u001b[0m (),\n\u001b[1;32m 18\u001b[0m {},\n\u001b[1;32m 19\u001b[0m )\n\u001b[1;32m 20\u001b[0m compressor_kernel_size \u001b[39m=\u001b[39m trial\u001b[39m.\u001b[39msuggest_int(\n\u001b[1;32m 21\u001b[0m \u001b[39m\"\u001b[39m\u001b[39mcompressor_kernel_size\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m64\u001b[39m, \u001b[39m257\u001b[39m, \u001b[39m64\u001b[39m\n\u001b[1;32m 22\u001b[0m )\n\u001b[1;32m 23\u001b[0m compressor_chunk_size \u001b[39m=\u001b[39m \u001b[39m128\u001b[39m\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/trial/_trial.py:405\u001b[0m, in \u001b[0;36mTrial.suggest_categorical\u001b[0;34m(self, name, choices)\u001b[0m\n\u001b[1;32m 354\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Suggest a value for the categorical parameter.\u001b[39;00m\n\u001b[1;32m 355\u001b[0m \n\u001b[1;32m 356\u001b[0m \u001b[39mThe value is sampled from ``choices``.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 400\u001b[0m \u001b[39m :ref:`configurations` tutorial describes more details and flexible usages.\u001b[39;00m\n\u001b[1;32m 401\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[1;32m 402\u001b[0m \u001b[39m# There is no need to call self._check_distribution because\u001b[39;00m\n\u001b[1;32m 403\u001b[0m \u001b[39m# CategoricalDistribution does not support dynamic value space.\u001b[39;00m\n\u001b[0;32m--> 405\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_suggest(name, CategoricalDistribution(choices\u001b[39m=\u001b[39;49mchoices))\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/trial/_trial.py:630\u001b[0m, in \u001b[0;36mTrial._suggest\u001b[0;34m(self, name, distribution)\u001b[0m\n\u001b[1;32m 628\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 629\u001b[0m study \u001b[39m=\u001b[39m pruners\u001b[39m.\u001b[39m_filter_study(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstudy, trial)\n\u001b[0;32m--> 630\u001b[0m param_value \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mstudy\u001b[39m.\u001b[39;49msampler\u001b[39m.\u001b[39;49msample_independent(\n\u001b[1;32m 631\u001b[0m study, trial, name, distribution\n\u001b[1;32m 632\u001b[0m )\n\u001b[1;32m 634\u001b[0m \u001b[39m# `param_value` is validated here (invalid value like `np.nan` raises ValueError).\u001b[39;00m\n\u001b[1;32m 635\u001b[0m param_value_in_internal_repr \u001b[39m=\u001b[39m distribution\u001b[39m.\u001b[39mto_internal_repr(param_value)\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/sampler.py:471\u001b[0m, in \u001b[0;36mTPESampler.sample_independent\u001b[0;34m(self, study, trial, param_name, param_distribution)\u001b[0m\n\u001b[1;32m 467\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 468\u001b[0m mpe_below \u001b[39m=\u001b[39m _ParzenEstimator(\n\u001b[1;32m 469\u001b[0m below, {param_name: param_distribution}, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_parzen_estimator_parameters\n\u001b[1;32m 470\u001b[0m )\n\u001b[0;32m--> 471\u001b[0m mpe_above \u001b[39m=\u001b[39m _ParzenEstimator(\n\u001b[1;32m 472\u001b[0m above, {param_name: param_distribution}, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_parzen_estimator_parameters\n\u001b[1;32m 473\u001b[0m )\n\u001b[1;32m 474\u001b[0m samples_below \u001b[39m=\u001b[39m mpe_below\u001b[39m.\u001b[39msample(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_rng, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_n_ei_candidates)\n\u001b[1;32m 475\u001b[0m log_likelihoods_below \u001b[39m=\u001b[39m mpe_below\u001b[39m.\u001b[39mlog_pdf(samples_below)\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py:75\u001b[0m, in \u001b[0;36m_ParzenEstimator.__init__\u001b[0;34m(self, observations, search_space, parameters, predetermined_weights)\u001b[0m\n\u001b[1;32m 71\u001b[0m weights \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mappend(weights, [parameters\u001b[39m.\u001b[39mprior_weight])\n\u001b[1;32m 72\u001b[0m weights \u001b[39m/\u001b[39m\u001b[39m=\u001b[39m weights\u001b[39m.\u001b[39msum()\n\u001b[1;32m 73\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_mixture_distribution \u001b[39m=\u001b[39m _MixtureOfProductDistribution(\n\u001b[1;32m 74\u001b[0m weights\u001b[39m=\u001b[39mweights,\n\u001b[0;32m---> 75\u001b[0m distributions\u001b[39m=\u001b[39m[\n\u001b[1;32m 76\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_calculate_distributions(\n\u001b[1;32m 77\u001b[0m transformed_observations[:, i], search_space[param], parameters\n\u001b[1;32m 78\u001b[0m )\n\u001b[1;32m 79\u001b[0m \u001b[39mfor\u001b[39;49;00m i, param \u001b[39min\u001b[39;49;00m \u001b[39menumerate\u001b[39;49m(search_space)\n\u001b[1;32m 80\u001b[0m ],\n\u001b[1;32m 81\u001b[0m )\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py:76\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 71\u001b[0m weights \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mappend(weights, [parameters\u001b[39m.\u001b[39mprior_weight])\n\u001b[1;32m 72\u001b[0m weights \u001b[39m/\u001b[39m\u001b[39m=\u001b[39m weights\u001b[39m.\u001b[39msum()\n\u001b[1;32m 73\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_mixture_distribution \u001b[39m=\u001b[39m _MixtureOfProductDistribution(\n\u001b[1;32m 74\u001b[0m weights\u001b[39m=\u001b[39mweights,\n\u001b[1;32m 75\u001b[0m distributions\u001b[39m=\u001b[39m[\n\u001b[0;32m---> 76\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_calculate_distributions(\n\u001b[1;32m 77\u001b[0m transformed_observations[:, i], search_space[param], parameters\n\u001b[1;32m 78\u001b[0m )\n\u001b[1;32m 79\u001b[0m \u001b[39mfor\u001b[39;00m i, param \u001b[39min\u001b[39;00m \u001b[39menumerate\u001b[39m(search_space)\n\u001b[1;32m 80\u001b[0m ],\n\u001b[1;32m 81\u001b[0m )\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py:154\u001b[0m, in \u001b[0;36m_ParzenEstimator._calculate_distributions\u001b[0;34m(self, transformed_observations, search_space, parameters)\u001b[0m\n\u001b[1;32m 147\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_calculate_distributions\u001b[39m(\n\u001b[1;32m 148\u001b[0m \u001b[39mself\u001b[39m,\n\u001b[1;32m 149\u001b[0m transformed_observations: np\u001b[39m.\u001b[39mndarray,\n\u001b[1;32m 150\u001b[0m search_space: BaseDistribution,\n\u001b[1;32m 151\u001b[0m parameters: _ParzenEstimatorParameters,\n\u001b[1;32m 152\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m _BatchedDistributions:\n\u001b[1;32m 153\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39misinstance\u001b[39m(search_space, CategoricalDistribution):\n\u001b[0;32m--> 154\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_calculate_categorical_distributions(\n\u001b[1;32m 155\u001b[0m transformed_observations, search_space\u001b[39m.\u001b[39;49mchoices, parameters\n\u001b[1;32m 156\u001b[0m )\n\u001b[1;32m 157\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m 158\u001b[0m \u001b[39massert\u001b[39;00m \u001b[39misinstance\u001b[39m(search_space, (FloatDistribution, IntDistribution))\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/optuna/samplers/_tpe/parzen_estimator.py:192\u001b[0m, in \u001b[0;36m_ParzenEstimator._calculate_categorical_distributions\u001b[0;34m(self, observations, choices, parameters)\u001b[0m\n\u001b[1;32m 186\u001b[0m weights \u001b[39m=\u001b[39m np\u001b[39m.\u001b[39mfull(\n\u001b[1;32m 187\u001b[0m shape\u001b[39m=\u001b[39m(\u001b[39mlen\u001b[39m(observations) \u001b[39m+\u001b[39m consider_prior, \u001b[39mlen\u001b[39m(choices)),\n\u001b[1;32m 188\u001b[0m fill_value\u001b[39m=\u001b[39mparameters\u001b[39m.\u001b[39mprior_weight \u001b[39m/\u001b[39m (\u001b[39mlen\u001b[39m(observations) \u001b[39m+\u001b[39m consider_prior),\n\u001b[1;32m 189\u001b[0m )\n\u001b[1;32m 191\u001b[0m weights[np\u001b[39m.\u001b[39marange(\u001b[39mlen\u001b[39m(observations)), observations\u001b[39m.\u001b[39mastype(\u001b[39mint\u001b[39m)] \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n\u001b[0;32m--> 192\u001b[0m weights \u001b[39m/\u001b[39m\u001b[39m=\u001b[39m weights\u001b[39m.\u001b[39;49msum(axis\u001b[39m=\u001b[39;49m\u001b[39m1\u001b[39;49m, keepdims\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n\u001b[1;32m 193\u001b[0m \u001b[39mreturn\u001b[39;00m _BatchedCategoricalDistributions(weights)\n", "File \u001b[0;32m~/Programming/Git_Projects/Aconity_ML_Test/.venv/lib/python3.11/site-packages/numpy/core/_methods.py:47\u001b[0m, in \u001b[0;36m_sum\u001b[0;34m(a, axis, dtype, out, keepdims, initial, where)\u001b[0m\n\u001b[1;32m 43\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_amin\u001b[39m(a, axis\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, out\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, keepdims\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m 44\u001b[0m initial\u001b[39m=\u001b[39m_NoValue, where\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m):\n\u001b[1;32m 45\u001b[0m \u001b[39mreturn\u001b[39;00m umr_minimum(a, axis, \u001b[39mNone\u001b[39;00m, out, keepdims, initial, where)\n\u001b[0;32m---> 47\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_sum\u001b[39m(a, axis\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, dtype\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, out\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, keepdims\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m 48\u001b[0m initial\u001b[39m=\u001b[39m_NoValue, where\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m):\n\u001b[1;32m 49\u001b[0m \u001b[39mreturn\u001b[39;00m umr_sum(a, axis, dtype, out, keepdims, initial, where)\n\u001b[1;32m 51\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_prod\u001b[39m(a, axis\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, dtype\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, out\u001b[39m=\u001b[39m\u001b[39mNone\u001b[39;00m, keepdims\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m,\n\u001b[1;32m 52\u001b[0m initial\u001b[39m=\u001b[39m_NoValue, where\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m):\n", "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], "source": [ "storage_name = \"sqlite:///optuna.sql\"\n", "storage_name = \"mysql+pymysql://root:Ch31121992@192.168.1.10:3306/optuna_db\"\n", "study_name = \"Experiment 2\"\n", "study = optuna.create_study(\n", " study_name=study_name,\n", " storage=storage_name,\n", " direction=\"minimize\",\n", " pruner=HyperbandPruner(),\n", " load_if_exists=True,\n", ")\n", "study.optimize(\n", " objective,\n", " n_trials=None,\n", " timeout=None,\n", ")" ] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.3" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }