Cian-H/Aconity_ML_Expt1
1
0
Fork 0
mirror of https://github.com/Cian-H/Aconity_ML_Expt1.git synced 2025-12-22 20:51:58 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
Aconity_ML_Expt1/data_postprocessing.ipynb
Cian Hughes 9781c9f050 Initial Commit
2023-08-03 21:33:03 +01:00

1013 lines
76 KiB
Plaintext
Raw Permalink Blame History

{
"cells": [
{
"cell_type": "raw",
"metadata": {},
"source": [
"This sheet will contain functions for processing sample_X and sample_y into a format better suited for the neural network."
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/cianh/Programming/Git_Projects/Aconity_ML_Expt1/.venv/lib/python3.11/site-packages/distributed/node.py:182: UserWarning: Port 8787 is already in use.\n",
"Perhaps you already have a cluster running?\n",
"Hosting the HTTP server on port 42341 instead\n",
" warnings.warn(\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
" <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-c75affb4-3164-11ee-9498-94e70ba5618f</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
"\n",
" <tr>\n",
" \n",
" <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
" <td style=\"text-align: left;\"><strong>Cluster type:</strong> distributed.LocalCluster</td>\n",
" \n",
" </tr>\n",
"\n",
" \n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:42341/status\" target=\"_blank\">http://127.0.0.1:42341/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" \n",
"\n",
" </table>\n",
"\n",
" \n",
"\n",
" \n",
" <details>\n",
" <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
" <div class=\"jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\">\n",
" </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px; margin-top: 0px;\">LocalCluster</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">c08f98d9</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:42341/status\" target=\"_blank\">http://127.0.0.1:42341/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Workers:</strong> 12\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads:</strong> 12\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total memory:</strong> 15.27 GiB\n",
" </td>\n",
" </tr>\n",
" \n",
" <tr>\n",
" <td style=\"text-align: left;\"><strong>Status:</strong> running</td>\n",
" <td style=\"text-align: left;\"><strong>Using processes:</strong> True</td>\n",
"</tr>\n",
"\n",
" \n",
" </table>\n",
"\n",
" <details>\n",
" <summary style=\"margin-bottom: 20px;\">\n",
" <h3 style=\"display: inline;\">Scheduler Info</h3>\n",
" </summary>\n",
"\n",
" <div style=\"\">\n",
" <div>\n",
" <div style=\"width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <h3 style=\"margin-bottom: 0px;\">Scheduler</h3>\n",
" <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Scheduler-3b57d4c0-ee42-41d2-8f9e-8b3c447dbbba</p>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm:</strong> tcp://127.0.0.1:41583\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Workers:</strong> 12\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard:</strong> <a href=\"http://127.0.0.1:42341/status\" target=\"_blank\">http://127.0.0.1:42341/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads:</strong> 12\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Started:</strong> Just now\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total memory:</strong> 15.27 GiB\n",
" </td>\n",
" </tr>\n",
" </table>\n",
" </div>\n",
" </div>\n",
"\n",
" <details style=\"margin-left: 48px;\">\n",
" <summary style=\"margin-bottom: 20px;\">\n",
" <h3 style=\"display: inline;\">Workers</h3>\n",
" </summary>\n",
"\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 0</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:36129\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:45385/status\" target=\"_blank\">http://127.0.0.1:45385/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:34663\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-nvlgjdsy\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 1</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:44549\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:35385/status\" target=\"_blank\">http://127.0.0.1:35385/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:43857\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-vq7_1l_n\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 2</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:39469\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:34561/status\" target=\"_blank\">http://127.0.0.1:34561/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:34253\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-6d09l2i_\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 3</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:46873\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:43591/status\" target=\"_blank\">http://127.0.0.1:43591/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:43029\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-ufai363_\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 4</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:44661\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:38421/status\" target=\"_blank\">http://127.0.0.1:38421/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:33371\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-pg33jmms\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 5</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:39485\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:38259/status\" target=\"_blank\">http://127.0.0.1:38259/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:43601\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-zjadt63s\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 6</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:36823\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:33121/status\" target=\"_blank\">http://127.0.0.1:33121/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:44609\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-2rplwyh_\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 7</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:44023\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:38183/status\" target=\"_blank\">http://127.0.0.1:38183/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:41607\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-pqn109wo\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 8</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:37553\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:45271/status\" target=\"_blank\">http://127.0.0.1:45271/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:35725\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-ns7d8c9m\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 9</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:36985\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:45069/status\" target=\"_blank\">http://127.0.0.1:45069/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:36523\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-6nhixvcl\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 10</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:46097\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:44731/status\" target=\"_blank\">http://127.0.0.1:44731/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:37565\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-n9rygn28\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
" <div style=\"margin-bottom: 20px;\">\n",
" <div style=\"width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;\"> </div>\n",
" <div style=\"margin-left: 48px;\">\n",
" <details>\n",
" <summary>\n",
" <h4 style=\"margin-bottom: 0px; display: inline;\">Worker: 11</h4>\n",
" </summary>\n",
" <table style=\"width: 100%; text-align: left;\">\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Comm: </strong> tcp://127.0.0.1:42077\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Total threads: </strong> 1\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Dashboard: </strong> <a href=\"http://127.0.0.1:37763/status\" target=\"_blank\">http://127.0.0.1:37763/status</a>\n",
" </td>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Memory: </strong> 1.27 GiB\n",
" </td>\n",
" </tr>\n",
" <tr>\n",
" <td style=\"text-align: left;\">\n",
" <strong>Nanny: </strong> tcp://127.0.0.1:34247\n",
" </td>\n",
" <td style=\"text-align: left;\"></td>\n",
" </tr>\n",
" <tr>\n",
" <td colspan=\"2\" style=\"text-align: left;\">\n",
" <strong>Local directory: </strong> /tmp/dask-scratch-space/worker-0arh8rvc\n",
" </td>\n",
" </tr>\n",
"\n",
" \n",
"\n",
" \n",
"\n",
" </table>\n",
" </details>\n",
" </div>\n",
" </div>\n",
" \n",
"\n",
" </details>\n",
"</div>\n",
"\n",
" </details>\n",
" </div>\n",
"</div>\n",
" </details>\n",
" \n",
"\n",
" </div>\n",
"</div>"
],
"text/plain": [
"<Client: 'tcp://127.0.0.1:41583' processes=12 threads=12, memory=15.27 GiB>"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from pathlib import Path\n",
"from dask.distributed import Client, LocalCluster, as_completed\n",
"import dask.dataframe as dd\n",
"import pandas as pd\n",
"import pickle\n",
"from tqdm.auto import tqdm\n",
"\n",
"# Prepare a dask cluster and client\n",
"cluster = LocalCluster(n_workers=12, threads_per_worker=1)\n",
"client = Client(cluster)\n",
"client"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# Load X and y for training\n",
"samples = list(range(1, 82))\n",
"\n",
"X = {i: dd.read_parquet(f\"sample_X/{i}\") for i in samples}\n",
"\n",
"with open(\"sample_y.pkl\", \"rb\") as f:\n",
" y = pickle.load(f)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def separate_params(d):\n",
" for sample, df in d.items():\n",
" params = df[\n",
" [\n",
" \"Laser power, P (W)\",\n",
" \"Scan speed, V (mm/sec)\",\n",
" \"Spot size, F (µm)\",\n",
" \"Hatch spacing, H (µm)\",\n",
" ]\n",
" ]\n",
" params = params.rename(\n",
" columns={\n",
" \"Laser power, P (W)\": \"laser_power\",\n",
" \"Scan speed, V (mm/sec)\": \"scan_speed\",\n",
" \"Spot size, F (µm)\": \"spot_size\",\n",
" \"Hatch spacing, H (µm)\": \"hatch_spacing\",\n",
" }\n",
" )\n",
" yield sample, params\n",
"\n",
"\n",
"params = dict(separate_params(y))"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def separate_composition(d):\n",
" for sample, df in d.items():\n",
" df = df[[\"Ni (Norm)\", \"Ti (Norm)\"]]\n",
" df = df.rename(columns={\"Ni (Norm)\": \"Ni\", \"Ti (Norm)\": \"Ti\"})\n",
" yield sample, df\n",
"\n",
"\n",
"composition = dict(separate_composition(y))"
]
},
{
"cell_type": "code",
"execution_count": 135,
"metadata": {},
"outputs": [],
"source": [
"with open(\"sample_X.pkl\", \"wb\") as f:\n",
" pickle.dump(\n",
" {\n",
" int(sample.stem): (\n",
" params[int(sample.stem)],\n",
" dd.read_parquet(sample),\n",
" )\n",
" for sample in Path(\"sample_X\").iterdir()\n",
" },\n",
" f,\n",
" )\n",
"\n",
"with open(\"sample_y.pkl\", \"wb\") as f:\n",
" pickle.dump(composition, f)"
]
},
{
"cell_type": "code",
"execution_count": 100,
"metadata": {},
"outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "2dd118d845c84669ac69a8cbac359301",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
" 0%| | 0/7 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 853), ('color', 499), ('duration', 853), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 853), ('worker', 499), ('worker_thread', 499), ('y', 853)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 858), ('color', 496), ('duration', 858), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 858), ('worker', 496), ('worker_thread', 496), ('y', 858)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 861), ('color', 499), ('duration', 861), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 861), ('worker', 499), ('worker_thread', 499), ('y', 861)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 862), ('color', 500), ('duration', 862), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 862), ('worker', 500), ('worker_thread', 500), ('y', 862)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 876), ('color', 486), ('duration', 876), ('duration_text', 486), ('key', 486), ('name', 486), ('start', 876), ('worker', 486), ('worker_thread', 486), ('y', 876)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 877), ('color', 487), ('duration', 877), ('duration_text', 487), ('key', 487), ('name', 487), ('start', 877), ('worker', 487), ('worker_thread', 487), ('y', 877)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 880), ('color', 490), ('duration', 880), ('duration_text', 490), ('key', 490), ('name', 490), ('start', 880), ('worker', 490), ('worker_thread', 490), ('y', 880)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 900), ('color', 490), ('duration', 900), ('duration_text', 490), ('key', 490), ('name', 490), ('start', 900), ('worker', 490), ('worker_thread', 490), ('y', 900)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 909), ('color', 499), ('duration', 909), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 909), ('worker', 499), ('worker_thread', 499), ('y', 909)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 910), ('color', 500), ('duration', 910), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 910), ('worker', 500), ('worker_thread', 500), ('y', 910)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 911), ('color', 499), ('duration', 911), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 911), ('worker', 499), ('worker_thread', 499), ('y', 911)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 917), ('color', 495), ('duration', 917), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 917), ('worker', 495), ('worker_thread', 495), ('y', 917)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 920), ('color', 498), ('duration', 920), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 920), ('worker', 498), ('worker_thread', 498), ('y', 920)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 935), ('color', 487), ('duration', 935), ('duration_text', 487), ('key', 487), ('name', 487), ('start', 935), ('worker', 487), ('worker_thread', 487), ('y', 935)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 945), ('color', 497), ('duration', 945), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 945), ('worker', 497), ('worker_thread', 497), ('y', 945)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 946), ('color', 498), ('duration', 946), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 946), ('worker', 498), ('worker_thread', 498), ('y', 946)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 948), ('color', 500), ('duration', 948), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 948), ('worker', 500), ('worker_thread', 500), ('y', 948)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 953), ('color', 495), ('duration', 953), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 953), ('worker', 495), ('worker_thread', 495), ('y', 953)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 964), ('color', 494), ('duration', 964), ('duration_text', 494), ('key', 494), ('name', 494), ('start', 964), ('worker', 494), ('worker_thread', 494), ('y', 964)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 967), ('color', 497), ('duration', 967), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 967), ('worker', 497), ('worker_thread', 497), ('y', 967)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 974), ('color', 496), ('duration', 974), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 974), ('worker', 496), ('worker_thread', 496), ('y', 974)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 982), ('color', 496), ('duration', 982), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 982), ('worker', 496), ('worker_thread', 496), ('y', 982)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 984), ('color', 498), ('duration', 984), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 984), ('worker', 498), ('worker_thread', 498), ('y', 984)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 996), ('color', 490), ('duration', 996), ('duration_text', 490), ('key', 490), ('name', 490), ('start', 996), ('worker', 490), ('worker_thread', 490), ('y', 996)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 475), ('duration', 1000), ('duration_text', 475), ('key', 475), ('name', 475), ('start', 1000), ('worker', 475), ('worker_thread', 475), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 480), ('duration', 1000), ('duration_text', 480), ('key', 480), ('name', 480), ('start', 1000), ('worker', 480), ('worker_thread', 480), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 487), ('duration', 1000), ('duration_text', 487), ('key', 487), ('name', 487), ('start', 1000), ('worker', 487), ('worker_thread', 487), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 496), ('duration', 1000), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 1000), ('worker', 496), ('worker_thread', 496), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 495), ('duration', 1000), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 1000), ('worker', 495), ('worker_thread', 495), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 500), ('duration', 1000), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 1000), ('worker', 500), ('worker_thread', 500), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 496), ('duration', 1000), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 1000), ('worker', 496), ('worker_thread', 496), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 495), ('duration', 1000), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 1000), ('worker', 495), ('worker_thread', 495), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 490), ('duration', 1000), ('duration_text', 490), ('key', 490), ('name', 490), ('start', 1000), ('worker', 490), ('worker_thread', 490), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 489), ('duration', 1000), ('duration_text', 489), ('key', 489), ('name', 489), ('start', 1000), ('worker', 489), ('worker_thread', 489), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 491), ('duration', 1000), ('duration_text', 491), ('key', 491), ('name', 491), ('start', 1000), ('worker', 491), ('worker_thread', 491), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 500), ('duration', 1000), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 1000), ('worker', 500), ('worker_thread', 500), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 490), ('duration', 1000), ('duration_text', 490), ('key', 490), ('name', 490), ('start', 1000), ('worker', 490), ('worker_thread', 490), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 491), ('duration', 1000), ('duration_text', 491), ('key', 491), ('name', 491), ('start', 1000), ('worker', 491), ('worker_thread', 491), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 494), ('duration', 1000), ('duration_text', 494), ('key', 494), ('name', 494), ('start', 1000), ('worker', 494), ('worker_thread', 494), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 484), ('duration', 1000), ('duration_text', 484), ('key', 484), ('name', 484), ('start', 1000), ('worker', 484), ('worker_thread', 484), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 486), ('duration', 1000), ('duration_text', 486), ('key', 486), ('name', 486), ('start', 1000), ('worker', 486), ('worker_thread', 486), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 495), ('duration', 1000), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 1000), ('worker', 495), ('worker_thread', 495), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 500), ('duration', 1000), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 1000), ('worker', 500), ('worker_thread', 500), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 500), ('duration', 1000), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 1000), ('worker', 500), ('worker_thread', 500), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 500), ('duration', 1000), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 1000), ('worker', 500), ('worker_thread', 500), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 489), ('duration', 1000), ('duration_text', 489), ('key', 489), ('name', 489), ('start', 1000), ('worker', 489), ('worker_thread', 489), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 494), ('duration', 1000), ('duration_text', 494), ('key', 494), ('name', 494), ('start', 1000), ('worker', 494), ('worker_thread', 494), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 496), ('duration', 1000), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 1000), ('worker', 496), ('worker_thread', 496), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 485), ('duration', 1000), ('duration_text', 485), ('key', 485), ('name', 485), ('start', 1000), ('worker', 485), ('worker_thread', 485), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 486), ('duration', 1000), ('duration_text', 486), ('key', 486), ('name', 486), ('start', 1000), ('worker', 486), ('worker_thread', 486), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 489), ('duration', 1000), ('duration_text', 489), ('key', 489), ('name', 489), ('start', 1000), ('worker', 489), ('worker_thread', 489), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 493), ('duration', 1000), ('duration_text', 493), ('key', 493), ('name', 493), ('start', 1000), ('worker', 493), ('worker_thread', 493), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 495), ('duration', 1000), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 1000), ('worker', 495), ('worker_thread', 495), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 496), ('duration', 1000), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 1000), ('worker', 496), ('worker_thread', 496), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 494), ('duration', 1000), ('duration_text', 494), ('key', 494), ('name', 494), ('start', 1000), ('worker', 494), ('worker_thread', 494), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 496), ('duration', 1000), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 1000), ('worker', 496), ('worker_thread', 496), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 493), ('duration', 1000), ('duration_text', 493), ('key', 493), ('name', 493), ('start', 1000), ('worker', 493), ('worker_thread', 493), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 495), ('duration', 1000), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 1000), ('worker', 495), ('worker_thread', 495), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 496), ('duration', 1000), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 1000), ('worker', 496), ('worker_thread', 496), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 486), ('duration', 1000), ('duration_text', 486), ('key', 486), ('name', 486), ('start', 1000), ('worker', 486), ('worker_thread', 486), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 500), ('duration', 1000), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 1000), ('worker', 500), ('worker_thread', 500), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 488), ('duration', 1000), ('duration_text', 488), ('key', 488), ('name', 488), ('start', 1000), ('worker', 488), ('worker_thread', 488), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 492), ('duration', 1000), ('duration_text', 492), ('key', 492), ('name', 492), ('start', 1000), ('worker', 492), ('worker_thread', 492), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 490), ('duration', 1000), ('duration_text', 490), ('key', 490), ('name', 490), ('start', 1000), ('worker', 490), ('worker_thread', 490), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 477), ('duration', 1000), ('duration_text', 477), ('key', 477), ('name', 477), ('start', 1000), ('worker', 477), ('worker_thread', 477), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 479), ('duration', 1000), ('duration_text', 479), ('key', 479), ('name', 479), ('start', 1000), ('worker', 479), ('worker_thread', 479), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 487), ('duration', 1000), ('duration_text', 487), ('key', 487), ('name', 487), ('start', 1000), ('worker', 487), ('worker_thread', 487), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 477), ('duration', 1000), ('duration_text', 477), ('key', 477), ('name', 477), ('start', 1000), ('worker', 477), ('worker_thread', 477), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 484), ('duration', 1000), ('duration_text', 484), ('key', 484), ('name', 484), ('start', 1000), ('worker', 484), ('worker_thread', 484), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 495), ('duration', 1000), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 1000), ('worker', 495), ('worker_thread', 495), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 500), ('duration', 1000), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 1000), ('worker', 500), ('worker_thread', 500), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 500), ('duration', 1000), ('duration_text', 500), ('key', 500), ('name', 500), ('start', 1000), ('worker', 500), ('worker_thread', 500), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 493), ('duration', 1000), ('duration_text', 493), ('key', 493), ('name', 493), ('start', 1000), ('worker', 493), ('worker_thread', 493), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 490), ('duration', 1000), ('duration_text', 490), ('key', 490), ('name', 490), ('start', 1000), ('worker', 490), ('worker_thread', 490), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 492), ('duration', 1000), ('duration_text', 492), ('key', 492), ('name', 492), ('start', 1000), ('worker', 492), ('worker_thread', 492), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 494), ('duration', 1000), ('duration_text', 494), ('key', 494), ('name', 494), ('start', 1000), ('worker', 494), ('worker_thread', 494), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 490), ('duration', 1000), ('duration_text', 490), ('key', 490), ('name', 490), ('start', 1000), ('worker', 490), ('worker_thread', 490), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 491), ('duration', 1000), ('duration_text', 491), ('key', 491), ('name', 491), ('start', 1000), ('worker', 491), ('worker_thread', 491), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 494), ('duration', 1000), ('duration_text', 494), ('key', 494), ('name', 494), ('start', 1000), ('worker', 494), ('worker_thread', 494), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 487), ('duration', 1000), ('duration_text', 487), ('key', 487), ('name', 487), ('start', 1000), ('worker', 487), ('worker_thread', 487), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 488), ('duration', 1000), ('duration_text', 488), ('key', 488), ('name', 488), ('start', 1000), ('worker', 488), ('worker_thread', 488), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 490), ('duration', 1000), ('duration_text', 490), ('key', 490), ('name', 490), ('start', 1000), ('worker', 490), ('worker_thread', 490), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 493), ('duration', 1000), ('duration_text', 493), ('key', 493), ('name', 493), ('start', 1000), ('worker', 493), ('worker_thread', 493), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 495), ('duration', 1000), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 1000), ('worker', 495), ('worker_thread', 495), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 476), ('duration', 1000), ('duration_text', 476), ('key', 476), ('name', 476), ('start', 1000), ('worker', 476), ('worker_thread', 476), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 481), ('duration', 1000), ('duration_text', 481), ('key', 481), ('name', 481), ('start', 1000), ('worker', 481), ('worker_thread', 481), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 491), ('duration', 1000), ('duration_text', 491), ('key', 491), ('name', 491), ('start', 1000), ('worker', 491), ('worker_thread', 491), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 492), ('duration', 1000), ('duration_text', 492), ('key', 492), ('name', 492), ('start', 1000), ('worker', 492), ('worker_thread', 492), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 496), ('duration', 1000), ('duration_text', 496), ('key', 496), ('name', 496), ('start', 1000), ('worker', 496), ('worker_thread', 496), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 498), ('duration', 1000), ('duration_text', 498), ('key', 498), ('name', 498), ('start', 1000), ('worker', 498), ('worker_thread', 498), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 491), ('duration', 1000), ('duration_text', 491), ('key', 491), ('name', 491), ('start', 1000), ('worker', 491), ('worker_thread', 491), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 495), ('duration', 1000), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 1000), ('worker', 495), ('worker_thread', 495), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 497), ('duration', 1000), ('duration_text', 497), ('key', 497), ('name', 497), ('start', 1000), ('worker', 497), ('worker_thread', 497), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 495), ('duration', 1000), ('duration_text', 495), ('key', 495), ('name', 495), ('start', 1000), ('worker', 495), ('worker_thread', 495), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 499), ('duration', 1000), ('duration_text', 499), ('key', 499), ('name', 499), ('start', 1000), ('worker', 499), ('worker_thread', 499), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 476), ('duration', 1000), ('duration_text', 476), ('key', 476), ('name', 476), ('start', 1000), ('worker', 476), ('worker_thread', 476), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 478), ('duration', 1000), ('duration_text', 478), ('key', 478), ('name', 478), ('start', 1000), ('worker', 478), ('worker_thread', 478), ('y', 1000)\n",
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 483), ('duration', 1000), ('duration_text', 483), ('key', 483), ('name', 483), ('start', 1000), ('worker', 483), ('worker_thread', 483), ('y', 1000)\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"BokehUserWarning: ColumnDataSource's columns must be of the same length. Current lengths: ('alpha', 1000), ('color', 488), ('duration', 1000), ('duration_text', 488), ('key', 488), ('name', 488), ('start', 1000), ('worker', 488), ('worker_thread', 488), ('y', 1000)\n"
]
}
],
"source": [
"def renormalize_pyrometry(df):\n",
" # temps are usually centered around roughly 1000mV\n",
" df[\"t\"] = (df[\"t\"] / 1000.0) - 1 # adjust to be between -1 and 1 is most cases\n",
" df[\"x\"] = df[\"x\"] / 10.0 # Conversion to cm brings to closer to normalized range\n",
" df[\"y\"] = df[\"y\"] / 10.0\n",
" df[\"z\"] = df[\"z\"] / 10.0\n",
" df[\"time\"] = df[\"time\"] / (60 * 60) # convert to hours\n",
" return df\n",
"\n",
"\n",
"futures = [\n",
" renormalize_pyrometry(dd.read_parquet(sample)).to_parquet(\n",
" Path(\"sample_X_new\") / sample.name,\n",
" compression=\"lz4\",\n",
" write_metadata_file=True,\n",
" compute=False,\n",
" )\n",
" for sample in Path(\"sample_X\").iterdir()\n",
"]\n",
"\n",
"for i in tqdm(list(range(0, len(futures), 12))):\n",
" dd.compute(*futures[i : i + 12])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.3"
},
"orig_nbformat": 4
},
"nbformat": 4,
"nbformat_minor": 2
}
Reference in New Issue View Git Blame Copy Permalink