Housekeeping and polish based on test discoveries

config.toml

@@ -25,6 +25,7 @@ CLI = true
 [debug]
 DEBUG = true
 CLI = false
+CLEANUP_IMAGES = false
 
 [test]
 TESTING = true

poetry.lock

@@ -883,13 +883,13 @@ files = [
 
 [[package]]
 name = "selenium"
-version = "4.14.0"
+version = "4.15.1"
 description = ""
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "selenium-4.14.0-py3-none-any.whl", hash = "sha256:be9824a9354a7fe288e3fad9ceb6a9c65ddc7c44545d23ad0ebf4ce202b19893"},
+    {file = "selenium-4.15.1-py3-none-any.whl", hash = "sha256:e3a4ebdcc3eed27eec69f8000d798923dbf4897c97cc6441eb88a1386809170d"},
-    {file = "selenium-4.14.0.tar.gz", hash = "sha256:0d14b0d9842366f38fb5f8f842cf7c042bcfa062affc6a0a86e4d634bdd0fe54"},
+    {file = "selenium-4.15.1.tar.gz", hash = "sha256:8f0436b5949f1d4aa742f3dff0d748b955c371be92db8b6b008bf9c9ca227de7"},
 ]
 
 [package.dependencies]
@@ -987,13 +987,13 @@ files = [
 
 [[package]]
 name = "trio"
-version = "0.22.2"
+version = "0.23.0"
 description = "A friendly Python library for async concurrency and I/O"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "trio-0.22.2-py3-none-any.whl", hash = "sha256:f43da357620e5872b3d940a2e3589aa251fd3f881b65a608d742e00809b1ec38"},
+    {file = "trio-0.23.0-py3-none-any.whl", hash = "sha256:213cd69a05962b1ba24d48caf08f7e7acf02bf1ebfac17c06d1248497f05795e"},
-    {file = "trio-0.22.2.tar.gz", hash = "sha256:3887cf18c8bcc894433420305468388dac76932e9668afa1c49aa3806b6accb3"},
+    {file = "trio-0.23.0.tar.gz", hash = "sha256:662cfe10018018607a8e7ee191c274bcffbf9056be60b3ccb4f1790df98fc0a3"},
 ]
 
 [package.dependencies]
@@ -1001,7 +1001,7 @@ attrs = ">=20.1.0"
 cffi = {version = ">=1.14", markers = "os_name == \"nt\" and implementation_name != \"pypy\""}
 idna = "*"
 outcome = "*"
-sniffio = "*"
+sniffio = ">=1.3.0"
 sortedcontainers = "*"
 
 [[package]]

pyproject.toml

@@ -73,3 +73,11 @@ lines-after-imports = 2
 quote-style = "double"
 indent-style = "space"
 line-ending = "auto"
+
+[tool.pytest.ini_options]
+testpaths = [
+    "tests",
+]
+pythonpath = [
+    ".venv/bin/python",
+]

src/docker/validate.dockerfile

@@ -2,10 +2,12 @@ FROM quay.io/coreos/ignition-validate:release AS ignition-validate
 FROM alpine:latest as base
 
 ARG CWD_MOUNTDIR
+ARG BUILD_DIR
 ENV CWD_MOUNTDIR=$CWD_MOUNTDIR
+ENV BUILD_DIR=$BUILD_DIR
 
 COPY --from=ignition-validate . .
-COPY src/scripts/installs.sh /installs.sh
+COPY src/scripts/validate_installs.sh /installs.sh
 
 RUN /installs.sh
 

src/node_deployer/autoignition.py

@@ -76,7 +76,7 @@ def convert_json_via_fuelignition(
     image_file = container.exec_run("ls /home/seluser/Downloads/").output.decode().split()[0]
     # Finally, fetch the image file from the container
     client_image_path = f"/home/seluser/Downloads/{image_file}"
-    host_image_path = config.SRC_DIR / img_path
+    host_image_path = config.PROJECT_ROOT / img_path
     if host_image_path.exists():
         host_image_path.unlink()
     filestream = container.get_archive(client_image_path)[0]
@@ -211,6 +211,8 @@ def json_to_img(
                 config.CWD_MOUNT,
             ],
         )
+        while config.SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode():
+            time.sleep(0.1)
         fuelignition_image = build_fuelignition()
         fuelignition_container = config.CLIENT.containers.run(
             fuelignition_image,
@@ -218,11 +220,7 @@ def json_to_img(
             remove=True,
             network_mode=f"container:{selenium_container.id}",
         )
-        # Wait for the containers to finish starting up
+        # Wait for the container to finish starting up
-        while config.SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode():
-            time.sleep(0.1)
-            for event in config.CLIENT.events(decode=True):
-                print(event)
         while not fnmatch(
             fuelignition_container.logs().decode().strip().split("\n")[-1].strip(),
             config.FUELIGNITION_INIT_MESSAGE,

src/node_deployer/create_disk.py

@@ -47,7 +47,10 @@ def validation_result() -> str:
         path=".",
         dockerfile=str(dockerfile),
         tag="validate",
-        buildargs={"CWD_MOUNTDIR": str(config.CWD_MOUNTDIR)},
+        buildargs={
+            "CWD_MOUNTDIR": str(config.CWD_MOUNTDIR),
+            "BUILD_DIR": str(config.BUILD_DIR.relative_to(config.PROJECT_ROOT)),
+        },
         rm=config.CLEANUP_IMAGES,
         pull=True,
         quiet=True,
@@ -57,7 +60,7 @@ def validation_result() -> str:
         mounts=[
             config.CWD_MOUNT,
         ],
-        remove=True,
+        remove=config.CLEANUP_IMAGES,
     )
     if config.CLEANUP_IMAGES:
         image.remove(force=True)

src/node_deployer/create_img.py

@@ -31,7 +31,7 @@ def apply_ignition_settings(
     template: dict,
     hostname: str,
     password: str,
-    swarm_config: str,
+    swarm_config: dict,
 ) -> dict:
     """Applies the specified ignition settings to the given template
 
@@ -46,8 +46,8 @@ def apply_ignition_settings(
     """
     ignition_config = template.copy()
     ignition_config["hostname"] = hostname
+    ignition_config["login"]["users"][0]["passwd"] = password
     if password:
-        ignition_config["login"]["users"][0]["passwd"] = password
         ignition_config["login"]["users"][0]["hash_type"] = "bcrypt"
     elif not config.TESTING:
         raise ValueError("Password must be specified")
@@ -66,7 +66,7 @@ def apply_ignition_settings(
             "source_type": "data",
             "mode": 420,
             "overwrite": True,
-            "data_content": swarm_config,
+            "data_content": json.dumps(swarm_config),
         },
         {
             "path": "/root/join_swarm.sh",
@@ -197,13 +197,11 @@ def create_img(
         password = ""
     
     # get swarm configuration as JSON
-    swarm_config = json.dumps(
+    swarm_config = {
-        {
+        "SWITCH_IP_ADDRESS": str(switch_ip),
-            "SWITCH_IP_ADDRESS": str(switch_ip),
+        "SWITCH_PORT": switch_port,
-            "SWITCH_PORT": switch_port,
+        "SWARM_TOKEN": swarm_token,
-            "SWARM_TOKEN": swarm_token,
+    }
-        }
-    )
 
     # Create ignition configuration
     ignition_config = apply_ignition_settings(

src/node_deployer/debug.py

@@ -7,6 +7,11 @@ import typer
 from .config import config
 
 
+def get_debug_f(f: Callable) -> Callable:
+    import snoop # type: ignore
+    return wraps(f)(snoop.snoop(**config.snoop["snoop"])(f))
+
+
 def debug_guard(f: Callable) -> Callable:
     """A decorator that contextually enables debug mode for the decorated function
 
@@ -35,9 +40,10 @@ def debug_guard(f: Callable) -> Callable:
         **kwargs,
     ) -> Callable:
         typer.echo(f"Debug mode enabled: {inspect.stack()[1].filename}")
+        debug_f = get_debug_f(f)
         if kwargs.get("debug", False):
             # Snoop depth is set to compensate for wrapper stack frames
-            return snoop.snoop(**config.snoop["snoop"])(f)(*args, **kwargs) # noqa: F821 #* ss is installed in debug_mode
+            return debug_f(*args, **kwargs) # noqa: F821 #* ss is installed in debug_mode
         else:
             return f(*args, **kwargs)
 

src/node_deployer/utils.py

@@ -1,6 +1,7 @@
 from functools import wraps
 from pathlib import Path
 from typing import Callable
+import docker
 
 from .config import config
 
@@ -51,14 +52,22 @@ def next_free_tcp_port(port: int) -> int:
     Returns:
         int: The next free port
     """
-    containers = config.CLIENT.containers.list(all=True)
     ports = []
-    for container in containers:
+    try:
-        port_values = container.ports.values()
+        containers = config.CLIENT.containers.list(all=True)
-        if not port_values:
+        ports = []
-            continue
+        for container in containers:
-        for x in list(container.ports.values())[0]:
+            port_values = container.ports.values()
-            ports.append(int(x["HostPort"]))
+            if not port_values:
+                continue
+            for x in list(container.ports.values())[0]:
+                ports.append(int(x["HostPort"]))
+    except docker.errors.NotFound:  # type: ignore
+        #* This error is raised if container list changes between getting the list and
+        #* getting the ports. If this happens, just try again
+        return next_free_tcp_port(port)
+    if not ports:
+        return port
     ports = set(ports)
     while port in ports:
         port += 1

src/scripts/validate.sh

@@ -1,2 +1,2 @@
 ${CWD_MOUNTDIR}/src/scripts/fetch_config.sh
-/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/build/config.ign
+/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/${BUILD_DIR}/config.ign

tests/data/node_deployer/fuelignition.json

@@ -2,7 +2,8 @@
     "login": {
         "users": [
             {
-                "name": "root"
+                "name": "root",
+                "passwd": ""
             }
         ]
     },

tests/data/node_deployer/test_args.toml

@@ -0,0 +1,11 @@
+[create_img.create_img]
+hostname = "test_hostname"
+password = ""
+switch_ip = "192.168.1.1"
+switch_port = 42
+swarm_token = "SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG"
+
+[create_img.apply_ignition_settings]
+hostname = "test_hostname"
+password = ""
+swarm_config = {SWITCH_IP_ADDRESS="192.168.1.1", SWITCH_PORT=42, SWARM_TOKEN="SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG"}

tests/test_node_deployer.py

@@ -0,0 +1,88 @@
+import atexit
+import filecmp
+from pathlib import Path
+import pickle
+import shutil
+
+from node_deployer.config import config
+import tomllib
+
+
+config.update_config("test")
+config.BUILD_DIR = config.BUILD_DIR / "tests"
+atexit.register(lambda: shutil.rmtree(config.BUILD_DIR, ignore_errors=True))
+
+from node_deployer import autoignition, create_img  # noqa: E402
+
+
+with open(config.PROJECT_ROOT / "tests/data/node_deployer/test_args.toml", "rb") as f:
+    TEST_PARAMS = tomllib.load(f)
+
+
+class TestAutoignition:
+    def test_json_to_img(self, tmp_path: Path):
+        tmp_path.mkdir(parents=True, exist_ok=True)
+        autoignition.json_to_img(
+            config.PROJECT_ROOT / "tests/data/node_deployer/fuelignition.json",
+            tmp_path / "ignition.img",
+        )
+        assert filecmp.cmp(
+            config.PROJECT_ROOT / "tests/data/node_deployer/ignition.img",
+            tmp_path / "ignition.img",
+        )
+
+
+class TestCreateImg:
+    def test_load_template(self):
+        template = create_img.load_template()
+        with open(
+            config.PROJECT_ROOT / "tests/data/node_deployer/create_img/load_template.pkl", "rb"
+        ) as f:
+            assert pickle.load(f) == template
+
+    def test_apply_ignition_settings(self):
+        with open(
+            config.PROJECT_ROOT / "tests/data/node_deployer/create_img/load_template.pkl",
+            mode="rb",
+        ) as f:
+            template = pickle.load(f)
+        test_result = create_img.apply_ignition_settings(
+            template,
+            **TEST_PARAMS["create_img"]["apply_ignition_settings"],
+        )
+        with open(
+            config.PROJECT_ROOT / "tests/data/node_deployer/create_img/apply_ignition_settings.pkl",
+            mode="rb",
+        ) as f:
+            expected = pickle.load(f)
+
+        assert expected == test_result
+
+    def test_create_img(self, tmp_path: Path):
+        tmp_path.mkdir(parents=True, exist_ok=True)
+        create_img.create_img(
+            **TEST_PARAMS["create_img"]["create_img"],
+            img_path=tmp_path / "ignition.img",
+        )
+        assert filecmp.cmp(
+            tmp_path / "ignition.img",
+            config.PROJECT_ROOT / "tests/data/node_deployer/ignition.img",
+        )
+
+
+# class TestWriteDisk:
+#     def init(self):
+#         test_target = config.BUILD_DIR / "ignition.img"
+#         if not test_target.exists():
+#             test_target.write_bytes(
+#                 Path(config.PROJECT_ROOT / "tests/data/node_deployer/ignition.img").read_bytes()
+#             )
+
+#     def test_validation_result(self):
+#         raise NotImplementedError
+
+#     def test_filter_validation_result(self):
+#         raise NotImplementedError
+
+#     def test_validate(self):
+#         raise NotImplementedError

tests/test_node_deployer/__init__.py

@@ -1,5 +0,0 @@
-from node_deployer.config import config
-
-
-config.update_config("test")
-config.BUILD_DIR = config.BUILD_DIR / "tests"

tests/test_node_deployer/data/config.ign

@@ -1,76 +0,0 @@
-{
-  "ignition": {
-    "version": "3.2.0"
-  },
-  "passwd": {
-    "users": [
-      {
-        "name": "root",
-        "passwordHash": "$2a$08$YdxsvKFRc1q2S1lmVixm5Oel3Y2oCNAsx9Sh2Dx4pL/udApPzUQu6"
-      }
-    ]
-  },
-  "storage": {
-    "files": [
-      {
-        "path": "/root/join_swarm.json",
-        "mode": 420,
-        "overwrite": true,
-        "contents": {
-          "source": "data:text/plain;charset=utf-8;base64,eyJTV0lUQ0hfSVBfQUREUkVTUyI6ICIxOTIuMTY4LjEuMSIsICJTV0lUQ0hfUE9SVCI6IDQyLCAiU1dBUk1fVE9LRU4iOiAiU1dNVEtOLTEtVEhJU0lTQVRFU1RTV0FSTVRPS0VORk9SVEVTVElOR1BVUlBPU0VTQU5EVEhBVE1FQU5TSVRORUVEU1RPQkVRVUlURUxPTkcifQ=="
-        }
-      },
-      {
-        "path": "/root/join_swarm.sh",
-        "mode": 420,
-        "overwrite": true,
-        "contents": {
-          "source": "data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2gKCmlmIFtbICRFVUlEIC1uZSAwIF1dOyB0aGVuCiAgIGVjaG8gIlRoaXMgc2NyaXB0IG11c3QgYmUgcnVuIGFzIHJvb3QiIAogICBleGl0IDEKZmkKCiMgTG9hZCB0aGUgY29uZmlnIGZpbGUgaW50byB2YXJpYWJsZXMKZXZhbCAiJChqcSAtciAndG9fZW50cmllc1tdIHwgImV4cG9ydCBcKC5rZXkpPVwoLnZhbHVlIHwgQHNoKSInIC9yb290L2pvaW5fc3dhcm0uanNvbikiCgppZiBbWyAkKGRvY2tlciBpbmZvIHwgZ3JlcCBTd2FybSB8IGF3ayAne3ByaW50ICQyfScpID09ICJpbmFjdGl2ZSIgXV07IHRoZW4KICAgIGRvY2tlciBzd2FybSBqb2luIC0tdG9rZW4gJFNXQVJNX1RPS0VOIFskU1dJVENIX0lQX0FERFJFU1NdOiRTV0lUQ0hfUE9SVAplbHNlCiAgICBlY2hvICJUaGlzIG5vZGUgaXMgYWxyZWFkeSBwYXJ0IG9mIGEgc3dhcm0iCiAgICBkb2NrZXIgaW5mbyAtZiBqc29uIHwganEgLlN3YXJtCmZpCg=="
-        }
-      },
-      {
-        "path": "/etc/hostname",
-        "mode": 420,
-        "overwrite": true,
-        "contents": {
-          "source": "data:,test_hostname"
-        }
-      },
-      {
-        "path": "/etc/NetworkManager/system-connections/eth0.nmconnection",
-        "mode": 384,
-        "overwrite": true,
-        "contents": {
-          "source": "data:text/plain;charset=utf-8;base64,Cltjb25uZWN0aW9uXQppZD1ldGgwCnR5cGU9ZXRoZXJuZXQKaW50ZXJmYWNlLW5hbWU9ZXRoMAoKW2lwdjRdCmRucy1zZWFyY2g9Cm1ldGhvZD1hdXRvCgpbaXB2Nl0KZG5zLXNlYXJjaD0KYWRkci1nZW4tbW9kZT1ldWk2NAptZXRob2Q9YXV0bwo=",
-          "human_read": "\n[connection]\nid=eth0\ntype=ethernet\ninterface-name=eth0\n\n[ipv4]\ndns-search=\nmethod=auto\n\n[ipv6]\ndns-search=\naddr-gen-mode=eui64\nmethod=auto\n"
-        }
-      },
-      {
-        "path": "/etc/NetworkManager/conf.d/noauto.conf",
-        "mode": 420,
-        "overwrite": true,
-        "contents": {
-          "source": "data:text/plain;charset=utf-8;base64,W21haW5dCiMgRG8gbm90IGRvIGF1dG9tYXRpYyAoREhDUC9TTEFBQykgY29uZmlndXJhdGlvbiBvbiBldGhlcm5ldCBkZXZpY2VzCiMgd2l0aCBubyBvdGhlciBtYXRjaGluZyBjb25uZWN0aW9ucy4Kbm8tYXV0by1kZWZhdWx0PSoK",
-          "human_read": "[main]\n# Do not do automatic (DHCP/SLAAC) configuration on ethernet devices\n# with no other matching connections.\nno-auto-default=*\n"
-        }
-      }
-    ]
-  },
-  "systemd": {
-    "units": [
-      {
-        "name": "cockpit.socket.service",
-        "enabled": true
-      },
-      {
-        "name": "docker.service",
-        "enabled": true
-      },
-      {
-        "name": "join_swarm.service",
-        "enabled": false,
-        "contents": "[Unit]\nDescription=Ensure that node joins a swarm on startup\n\n[Service]\nExecStart=/root/join_swarm.sh\n\n[Install]\nWantedBy=multi-user.target"
-      }
-    ]
-  }
-}

tests/test_node_deployer/test_autoignition.py

@@ -1,16 +0,0 @@
-import filecmp
-
-from node_deployer import autoignition
-from node_deployer.config import config
-
-
-class TestAutoignition:
-    def test_json_to_img(self, tmpdir):
-        autoignition.json_to_img(
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/fuelignition.json",
-            tmpdir / "ignition.img",
-        )
-        assert filecmp.cmp(
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/ignition.img",
-            tmpdir / "ignition.img",
-        )

tests/test_node_deployer/test_create_img.py

@@ -1,50 +0,0 @@
-import filecmp
-import pickle
-
-from node_deployer import create_img
-from node_deployer.config import config
-
-
-class TestCreateImg:
-    def test_load_template(self):
-        template = create_img.load_template()
-        with open(
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/create_img/load_template.pkl", "rb"
-        ) as f:
-            assert pickle.load(f) == template
-
-    def test_apply_ignition_settings(self):
-        with open(
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/create_img/load_template.pkl",
-            mode="rb",
-        ) as f:
-            template = pickle.load(f)
-        test_result = create_img.apply_ignition_settings(
-            template,
-            "test_hostname",
-            "",
-            {
-                "SWITCH_IP_ADDRESS": "192.168.1.1",
-                "SWITCH_PORT": 42,
-                "SWARM_TOKEN": "SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG",  # noqa: E501
-            },
-        )
-        with open(
-            config.PROJECT_ROOT
-            / "tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl",
-            mode="rb",
-        ) as f:
-            assert pickle.load(f) == test_result
-
-    def test_create_img(self, tmpdir):
-        create_img.create_img(
-            hostname="test_hostname",
-            password="",
-            switch_ip="192.168.1.1",
-            switch_port=42,
-            img_path=tmpdir / "ignition.img",
-        )
-        assert filecmp.cmp(
-            tmpdir / "ignition.img",
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/ignition.img",
-        )