Housekeeping and polish based on test discoveries

2025-12-22 22:22:02 +00:00 · 2023-11-03 16:52:12 +00:00
parent e279c23745
commit 2f452892b0
22 changed files with 163 additions and 185 deletions
--- a/config.toml
+++ b/config.toml
@@ -25,6 +25,7 @@ CLI = true
 [debug]
 DEBUG = true
 CLI = false
+CLEANUP_IMAGES = false

 [test]
 TESTING = true
--- a/poetry.lock
+++ b/poetry.lock
@@ -883,13 +883,13 @@ files = [

 [[package]]
 name = "selenium"
-version = "4.14.0"
+version = "4.15.1"
 description = ""
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "selenium-4.14.0-py3-none-any.whl", hash = "sha256:be9824a9354a7fe288e3fad9ceb6a9c65ddc7c44545d23ad0ebf4ce202b19893"},
-    {file = "selenium-4.14.0.tar.gz", hash = "sha256:0d14b0d9842366f38fb5f8f842cf7c042bcfa062affc6a0a86e4d634bdd0fe54"},
+    {file = "selenium-4.15.1-py3-none-any.whl", hash = "sha256:e3a4ebdcc3eed27eec69f8000d798923dbf4897c97cc6441eb88a1386809170d"},
+    {file = "selenium-4.15.1.tar.gz", hash = "sha256:8f0436b5949f1d4aa742f3dff0d748b955c371be92db8b6b008bf9c9ca227de7"},
 ]

 [package.dependencies]
@@ -987,13 +987,13 @@ files = [

 [[package]]
 name = "trio"
-version = "0.22.2"
+version = "0.23.0"
 description = "A friendly Python library for async concurrency and I/O"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "trio-0.22.2-py3-none-any.whl", hash = "sha256:f43da357620e5872b3d940a2e3589aa251fd3f881b65a608d742e00809b1ec38"},
-    {file = "trio-0.22.2.tar.gz", hash = "sha256:3887cf18c8bcc894433420305468388dac76932e9668afa1c49aa3806b6accb3"},
+    {file = "trio-0.23.0-py3-none-any.whl", hash = "sha256:213cd69a05962b1ba24d48caf08f7e7acf02bf1ebfac17c06d1248497f05795e"},
+    {file = "trio-0.23.0.tar.gz", hash = "sha256:662cfe10018018607a8e7ee191c274bcffbf9056be60b3ccb4f1790df98fc0a3"},
 ]

 [package.dependencies]
@@ -1001,7 +1001,7 @@ attrs = ">=20.1.0"
 cffi = {version = ">=1.14", markers = "os_name == \"nt\" and implementation_name != \"pypy\""}
 idna = "*"
 outcome = "*"
-sniffio = "*"
+sniffio = ">=1.3.0"
 sortedcontainers = "*"

 [[package]]
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -73,3 +73,11 @@ lines-after-imports = 2
 quote-style = "double"
 indent-style = "space"
 line-ending = "auto"
+
+[tool.pytest.ini_options]
+testpaths = [
+    "tests",
+]
+pythonpath = [
+    ".venv/bin/python",
+]
--- a/src/docker/validate.dockerfile
+++ b/src/docker/validate.dockerfile
@@ -2,10 +2,12 @@ FROM quay.io/coreos/ignition-validate:release AS ignition-validate
 FROM alpine:latest as base

 ARG CWD_MOUNTDIR
+ARG BUILD_DIR
 ENV CWD_MOUNTDIR=$CWD_MOUNTDIR
+ENV BUILD_DIR=$BUILD_DIR

 COPY --from=ignition-validate . .
-COPY src/scripts/installs.sh /installs.sh
+COPY src/scripts/validate_installs.sh /installs.sh

 RUN /installs.sh

--- a/src/node_deployer/autoignition.py
+++ b/src/node_deployer/autoignition.py
@@ -76,7 +76,7 @@ def convert_json_via_fuelignition(
    image_file = container.exec_run("ls /home/seluser/Downloads/").output.decode().split()[0]
    # Finally, fetch the image file from the container
    client_image_path = f"/home/seluser/Downloads/{image_file}"
-    host_image_path = config.SRC_DIR / img_path
+    host_image_path = config.PROJECT_ROOT / img_path
    if host_image_path.exists():
        host_image_path.unlink()
    filestream = container.get_archive(client_image_path)[0]
@@ -211,6 +211,8 @@ def json_to_img(
                config.CWD_MOUNT,
            ],
        )
+        while config.SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode():
+            time.sleep(0.1)
        fuelignition_image = build_fuelignition()
        fuelignition_container = config.CLIENT.containers.run(
            fuelignition_image,
@@ -218,11 +220,7 @@ def json_to_img(
            remove=True,
            network_mode=f"container:{selenium_container.id}",
        )
-        # Wait for the containers to finish starting up
-        while config.SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode():
-            time.sleep(0.1)
-            for event in config.CLIENT.events(decode=True):
-                print(event)
+        # Wait for the container to finish starting up
        while not fnmatch(
            fuelignition_container.logs().decode().strip().split("\n")[-1].strip(),
            config.FUELIGNITION_INIT_MESSAGE,
--- a/src/node_deployer/create_disk.py
+++ b/src/node_deployer/create_disk.py
@@ -47,7 +47,10 @@ def validation_result() -> str:
        path=".",
        dockerfile=str(dockerfile),
        tag="validate",
-        buildargs={"CWD_MOUNTDIR": str(config.CWD_MOUNTDIR)},
+        buildargs={
+            "CWD_MOUNTDIR": str(config.CWD_MOUNTDIR),
+            "BUILD_DIR": str(config.BUILD_DIR.relative_to(config.PROJECT_ROOT)),
+        },
        rm=config.CLEANUP_IMAGES,
        pull=True,
        quiet=True,
@@ -57,7 +60,7 @@ def validation_result() -> str:
        mounts=[
            config.CWD_MOUNT,
        ],
-        remove=True,
+        remove=config.CLEANUP_IMAGES,
    )
    if config.CLEANUP_IMAGES:
        image.remove(force=True)
--- a/src/node_deployer/create_img.py
+++ b/src/node_deployer/create_img.py
@@ -31,7 +31,7 @@ def apply_ignition_settings(
    template: dict,
    hostname: str,
    password: str,
-    swarm_config: str,
+    swarm_config: dict,
 ) -> dict:
    """Applies the specified ignition settings to the given template

@@ -46,8 +46,8 @@ def apply_ignition_settings(
    """
    ignition_config = template.copy()
    ignition_config["hostname"] = hostname
-    if password:
    ignition_config["login"]["users"][0]["passwd"] = password
+    if password:
        ignition_config["login"]["users"][0]["hash_type"] = "bcrypt"
    elif not config.TESTING:
        raise ValueError("Password must be specified")
@@ -66,7 +66,7 @@ def apply_ignition_settings(
            "source_type": "data",
            "mode": 420,
            "overwrite": True,
-            "data_content": swarm_config,
+            "data_content": json.dumps(swarm_config),
        },
        {
            "path": "/root/join_swarm.sh",
@@ -197,13 +197,11 @@ def create_img(
        password = ""
    
    # get swarm configuration as JSON
-    swarm_config = json.dumps(
-        {
+    swarm_config = {
        "SWITCH_IP_ADDRESS": str(switch_ip),
        "SWITCH_PORT": switch_port,
        "SWARM_TOKEN": swarm_token,
    }
-    )

    # Create ignition configuration
    ignition_config = apply_ignition_settings(
--- a/src/node_deployer/debug.py
+++ b/src/node_deployer/debug.py
@@ -7,6 +7,11 @@ import typer
 from .config import config


+def get_debug_f(f: Callable) -> Callable:
+    import snoop # type: ignore
+    return wraps(f)(snoop.snoop(**config.snoop["snoop"])(f))
+
+
 def debug_guard(f: Callable) -> Callable:
    """A decorator that contextually enables debug mode for the decorated function

@@ -35,9 +40,10 @@ def debug_guard(f: Callable) -> Callable:
        **kwargs,
    ) -> Callable:
        typer.echo(f"Debug mode enabled: {inspect.stack()[1].filename}")
+        debug_f = get_debug_f(f)
        if kwargs.get("debug", False):
            # Snoop depth is set to compensate for wrapper stack frames
-            return snoop.snoop(**config.snoop["snoop"])(f)(*args, **kwargs) # noqa: F821 #* ss is installed in debug_mode
+            return debug_f(*args, **kwargs) # noqa: F821 #* ss is installed in debug_mode
        else:
            return f(*args, **kwargs)

--- a/src/node_deployer/utils.py
+++ b/src/node_deployer/utils.py
@@ -1,6 +1,7 @@
 from functools import wraps
 from pathlib import Path
 from typing import Callable
+import docker

 from .config import config

@@ -51,6 +52,8 @@ def next_free_tcp_port(port: int) -> int:
    Returns:
        int: The next free port
    """
+    ports = []
+    try:
        containers = config.CLIENT.containers.list(all=True)
        ports = []
        for container in containers:
@@ -59,6 +62,12 @@ def next_free_tcp_port(port: int) -> int:
                continue
            for x in list(container.ports.values())[0]:
                ports.append(int(x["HostPort"]))
+    except docker.errors.NotFound:  # type: ignore
+        #* This error is raised if container list changes between getting the list and
+        #* getting the ports. If this happens, just try again
+        return next_free_tcp_port(port)
+    if not ports:
+        return port
    ports = set(ports)
    while port in ports:
        port += 1
--- a/src/scripts/validate.sh
+++ b/src/scripts/validate.sh
@@ -1,2 +1,2 @@
 ${CWD_MOUNTDIR}/src/scripts/fetch_config.sh
-/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/build/config.ign
+/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/${BUILD_DIR}/config.ign
--- a/src/scripts/validate_installs.sh
+++ b/src/scripts/validate_installs.sh
--- a/tests/init.py
+++ b/tests/init.py
--- a/tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl
+++ b/tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl
--- a/tests/test_node_deployer/data/create_img/load_template.pkl
+++ b/tests/test_node_deployer/data/create_img/load_template.pkl
--- a/tests/test_node_deployer/data/fuelignition.json
+++ b/tests/test_node_deployer/data/fuelignition.json
@@ -2,7 +2,8 @@
    "login": {
        "users": [
            {
-                "name": "root"
+                "name": "root",
+                "passwd": ""
            }
        ]
    },
--- a/tests/test_node_deployer/data/ignition.img
+++ b/tests/test_node_deployer/data/ignition.img
--- a/tests/data/node_deployer/test_args.toml
+++ b/tests/data/node_deployer/test_args.toml
@@ -0,0 +1,11 @@
+[create_img.create_img]
+hostname = "test_hostname"
+password = ""
+switch_ip = "192.168.1.1"
+switch_port = 42
+swarm_token = "SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG"
+
+[create_img.apply_ignition_settings]
+hostname = "test_hostname"
+password = ""
+swarm_config = {SWITCH_IP_ADDRESS="192.168.1.1", SWITCH_PORT=42, SWARM_TOKEN="SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG"}
--- a/tests/test_node_deployer.py
+++ b/tests/test_node_deployer.py
@@ -0,0 +1,88 @@
+import atexit
+import filecmp
+from pathlib import Path
+import pickle
+import shutil
+
+from node_deployer.config import config
+import tomllib
+
+
+config.update_config("test")
+config.BUILD_DIR = config.BUILD_DIR / "tests"
+atexit.register(lambda: shutil.rmtree(config.BUILD_DIR, ignore_errors=True))
+
+from node_deployer import autoignition, create_img  # noqa: E402
+
+
+with open(config.PROJECT_ROOT / "tests/data/node_deployer/test_args.toml", "rb") as f:
+    TEST_PARAMS = tomllib.load(f)
+
+
+class TestAutoignition:
+    def test_json_to_img(self, tmp_path: Path):
+        tmp_path.mkdir(parents=True, exist_ok=True)
+        autoignition.json_to_img(
+            config.PROJECT_ROOT / "tests/data/node_deployer/fuelignition.json",
+            tmp_path / "ignition.img",
+        )
+        assert filecmp.cmp(
+            config.PROJECT_ROOT / "tests/data/node_deployer/ignition.img",
+            tmp_path / "ignition.img",
+        )
+
+
+class TestCreateImg:
+    def test_load_template(self):
+        template = create_img.load_template()
+        with open(
+            config.PROJECT_ROOT / "tests/data/node_deployer/create_img/load_template.pkl", "rb"
+        ) as f:
+            assert pickle.load(f) == template
+
+    def test_apply_ignition_settings(self):
+        with open(
+            config.PROJECT_ROOT / "tests/data/node_deployer/create_img/load_template.pkl",
+            mode="rb",
+        ) as f:
+            template = pickle.load(f)
+        test_result = create_img.apply_ignition_settings(
+            template,
+            **TEST_PARAMS["create_img"]["apply_ignition_settings"],
+        )
+        with open(
+            config.PROJECT_ROOT / "tests/data/node_deployer/create_img/apply_ignition_settings.pkl",
+            mode="rb",
+        ) as f:
+            expected = pickle.load(f)
+
+        assert expected == test_result
+
+    def test_create_img(self, tmp_path: Path):
+        tmp_path.mkdir(parents=True, exist_ok=True)
+        create_img.create_img(
+            **TEST_PARAMS["create_img"]["create_img"],
+            img_path=tmp_path / "ignition.img",
+        )
+        assert filecmp.cmp(
+            tmp_path / "ignition.img",
+            config.PROJECT_ROOT / "tests/data/node_deployer/ignition.img",
+        )
+
+
+# class TestWriteDisk:
+#     def init(self):
+#         test_target = config.BUILD_DIR / "ignition.img"
+#         if not test_target.exists():
+#             test_target.write_bytes(
+#                 Path(config.PROJECT_ROOT / "tests/data/node_deployer/ignition.img").read_bytes()
+#             )
+
+#     def test_validation_result(self):
+#         raise NotImplementedError
+
+#     def test_filter_validation_result(self):
+#         raise NotImplementedError
+
+#     def test_validate(self):
+#         raise NotImplementedError
--- a/tests/test_node_deployer/init.py
+++ b/tests/test_node_deployer/init.py
@@ -1,5 +0,0 @@
-from node_deployer.config import config
-
-
-config.update_config("test")
-config.BUILD_DIR = config.BUILD_DIR / "tests"
--- a/tests/test_node_deployer/data/config.ign
+++ b/tests/test_node_deployer/data/config.ign
@@ -1,76 +0,0 @@
-{
-  "ignition": {
-    "version": "3.2.0"
-  },
-  "passwd": {
-    "users": [
-      {
-        "name": "root",
-        "passwordHash": "$2a$08$YdxsvKFRc1q2S1lmVixm5Oel3Y2oCNAsx9Sh2Dx4pL/udApPzUQu6"
-      }
-    ]
-  },
-  "storage": {
-    "files": [
-      {
-        "path": "/root/join_swarm.json",
-        "mode": 420,
-        "overwrite": true,
-        "contents": {
-          "source": "data:text/plain;charset=utf-8;base64,eyJTV0lUQ0hfSVBfQUREUkVTUyI6ICIxOTIuMTY4LjEuMSIsICJTV0lUQ0hfUE9SVCI6IDQyLCAiU1dBUk1fVE9LRU4iOiAiU1dNVEtOLTEtVEhJU0lTQVRFU1RTV0FSTVRPS0VORk9SVEVTVElOR1BVUlBPU0VTQU5EVEhBVE1FQU5TSVRORUVEU1RPQkVRVUlURUxPTkcifQ=="
-        }
-      },
-      {
-        "path": "/root/join_swarm.sh",
-        "mode": 420,
-        "overwrite": true,
-        "contents": {
-          "source": "data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2gKCmlmIFtbICRFVUlEIC1uZSAwIF1dOyB0aGVuCiAgIGVjaG8gIlRoaXMgc2NyaXB0IG11c3QgYmUgcnVuIGFzIHJvb3QiIAogICBleGl0IDEKZmkKCiMgTG9hZCB0aGUgY29uZmlnIGZpbGUgaW50byB2YXJpYWJsZXMKZXZhbCAiJChqcSAtciAndG9fZW50cmllc1tdIHwgImV4cG9ydCBcKC5rZXkpPVwoLnZhbHVlIHwgQHNoKSInIC9yb290L2pvaW5fc3dhcm0uanNvbikiCgppZiBbWyAkKGRvY2tlciBpbmZvIHwgZ3JlcCBTd2FybSB8IGF3ayAne3ByaW50ICQyfScpID09ICJpbmFjdGl2ZSIgXV07IHRoZW4KICAgIGRvY2tlciBzd2FybSBqb2luIC0tdG9rZW4gJFNXQVJNX1RPS0VOIFskU1dJVENIX0lQX0FERFJFU1NdOiRTV0lUQ0hfUE9SVAplbHNlCiAgICBlY2hvICJUaGlzIG5vZGUgaXMgYWxyZWFkeSBwYXJ0IG9mIGEgc3dhcm0iCiAgICBkb2NrZXIgaW5mbyAtZiBqc29uIHwganEgLlN3YXJtCmZpCg=="
-        }
-      },
-      {
-        "path": "/etc/hostname",
-        "mode": 420,
-        "overwrite": true,
-        "contents": {
-          "source": "data:,test_hostname"
-        }
-      },
-      {
-        "path": "/etc/NetworkManager/system-connections/eth0.nmconnection",
-        "mode": 384,
-        "overwrite": true,
-        "contents": {
-          "source": "data:text/plain;charset=utf-8;base64,Cltjb25uZWN0aW9uXQppZD1ldGgwCnR5cGU9ZXRoZXJuZXQKaW50ZXJmYWNlLW5hbWU9ZXRoMAoKW2lwdjRdCmRucy1zZWFyY2g9Cm1ldGhvZD1hdXRvCgpbaXB2Nl0KZG5zLXNlYXJjaD0KYWRkci1nZW4tbW9kZT1ldWk2NAptZXRob2Q9YXV0bwo=",
-          "human_read": "\n[connection]\nid=eth0\ntype=ethernet\ninterface-name=eth0\n\n[ipv4]\ndns-search=\nmethod=auto\n\n[ipv6]\ndns-search=\naddr-gen-mode=eui64\nmethod=auto\n"
-        }
-      },
-      {
-        "path": "/etc/NetworkManager/conf.d/noauto.conf",
-        "mode": 420,
-        "overwrite": true,
-        "contents": {
-          "source": "data:text/plain;charset=utf-8;base64,W21haW5dCiMgRG8gbm90IGRvIGF1dG9tYXRpYyAoREhDUC9TTEFBQykgY29uZmlndXJhdGlvbiBvbiBldGhlcm5ldCBkZXZpY2VzCiMgd2l0aCBubyBvdGhlciBtYXRjaGluZyBjb25uZWN0aW9ucy4Kbm8tYXV0by1kZWZhdWx0PSoK",
-          "human_read": "[main]\n# Do not do automatic (DHCP/SLAAC) configuration on ethernet devices\n# with no other matching connections.\nno-auto-default=*\n"
-        }
-      }
-    ]
-  },
-  "systemd": {
-    "units": [
-      {
-        "name": "cockpit.socket.service",
-        "enabled": true
-      },
-      {
-        "name": "docker.service",
-        "enabled": true
-      },
-      {
-        "name": "join_swarm.service",
-        "enabled": false,
-        "contents": "[Unit]\nDescription=Ensure that node joins a swarm on startup\n\n[Service]\nExecStart=/root/join_swarm.sh\n\n[Install]\nWantedBy=multi-user.target"
-      }
-    ]
-  }
-}
--- a/tests/test_node_deployer/test_autoignition.py
+++ b/tests/test_node_deployer/test_autoignition.py
@@ -1,16 +0,0 @@
-import filecmp
-
-from node_deployer import autoignition
-from node_deployer.config import config
-
-
-class TestAutoignition:
-    def test_json_to_img(self, tmpdir):
-        autoignition.json_to_img(
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/fuelignition.json",
-            tmpdir / "ignition.img",
-        )
-        assert filecmp.cmp(
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/ignition.img",
-            tmpdir / "ignition.img",
-        )
--- a/tests/test_node_deployer/test_create_img.py
+++ b/tests/test_node_deployer/test_create_img.py
@@ -1,50 +0,0 @@
-import filecmp
-import pickle
-
-from node_deployer import create_img
-from node_deployer.config import config
-
-
-class TestCreateImg:
-    def test_load_template(self):
-        template = create_img.load_template()
-        with open(
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/create_img/load_template.pkl", "rb"
-        ) as f:
-            assert pickle.load(f) == template
-
-    def test_apply_ignition_settings(self):
-        with open(
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/create_img/load_template.pkl",
-            mode="rb",
-        ) as f:
-            template = pickle.load(f)
-        test_result = create_img.apply_ignition_settings(
-            template,
-            "test_hostname",
-            "",
-            {
-                "SWITCH_IP_ADDRESS": "192.168.1.1",
-                "SWITCH_PORT": 42,
-                "SWARM_TOKEN": "SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG",  # noqa: E501
-            },
-        )
-        with open(
-            config.PROJECT_ROOT
-            / "tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl",
-            mode="rb",
-        ) as f:
-            assert pickle.load(f) == test_result
-
-    def test_create_img(self, tmpdir):
-        create_img.create_img(
-            hostname="test_hostname",
-            password="",
-            switch_ip="192.168.1.1",
-            switch_port=42,
-            img_path=tmpdir / "ignition.img",
-        )
-        assert filecmp.cmp(
-            tmpdir / "ignition.img",
-            config.PROJECT_ROOT / "tests/test_node_deployer/data/ignition.img",
-        )