diff --git a/config.toml b/config.toml index bc2f812..77a0dcc 100644 --- a/config.toml +++ b/config.toml @@ -25,6 +25,7 @@ CLI = true [debug] DEBUG = true CLI = false +CLEANUP_IMAGES = false [test] TESTING = true diff --git a/poetry.lock b/poetry.lock index 66f4f6b..b221365 100644 --- a/poetry.lock +++ b/poetry.lock @@ -883,13 +883,13 @@ files = [ [[package]] name = "selenium" -version = "4.14.0" +version = "4.15.1" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "selenium-4.14.0-py3-none-any.whl", hash = "sha256:be9824a9354a7fe288e3fad9ceb6a9c65ddc7c44545d23ad0ebf4ce202b19893"}, - {file = "selenium-4.14.0.tar.gz", hash = "sha256:0d14b0d9842366f38fb5f8f842cf7c042bcfa062affc6a0a86e4d634bdd0fe54"}, + {file = "selenium-4.15.1-py3-none-any.whl", hash = "sha256:e3a4ebdcc3eed27eec69f8000d798923dbf4897c97cc6441eb88a1386809170d"}, + {file = "selenium-4.15.1.tar.gz", hash = "sha256:8f0436b5949f1d4aa742f3dff0d748b955c371be92db8b6b008bf9c9ca227de7"}, ] [package.dependencies] @@ -987,13 +987,13 @@ files = [ [[package]] name = "trio" -version = "0.22.2" +version = "0.23.0" description = "A friendly Python library for async concurrency and I/O" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "trio-0.22.2-py3-none-any.whl", hash = "sha256:f43da357620e5872b3d940a2e3589aa251fd3f881b65a608d742e00809b1ec38"}, - {file = "trio-0.22.2.tar.gz", hash = "sha256:3887cf18c8bcc894433420305468388dac76932e9668afa1c49aa3806b6accb3"}, + {file = "trio-0.23.0-py3-none-any.whl", hash = "sha256:213cd69a05962b1ba24d48caf08f7e7acf02bf1ebfac17c06d1248497f05795e"}, + {file = "trio-0.23.0.tar.gz", hash = "sha256:662cfe10018018607a8e7ee191c274bcffbf9056be60b3ccb4f1790df98fc0a3"}, ] [package.dependencies] @@ -1001,7 +1001,7 @@ attrs = ">=20.1.0" cffi = {version = ">=1.14", markers = "os_name == \"nt\" and implementation_name != \"pypy\""} idna = "*" outcome = "*" -sniffio = "*" +sniffio = ">=1.3.0" sortedcontainers = "*" [[package]] diff --git a/pyproject.toml b/pyproject.toml index 360babc..d5749cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,4 +72,12 @@ lines-after-imports = 2 [tool.ruff.format] quote-style = "double" indent-style = "space" -line-ending = "auto" \ No newline at end of file +line-ending = "auto" + +[tool.pytest.ini_options] +testpaths = [ + "tests", +] +pythonpath = [ + ".venv/bin/python", +] \ No newline at end of file diff --git a/src/docker/validate.dockerfile b/src/docker/validate.dockerfile index 863a454..afcf5b7 100644 --- a/src/docker/validate.dockerfile +++ b/src/docker/validate.dockerfile @@ -2,10 +2,12 @@ FROM quay.io/coreos/ignition-validate:release AS ignition-validate FROM alpine:latest as base ARG CWD_MOUNTDIR +ARG BUILD_DIR ENV CWD_MOUNTDIR=$CWD_MOUNTDIR +ENV BUILD_DIR=$BUILD_DIR COPY --from=ignition-validate . . -COPY src/scripts/installs.sh /installs.sh +COPY src/scripts/validate_installs.sh /installs.sh RUN /installs.sh diff --git a/src/node_deployer/autoignition.py b/src/node_deployer/autoignition.py index 97439d7..9d85cb2 100644 --- a/src/node_deployer/autoignition.py +++ b/src/node_deployer/autoignition.py @@ -76,7 +76,7 @@ def convert_json_via_fuelignition( image_file = container.exec_run("ls /home/seluser/Downloads/").output.decode().split()[0] # Finally, fetch the image file from the container client_image_path = f"/home/seluser/Downloads/{image_file}" - host_image_path = config.SRC_DIR / img_path + host_image_path = config.PROJECT_ROOT / img_path if host_image_path.exists(): host_image_path.unlink() filestream = container.get_archive(client_image_path)[0] @@ -211,6 +211,8 @@ def json_to_img( config.CWD_MOUNT, ], ) + while config.SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode(): + time.sleep(0.1) fuelignition_image = build_fuelignition() fuelignition_container = config.CLIENT.containers.run( fuelignition_image, @@ -218,11 +220,7 @@ def json_to_img( remove=True, network_mode=f"container:{selenium_container.id}", ) - # Wait for the containers to finish starting up - while config.SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode(): - time.sleep(0.1) - for event in config.CLIENT.events(decode=True): - print(event) + # Wait for the container to finish starting up while not fnmatch( fuelignition_container.logs().decode().strip().split("\n")[-1].strip(), config.FUELIGNITION_INIT_MESSAGE, diff --git a/src/node_deployer/create_disk.py b/src/node_deployer/create_disk.py index bd9ff20..30d4fcf 100644 --- a/src/node_deployer/create_disk.py +++ b/src/node_deployer/create_disk.py @@ -47,7 +47,10 @@ def validation_result() -> str: path=".", dockerfile=str(dockerfile), tag="validate", - buildargs={"CWD_MOUNTDIR": str(config.CWD_MOUNTDIR)}, + buildargs={ + "CWD_MOUNTDIR": str(config.CWD_MOUNTDIR), + "BUILD_DIR": str(config.BUILD_DIR.relative_to(config.PROJECT_ROOT)), + }, rm=config.CLEANUP_IMAGES, pull=True, quiet=True, @@ -57,7 +60,7 @@ def validation_result() -> str: mounts=[ config.CWD_MOUNT, ], - remove=True, + remove=config.CLEANUP_IMAGES, ) if config.CLEANUP_IMAGES: image.remove(force=True) diff --git a/src/node_deployer/create_img.py b/src/node_deployer/create_img.py index ce3f555..5a4e67f 100644 --- a/src/node_deployer/create_img.py +++ b/src/node_deployer/create_img.py @@ -31,7 +31,7 @@ def apply_ignition_settings( template: dict, hostname: str, password: str, - swarm_config: str, + swarm_config: dict, ) -> dict: """Applies the specified ignition settings to the given template @@ -46,8 +46,8 @@ def apply_ignition_settings( """ ignition_config = template.copy() ignition_config["hostname"] = hostname + ignition_config["login"]["users"][0]["passwd"] = password if password: - ignition_config["login"]["users"][0]["passwd"] = password ignition_config["login"]["users"][0]["hash_type"] = "bcrypt" elif not config.TESTING: raise ValueError("Password must be specified") @@ -66,7 +66,7 @@ def apply_ignition_settings( "source_type": "data", "mode": 420, "overwrite": True, - "data_content": swarm_config, + "data_content": json.dumps(swarm_config), }, { "path": "/root/join_swarm.sh", @@ -197,13 +197,11 @@ def create_img( password = "" # get swarm configuration as JSON - swarm_config = json.dumps( - { - "SWITCH_IP_ADDRESS": str(switch_ip), - "SWITCH_PORT": switch_port, - "SWARM_TOKEN": swarm_token, - } - ) + swarm_config = { + "SWITCH_IP_ADDRESS": str(switch_ip), + "SWITCH_PORT": switch_port, + "SWARM_TOKEN": swarm_token, + } # Create ignition configuration ignition_config = apply_ignition_settings( diff --git a/src/node_deployer/debug.py b/src/node_deployer/debug.py index 5449fba..437bf6b 100644 --- a/src/node_deployer/debug.py +++ b/src/node_deployer/debug.py @@ -7,6 +7,11 @@ import typer from .config import config +def get_debug_f(f: Callable) -> Callable: + import snoop # type: ignore + return wraps(f)(snoop.snoop(**config.snoop["snoop"])(f)) + + def debug_guard(f: Callable) -> Callable: """A decorator that contextually enables debug mode for the decorated function @@ -35,9 +40,10 @@ def debug_guard(f: Callable) -> Callable: **kwargs, ) -> Callable: typer.echo(f"Debug mode enabled: {inspect.stack()[1].filename}") + debug_f = get_debug_f(f) if kwargs.get("debug", False): # Snoop depth is set to compensate for wrapper stack frames - return snoop.snoop(**config.snoop["snoop"])(f)(*args, **kwargs) # noqa: F821 #* ss is installed in debug_mode + return debug_f(*args, **kwargs) # noqa: F821 #* ss is installed in debug_mode else: return f(*args, **kwargs) diff --git a/src/node_deployer/utils.py b/src/node_deployer/utils.py index f942230..6ee7740 100644 --- a/src/node_deployer/utils.py +++ b/src/node_deployer/utils.py @@ -1,6 +1,7 @@ from functools import wraps from pathlib import Path from typing import Callable +import docker from .config import config @@ -51,14 +52,22 @@ def next_free_tcp_port(port: int) -> int: Returns: int: The next free port """ - containers = config.CLIENT.containers.list(all=True) ports = [] - for container in containers: - port_values = container.ports.values() - if not port_values: - continue - for x in list(container.ports.values())[0]: - ports.append(int(x["HostPort"])) + try: + containers = config.CLIENT.containers.list(all=True) + ports = [] + for container in containers: + port_values = container.ports.values() + if not port_values: + continue + for x in list(container.ports.values())[0]: + ports.append(int(x["HostPort"])) + except docker.errors.NotFound: # type: ignore + #* This error is raised if container list changes between getting the list and + #* getting the ports. If this happens, just try again + return next_free_tcp_port(port) + if not ports: + return port ports = set(ports) while port in ports: port += 1 diff --git a/src/scripts/validate.sh b/src/scripts/validate.sh index 7202397..a6a34ad 100755 --- a/src/scripts/validate.sh +++ b/src/scripts/validate.sh @@ -1,2 +1,2 @@ ${CWD_MOUNTDIR}/src/scripts/fetch_config.sh -/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/build/config.ign \ No newline at end of file +/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/${BUILD_DIR}/config.ign \ No newline at end of file diff --git a/src/scripts/installs.sh b/src/scripts/validate_installs.sh similarity index 100% rename from src/scripts/installs.sh rename to src/scripts/validate_installs.sh diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl b/tests/data/node_deployer/create_img/apply_ignition_settings.pkl similarity index 64% rename from tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl rename to tests/data/node_deployer/create_img/apply_ignition_settings.pkl index 46d6001..82f18df 100644 Binary files a/tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl and b/tests/data/node_deployer/create_img/apply_ignition_settings.pkl differ diff --git a/tests/test_node_deployer/data/create_img/load_template.pkl b/tests/data/node_deployer/create_img/load_template.pkl similarity index 100% rename from tests/test_node_deployer/data/create_img/load_template.pkl rename to tests/data/node_deployer/create_img/load_template.pkl diff --git a/tests/test_node_deployer/data/fuelignition.json b/tests/data/node_deployer/fuelignition.json similarity index 97% rename from tests/test_node_deployer/data/fuelignition.json rename to tests/data/node_deployer/fuelignition.json index 0089c6f..f118b4f 100644 --- a/tests/test_node_deployer/data/fuelignition.json +++ b/tests/data/node_deployer/fuelignition.json @@ -2,7 +2,8 @@ "login": { "users": [ { - "name": "root" + "name": "root", + "passwd": "" } ] }, diff --git a/tests/test_node_deployer/data/ignition.img b/tests/data/node_deployer/ignition.img similarity index 99% rename from tests/test_node_deployer/data/ignition.img rename to tests/data/node_deployer/ignition.img index a3262ee..dba5896 100644 Binary files a/tests/test_node_deployer/data/ignition.img and b/tests/data/node_deployer/ignition.img differ diff --git a/tests/data/node_deployer/test_args.toml b/tests/data/node_deployer/test_args.toml new file mode 100644 index 0000000..2d4fea1 --- /dev/null +++ b/tests/data/node_deployer/test_args.toml @@ -0,0 +1,11 @@ +[create_img.create_img] +hostname = "test_hostname" +password = "" +switch_ip = "192.168.1.1" +switch_port = 42 +swarm_token = "SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG" + +[create_img.apply_ignition_settings] +hostname = "test_hostname" +password = "" +swarm_config = {SWITCH_IP_ADDRESS="192.168.1.1", SWITCH_PORT=42, SWARM_TOKEN="SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG"} \ No newline at end of file diff --git a/tests/test_node_deployer.py b/tests/test_node_deployer.py new file mode 100644 index 0000000..f033f75 --- /dev/null +++ b/tests/test_node_deployer.py @@ -0,0 +1,88 @@ +import atexit +import filecmp +from pathlib import Path +import pickle +import shutil + +from node_deployer.config import config +import tomllib + + +config.update_config("test") +config.BUILD_DIR = config.BUILD_DIR / "tests" +atexit.register(lambda: shutil.rmtree(config.BUILD_DIR, ignore_errors=True)) + +from node_deployer import autoignition, create_img # noqa: E402 + + +with open(config.PROJECT_ROOT / "tests/data/node_deployer/test_args.toml", "rb") as f: + TEST_PARAMS = tomllib.load(f) + + +class TestAutoignition: + def test_json_to_img(self, tmp_path: Path): + tmp_path.mkdir(parents=True, exist_ok=True) + autoignition.json_to_img( + config.PROJECT_ROOT / "tests/data/node_deployer/fuelignition.json", + tmp_path / "ignition.img", + ) + assert filecmp.cmp( + config.PROJECT_ROOT / "tests/data/node_deployer/ignition.img", + tmp_path / "ignition.img", + ) + + +class TestCreateImg: + def test_load_template(self): + template = create_img.load_template() + with open( + config.PROJECT_ROOT / "tests/data/node_deployer/create_img/load_template.pkl", "rb" + ) as f: + assert pickle.load(f) == template + + def test_apply_ignition_settings(self): + with open( + config.PROJECT_ROOT / "tests/data/node_deployer/create_img/load_template.pkl", + mode="rb", + ) as f: + template = pickle.load(f) + test_result = create_img.apply_ignition_settings( + template, + **TEST_PARAMS["create_img"]["apply_ignition_settings"], + ) + with open( + config.PROJECT_ROOT / "tests/data/node_deployer/create_img/apply_ignition_settings.pkl", + mode="rb", + ) as f: + expected = pickle.load(f) + + assert expected == test_result + + def test_create_img(self, tmp_path: Path): + tmp_path.mkdir(parents=True, exist_ok=True) + create_img.create_img( + **TEST_PARAMS["create_img"]["create_img"], + img_path=tmp_path / "ignition.img", + ) + assert filecmp.cmp( + tmp_path / "ignition.img", + config.PROJECT_ROOT / "tests/data/node_deployer/ignition.img", + ) + + +# class TestWriteDisk: +# def init(self): +# test_target = config.BUILD_DIR / "ignition.img" +# if not test_target.exists(): +# test_target.write_bytes( +# Path(config.PROJECT_ROOT / "tests/data/node_deployer/ignition.img").read_bytes() +# ) + +# def test_validation_result(self): +# raise NotImplementedError + +# def test_filter_validation_result(self): +# raise NotImplementedError + +# def test_validate(self): +# raise NotImplementedError diff --git a/tests/test_node_deployer/__init__.py b/tests/test_node_deployer/__init__.py deleted file mode 100644 index 2367253..0000000 --- a/tests/test_node_deployer/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from node_deployer.config import config - - -config.update_config("test") -config.BUILD_DIR = config.BUILD_DIR / "tests" diff --git a/tests/test_node_deployer/data/config.ign b/tests/test_node_deployer/data/config.ign deleted file mode 100644 index dc780cc..0000000 --- a/tests/test_node_deployer/data/config.ign +++ /dev/null @@ -1,76 +0,0 @@ -{ - "ignition": { - "version": "3.2.0" - }, - "passwd": { - "users": [ - { - "name": "root", - "passwordHash": "$2a$08$YdxsvKFRc1q2S1lmVixm5Oel3Y2oCNAsx9Sh2Dx4pL/udApPzUQu6" - } - ] - }, - "storage": { - "files": [ - { - "path": "/root/join_swarm.json", - "mode": 420, - "overwrite": true, - "contents": { - "source": "data:text/plain;charset=utf-8;base64,eyJTV0lUQ0hfSVBfQUREUkVTUyI6ICIxOTIuMTY4LjEuMSIsICJTV0lUQ0hfUE9SVCI6IDQyLCAiU1dBUk1fVE9LRU4iOiAiU1dNVEtOLTEtVEhJU0lTQVRFU1RTV0FSTVRPS0VORk9SVEVTVElOR1BVUlBPU0VTQU5EVEhBVE1FQU5TSVRORUVEU1RPQkVRVUlURUxPTkcifQ==" - } - }, - { - "path": "/root/join_swarm.sh", - "mode": 420, - "overwrite": true, - "contents": { - "source": "data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2gKCmlmIFtbICRFVUlEIC1uZSAwIF1dOyB0aGVuCiAgIGVjaG8gIlRoaXMgc2NyaXB0IG11c3QgYmUgcnVuIGFzIHJvb3QiIAogICBleGl0IDEKZmkKCiMgTG9hZCB0aGUgY29uZmlnIGZpbGUgaW50byB2YXJpYWJsZXMKZXZhbCAiJChqcSAtciAndG9fZW50cmllc1tdIHwgImV4cG9ydCBcKC5rZXkpPVwoLnZhbHVlIHwgQHNoKSInIC9yb290L2pvaW5fc3dhcm0uanNvbikiCgppZiBbWyAkKGRvY2tlciBpbmZvIHwgZ3JlcCBTd2FybSB8IGF3ayAne3ByaW50ICQyfScpID09ICJpbmFjdGl2ZSIgXV07IHRoZW4KICAgIGRvY2tlciBzd2FybSBqb2luIC0tdG9rZW4gJFNXQVJNX1RPS0VOIFskU1dJVENIX0lQX0FERFJFU1NdOiRTV0lUQ0hfUE9SVAplbHNlCiAgICBlY2hvICJUaGlzIG5vZGUgaXMgYWxyZWFkeSBwYXJ0IG9mIGEgc3dhcm0iCiAgICBkb2NrZXIgaW5mbyAtZiBqc29uIHwganEgLlN3YXJtCmZpCg==" - } - }, - { - "path": "/etc/hostname", - "mode": 420, - "overwrite": true, - "contents": { - "source": "data:,test_hostname" - } - }, - { - "path": "/etc/NetworkManager/system-connections/eth0.nmconnection", - "mode": 384, - "overwrite": true, - "contents": { - "source": "data:text/plain;charset=utf-8;base64,Cltjb25uZWN0aW9uXQppZD1ldGgwCnR5cGU9ZXRoZXJuZXQKaW50ZXJmYWNlLW5hbWU9ZXRoMAoKW2lwdjRdCmRucy1zZWFyY2g9Cm1ldGhvZD1hdXRvCgpbaXB2Nl0KZG5zLXNlYXJjaD0KYWRkci1nZW4tbW9kZT1ldWk2NAptZXRob2Q9YXV0bwo=", - "human_read": "\n[connection]\nid=eth0\ntype=ethernet\ninterface-name=eth0\n\n[ipv4]\ndns-search=\nmethod=auto\n\n[ipv6]\ndns-search=\naddr-gen-mode=eui64\nmethod=auto\n" - } - }, - { - "path": "/etc/NetworkManager/conf.d/noauto.conf", - "mode": 420, - "overwrite": true, - "contents": { - "source": "data:text/plain;charset=utf-8;base64,W21haW5dCiMgRG8gbm90IGRvIGF1dG9tYXRpYyAoREhDUC9TTEFBQykgY29uZmlndXJhdGlvbiBvbiBldGhlcm5ldCBkZXZpY2VzCiMgd2l0aCBubyBvdGhlciBtYXRjaGluZyBjb25uZWN0aW9ucy4Kbm8tYXV0by1kZWZhdWx0PSoK", - "human_read": "[main]\n# Do not do automatic (DHCP/SLAAC) configuration on ethernet devices\n# with no other matching connections.\nno-auto-default=*\n" - } - } - ] - }, - "systemd": { - "units": [ - { - "name": "cockpit.socket.service", - "enabled": true - }, - { - "name": "docker.service", - "enabled": true - }, - { - "name": "join_swarm.service", - "enabled": false, - "contents": "[Unit]\nDescription=Ensure that node joins a swarm on startup\n\n[Service]\nExecStart=/root/join_swarm.sh\n\n[Install]\nWantedBy=multi-user.target" - } - ] - } -} \ No newline at end of file diff --git a/tests/test_node_deployer/test_autoignition.py b/tests/test_node_deployer/test_autoignition.py deleted file mode 100644 index d7b0deb..0000000 --- a/tests/test_node_deployer/test_autoignition.py +++ /dev/null @@ -1,16 +0,0 @@ -import filecmp - -from node_deployer import autoignition -from node_deployer.config import config - - -class TestAutoignition: - def test_json_to_img(self, tmpdir): - autoignition.json_to_img( - config.PROJECT_ROOT / "tests/test_node_deployer/data/fuelignition.json", - tmpdir / "ignition.img", - ) - assert filecmp.cmp( - config.PROJECT_ROOT / "tests/test_node_deployer/data/ignition.img", - tmpdir / "ignition.img", - ) diff --git a/tests/test_node_deployer/test_create_img.py b/tests/test_node_deployer/test_create_img.py deleted file mode 100644 index 87f07c6..0000000 --- a/tests/test_node_deployer/test_create_img.py +++ /dev/null @@ -1,50 +0,0 @@ -import filecmp -import pickle - -from node_deployer import create_img -from node_deployer.config import config - - -class TestCreateImg: - def test_load_template(self): - template = create_img.load_template() - with open( - config.PROJECT_ROOT / "tests/test_node_deployer/data/create_img/load_template.pkl", "rb" - ) as f: - assert pickle.load(f) == template - - def test_apply_ignition_settings(self): - with open( - config.PROJECT_ROOT / "tests/test_node_deployer/data/create_img/load_template.pkl", - mode="rb", - ) as f: - template = pickle.load(f) - test_result = create_img.apply_ignition_settings( - template, - "test_hostname", - "", - { - "SWITCH_IP_ADDRESS": "192.168.1.1", - "SWITCH_PORT": 42, - "SWARM_TOKEN": "SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG", # noqa: E501 - }, - ) - with open( - config.PROJECT_ROOT - / "tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl", - mode="rb", - ) as f: - assert pickle.load(f) == test_result - - def test_create_img(self, tmpdir): - create_img.create_img( - hostname="test_hostname", - password="", - switch_ip="192.168.1.1", - switch_port=42, - img_path=tmpdir / "ignition.img", - ) - assert filecmp.cmp( - tmpdir / "ignition.img", - config.PROJECT_ROOT / "tests/test_node_deployer/data/ignition.img", - )