From 2f452892b081d84f4db85c97f7a7033420fdc1a9 Mon Sep 17 00:00:00 2001 From: Cian Hughes Date: Fri, 3 Nov 2023 16:52:12 +0000 Subject: [PATCH] Housekeeping and polish based on test discoveries --- config.toml | 1 + poetry.lock | 16 ++-- pyproject.toml | 10 +- src/docker/validate.dockerfile | 4 +- src/node_deployer/autoignition.py | 10 +- src/node_deployer/create_disk.py | 7 +- src/node_deployer/create_img.py | 18 ++-- src/node_deployer/debug.py | 8 +- src/node_deployer/utils.py | 23 +++-- src/scripts/validate.sh | 2 +- .../{installs.sh => validate_installs.sh} | 0 tests/__init__.py | 0 .../create_img/apply_ignition_settings.pkl | Bin 1343 -> 1370 bytes .../create_img/load_template.pkl | Bin .../node_deployer}/fuelignition.json | 3 +- .../data => data/node_deployer}/ignition.img | Bin 20480000 -> 20480000 bytes tests/data/node_deployer/test_args.toml | 11 +++ tests/test_node_deployer.py | 88 ++++++++++++++++++ tests/test_node_deployer/__init__.py | 5 - tests/test_node_deployer/data/config.ign | 76 --------------- tests/test_node_deployer/test_autoignition.py | 16 ---- tests/test_node_deployer/test_create_img.py | 50 ---------- 22 files changed, 163 insertions(+), 185 deletions(-) rename src/scripts/{installs.sh => validate_installs.sh} (100%) delete mode 100644 tests/__init__.py rename tests/{test_node_deployer/data => data/node_deployer}/create_img/apply_ignition_settings.pkl (64%) rename tests/{test_node_deployer/data => data/node_deployer}/create_img/load_template.pkl (100%) rename tests/{test_node_deployer/data => data/node_deployer}/fuelignition.json (97%) rename tests/{test_node_deployer/data => data/node_deployer}/ignition.img (99%) create mode 100644 tests/data/node_deployer/test_args.toml create mode 100644 tests/test_node_deployer.py delete mode 100644 tests/test_node_deployer/__init__.py delete mode 100644 tests/test_node_deployer/data/config.ign delete mode 100644 tests/test_node_deployer/test_autoignition.py delete mode 100644 tests/test_node_deployer/test_create_img.py diff --git a/config.toml b/config.toml index bc2f812..77a0dcc 100644 --- a/config.toml +++ b/config.toml @@ -25,6 +25,7 @@ CLI = true [debug] DEBUG = true CLI = false +CLEANUP_IMAGES = false [test] TESTING = true diff --git a/poetry.lock b/poetry.lock index 66f4f6b..b221365 100644 --- a/poetry.lock +++ b/poetry.lock @@ -883,13 +883,13 @@ files = [ [[package]] name = "selenium" -version = "4.14.0" +version = "4.15.1" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "selenium-4.14.0-py3-none-any.whl", hash = "sha256:be9824a9354a7fe288e3fad9ceb6a9c65ddc7c44545d23ad0ebf4ce202b19893"}, - {file = "selenium-4.14.0.tar.gz", hash = "sha256:0d14b0d9842366f38fb5f8f842cf7c042bcfa062affc6a0a86e4d634bdd0fe54"}, + {file = "selenium-4.15.1-py3-none-any.whl", hash = "sha256:e3a4ebdcc3eed27eec69f8000d798923dbf4897c97cc6441eb88a1386809170d"}, + {file = "selenium-4.15.1.tar.gz", hash = "sha256:8f0436b5949f1d4aa742f3dff0d748b955c371be92db8b6b008bf9c9ca227de7"}, ] [package.dependencies] @@ -987,13 +987,13 @@ files = [ [[package]] name = "trio" -version = "0.22.2" +version = "0.23.0" description = "A friendly Python library for async concurrency and I/O" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "trio-0.22.2-py3-none-any.whl", hash = "sha256:f43da357620e5872b3d940a2e3589aa251fd3f881b65a608d742e00809b1ec38"}, - {file = "trio-0.22.2.tar.gz", hash = "sha256:3887cf18c8bcc894433420305468388dac76932e9668afa1c49aa3806b6accb3"}, + {file = "trio-0.23.0-py3-none-any.whl", hash = "sha256:213cd69a05962b1ba24d48caf08f7e7acf02bf1ebfac17c06d1248497f05795e"}, + {file = "trio-0.23.0.tar.gz", hash = "sha256:662cfe10018018607a8e7ee191c274bcffbf9056be60b3ccb4f1790df98fc0a3"}, ] [package.dependencies] @@ -1001,7 +1001,7 @@ attrs = ">=20.1.0" cffi = {version = ">=1.14", markers = "os_name == \"nt\" and implementation_name != \"pypy\""} idna = "*" outcome = "*" -sniffio = "*" +sniffio = ">=1.3.0" sortedcontainers = "*" [[package]] diff --git a/pyproject.toml b/pyproject.toml index 360babc..d5749cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,4 +72,12 @@ lines-after-imports = 2 [tool.ruff.format] quote-style = "double" indent-style = "space" -line-ending = "auto" \ No newline at end of file +line-ending = "auto" + +[tool.pytest.ini_options] +testpaths = [ + "tests", +] +pythonpath = [ + ".venv/bin/python", +] \ No newline at end of file diff --git a/src/docker/validate.dockerfile b/src/docker/validate.dockerfile index 863a454..afcf5b7 100644 --- a/src/docker/validate.dockerfile +++ b/src/docker/validate.dockerfile @@ -2,10 +2,12 @@ FROM quay.io/coreos/ignition-validate:release AS ignition-validate FROM alpine:latest as base ARG CWD_MOUNTDIR +ARG BUILD_DIR ENV CWD_MOUNTDIR=$CWD_MOUNTDIR +ENV BUILD_DIR=$BUILD_DIR COPY --from=ignition-validate . . -COPY src/scripts/installs.sh /installs.sh +COPY src/scripts/validate_installs.sh /installs.sh RUN /installs.sh diff --git a/src/node_deployer/autoignition.py b/src/node_deployer/autoignition.py index 97439d7..9d85cb2 100644 --- a/src/node_deployer/autoignition.py +++ b/src/node_deployer/autoignition.py @@ -76,7 +76,7 @@ def convert_json_via_fuelignition( image_file = container.exec_run("ls /home/seluser/Downloads/").output.decode().split()[0] # Finally, fetch the image file from the container client_image_path = f"/home/seluser/Downloads/{image_file}" - host_image_path = config.SRC_DIR / img_path + host_image_path = config.PROJECT_ROOT / img_path if host_image_path.exists(): host_image_path.unlink() filestream = container.get_archive(client_image_path)[0] @@ -211,6 +211,8 @@ def json_to_img( config.CWD_MOUNT, ], ) + while config.SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode(): + time.sleep(0.1) fuelignition_image = build_fuelignition() fuelignition_container = config.CLIENT.containers.run( fuelignition_image, @@ -218,11 +220,7 @@ def json_to_img( remove=True, network_mode=f"container:{selenium_container.id}", ) - # Wait for the containers to finish starting up - while config.SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode(): - time.sleep(0.1) - for event in config.CLIENT.events(decode=True): - print(event) + # Wait for the container to finish starting up while not fnmatch( fuelignition_container.logs().decode().strip().split("\n")[-1].strip(), config.FUELIGNITION_INIT_MESSAGE, diff --git a/src/node_deployer/create_disk.py b/src/node_deployer/create_disk.py index bd9ff20..30d4fcf 100644 --- a/src/node_deployer/create_disk.py +++ b/src/node_deployer/create_disk.py @@ -47,7 +47,10 @@ def validation_result() -> str: path=".", dockerfile=str(dockerfile), tag="validate", - buildargs={"CWD_MOUNTDIR": str(config.CWD_MOUNTDIR)}, + buildargs={ + "CWD_MOUNTDIR": str(config.CWD_MOUNTDIR), + "BUILD_DIR": str(config.BUILD_DIR.relative_to(config.PROJECT_ROOT)), + }, rm=config.CLEANUP_IMAGES, pull=True, quiet=True, @@ -57,7 +60,7 @@ def validation_result() -> str: mounts=[ config.CWD_MOUNT, ], - remove=True, + remove=config.CLEANUP_IMAGES, ) if config.CLEANUP_IMAGES: image.remove(force=True) diff --git a/src/node_deployer/create_img.py b/src/node_deployer/create_img.py index ce3f555..5a4e67f 100644 --- a/src/node_deployer/create_img.py +++ b/src/node_deployer/create_img.py @@ -31,7 +31,7 @@ def apply_ignition_settings( template: dict, hostname: str, password: str, - swarm_config: str, + swarm_config: dict, ) -> dict: """Applies the specified ignition settings to the given template @@ -46,8 +46,8 @@ def apply_ignition_settings( """ ignition_config = template.copy() ignition_config["hostname"] = hostname + ignition_config["login"]["users"][0]["passwd"] = password if password: - ignition_config["login"]["users"][0]["passwd"] = password ignition_config["login"]["users"][0]["hash_type"] = "bcrypt" elif not config.TESTING: raise ValueError("Password must be specified") @@ -66,7 +66,7 @@ def apply_ignition_settings( "source_type": "data", "mode": 420, "overwrite": True, - "data_content": swarm_config, + "data_content": json.dumps(swarm_config), }, { "path": "/root/join_swarm.sh", @@ -197,13 +197,11 @@ def create_img( password = "" # get swarm configuration as JSON - swarm_config = json.dumps( - { - "SWITCH_IP_ADDRESS": str(switch_ip), - "SWITCH_PORT": switch_port, - "SWARM_TOKEN": swarm_token, - } - ) + swarm_config = { + "SWITCH_IP_ADDRESS": str(switch_ip), + "SWITCH_PORT": switch_port, + "SWARM_TOKEN": swarm_token, + } # Create ignition configuration ignition_config = apply_ignition_settings( diff --git a/src/node_deployer/debug.py b/src/node_deployer/debug.py index 5449fba..437bf6b 100644 --- a/src/node_deployer/debug.py +++ b/src/node_deployer/debug.py @@ -7,6 +7,11 @@ import typer from .config import config +def get_debug_f(f: Callable) -> Callable: + import snoop # type: ignore + return wraps(f)(snoop.snoop(**config.snoop["snoop"])(f)) + + def debug_guard(f: Callable) -> Callable: """A decorator that contextually enables debug mode for the decorated function @@ -35,9 +40,10 @@ def debug_guard(f: Callable) -> Callable: **kwargs, ) -> Callable: typer.echo(f"Debug mode enabled: {inspect.stack()[1].filename}") + debug_f = get_debug_f(f) if kwargs.get("debug", False): # Snoop depth is set to compensate for wrapper stack frames - return snoop.snoop(**config.snoop["snoop"])(f)(*args, **kwargs) # noqa: F821 #* ss is installed in debug_mode + return debug_f(*args, **kwargs) # noqa: F821 #* ss is installed in debug_mode else: return f(*args, **kwargs) diff --git a/src/node_deployer/utils.py b/src/node_deployer/utils.py index f942230..6ee7740 100644 --- a/src/node_deployer/utils.py +++ b/src/node_deployer/utils.py @@ -1,6 +1,7 @@ from functools import wraps from pathlib import Path from typing import Callable +import docker from .config import config @@ -51,14 +52,22 @@ def next_free_tcp_port(port: int) -> int: Returns: int: The next free port """ - containers = config.CLIENT.containers.list(all=True) ports = [] - for container in containers: - port_values = container.ports.values() - if not port_values: - continue - for x in list(container.ports.values())[0]: - ports.append(int(x["HostPort"])) + try: + containers = config.CLIENT.containers.list(all=True) + ports = [] + for container in containers: + port_values = container.ports.values() + if not port_values: + continue + for x in list(container.ports.values())[0]: + ports.append(int(x["HostPort"])) + except docker.errors.NotFound: # type: ignore + #* This error is raised if container list changes between getting the list and + #* getting the ports. If this happens, just try again + return next_free_tcp_port(port) + if not ports: + return port ports = set(ports) while port in ports: port += 1 diff --git a/src/scripts/validate.sh b/src/scripts/validate.sh index 7202397..a6a34ad 100755 --- a/src/scripts/validate.sh +++ b/src/scripts/validate.sh @@ -1,2 +1,2 @@ ${CWD_MOUNTDIR}/src/scripts/fetch_config.sh -/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/build/config.ign \ No newline at end of file +/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/${BUILD_DIR}/config.ign \ No newline at end of file diff --git a/src/scripts/installs.sh b/src/scripts/validate_installs.sh similarity index 100% rename from src/scripts/installs.sh rename to src/scripts/validate_installs.sh diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl b/tests/data/node_deployer/create_img/apply_ignition_settings.pkl similarity index 64% rename from tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl rename to tests/data/node_deployer/create_img/apply_ignition_settings.pkl index 46d6001f7431be859a33a08fd9f39b5ace6f8c14..82f18df43ce310bb32bab3c55f04f2da195daa5e 100644 GIT binary patch delta 243 zcmdnbb&HFofn}=yL>2`ZjUJY~#N5;=JuF4}`6W|&*a{Mhi_24{^e{{*ooJlNCDg;> z;^7=HB|~K5vLr*f9_Gr_;wh!IQ{ei9vhp+Y;)}}@i*ogfQ;W(nlYxe@r{*Ol<)o%e z*{s8u%jnfJt6C{I+%v@4Bi=J0-qFP+$Tc`v$x1=V(9%fH(9A;5P|r|FM**fLz&|Ji zC~IN_7IF;ojSunncJ%{kn0${ZKv$_2?1Bu_9$|futM!qckTE60Ji{Wx(sv1ChE+#~ K_2ziy2aEv3h*75i delta 216 zcmcb`wV#Wnfn}=6L>2}99+tes+|(&OEJgYGB~ywgI%IMQ^su;iI0sD05Sq9-NkyiI zxiYnQN@?vBjSRLPp{)GOy!hhs#G+ii;?$zD%;eN588VyQ7;_n2fhu|ggTp;ToIT<_ z1L7TBT!LJKgQxUx8(JFa8Jbz>8R{7Vc~BJr{y`yAytP1l#~|PM5Pxr1zbQQdlm9XW u7)~h#TW8oKtPgUQKC-15Q!3Pm zJh93G2T_=T80>^yup8p=1b8A`2?=--JQ=QnB=8`>EQpW-3Fd%68ZwZDdB{N?7N7t! z6rltPs8EIqR6&D1uow2hQ(!+l6?8Dbgc?{-2OApTz#=rk1rK~^K^y2Abf60X^q>y| zSb`9i;c4)6cm`YzE3gXBglECC;Tm`jTnh)`x$r!AK3oScfEU7x;Cgs5+yFPiOW>vO zGB^Z7xCvekuYgy=tKik}8h9CN);JffWxC_1ycf$|hhwvl#G5iF63O|FN z!!O{M@GJN={O0J1)qA!aJapdZwu|uP2zD&C9Djdzpm)6Oi`|w#Jb!Xz4i8U^A}dyy zE~$*G6qrW6sPgrKB8y5>Eh!<{Nd&T#FOlASsXCKg6npbU(j~%u?G-UzR0Xt~EreMi z=_qWCSDI{H72{c1nQ`*Y+W&H@8051hF;K-uT4CH$L6uVqE91VYLb0UE3q?lF%S{X> zswGu)^D@h;ipwu3j9MxvF%cbkRb*4RE)`WdFDt5u0Tv2PRYpTaR+bh@P1C6t5{cpQ ziP4S={&&{h#(&M3gkrzyxxFlxZkU{%5j@u;slXtDoWgJ-5!jwqOb$qjt>r_W(IvGf zI7ypG>Y^^~wuvjZ^vb+#a=B2i@C=by){GYHs;Ahdul9(PS|)Spz7Z`rB-yfwz`K^@ zGGtcFXkK$ABRKPRNs2c#fexo(JETQ zz#^xG#sRaK3`{5K*Q9t|LtE2xU6T!JWNu&+Pn|KPcvus7Gb6-0TBX@uPzSB7kN$J2 zOXgMvc402p$`pxDgm`FhaVE>P`?zV{jMf^a$MkeR*EgJ|BiQY>=I};2Og1x;+^=$h zYdU<}@U%X?nHEjErofkKTq-n*{BF!Vs)tG6it-_y1Q7~hy=b<{EEC7cLBG^uZAEf9 ztw`*O%IqR@a!V`EWQahQ_dhpSJ~Q9}gL8XCU;@mvgLZa0m8%rgbgqKkr>BObx=kFr zv?}IEg>efNCMC+LG>%uUmUrk8JAJg*j%u>)8gn^!Wo<99CsnS|$5Sb~GyR(em&5WBx?P?TCIhV00TWwES4Q_mJL~46 z=pP84))wqUVtO({|Au!?jBeTxSlwWMqty+3-LvSSjSer}FxoV{b#ip!Sys?shQ}sH zH?O!g$sP#495W>)u2ryFm2^+56f$&qaqU&hIX7F5HH`A<&*zh^Oqp{BwMDrmcyiAS z*=0;7ta5f-@K}u3=xeiVqgIjo*j@t%n9I3LMw4R8*-FmE125{cOr~0v@r}BI^|9=M zQI0pY5;~WgTrSxT=z|zsAzsBE8iQ71=x!WcHcU?a`_cDJjUsE4IC|~W=&tn}&-^%V z@6sRVVCZcciRYaDp8#vo~U8}PleQhh4KDc%B2!0E{gWtm+;E(VpxEKBm ze}TWk-{9}?5BTSGTQ{$}9%!tm@c* zdj^Mk)5_}nj#hI@yV+^N8WBVYVMNe`X0)IcZHPji z*F+3)ByciLK|7L2p#z;*h%PL`sW=U%;|wfD8fW4xoQ-pEF3v*+=i>rgh$XlP7vmCi z<5KisDK5k1xB|V%;!0eFt8opk#dWwIH{eFxgqv{-ZpCf59eub1cj7MGjeBq}?!*09 zh6nH<`tcB!<6#Wo5j={=@Hn2plXwa%@HAH989a;U@H}3?i+Bky;}yJ$RalMJ@Hz(Z z2HwP5cpLBFUA%`icpo2N2*dagAK_!H#X79VC-@Yf;d6X}FYy&NU?aZ9H~1DK10(58 zC7GVValLug^1?_mG_^f$MndseQ)_d!yTF>5-B@h7dkd|K(ObKb%@kM$uV~0m#ln%M z)>tH*)e5b?#Y)ih8oFtubib!ITw`?V3lt;lZZSe;r4gQ;NCabE&U%ZcHY9_lE6M%; zyr