diff --git a/config.toml b/config.toml index 878b42a..bc2f812 100644 --- a/config.toml +++ b/config.toml @@ -11,6 +11,7 @@ CLIENT_STDOUT = true CLEANUP_IMAGES = true CLI = false DEBUG = false +TESTING = false [local] FUELIGNITION_URL = "http://localhost:3000/fuel-ignition/edit" @@ -25,6 +26,10 @@ CLI = true DEBUG = true CLI = false +[test] +TESTING = true +CLEANUP_IMAGES = false + [default.snoop.install] snoop = "ss" out = "snoop.log" diff --git a/pyproject.toml b/pyproject.toml index ef4efa5..360babc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "" authors = ["Cian Hughes "] readme = "README.md" packages = [ - {include = "node_deployer", from = "src"} + {include = "node_deployer", from = "src"}, ] [tool.poetry.dependencies] diff --git a/src/node_deployer/__init__.py b/src/node_deployer/__init__.py index e69de29..18cf1c3 100644 --- a/src/node_deployer/__init__.py +++ b/src/node_deployer/__init__.py @@ -0,0 +1,13 @@ +from . import ( + config, + autoignition, + create_img, + create_disk, +) + +__all__ = [ + "config", + "autoignition", + "create_img", + "create_disk", +] \ No newline at end of file diff --git a/src/node_deployer/autoignition.py b/src/node_deployer/autoignition.py index 247fd74..97439d7 100644 --- a/src/node_deployer/autoignition.py +++ b/src/node_deployer/autoignition.py @@ -10,23 +10,26 @@ import git from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support.ui import WebDriverWait # type: ignore import typer -from .config import config from .cli import cli_spinner +from .config import config from .debug import debug_guard -from .utils import ensure_build_dir +from .utils import ensure_build_dir, next_free_tcp_port -def create_driver() -> webdriver.Remote: +def create_driver(port: int) -> webdriver.Remote: """Creates a selenium webdriver instance + Args: + port (int): The port to connect to + Returns: webdriver.Remote: The created webdriver instance """ driver = webdriver.Remote( - "http://127.0.0.1:4444", + f"http://127.0.0.1:{port}", options=webdriver.FirefoxOptions(), ) driver.implicitly_wait(10) @@ -34,7 +37,7 @@ def create_driver() -> webdriver.Remote: def convert_json_via_fuelignition( - container: docker.models.containers.Container, + container: docker.models.containers.Container, # type: ignore driver: webdriver.Remote, fuelignition_json: Path, img_path: Path, @@ -88,7 +91,7 @@ def convert_json_via_fuelignition( f.write(container_image.read()) -def build_fuelignition() -> docker.models.images.Image: +def build_fuelignition() -> docker.models.images.Image: # type: ignore """Builds the fuel-ignition docker image Returns: @@ -196,12 +199,14 @@ def json_to_img( fuelignition_container = None fuelignition_image = None try: + driver_port = next_free_tcp_port(4444) # Initialise containers selenium_container = config.CLIENT.containers.run( "selenium/standalone-firefox:latest", detach=True, remove=True, - ports={4444: 4444, 7900: 7900}, + network_mode="bridge", + ports={4444: driver_port}, mounts=[ config.CWD_MOUNT, ], @@ -224,7 +229,7 @@ def json_to_img( ): time.sleep(0.1) # Now, create the webdriver and convert the json to an img - driver = create_driver() + driver = create_driver(driver_port) convert_json_via_fuelignition(selenium_container, driver, json_path, img_path) driver.quit() except Exception as e: diff --git a/src/node_deployer/config.py b/src/node_deployer/config.py index 1983578..060e173 100644 --- a/src/node_deployer/config.py +++ b/src/node_deployer/config.py @@ -8,7 +8,14 @@ import tomllib CLIENT = docker.from_env(version="auto") MAX_PORT: int = 65535 -PROJECT_ROOT: Path = Path(__file__).parent.parent.parent.absolute() + +def __get_project_root(): + r = Path(__file__) + while r.name != "src": + r = r.parent + return r.parent + +PROJECT_ROOT: Path = __get_project_root() ConfigLabel = Union[str, list[str]] # After PEP695 support: type ConfigLabel = str | list[str] @@ -63,19 +70,19 @@ class Config(SimpleNamespace): for k, v in config.items(): match k: case "SRC_DIR" | "BUILD_DIR": - config[k] = Path(v).absolute() + config[k] = Path(PROJECT_ROOT / v).absolute() case "CWD_MOUNTDIR": config[k] = Path(v) # Then, get required paths from config or globals if not present - build_dir = config.get("BUILD_DIR", self.BUILD_DIR) - cwd_mountdir = config.get("CWD_MOUNTDIR", self.CWD_MOUNTDIR) - src_dir = config.get("SRC_DIR", self.SRC_DIR) + build_dir = Path(config.get("BUILD_DIR", self.BUILD_DIR)).absolute() + cwd_mountdir = Path(config.get("CWD_MOUNTDIR", self.CWD_MOUNTDIR)) + src_dir = Path(config.get("SRC_DIR", self.SRC_DIR)).absolute() # Finally, construct the secondary parameters config["FUELIGNITION_BUILD_DIR"] = build_dir / config.get( "FUELIGNITION_BUILD_DIR", self.FUELIGNITION_BUILD_DIR ) config["DOCKERFILE_DIR"] = src_dir / config.get("DOCKERFILE_DIR", self.DOCKERFILE_DIR) - config["CWD_MOUNT"] = docker.types.Mount( + config["CWD_MOUNT"] = docker.types.Mount( # type: ignore <- I really wish docker-py had typeshed stubs target=str(cwd_mountdir), source=str(PROJECT_ROOT), type="bind", diff --git a/src/node_deployer/create_disk.py b/src/node_deployer/create_disk.py index 7ec2d2e..bd9ff20 100644 --- a/src/node_deployer/create_disk.py +++ b/src/node_deployer/create_disk.py @@ -1,6 +1,6 @@ from fnmatch import fnmatch import ipaddress -from typing import Annotated, Optional, Union +from typing import Annotated, Optional from docker.types import Mount import typer @@ -14,7 +14,7 @@ from .utils import ensure_build_dir # When PEP695 is supported this line should be: # type IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address -IPAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address] +IPAddress = ipaddress._IPAddressBase def filter_validation_response(response: str) -> str: @@ -89,6 +89,7 @@ def write_disk(disk: str) -> None: mounts=[config.CWD_MOUNT, Mount("/ignition_disk", disk, type="bind")], privileged=True, command=f"dd if={config.CWD_MOUNTDIR}/build/ignition.img of=/ignition_disk", + remove=config.CLEANUP_CONTAINERS, ) diff --git a/src/node_deployer/create_img.py b/src/node_deployer/create_img.py index a2f28d4..ce3f555 100644 --- a/src/node_deployer/create_img.py +++ b/src/node_deployer/create_img.py @@ -1,7 +1,7 @@ import ipaddress import json from pathlib import Path -from typing import Annotated, Optional, Union +from typing import Annotated, Optional import typer @@ -13,7 +13,7 @@ from .utils import ensure_build_dir # When PEP695 is supported this line should be: # type IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address -IPAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address] +IPAddress = ipaddress._IPAddressBase def load_template() -> dict: @@ -33,7 +33,7 @@ def apply_ignition_settings( password: str, swarm_config: str, ) -> dict: - """Applies the specified ignition settings to the template + """Applies the specified ignition settings to the given template Args: template (dict): The template to apply the settings to @@ -46,7 +46,11 @@ def apply_ignition_settings( """ ignition_config = template.copy() ignition_config["hostname"] = hostname - ignition_config["login"]["users"][0]["passwd"] = password + if password: + ignition_config["login"]["users"][0]["passwd"] = password + ignition_config["login"]["users"][0]["hash_type"] = "bcrypt" + elif not config.TESTING: + raise ValueError("Password must be specified") # Add files that will define a service to ensure that the node joins the swarm with open(config.SRC_DIR / "templates/join_swarm.sh", "r") as f1, open( @@ -186,6 +190,12 @@ def create_img( Enable debug mode. Defaults to False. """ + # Guards against the user not specifying a password + if password is None and not config.TESTING: + raise typer.BadParameter("Password must be specified") + elif password is None: + password = "" + # get swarm configuration as JSON swarm_config = json.dumps( { @@ -195,10 +205,6 @@ def create_img( } ) - # Guards against the user not specifying a password - if password is None: - raise typer.BadParameter("Password must be specified") - # Create ignition configuration ignition_config = apply_ignition_settings( load_template(), diff --git a/src/node_deployer/node_deployer.py b/src/node_deployer/node_deployer.py old mode 100755 new mode 100644 index 29b9c9a..387eddc --- a/src/node_deployer/node_deployer.py +++ b/src/node_deployer/node_deployer.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + from typing import Any, Dict import typer diff --git a/src/node_deployer/utils.py b/src/node_deployer/utils.py index 4da6c04..f942230 100644 --- a/src/node_deployer/utils.py +++ b/src/node_deployer/utils.py @@ -14,6 +14,7 @@ def ensure_build_dir(f: Callable) -> Callable: Returns: Callable: The decorated function """ + @wraps(f) def wrapper(*args, **kwargs): Path(config.BUILD_DIR).mkdir(exist_ok=True, parents=True) @@ -24,8 +25,9 @@ def ensure_build_dir(f: Callable) -> Callable: class Singleton(type): """A singleton metaclass""" + _instance = None - + def __call__(cls, *args, **kwargs): """Creates a new instance of the class if one does not already exist @@ -35,3 +37,32 @@ class Singleton(type): if cls._instance is None: cls._instance = super().__call__(*args, **kwargs) return cls._instance + + +def next_free_tcp_port(port: int) -> int: + """Finds the next free port after the specified port + + Args: + port (int): The port to start searching from + + Raises: + ValueError: If no free ports are found + + Returns: + int: The next free port + """ + containers = config.CLIENT.containers.list(all=True) + ports = [] + for container in containers: + port_values = container.ports.values() + if not port_values: + continue + for x in list(container.ports.values())[0]: + ports.append(int(x["HostPort"])) + ports = set(ports) + while port in ports: + port += 1 + if port > 65535: + raise ValueError("No free ports") + return port + \ No newline at end of file diff --git a/src/templates/fuelignition.json b/src/templates/fuelignition.json index 56a539c..c87a2e4 100644 --- a/src/templates/fuelignition.json +++ b/src/templates/fuelignition.json @@ -2,8 +2,7 @@ "login": { "users": [ { - "name": "root", - "hash_type": "bcrypt" + "name": "root" } ] }, diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_node_deployer/__init__.py b/tests/test_node_deployer/__init__.py new file mode 100644 index 0000000..2367253 --- /dev/null +++ b/tests/test_node_deployer/__init__.py @@ -0,0 +1,5 @@ +from node_deployer.config import config + + +config.update_config("test") +config.BUILD_DIR = config.BUILD_DIR / "tests" diff --git a/tests/test_node_deployer/data/config.ign b/tests/test_node_deployer/data/config.ign new file mode 100644 index 0000000..dc780cc --- /dev/null +++ b/tests/test_node_deployer/data/config.ign @@ -0,0 +1,76 @@ +{ + "ignition": { + "version": "3.2.0" + }, + "passwd": { + "users": [ + { + "name": "root", + "passwordHash": "$2a$08$YdxsvKFRc1q2S1lmVixm5Oel3Y2oCNAsx9Sh2Dx4pL/udApPzUQu6" + } + ] + }, + "storage": { + "files": [ + { + "path": "/root/join_swarm.json", + "mode": 420, + "overwrite": true, + "contents": { + "source": "data:text/plain;charset=utf-8;base64,eyJTV0lUQ0hfSVBfQUREUkVTUyI6ICIxOTIuMTY4LjEuMSIsICJTV0lUQ0hfUE9SVCI6IDQyLCAiU1dBUk1fVE9LRU4iOiAiU1dNVEtOLTEtVEhJU0lTQVRFU1RTV0FSTVRPS0VORk9SVEVTVElOR1BVUlBPU0VTQU5EVEhBVE1FQU5TSVRORUVEU1RPQkVRVUlURUxPTkcifQ==" + } + }, + { + "path": "/root/join_swarm.sh", + "mode": 420, + "overwrite": true, + "contents": { + "source": "data:text/plain;charset=utf-8;base64,IyEvYmluL2Jhc2gKCmlmIFtbICRFVUlEIC1uZSAwIF1dOyB0aGVuCiAgIGVjaG8gIlRoaXMgc2NyaXB0IG11c3QgYmUgcnVuIGFzIHJvb3QiIAogICBleGl0IDEKZmkKCiMgTG9hZCB0aGUgY29uZmlnIGZpbGUgaW50byB2YXJpYWJsZXMKZXZhbCAiJChqcSAtciAndG9fZW50cmllc1tdIHwgImV4cG9ydCBcKC5rZXkpPVwoLnZhbHVlIHwgQHNoKSInIC9yb290L2pvaW5fc3dhcm0uanNvbikiCgppZiBbWyAkKGRvY2tlciBpbmZvIHwgZ3JlcCBTd2FybSB8IGF3ayAne3ByaW50ICQyfScpID09ICJpbmFjdGl2ZSIgXV07IHRoZW4KICAgIGRvY2tlciBzd2FybSBqb2luIC0tdG9rZW4gJFNXQVJNX1RPS0VOIFskU1dJVENIX0lQX0FERFJFU1NdOiRTV0lUQ0hfUE9SVAplbHNlCiAgICBlY2hvICJUaGlzIG5vZGUgaXMgYWxyZWFkeSBwYXJ0IG9mIGEgc3dhcm0iCiAgICBkb2NrZXIgaW5mbyAtZiBqc29uIHwganEgLlN3YXJtCmZpCg==" + } + }, + { + "path": "/etc/hostname", + "mode": 420, + "overwrite": true, + "contents": { + "source": "data:,test_hostname" + } + }, + { + "path": "/etc/NetworkManager/system-connections/eth0.nmconnection", + "mode": 384, + "overwrite": true, + "contents": { + "source": "data:text/plain;charset=utf-8;base64,Cltjb25uZWN0aW9uXQppZD1ldGgwCnR5cGU9ZXRoZXJuZXQKaW50ZXJmYWNlLW5hbWU9ZXRoMAoKW2lwdjRdCmRucy1zZWFyY2g9Cm1ldGhvZD1hdXRvCgpbaXB2Nl0KZG5zLXNlYXJjaD0KYWRkci1nZW4tbW9kZT1ldWk2NAptZXRob2Q9YXV0bwo=", + "human_read": "\n[connection]\nid=eth0\ntype=ethernet\ninterface-name=eth0\n\n[ipv4]\ndns-search=\nmethod=auto\n\n[ipv6]\ndns-search=\naddr-gen-mode=eui64\nmethod=auto\n" + } + }, + { + "path": "/etc/NetworkManager/conf.d/noauto.conf", + "mode": 420, + "overwrite": true, + "contents": { + "source": "data:text/plain;charset=utf-8;base64,W21haW5dCiMgRG8gbm90IGRvIGF1dG9tYXRpYyAoREhDUC9TTEFBQykgY29uZmlndXJhdGlvbiBvbiBldGhlcm5ldCBkZXZpY2VzCiMgd2l0aCBubyBvdGhlciBtYXRjaGluZyBjb25uZWN0aW9ucy4Kbm8tYXV0by1kZWZhdWx0PSoK", + "human_read": "[main]\n# Do not do automatic (DHCP/SLAAC) configuration on ethernet devices\n# with no other matching connections.\nno-auto-default=*\n" + } + } + ] + }, + "systemd": { + "units": [ + { + "name": "cockpit.socket.service", + "enabled": true + }, + { + "name": "docker.service", + "enabled": true + }, + { + "name": "join_swarm.service", + "enabled": false, + "contents": "[Unit]\nDescription=Ensure that node joins a swarm on startup\n\n[Service]\nExecStart=/root/join_swarm.sh\n\n[Install]\nWantedBy=multi-user.target" + } + ] + } +} \ No newline at end of file diff --git a/tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl b/tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl new file mode 100644 index 0000000..46d6001 Binary files /dev/null and b/tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl differ diff --git a/tests/test_node_deployer/data/create_img/load_template.pkl b/tests/test_node_deployer/data/create_img/load_template.pkl new file mode 100644 index 0000000..f0190a9 Binary files /dev/null and b/tests/test_node_deployer/data/create_img/load_template.pkl differ diff --git a/tests/test_node_deployer/data/fuelignition.json b/tests/test_node_deployer/data/fuelignition.json new file mode 100644 index 0000000..0089c6f --- /dev/null +++ b/tests/test_node_deployer/data/fuelignition.json @@ -0,0 +1,65 @@ +{ + "login": { + "users": [ + { + "name": "root" + } + ] + }, + "network": { + "interfaces": [ + { + "name": "eth0", + "ipv4": { + "network_type": "DHCP", + "auto_dns_enabled": true + }, + "ipv6": { + "network_type": "DHCP", + "auto_dns_enabled": true + } + } + ] + }, + "systemd": { + "units": [ + { + "name": "cockpit.socket.service", + "enabled": "yes" + }, + { + "name": "docker.service", + "enabled": "yes" + }, + { + "name": "join_swarm.service", + "enabled": true, + "contents": "[Unit]\nDescription=Ensure that node joins a swarm on startup\n\n[Service]\nExecStart=/root/join_swarm.sh\n\n[Install]\nWantedBy=multi-user.target" + } + ] + }, + "package": { + "install": [ + "patterns-microos-cockpit, docker, jq" + ] + }, + "hostname": "test_hostname", + "storage": { + "files": [ + { + "path": "/root/join_swarm.json", + "source_type": "data", + "mode": 420, + "overwrite": true, + "data_content": "{\"SWITCH_IP_ADDRESS\": \"192.168.1.1\", \"SWITCH_PORT\": 42, \"SWARM_TOKEN\": \"SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG\"}" + }, + { + "path": "/root/join_swarm.sh", + "source_type": "data", + "mode": 420, + "overwrite": true, + "data_content": "#!/bin/bash\n\nif [[ $EUID -ne 0 ]]; then\n echo \"This script must be run as root\" \n exit 1\nfi\n\n# Load the config file into variables\neval \"$(jq -r 'to_entries[] | \"export \\(.key)=\\(.value | @sh)\"' /root/join_swarm.json)\"\n\nif [[ $(docker info | grep Swarm | awk '{print $2}') == \"inactive\" ]]; then\n docker swarm join --token $SWARM_TOKEN [$SWITCH_IP_ADDRESS]:$SWITCH_PORT\nelse\n echo \"This node is already part of a swarm\"\n docker info -f json | jq .Swarm\nfi\n" + } + ] + } +} \ No newline at end of file diff --git a/tests/test_node_deployer/data/ignition.img b/tests/test_node_deployer/data/ignition.img new file mode 100644 index 0000000..a3262ee Binary files /dev/null and b/tests/test_node_deployer/data/ignition.img differ diff --git a/tests/test_node_deployer/test_autoignition.py b/tests/test_node_deployer/test_autoignition.py new file mode 100644 index 0000000..d7b0deb --- /dev/null +++ b/tests/test_node_deployer/test_autoignition.py @@ -0,0 +1,16 @@ +import filecmp + +from node_deployer import autoignition +from node_deployer.config import config + + +class TestAutoignition: + def test_json_to_img(self, tmpdir): + autoignition.json_to_img( + config.PROJECT_ROOT / "tests/test_node_deployer/data/fuelignition.json", + tmpdir / "ignition.img", + ) + assert filecmp.cmp( + config.PROJECT_ROOT / "tests/test_node_deployer/data/ignition.img", + tmpdir / "ignition.img", + ) diff --git a/tests/test_node_deployer/test_create_img.py b/tests/test_node_deployer/test_create_img.py new file mode 100644 index 0000000..87f07c6 --- /dev/null +++ b/tests/test_node_deployer/test_create_img.py @@ -0,0 +1,50 @@ +import filecmp +import pickle + +from node_deployer import create_img +from node_deployer.config import config + + +class TestCreateImg: + def test_load_template(self): + template = create_img.load_template() + with open( + config.PROJECT_ROOT / "tests/test_node_deployer/data/create_img/load_template.pkl", "rb" + ) as f: + assert pickle.load(f) == template + + def test_apply_ignition_settings(self): + with open( + config.PROJECT_ROOT / "tests/test_node_deployer/data/create_img/load_template.pkl", + mode="rb", + ) as f: + template = pickle.load(f) + test_result = create_img.apply_ignition_settings( + template, + "test_hostname", + "", + { + "SWITCH_IP_ADDRESS": "192.168.1.1", + "SWITCH_PORT": 42, + "SWARM_TOKEN": "SWMTKN-1-THISISATESTSWARMTOKENFORTESTINGPURPOSESANDTHATMEANSITNEEDSTOBEQUITELONG", # noqa: E501 + }, + ) + with open( + config.PROJECT_ROOT + / "tests/test_node_deployer/data/create_img/apply_ignition_settings.pkl", + mode="rb", + ) as f: + assert pickle.load(f) == test_result + + def test_create_img(self, tmpdir): + create_img.create_img( + hostname="test_hostname", + password="", + switch_ip="192.168.1.1", + switch_port=42, + img_path=tmpdir / "ignition.img", + ) + assert filecmp.cmp( + tmpdir / "ignition.img", + config.PROJECT_ROOT / "tests/test_node_deployer/data/ignition.img", + )