Fully working end-to-end locally with validation

autoignition.py

@@ -1,11 +1,9 @@
 from fnmatch import fnmatch
 import io
-from pathlib import Path
 import tarfile
 import time
 from typing import Annotated
 
-import docker
 import git
 from selenium import webdriver
 from selenium.webdriver.common.by import By
@@ -13,17 +11,19 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 import typer
 
-from debug import debug_mode
+from config import (
-
+    CLEANUP_IMAGES,
-
+    CLIENT,
-CLIENT = docker.from_env(version="auto")
+    CWD_MOUNTDIR,
-SELENIUM_INIT_MESSAGE = "INFO [Standalone.execute] - Started Selenium Standalone"
+    DOCKERFILE_DIR,
-FUELIGNITION_INIT_MESSAGE = "ready in *ms."
+    FUELIGNITION_BUILD_DIR,
-FUELIGNITION_BUILD_DIR = Path("build/fuel-ignition")
+    FUELIGNITION_INIT_MESSAGE,
-FUELIGNITION_URL = (
+    FUELIGNITION_URL,
-    "http://localhost:3000/fuel-ignition/edit"  # "https://opensuse.github.io/fuel-ignition/edit"
+    ROOT_DIR,
+    SELENIUM_INIT_MESSAGE,
 )
-CWD_MOUNTDIR = Path("/host_cwd")
+from debug import debug_mode
+import docker
 
 
 def create_driver():
@@ -62,7 +62,7 @@ def convert_json_via_fuelignition(container, driver, fuelignition_json, img_path
     image_file = container.exec_run("ls /home/seluser/Downloads/").output.decode().split()[0]
     # Finally, fetch the image file from the container
     client_image_path = f"/home/seluser/Downloads/{image_file}"
-    host_image_path = Path().cwd() / img_path
+    host_image_path = ROOT_DIR / img_path
     if host_image_path.exists():
         host_image_path.unlink()
     filestream = container.get_archive(client_image_path)[0]
@@ -103,21 +103,24 @@ def build_fuelignition():
     root_container = (engine_version[0] > 9) or (engine_version[0] == 9 and engine_version[1] >= 3)
     dockerfile = "Dockerfile"
     if root_container:
-        dockerfile = "../../templates/patched.dockerfile"
+        dockerfile = DOCKERFILE_DIR / "fuel-ignition.dockerfile"
-    CLIENT.images.build(
+    image = CLIENT.images.build(
         path=str(FUELIGNITION_BUILD_DIR),
-        dockerfile=dockerfile,
+        dockerfile=str(dockerfile),
         tag="fuel-ignition",
         network_mode="host",
         buildargs={"CONTAINER_USERID": "1000"},
-        rm=True,
+        pull=True,
-        quiet=False,
+        quiet=True,
+        rm=CLEANUP_IMAGES,
     )
+    return image
 
 
 def json_to_img(fuelignition_json: str, img_path: str) -> None:
     selenium_container = None
     fuelignition_container = None
+    fuelignition_image = None
     try:
         # Initialise containers
         selenium_container = CLIENT.containers.run(
@@ -128,14 +131,14 @@ def json_to_img(fuelignition_json: str, img_path: str) -> None:
             mounts=[
                 docker.types.Mount(
                     target=str(CWD_MOUNTDIR),
-                    source=str(Path.cwd().absolute()),
+                    source=str(ROOT_DIR),
                     type="bind",
                 )
             ],
         )
-        build_fuelignition()
+        fuelignition_image = build_fuelignition()
         fuelignition_container = CLIENT.containers.run(
-            "fuel-ignition",
+            fuelignition_image,
             detach=True,
             remove=True,
             network_mode=f"container:{selenium_container.id}",
@@ -143,6 +146,8 @@ def json_to_img(fuelignition_json: str, img_path: str) -> None:
         # Wait for the containers to finish starting up
         while SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode():
             time.sleep(0.1)
+            for event in CLIENT.events(decode=True):
+                print(event)
         while not fnmatch(
             fuelignition_container.logs().decode().strip().split("\n")[-1].strip(),
             FUELIGNITION_INIT_MESSAGE,
@@ -156,9 +161,15 @@ def json_to_img(fuelignition_json: str, img_path: str) -> None:
         raise e
     finally:
         if selenium_container is not None:
+            selenium_image = selenium_container.image
             selenium_container.kill()
+            if CLEANUP_IMAGES:
+                selenium_image.remove(force=True)
         if fuelignition_container is not None:
             fuelignition_container.kill()
+        if fuelignition_image is not None:
+            if CLEANUP_IMAGES:
+                fuelignition_image.remove(force=True)
 
 
 def main(

client_stdout.py

@@ -0,0 +1,11 @@
+import asyncio
+import sys
+
+
+async def stdout_pipe(client):
+    events = client.events(decode=True)
+    while True:
+        for event in events:
+            sys.stdout.write(event)
+        sys.stdout.flush()
+        await asyncio.sleep(0.1)

config.py

@@ -0,0 +1,33 @@
+import asyncio
+from pathlib import Path
+
+import tomllib
+
+from client_stdout import stdout_pipe
+import docker
+
+
+def get_config(config: str = "default") -> dict:
+    with open("config.toml", "rb") as f:
+        configs: dict = tomllib.load(f)
+    out_config: dict = configs["default"]
+    out_config.update(configs[config])
+    return out_config
+
+
+def apply_config(config: dict) -> None:
+    config["CLIENT"] = docker.from_env(version="auto")
+    config["ROOT_DIR"] = Path(config["ROOT_DIR"]).absolute()
+    config["BUILD_DIR"] = Path(config["BUILD_DIR"]).absolute()
+    config["DOCKERFILE_DIR"] = Path(config["DOCKERFILE_DIR"]).absolute()
+    config["CWD_MOUNTDIR"] = Path(config["CWD_MOUNTDIR"])
+    config["FUELIGNITION_BUILD_DIR"] = config["BUILD_DIR"] / config["FUELIGNITION_BUILD_DIR"]
+    if config["CLIENT_STDOUT"]:
+        asyncio.run(stdout_pipe(config["CLIENT"]))
+    globals().update(config)
+    
+
+def init(config: str = "default") -> None:
+    apply_config(get_config(config))
+
+init()

config.toml

@@ -0,0 +1,17 @@
+[default]
+ROOT_DIR = "."
+BUILD_DIR = "build"
+DOCKERFILE_DIR = "docker"
+SELENIUM_INIT_MESSAGE = "INFO [Standalone.execute] - Started Selenium Standalone"
+FUELIGNITION_INIT_MESSAGE = "ready in *ms."
+FUELIGNITION_URL = "http://localhost:3000/fuel-ignition/edit"
+FUELIGNITION_BUILD_DIR = "fuel-ignition"
+CWD_MOUNTDIR = "/host_cwd"
+CLIENT_STDOUT = true
+CLEANUP_IMAGES = false
+
+[local]
+FUELIGNITION_URL = "http://localhost:3000/fuel-ignition/edit"
+
+[remote]
+FUELIGNITION_URL = "https://opensuse.github.io/fuel-ignition/edit"

create_img.py

@@ -0,0 +1,129 @@
+import ipaddress
+import json
+from typing import Annotated
+
+import typer
+
+from autoignition import json_to_img
+from debug import debug_mode
+
+
+MAX_PORT: int = 65535
+
+
+def load_template() -> dict:
+    with open("templates/fuelignition.json", "r") as f:
+        out = json.load(f)
+    return out
+
+
+def apply_ignition_settings(
+    template: dict,
+    hostname: str,
+    password: str,
+    swarm_config: str,
+) -> dict:
+    ignition_config = template.copy()
+    ignition_config["hostname"] = hostname
+    ignition_config["login"]["users"][0]["passwd"] = password
+
+    # Add files that will define a service to ensure that the node joins the swarm
+    with open("templates/join_swarm.sh", "r") as f1, open(
+        "templates/join_swarm.service", "r"
+    ) as f2:
+        swarm_script, swarm_service = f1.read(), f2.read()
+
+    ignition_config["storage"] = ignition_config.get("storage", {})
+    ignition_config["storage"]["files"] = ignition_config["storage"].get("files", [])
+    ignition_config["storage"]["files"] += [
+        {
+            "path": "/root/join_swarm.json",
+            "source_type": "data",
+            "mode": 420,
+            "overwrite": True,
+            "data_content": swarm_config,
+        },
+        {
+            "path": "/root/join_swarm.sh",
+            "source_type": "data",
+            "mode": 420,
+            "overwrite": True,
+            "data_content": swarm_script,
+        },
+    ]
+
+    ignition_config["systemd"] = ignition_config.get("systemd", {})
+    ignition_config["systemd"]["units"] = ignition_config["systemd"].get("units", [])
+    ignition_config["systemd"]["units"] += [
+        {
+            "name": "join_swarm.service",
+            "enabled": True,
+            "contents": swarm_service,
+        },
+    ]
+
+    return ignition_config
+
+
+def create_img(
+    hostname: str, password: str, switch_ip_address: str, switch_port: str, swarm_token: str
+) -> None:
+    switch_ip_address = ipaddress.ip_address(switch_ip_address)
+    if switch_port > MAX_PORT:
+        raise ValueError(f"Port must be less than {MAX_PORT}")
+
+    # get swarm configuration as JSON
+    swarm_config = json.dumps(
+        {
+            "SWITCH_IP_ADDRESS": str(switch_ip_address),
+            "SWITCH_PORT": switch_port,
+            "SWARM_TOKEN": swarm_token,
+        }
+    )
+
+    # Create ignition configuration
+    ignition_config = load_template()
+    ignition_config = apply_ignition_settings(
+        ignition_config,
+        hostname,
+        password,
+        swarm_config,
+    )
+
+    # export ignition configuration
+    with open("build/fuelignition.json", "w") as f:
+        json.dump(ignition_config, f, indent=4)
+
+    # convert ignition configuration to image
+    json_to_img("build/fuelignition.json", "build/ignition.img")
+
+
+def main(
+    hostname: Annotated[str, typer.Option(help="Hostname for the new node", prompt=True)],
+    password: Annotated[
+        str,
+        typer.Option(
+            help="Password for the root user on the new node",
+            prompt=True,
+            confirmation_prompt=True,
+            hide_input=True,
+        ),
+    ],
+    switch_ip_address: Annotated[
+        str, typer.Option(help="IP address of the switch to connect to", prompt=True)
+    ],
+    switch_port: Annotated[int, typer.Option(help="Port on the switch to connect to", prompt=True)],
+    swarm_token: Annotated[
+        str, typer.Option(help="Swarm token for connecting to the swarm", prompt=True)
+    ],
+    debug: Annotated[bool, typer.Option(help="Enable debug mode")] = False,
+) -> None:
+    debug_mode(debug)
+    f = create_img
+    if debug:
+        f = ss(f)  # noqa: F821, # type: ignore #? ss is installed in debug_mode
+    f(hostname, password, switch_ip_address, switch_port, swarm_token)
+
+
+if __name__ == "__main__":
+    typer.run(main)

docker/validate.dockerfile

@@ -0,0 +1,12 @@
+FROM quay.io/coreos/ignition-validate:release AS ignition-validate
+FROM alpine:latest as base
+
+ARG CWD_MOUNTDIR
+ENV CWD_MOUNTDIR=$CWD_MOUNTDIR
+
+COPY --from=ignition-validate . .
+COPY scripts/installs.sh /installs.sh
+
+RUN /installs.sh
+
+CMD $CWD_MOUNTDIR/scripts/validate.sh

main.py

@@ -1,101 +1,62 @@
-import ipaddress
+from fnmatch import fnmatch
-import json
 from typing import Annotated
 
 import typer
 
-from autoignition import json_to_img
+from config import (
+    CLEANUP_IMAGES,
+    CLIENT,
+    CWD_MOUNTDIR,
+    DOCKERFILE_DIR,
+    ROOT_DIR,
+)
+from create_img import create_img
 from debug import debug_mode
+import docker
 
 
-MAX_PORT: int = 65535
+def filter_validation_response(response: str) -> str:
+    return "\n".join(
+        filter(
+            # Filter out the warning about unused key human_readable, this always exists in
+            # configurations produced by fuel-ignition
+            lambda x: not fnmatch(x.strip(), "warning at*Unused key human_read"),
+            response.split("\n"),
+        )
+    ).strip()
 
 
-def load_template() -> dict:
+def validation_result() -> str:
-    with open("templates/fuelignition.json", "r") as f:
+    dockerfile = DOCKERFILE_DIR / "validate.dockerfile"
-        out = json.load(f)
+    image = CLIENT.images.build(
-    return out
+        path=".",
-
+        dockerfile=str(dockerfile),
-
+        tag="validate",
-def apply_ignition_settings(
+        buildargs={"CWD_MOUNTDIR": str(CWD_MOUNTDIR)},
-    template: dict,
+        rm=CLEANUP_IMAGES,
-    hostname: str,
+        pull=True,
-    password: str,
+        quiet=True,
-    swarm_config: str,
-) -> dict:
-    ignition_config = template.copy()
-    ignition_config["hostname"] = hostname
-    ignition_config["login"]["users"][0]["passwd"] = password
-
-    # Add files that will define a service to ensure that the node joins the swarm
-    with open("templates/join_swarm.sh", "r") as f1, open(
-        "templates/join_swarm.service", "r"
-    ) as f2:
-        swarm_script, swarm_service = f1.read(), f2.read()
-
-    ignition_config["storage"] = ignition_config.get("storage", {})
-    ignition_config["storage"]["files"] = ignition_config["storage"].get("files", [])
-    ignition_config["storage"]["files"] += [
-        {
-            "path": "/root/join_swarm.json",
-            "source_type": "data",
-            "mode": 420,
-            "overwrite": True,
-            "data_content": swarm_config,
-        },
-        {
-            "path": "/root/join_swarm.sh",
-            "source_type": "data",
-            "mode": 420,
-            "overwrite": True,
-            "data_content": swarm_script,
-        },
-    ]
-
-    ignition_config["systemd"] = ignition_config.get("systemd", {})
-    ignition_config["systemd"]["units"] = ignition_config["systemd"].get("units", [])
-    ignition_config["systemd"]["units"] += [
-        {
-            "name": "join_swarm.service",
-            "enabled": True,
-            "contents": swarm_service,
-        },
-    ]
-
-    return ignition_config
-
-
-def create_img(
-    hostname: str, password: str, switch_ip_address: str, switch_port: str, swarm_token: str
-) -> None:
-    switch_ip_address = ipaddress.ip_address(switch_ip_address)
-    if switch_port > MAX_PORT:
-        raise ValueError(f"Port must be less than {MAX_PORT}")
-
-    # get swarm configuration as JSON
-    swarm_config = json.dumps(
-        {
-            "SWITCH_IP_ADDRESS": str(switch_ip_address),
-            "SWITCH_PORT": switch_port,
-            "SWARM_TOKEN": swarm_token,
-        }
     )
-
+    response = CLIENT.containers.run(
-    # Create ignition configuration
+        image,
-    ignition_config = load_template()
+        mounts=[
-    ignition_config = apply_ignition_settings(
+            docker.types.Mount(
-        ignition_config,
+                target=str(CWD_MOUNTDIR),
-        hostname,
+                source=str(ROOT_DIR),
-        password,
+                type="bind",
-        swarm_config,
+            )
+        ],
+        remove=True,
     )
+    if CLEANUP_IMAGES:
+        image.remove(force=True)
+    return response
 
-    # export ignition configuration
-    with open("build/fuelignition.json", "w") as f:
-        json.dump(ignition_config, f, indent=4)
 
-    # convert ignition configuration to image
+def validate() -> (bool, str):
-    json_to_img("build/fuelignition.json", "build/ignition.img")
+    response = validation_result().decode()
+    response = filter_validation_response(response)
+    return (not bool(response), response)
 
 
 def main(
@@ -119,10 +80,17 @@ def main(
     debug: Annotated[bool, typer.Option(help="Enable debug mode")] = False,
 ) -> None:
     debug_mode(debug)
-    f = create_img
+    # f = create_img
-    if debug:
+    # if debug:
-        f = ss(f)  # noqa: F821, # type: ignore #? ss is installed in debug_mode
+    #     f = ss(f)  # noqa: F821, # type: ignore #? ss is installed in debug_mode
-    f(hostname, password, switch_ip_address, switch_port, swarm_token)
+    # f(hostname, password, switch_ip_address, switch_port, swarm_token)
+    create_img(hostname, password, switch_ip_address, switch_port, swarm_token)
+    valid, response = validate()
+    if not valid:
+        print(response)
+        raise typer.Exit(1)
+    else:
+        print("Valid ignition image created!")
 
 
 if __name__ == "__main__":

poetry.lock

@@ -341,6 +341,41 @@ files = [
 [package.extras]
 tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"]
 
+[[package]]
+name = "fsspec"
+version = "2023.10.0"
+description = "File-system specification"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "fsspec-2023.10.0-py3-none-any.whl", hash = "sha256:346a8f024efeb749d2a5fca7ba8854474b1ff9af7c3faaf636a4548781136529"},
+    {file = "fsspec-2023.10.0.tar.gz", hash = "sha256:330c66757591df346ad3091a53bd907e15348c2ba17d63fd54f5c39c4457d2a5"},
+]
+
+[package.extras]
+abfs = ["adlfs"]
+adl = ["adlfs"]
+arrow = ["pyarrow (>=1)"]
+dask = ["dask", "distributed"]
+devel = ["pytest", "pytest-cov"]
+dropbox = ["dropbox", "dropboxdrivefs", "requests"]
+full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"]
+fuse = ["fusepy"]
+gcs = ["gcsfs"]
+git = ["pygit2"]
+github = ["requests"]
+gs = ["gcsfs"]
+gui = ["panel"]
+hdfs = ["pyarrow (>=1)"]
+http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"]
+libarchive = ["libarchive-c"]
+oci = ["ocifs"]
+s3 = ["s3fs"]
+sftp = ["paramiko"]
+smb = ["smbprotocol"]
+ssh = ["paramiko"]
+tqdm = ["tqdm"]
+
 [[package]]
 name = "gitdb"
 version = "4.0.11"
@@ -965,4 +1000,4 @@ h11 = ">=0.9.0,<1"
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.11"
-content-hash = "c9be19a0841acc152e4c99c50539946b8831ec94ce94d907c463f4630d05d2a0"
+content-hash = "80f545060bae202b15081561a4df464f1c1f7dae9aff52c76f2e70c144628a35"

pyproject.toml

@@ -13,6 +13,7 @@ mechanicalsoup = "^1.3.0"
 docker = "^6.1.3"
 requests = "^2.31.0"
 gitpython = "^3.1.40"
+fsspec = "^2023.10.0"
 
 
 [tool.poetry.group.dev.dependencies]

scripts/fetch_config.sh

@@ -0,0 +1 @@
+mcopy -n -i ${CWD_MOUNTDIR}/build/ignition.img ::ignition/config.ign ${CWD_MOUNTDIR}/build/config.ign

scripts/installs.sh

@@ -0,0 +1,2 @@
+apk update
+apk add mtools

scripts/validate.sh

@@ -0,0 +1,2 @@
+${CWD_MOUNTDIR}/scripts/fetch_config.sh
+/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/build/config.ign