Cian-H/I-Form_Server_Node_Deployer
1
0
Fork 0
mirror of https://github.com/Cian-H/I-Form_Server_Node_Deployer.git synced 2025-12-23 14:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fully working end-to-end locally with validation

Browse Source
This commit is contained in:
Cian Hughes
2023-10-26 17:29:42 +01:00
parent 2a8d3c21fe
commit c14c0742b0
13 changed files with 333 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
autoignition.py
View File

@@ -1,11 +1,9 @@
from fnmatch import fnmatch
import io
from pathlib import Path
import tarfile
import time
from typing import Annotated
import docker
import git
from selenium import webdriver
from selenium.webdriver.common.by import By
@@ -13,17 +11,19 @@ from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
import typer
from debug import debug_mode
CLIENT = docker.from_env(version="auto")
SELENIUM_INIT_MESSAGE = "INFO [Standalone.execute] - Started Selenium Standalone"
FUELIGNITION_INIT_MESSAGE = "ready in *ms."
FUELIGNITION_BUILD_DIR = Path("build/fuel-ignition")
FUELIGNITION_URL = (
"http://localhost:3000/fuel-ignition/edit" # "https://opensuse.github.io/fuel-ignition/edit"
from config import (
CLEANUP_IMAGES,
CLIENT,
CWD_MOUNTDIR,
DOCKERFILE_DIR,
FUELIGNITION_BUILD_DIR,
FUELIGNITION_INIT_MESSAGE,
FUELIGNITION_URL,
ROOT_DIR,
SELENIUM_INIT_MESSAGE,
)
CWD_MOUNTDIR = Path("/host_cwd")
from debug import debug_mode
import docker
def create_driver():
@@ -62,7 +62,7 @@ def convert_json_via_fuelignition(container, driver, fuelignition_json, img_path
image_file = container.exec_run("ls /home/seluser/Downloads/").output.decode().split()[0]
# Finally, fetch the image file from the container
client_image_path = f"/home/seluser/Downloads/{image_file}"
host_image_path = Path().cwd() / img_path
host_image_path = ROOT_DIR / img_path
if host_image_path.exists():
host_image_path.unlink()
filestream = container.get_archive(client_image_path)[0]
@@ -103,21 +103,24 @@ def build_fuelignition():
root_container = (engine_version[0] > 9) or (engine_version[0] == 9 and engine_version[1] >= 3)
dockerfile = "Dockerfile"
if root_container:
dockerfile = "../../templates/patched.dockerfile"
CLIENT.images.build(
dockerfile = DOCKERFILE_DIR / "fuel-ignition.dockerfile"
image = CLIENT.images.build(
path=str(FUELIGNITION_BUILD_DIR),
dockerfile=dockerfile,
dockerfile=str(dockerfile),
tag="fuel-ignition",
network_mode="host",
buildargs={"CONTAINER_USERID": "1000"},
rm=True,
quiet=False,
pull=True,
quiet=True,
rm=CLEANUP_IMAGES,
)
return image
def json_to_img(fuelignition_json: str, img_path: str) -> None:
selenium_container = None
fuelignition_container = None
fuelignition_image = None
try:
# Initialise containers
selenium_container = CLIENT.containers.run(
@@ -128,14 +131,14 @@ def json_to_img(fuelignition_json: str, img_path: str) -> None:
mounts=[
docker.types.Mount(
target=str(CWD_MOUNTDIR),
source=str(Path.cwd().absolute()),
source=str(ROOT_DIR),
type="bind",
)
],
)
build_fuelignition()
fuelignition_image = build_fuelignition()
fuelignition_container = CLIENT.containers.run(
"fuel-ignition",
fuelignition_image,
detach=True,
remove=True,
network_mode=f"container:{selenium_container.id}",
@@ -143,6 +146,8 @@ def json_to_img(fuelignition_json: str, img_path: str) -> None:
# Wait for the containers to finish starting up
while SELENIUM_INIT_MESSAGE not in selenium_container.logs().decode():
time.sleep(0.1)
for event in CLIENT.events(decode=True):
print(event)
while not fnmatch(
fuelignition_container.logs().decode().strip().split("\n")[-1].strip(),
FUELIGNITION_INIT_MESSAGE,
@@ -156,9 +161,15 @@ def json_to_img(fuelignition_json: str, img_path: str) -> None:
raise e
finally:
if selenium_container is not None:
selenium_image = selenium_container.image
selenium_container.kill()
if CLEANUP_IMAGES:
selenium_image.remove(force=True)
if fuelignition_container is not None:
fuelignition_container.kill()
if fuelignition_image is not None:
if CLEANUP_IMAGES:
fuelignition_image.remove(force=True)
def main(

11
client_stdout.py Normal file
View File

@@ -0,0 +1,11 @@
import asyncio
import sys
async def stdout_pipe(client):
events = client.events(decode=True)
while True:
for event in events:
sys.stdout.write(event)
sys.stdout.flush()
await asyncio.sleep(0.1)

33
config.py Normal file
View File

@@ -0,0 +1,33 @@
import asyncio
from pathlib import Path
import tomllib
from client_stdout import stdout_pipe
import docker
def get_config(config: str = "default") -> dict:
with open("config.toml", "rb") as f:
configs: dict = tomllib.load(f)
out_config: dict = configs["default"]
out_config.update(configs[config])
return out_config
def apply_config(config: dict) -> None:
config["CLIENT"] = docker.from_env(version="auto")
config["ROOT_DIR"] = Path(config["ROOT_DIR"]).absolute()
config["BUILD_DIR"] = Path(config["BUILD_DIR"]).absolute()
config["DOCKERFILE_DIR"] = Path(config["DOCKERFILE_DIR"]).absolute()
config["CWD_MOUNTDIR"] = Path(config["CWD_MOUNTDIR"])
config["FUELIGNITION_BUILD_DIR"] = config["BUILD_DIR"] / config["FUELIGNITION_BUILD_DIR"]
if config["CLIENT_STDOUT"]:
asyncio.run(stdout_pipe(config["CLIENT"]))
globals().update(config)
def init(config: str = "default") -> None:
apply_config(get_config(config))
init()

17
config.toml Normal file
View File

@@ -0,0 +1,17 @@
[default]
ROOT_DIR = "."
BUILD_DIR = "build"
DOCKERFILE_DIR = "docker"
SELENIUM_INIT_MESSAGE = "INFO [Standalone.execute] - Started Selenium Standalone"
FUELIGNITION_INIT_MESSAGE = "ready in *ms."
FUELIGNITION_URL = "http://localhost:3000/fuel-ignition/edit"
FUELIGNITION_BUILD_DIR = "fuel-ignition"
CWD_MOUNTDIR = "/host_cwd"
CLIENT_STDOUT = true
CLEANUP_IMAGES = false
[local]
FUELIGNITION_URL = "http://localhost:3000/fuel-ignition/edit"
[remote]
FUELIGNITION_URL = "https://opensuse.github.io/fuel-ignition/edit"

129
create_img.py Normal file
View File

@@ -0,0 +1,129 @@
import ipaddress
import json
from typing import Annotated
import typer
from autoignition import json_to_img
from debug import debug_mode
MAX_PORT: int = 65535
def load_template() -> dict:
with open("templates/fuelignition.json", "r") as f:
out = json.load(f)
return out
def apply_ignition_settings(
template: dict,
hostname: str,
password: str,
swarm_config: str,
) -> dict:
ignition_config = template.copy()
ignition_config["hostname"] = hostname
ignition_config["login"]["users"][0]["passwd"] = password
# Add files that will define a service to ensure that the node joins the swarm
with open("templates/join_swarm.sh", "r") as f1, open(
"templates/join_swarm.service", "r"
) as f2:
swarm_script, swarm_service = f1.read(), f2.read()
ignition_config["storage"] = ignition_config.get("storage", {})
ignition_config["storage"]["files"] = ignition_config["storage"].get("files", [])
ignition_config["storage"]["files"] += [
{
"path": "/root/join_swarm.json",
"source_type": "data",
"mode": 420,
"overwrite": True,
"data_content": swarm_config,
},
{
"path": "/root/join_swarm.sh",
"source_type": "data",
"mode": 420,
"overwrite": True,
"data_content": swarm_script,
},
]
ignition_config["systemd"] = ignition_config.get("systemd", {})
ignition_config["systemd"]["units"] = ignition_config["systemd"].get("units", [])
ignition_config["systemd"]["units"] += [
{
"name": "join_swarm.service",
"enabled": True,
"contents": swarm_service,
},
]
return ignition_config
def create_img(
hostname: str, password: str, switch_ip_address: str, switch_port: str, swarm_token: str
) -> None:
switch_ip_address = ipaddress.ip_address(switch_ip_address)
if switch_port > MAX_PORT:
raise ValueError(f"Port must be less than {MAX_PORT}")
# get swarm configuration as JSON
swarm_config = json.dumps(
{
"SWITCH_IP_ADDRESS": str(switch_ip_address),
"SWITCH_PORT": switch_port,
"SWARM_TOKEN": swarm_token,
}
)
# Create ignition configuration
ignition_config = load_template()
ignition_config = apply_ignition_settings(
ignition_config,
hostname,
password,
swarm_config,
)
# export ignition configuration
with open("build/fuelignition.json", "w") as f:
json.dump(ignition_config, f, indent=4)
# convert ignition configuration to image
json_to_img("build/fuelignition.json", "build/ignition.img")
def main(
hostname: Annotated[str, typer.Option(help="Hostname for the new node", prompt=True)],
password: Annotated[
str,
typer.Option(
help="Password for the root user on the new node",
prompt=True,
confirmation_prompt=True,
hide_input=True,
),
],
switch_ip_address: Annotated[
str, typer.Option(help="IP address of the switch to connect to", prompt=True)
],
switch_port: Annotated[int, typer.Option(help="Port on the switch to connect to", prompt=True)],
swarm_token: Annotated[
str, typer.Option(help="Swarm token for connecting to the swarm", prompt=True)
],
debug: Annotated[bool, typer.Option(help="Enable debug mode")] = False,
) -> None:
debug_mode(debug)
f = create_img
if debug:
f = ss(f) # noqa: F821, # type: ignore #? ss is installed in debug_mode
f(hostname, password, switch_ip_address, switch_port, swarm_token)
if __name__ == "__main__":
typer.run(main)

0
templates/patched.dockerfile → docker/fuel-ignition.dockerfile
View File

12
docker/validate.dockerfile Normal file
View File

@@ -0,0 +1,12 @@
FROM quay.io/coreos/ignition-validate:release AS ignition-validate
FROM alpine:latest as base
ARG CWD_MOUNTDIR
ENV CWD_MOUNTDIR=$CWD_MOUNTDIR
COPY --from=ignition-validate . .
COPY scripts/installs.sh /installs.sh
RUN /installs.sh
CMD $CWD_MOUNTDIR/scripts/validate.sh

148
main.py
View File

@@ -1,101 +1,62 @@
import ipaddress
import json
from fnmatch import fnmatch
from typing import Annotated
import typer
from autoignition import json_to_img
from config import (
CLEANUP_IMAGES,
CLIENT,
CWD_MOUNTDIR,
DOCKERFILE_DIR,
ROOT_DIR,
)
from create_img import create_img
from debug import debug_mode
import docker
MAX_PORT: int = 65535
def load_template() -> dict:
with open("templates/fuelignition.json", "r") as f:
out = json.load(f)
return out
def apply_ignition_settings(
template: dict,
hostname: str,
password: str,
swarm_config: str,
) -> dict:
ignition_config = template.copy()
ignition_config["hostname"] = hostname
ignition_config["login"]["users"][0]["passwd"] = password
# Add files that will define a service to ensure that the node joins the swarm
with open("templates/join_swarm.sh", "r") as f1, open(
"templates/join_swarm.service", "r"
) as f2:
swarm_script, swarm_service = f1.read(), f2.read()
ignition_config["storage"] = ignition_config.get("storage", {})
ignition_config["storage"]["files"] = ignition_config["storage"].get("files", [])
ignition_config["storage"]["files"] += [
{
"path": "/root/join_swarm.json",
"source_type": "data",
"mode": 420,
"overwrite": True,
"data_content": swarm_config,
},
{
"path": "/root/join_swarm.sh",
"source_type": "data",
"mode": 420,
"overwrite": True,
"data_content": swarm_script,
},
]
ignition_config["systemd"] = ignition_config.get("systemd", {})
ignition_config["systemd"]["units"] = ignition_config["systemd"].get("units", [])
ignition_config["systemd"]["units"] += [
{
"name": "join_swarm.service",
"enabled": True,
"contents": swarm_service,
},
]
return ignition_config
def create_img(
hostname: str, password: str, switch_ip_address: str, switch_port: str, swarm_token: str
) -> None:
switch_ip_address = ipaddress.ip_address(switch_ip_address)
if switch_port > MAX_PORT:
raise ValueError(f"Port must be less than {MAX_PORT}")
# get swarm configuration as JSON
swarm_config = json.dumps(
{
"SWITCH_IP_ADDRESS": str(switch_ip_address),
"SWITCH_PORT": switch_port,
"SWARM_TOKEN": swarm_token,
}
def filter_validation_response(response: str) -> str:
return "\n".join(
filter(
# Filter out the warning about unused key human_readable, this always exists in
# configurations produced by fuel-ignition
lambda x: not fnmatch(x.strip(), "warning at*Unused key human_read"),
response.split("\n"),
)
).strip()
# Create ignition configuration
ignition_config = load_template()
ignition_config = apply_ignition_settings(
ignition_config,
hostname,
password,
swarm_config,
def validation_result() -> str:
dockerfile = DOCKERFILE_DIR / "validate.dockerfile"
image = CLIENT.images.build(
path=".",
dockerfile=str(dockerfile),
tag="validate",
buildargs={"CWD_MOUNTDIR": str(CWD_MOUNTDIR)},
rm=CLEANUP_IMAGES,
pull=True,
quiet=True,
)
response = CLIENT.containers.run(
image,
mounts=[
docker.types.Mount(
target=str(CWD_MOUNTDIR),
source=str(ROOT_DIR),
type="bind",
)
],
remove=True,
)
if CLEANUP_IMAGES:
image.remove(force=True)
return response
# export ignition configuration
with open("build/fuelignition.json", "w") as f:
json.dump(ignition_config, f, indent=4)
# convert ignition configuration to image
json_to_img("build/fuelignition.json", "build/ignition.img")
def validate() -> (bool, str):
response = validation_result().decode()
response = filter_validation_response(response)
return (not bool(response), response)
def main(
@@ -119,10 +80,17 @@ def main(
debug: Annotated[bool, typer.Option(help="Enable debug mode")] = False,
) -> None:
debug_mode(debug)
f = create_img
if debug:
f = ss(f) # noqa: F821, # type: ignore #? ss is installed in debug_mode
f(hostname, password, switch_ip_address, switch_port, swarm_token)
# f = create_img
# if debug:
# f = ss(f) # noqa: F821, # type: ignore #? ss is installed in debug_mode
# f(hostname, password, switch_ip_address, switch_port, swarm_token)
create_img(hostname, password, switch_ip_address, switch_port, swarm_token)
valid, response = validate()
if not valid:
print(response)
raise typer.Exit(1)
else:
print("Valid ignition image created!")
if __name__ == "__main__":

37
poetry.lock generated
View File

@@ -341,6 +341,41 @@ files = [
[package.extras]
tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"]
[[package]]
name = "fsspec"
version = "2023.10.0"
description = "File-system specification"
optional = false
python-versions = ">=3.8"
files = [
{file = "fsspec-2023.10.0-py3-none-any.whl", hash = "sha256:346a8f024efeb749d2a5fca7ba8854474b1ff9af7c3faaf636a4548781136529"},
{file = "fsspec-2023.10.0.tar.gz", hash = "sha256:330c66757591df346ad3091a53bd907e15348c2ba17d63fd54f5c39c4457d2a5"},
]
[package.extras]
abfs = ["adlfs"]
adl = ["adlfs"]
arrow = ["pyarrow (>=1)"]
dask = ["dask", "distributed"]
devel = ["pytest", "pytest-cov"]
dropbox = ["dropbox", "dropboxdrivefs", "requests"]
full = ["adlfs", "aiohttp (!=4.0.0a0,!=4.0.0a1)", "dask", "distributed", "dropbox", "dropboxdrivefs", "fusepy", "gcsfs", "libarchive-c", "ocifs", "panel", "paramiko", "pyarrow (>=1)", "pygit2", "requests", "s3fs", "smbprotocol", "tqdm"]
fuse = ["fusepy"]
gcs = ["gcsfs"]
git = ["pygit2"]
github = ["requests"]
gs = ["gcsfs"]
gui = ["panel"]
hdfs = ["pyarrow (>=1)"]
http = ["aiohttp (!=4.0.0a0,!=4.0.0a1)", "requests"]
libarchive = ["libarchive-c"]
oci = ["ocifs"]
s3 = ["s3fs"]
sftp = ["paramiko"]
smb = ["smbprotocol"]
ssh = ["paramiko"]
tqdm = ["tqdm"]
[[package]]
name = "gitdb"
version = "4.0.11"
@@ -965,4 +1000,4 @@ h11 = ">=0.9.0,<1"
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "c9be19a0841acc152e4c99c50539946b8831ec94ce94d907c463f4630d05d2a0"
content-hash = "80f545060bae202b15081561a4df464f1c1f7dae9aff52c76f2e70c144628a35"

1
pyproject.toml
View File

@@ -13,6 +13,7 @@ mechanicalsoup = "^1.3.0"
docker = "^6.1.3"
requests = "^2.31.0"
gitpython = "^3.1.40"
fsspec = "^2023.10.0"
[tool.poetry.group.dev.dependencies]

1
scripts/fetch_config.sh Executable file
View File

@@ -0,0 +1 @@
mcopy -n -i ${CWD_MOUNTDIR}/build/ignition.img ::ignition/config.ign ${CWD_MOUNTDIR}/build/config.ign

2
scripts/installs.sh Executable file
View File

@@ -0,0 +1,2 @@
apk update
apk add mtools

2
scripts/validate.sh Executable file
View File

@@ -0,0 +1,2 @@
${CWD_MOUNTDIR}/scripts/fetch_config.sh
/usr/local/bin/ignition-validate ${CWD_MOUNTDIR}/build/config.ign