From 753742c868dc42bcd42f1f76ce9e398637305844 Mon Sep 17 00:00:00 2001 From: Cian Hughes Date: Mon, 8 Jan 2024 17:55:04 +0000 Subject: [PATCH] Added autogenerated portainer api --- .env | 9 +- .gitignore | 173 +++++++++++++++++++++++++++- README.md | 2 + deploy | 29 +++++ env_samples/dataverse.env | 91 +++++++++++++++ pyproject.toml | 22 ++++ src/portainer/build.py | 87 ++++++++++++++ src/portainer/pyproject.toml | 24 ++++ src/portainer_backup/build.py | 87 ++++++++++++++ src/portainer_backup/pyproject.toml | 22 ++++ stack.yaml | 29 +++++ 11 files changed, 573 insertions(+), 2 deletions(-) create mode 100644 README.md create mode 100755 deploy create mode 100644 env_samples/dataverse.env create mode 100644 pyproject.toml create mode 100644 src/portainer/build.py create mode 100644 src/portainer/pyproject.toml create mode 100644 src/portainer_backup/build.py create mode 100644 src/portainer_backup/pyproject.toml diff --git a/.env b/.env index c773396..ff80cfc 100644 --- a/.env +++ b/.env @@ -1 +1,8 @@ -COMPOSE_PROJECT_NAME=i-form_research_server_core \ No newline at end of file +COMPOSE_PROJECT_NAME=i-form_research_server_core + +#hostname=www.yourdataverse.org +hostname=locahost:8080 +#traefikhost=www.yourdataverse.org +traefikhost=localhost:8080 +# traefik email settings +useremail=cian.hughes@dcu.ie \ No newline at end of file diff --git a/.gitignore b/.gitignore index f2739a8..e535394 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ !.vscode/launch.json !.vscode/extensions.json !.vscode/*.code-snippets +.venv/ # Local History for Visual Studio Code .history/ @@ -30,4 +31,174 @@ # Ignore dotenv files in the project root .env -/[^/]*.env \ No newline at end of file +/[^/]*.env + +.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Ignore dynamically constructed swagger APIs +.swagger-codegen* +src/portainer/* +# These files do need to be included though: +!src/portainer/pyproject.toml +!src/portainer/build.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..a422fd5 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# Usage +To use this script, run the command `deploy` in the root directory. This will deploy the core server services to whatever docker swarm the machine is connected to, then deploy the web server to the same swarm via portainer. \ No newline at end of file diff --git a/deploy b/deploy new file mode 100755 index 0000000..846f397 --- /dev/null +++ b/deploy @@ -0,0 +1,29 @@ +#!/usr/bin/env poetry run python + +from typing import Optional +import typer # type: ignore +import subprocess +import portainer # type: ignore + + +def docker_deploy_core(stack_name: Optional[str] = "core"): + """Simply deploys the core services""" + subprocess.run(["docker", "stack", "deploy", "-c", "docker-compose.yaml", stack_name]) + + +def docker_deploy_stack(username: str, password: str, stack_name: Optional[str] = "stack"): + """Deploys the stack using the portainer api, to allow for complete control over the stack""" + breakpoint() #! TODO: Implement way of using portainer api to deploy stack + # subprocess.run(["docker", "stack", "deploy", "-c", "stack.yaml", stack_name]) + + +def docker_deploy_all(core_name: Optional[str] = "core", stack_name: Optional[str] = "stack"): + """Deploys the core services and the stack""" + docker_deploy_core(core_name) + docker_deploy_stack(stack_name) + + +if __name__ == "__main__": + # typer.run(docker_deploy_all) + # docker_deploy_core("core") + typer.run(docker_deploy_stack) diff --git a/env_samples/dataverse.env b/env_samples/dataverse.env new file mode 100644 index 0000000..11bc4e3 --- /dev/null +++ b/env_samples/dataverse.env @@ -0,0 +1,91 @@ +LOCAL_WAR=./dataverse.war +COMPOSE_FILE=distros/vanilla/docker-compose.yml + +# Activate Dataverse language pack by setting language code: +# en - English hu - Hungarian fr - French sl - Slovenian +# se - Swedish es - Spanish it - Italian ua - Ukrainian +# pt - Portuguese ru - Russian at - Austrian German +# br - Brazilian Portuguese ca - French Canadian +#MAINLANG=en +VERSION=5.13.allclouds +DOCKER_HUB=coronawhy/dataverse +CONFIGURATION_PATH=. +LOCAL_STORAGE=. +DOCROOT=. + +# Dataverse database settings +DATAVERSE_DB_HOST=postgres +DATAVERSE_DB_USER=dataverse +# DATAVERSE_DB_PASSWORD=dvnsecret +DATAVERSE_DB_NAME=dataverse + +SOLR_SERVICE_HOST=solr:8983 +SOLR_SERVICE_PORT=8983 +DATAVERSE_URL=localhost:8080 +DATAVERSE_SERVICE_HOST=localhost + +# Postgres settings +POSTGRES_USER=dataverse +# POSTGRES_PASSWORD=dvnsecret +POSTGRES_SERVER=postgres +POSTGRES_DATABASE=dataverse +POSTGRES_DB=dataverse + +# Domain configuration and init folder +#hostname=www.yourdataverse.org +hostname=locahost:8080 +#traefikhost=www.yourdataverse.org +traefikhost=localhost:8080 +INIT_SCRIPTS_FOLDER=/opt/payara/init.d + +# traefik email settings +useremail=youremail@domain.com + +# Webhook configuration to bundle external services +WEBHOOK=/opt/payara/triggers/external-services.py +#CESSDA=True +#CLARIN=True + +# DOI parameters +# https://guides.dataverse.org/en/latest/installation/config.html#doi-baseurlstring +#doi_authority=doi_authority +#doi_provider=doi_provider +#doi_shoulder=doi_shoulder +#doi_username=doi_username +#doi_password=doi_password +dataciterestapiurlstring=https\\:\/\/api.test.datacite.org +baseurlstring=https\:\/\/mds.test.datacite.org + +# AWS settings +# https://guides.dataverse.org/en/latest/installation/config.html#id90 +#aws_bucket_name=aws_bucket_name +#aws_s3_profile=aws_s3_profile +#aws_endpoint_url=aws_endpoint_url + +# Mail relay +# https://guides.dataverse.org/en/latest/developers/troubleshooting.html +#system_email=system_email +#mailhost=mailhost +#mailuser=mailuser +#no_reply_email=no_reply_email +#smtp_password=smtp_password +#smtp_port=smtp_port +#socket_port=socket_port + +# Federated authentification file +# https://guides.dataverse.org/en/latest/installation/shibboleth.html +#federated_json_file=federated_json_file + +# MinIO bucket 1 +# https://guides.dataverse.org/en/latest/installation/config.html#id87 +#bucketname_1=bucketname_1 +#minio_label_1=minio_label_1 +#minio_bucket_1=minio_bucket_1 +#minio_profile_1=minio_profile_1 + +# MinIO bucket 2 +# https://guides.dataverse.org/en/latest/installation/config.html#id87 +#bucketname_2=bucketname_2 +#minio_label_1=minio_label_2 +#minio_bucket_1=minio_bucket_2 +#minio_profile_1=minio_profile_2 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..57ed4a4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,22 @@ +[tool.poetry] +name = "deploy" +version = "0.1.0" +description = "" +authors = ["Cian Hughes "] +readme = "README.md" +packages = [ + { include = "deploy", from = "." }, + # { include = "portainer", from = "src" }, +] + +[tool.poetry.dependencies] +python = "^3.11" +typer = "^0.9.0" +portainer = {path = "src/portainer"} + +[tool.poetry.group.dev.dependencies] +snoop = "^0.4.3" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/src/portainer/build.py b/src/portainer/build.py new file mode 100644 index 0000000..6537c26 --- /dev/null +++ b/src/portainer/build.py @@ -0,0 +1,87 @@ +from typing import Dict, Any +from pathlib import Path +import requests # type: ignore +import docker # type: ignore + + +def portainer_login(username: str, password: str) -> Dict[str, Any]: + """Logs into portainer and returns a session and the portainer version""" + auth_login = {"username": username, "password": password} + auth_response = requests.post("http://127.0.0.1:9000/api/auth", json=auth_login) + auth_response.raise_for_status() + auth_token = auth_response.json()["jwt"] + session = requests.Session() + session.headers.update({"Authorization": f"Bearer {auth_token}"}) + portainer_response = session.get("http://127.0.0.1:9000/api/system/version") + portainer_response.raise_for_status() + portainer_version = portainer_response.json()["ServerVersion"] + return {"session": session, "version": portainer_version, "auth_token": auth_token} + + +def fetch_portainer_api_spec(version: str): # -> Dict[str, Any]: + """Gets and caches the portainer api spec for a given version""" + if not Path(f".cache/portainer_api/{version}/swagger.json").exists(): + Path(f".cache/portainer_api/{version}").mkdir(parents=True, exist_ok=True) + json_request = requests.get(f"https://api.swaggerhub.com/apis/portainer/portainer-ce/{version}/swagger.json") + json_request.raise_for_status() + with open(f".cache/portainer_api/{version}/swagger.json", "w") as f: + f.write(json_request.text) + # with open(f".cache/portainer_api/{version}/swagger.json") as f: + # return f.read() + + +# def init_portainer_api(version: str, auth_token: str): # -> openapi.Swagger: +# """ +# Returns a swagger api for portainer which follows the schema outlined here: +# https://app.swaggerhub.com/apis/portainer/portainer-ce/2.19.4 +# """ +# # spec = get_portainer_api_spec(version) +# # api = openapi.loads(spec) +# # api.validate() +# # return api +# app = App.create(f"https://api.swaggerhub.com/apis/portainer/portainer-ce/{version}/swagger.json") +# auth = Security(app) +# auth.update_with("ApiKeyAuth", auth_token) +# client = Client(auth) +# return app, client + +# def create_portainer_api(version: str, auth_token: str): +# spec = get_portainer_api_spec(version) +# api = openapi.loads(spec) +# api.validate() +# return api + +def build_portainer_api(version: str): + """Builds the portainer api""" + # Path("src/portainer_api").mkdir(parents=True, exist_ok=True) + docker_client = docker.from_env() + docker_client.containers.run( + "swaggerapi/swagger-codegen-cli", + "generate -i /schemas/swagger.json -l python -o /local -D packageName=portainer", + mounts = [ + docker.types.Mount( + target="/local", + source=str(Path().resolve()), + type="bind" + ), + docker.types.Mount( + target="/schemas", + source=str(Path(f".cache/portainer_api/{version}").resolve()), + type="bind" + ), + ], + auto_remove=True, + ) + + +def init_portainer_api(username: str, password: str): + portainer_session = portainer_login(username, password) + fetch_portainer_api_spec(portainer_session["version"]) + build_portainer_api(portainer_session["version"]) + +def build(): + init_portainer_api("admin", "%a7DtJkb&^fG%aZKGNvga5V&yBU$#UYBfGjU*pu!v2HS288&kwbR7Gpd@A5MjWr2") + +if __name__ == "__main__": + build() + # import portainer # noqa: F401 \ No newline at end of file diff --git a/src/portainer/pyproject.toml b/src/portainer/pyproject.toml new file mode 100644 index 0000000..c3250a8 --- /dev/null +++ b/src/portainer/pyproject.toml @@ -0,0 +1,24 @@ +[tool.poetry] +name = "portainer" +version = "0.1.0" +description = "" +authors = ["Cian Hughes "] + +[tool.poetry.dependencies] +python = "^3.11" +requests = "^2.31.0" +docker = "^7.0.0" + +[tool.poetry.build] +script = "build.py" +generate-setup-file = false + +[build-system] +requires = ["poetry-core", "setuptools", "requests", "docker"] +build-backend = "poetry.core.masonry.api" + +[project] +dynamic = ["dependencies"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} diff --git a/src/portainer_backup/build.py b/src/portainer_backup/build.py new file mode 100644 index 0000000..6537c26 --- /dev/null +++ b/src/portainer_backup/build.py @@ -0,0 +1,87 @@ +from typing import Dict, Any +from pathlib import Path +import requests # type: ignore +import docker # type: ignore + + +def portainer_login(username: str, password: str) -> Dict[str, Any]: + """Logs into portainer and returns a session and the portainer version""" + auth_login = {"username": username, "password": password} + auth_response = requests.post("http://127.0.0.1:9000/api/auth", json=auth_login) + auth_response.raise_for_status() + auth_token = auth_response.json()["jwt"] + session = requests.Session() + session.headers.update({"Authorization": f"Bearer {auth_token}"}) + portainer_response = session.get("http://127.0.0.1:9000/api/system/version") + portainer_response.raise_for_status() + portainer_version = portainer_response.json()["ServerVersion"] + return {"session": session, "version": portainer_version, "auth_token": auth_token} + + +def fetch_portainer_api_spec(version: str): # -> Dict[str, Any]: + """Gets and caches the portainer api spec for a given version""" + if not Path(f".cache/portainer_api/{version}/swagger.json").exists(): + Path(f".cache/portainer_api/{version}").mkdir(parents=True, exist_ok=True) + json_request = requests.get(f"https://api.swaggerhub.com/apis/portainer/portainer-ce/{version}/swagger.json") + json_request.raise_for_status() + with open(f".cache/portainer_api/{version}/swagger.json", "w") as f: + f.write(json_request.text) + # with open(f".cache/portainer_api/{version}/swagger.json") as f: + # return f.read() + + +# def init_portainer_api(version: str, auth_token: str): # -> openapi.Swagger: +# """ +# Returns a swagger api for portainer which follows the schema outlined here: +# https://app.swaggerhub.com/apis/portainer/portainer-ce/2.19.4 +# """ +# # spec = get_portainer_api_spec(version) +# # api = openapi.loads(spec) +# # api.validate() +# # return api +# app = App.create(f"https://api.swaggerhub.com/apis/portainer/portainer-ce/{version}/swagger.json") +# auth = Security(app) +# auth.update_with("ApiKeyAuth", auth_token) +# client = Client(auth) +# return app, client + +# def create_portainer_api(version: str, auth_token: str): +# spec = get_portainer_api_spec(version) +# api = openapi.loads(spec) +# api.validate() +# return api + +def build_portainer_api(version: str): + """Builds the portainer api""" + # Path("src/portainer_api").mkdir(parents=True, exist_ok=True) + docker_client = docker.from_env() + docker_client.containers.run( + "swaggerapi/swagger-codegen-cli", + "generate -i /schemas/swagger.json -l python -o /local -D packageName=portainer", + mounts = [ + docker.types.Mount( + target="/local", + source=str(Path().resolve()), + type="bind" + ), + docker.types.Mount( + target="/schemas", + source=str(Path(f".cache/portainer_api/{version}").resolve()), + type="bind" + ), + ], + auto_remove=True, + ) + + +def init_portainer_api(username: str, password: str): + portainer_session = portainer_login(username, password) + fetch_portainer_api_spec(portainer_session["version"]) + build_portainer_api(portainer_session["version"]) + +def build(): + init_portainer_api("admin", "%a7DtJkb&^fG%aZKGNvga5V&yBU$#UYBfGjU*pu!v2HS288&kwbR7Gpd@A5MjWr2") + +if __name__ == "__main__": + build() + # import portainer # noqa: F401 \ No newline at end of file diff --git a/src/portainer_backup/pyproject.toml b/src/portainer_backup/pyproject.toml new file mode 100644 index 0000000..7e714b3 --- /dev/null +++ b/src/portainer_backup/pyproject.toml @@ -0,0 +1,22 @@ +[tool.poetry] +name = "portainer" +version = "0.1.0" +description = "" +authors = ["Cian Hughes "] +# packages = [ +# { include = "src" }, +# ] + +[tool.poetry.dependencies] +python = "^3.11" +requests = "^2.31.0" +docker = "^7.0.0" +six = "^1.16.0" + +[tool.poetry.build] +script = "build.py" +generate-setup-file = false + +[build-system] +requires = ["poetry-core", "setuptools", "requests", "docker"] +build-backend = "poetry.core.masonry.api" diff --git a/stack.yaml b/stack.yaml index b4ba390..cbcba67 100644 --- a/stack.yaml +++ b/stack.yaml @@ -75,6 +75,11 @@ services: condition: on-failure networks: - i-form_research_server_stack + labels: + - "traefik.enable=true" + - "traefik.http.routers.go.rule=Path(`/`)" + - "traefik.http.services.go.loadbalancer.server.port=7474" + - "traefik.http.services.go.loadbalancer.server.port=7687" # # Dataverse requires a postgres database, so we'll add that here # postgres: # image: postgres:10.13 @@ -97,6 +102,10 @@ services: # - dataverse_triggers:/triggers # networks: # - i-form_research_server_stack + # labels: + # - "traefik.enable=true" + # - "traefik.http.routers.go.rule=Path(`/`)" + # - "traefik.http.services.go.loadbalancer.server.port=5432" # We also want a mysql database for elabftw mysql: image: mysql/mysql-server:latest @@ -138,6 +147,10 @@ services: condition: on-failure networks: - i-form_research_server_stack + labels: + - "traefik.enable=true" + - "traefik.http.routers.go.rule=Path(`/`)" + - "traefik.http.services.go.loadbalancer.server.port=3000" # The following service is a simple nginx server that hosts static websites nginx: image: nginx:latest @@ -150,6 +163,10 @@ services: condition: on-failure networks: - i-form_research_server_stack + labels: + - "traefik.enable=true" + - "traefik.http.routers.go.rule=Path(`/`)" + - "traefik.http.services.go.loadbalancer.server.port=80" # Then, we add neodash as a service that can be used to visualize the neo4j database # This should provide the real AI assisted punching power for this stack neodash: @@ -161,6 +178,10 @@ services: condition: on-failure networks: - i-form_research_server_stack + labels: + - "traefik.enable=true" + - "traefik.http.routers.go.rule=Path(`/`)" + - "traefik.http.services.go.loadbalancer.server.port=5005" # Lastly, we need a LIMS system, but we're not sure which one to use yet # As a test, we'll run senaite, with elabftw for lab notebook functionality senaite: @@ -171,6 +192,10 @@ services: - senaite:/data networks: - i-form_research_server_stack + labels: + - "traefik.enable=true" + - "traefik.http.routers.go.rule=Path(`/`)" + - "traefik.http.services.go.loadbalancer.server.port=8080" # We also need to add a service for the elabftw instance and its database elabftw: image: elabftw/elabimg:latest @@ -219,6 +244,10 @@ services: #- /etc/letsencrypt:/ssl networks: - i-form_research_server_stack + labels: + - "traefik.enable=true" + - "traefik.http.routers.go.rule=Path(`/`)" + - "traefik.http.services.go.loadbalancer.server.port=443" # # Lastly, we have to add several services to get dataverse to work # solr: # image: coronawhy/solr:8.9.0