Added autogenerated portainer api

.env

@@ -1 +1,8 @@
-COMPOSE_PROJECT_NAME=i-form_research_server_core
+COMPOSE_PROJECT_NAME=i-form_research_server_core
+
+#hostname=www.yourdataverse.org
+hostname=locahost:8080
+#traefikhost=www.yourdataverse.org
+traefikhost=localhost:8080
+# traefik email settings
+useremail=cian.hughes@dcu.ie

.gitignore

@@ -5,6 +5,7 @@
 !.vscode/launch.json
 !.vscode/extensions.json
 !.vscode/*.code-snippets
+.venv/
 
 # Local History for Visual Studio Code
 .history/
@@ -30,4 +31,174 @@
 
 # Ignore dotenv files in the project root
 .env
-/[^/]*.env
+/[^/]*.env
+
+.gitignore
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# Ignore dynamically constructed swagger APIs
+.swagger-codegen*
+src/portainer/*
+# These files do need to be included though:
+!src/portainer/pyproject.toml
+!src/portainer/build.py

README.md

@@ -0,0 +1,2 @@
+# Usage
+To use this script, run the command `deploy` in the root directory. This will deploy the core server services to whatever docker swarm the machine is connected to, then deploy the web server to the same swarm via portainer.

deploy

@@ -0,0 +1,29 @@
+#!/usr/bin/env poetry run python
+
+from typing import Optional
+import typer # type: ignore
+import subprocess
+import portainer # type: ignore
+
+
+def docker_deploy_core(stack_name: Optional[str] = "core"):
+    """Simply deploys the core services"""
+    subprocess.run(["docker", "stack", "deploy", "-c", "docker-compose.yaml", stack_name])
+
+
+def docker_deploy_stack(username: str, password: str, stack_name: Optional[str] = "stack"):
+    """Deploys the stack using the portainer api, to allow for complete control over the stack"""
+    breakpoint() #! TODO: Implement way of using portainer api to deploy stack
+    # subprocess.run(["docker", "stack", "deploy", "-c", "stack.yaml", stack_name])
+
+
+def docker_deploy_all(core_name: Optional[str] = "core", stack_name: Optional[str] = "stack"):
+    """Deploys the core services and the stack"""
+    docker_deploy_core(core_name)
+    docker_deploy_stack(stack_name)
+
+
+if __name__ == "__main__":
+    # typer.run(docker_deploy_all)
+    # docker_deploy_core("core")
+    typer.run(docker_deploy_stack)

env_samples/dataverse.env

@@ -0,0 +1,91 @@
+LOCAL_WAR=./dataverse.war
+COMPOSE_FILE=distros/vanilla/docker-compose.yml
+
+# Activate Dataverse language pack by setting language code:
+# en - English          hu - Hungarian          fr - French     sl - Slovenian
+# se - Swedish          es - Spanish            it - Italian    ua - Ukrainian
+# pt - Portuguese       ru - Russian            at - Austrian German
+# br - Brazilian Portuguese     ca - French Canadian
+#MAINLANG=en
+VERSION=5.13.allclouds
+DOCKER_HUB=coronawhy/dataverse
+CONFIGURATION_PATH=.
+LOCAL_STORAGE=.
+DOCROOT=.
+
+# Dataverse database settings
+DATAVERSE_DB_HOST=postgres
+DATAVERSE_DB_USER=dataverse
+# DATAVERSE_DB_PASSWORD=dvnsecret
+DATAVERSE_DB_NAME=dataverse
+
+SOLR_SERVICE_HOST=solr:8983
+SOLR_SERVICE_PORT=8983
+DATAVERSE_URL=localhost:8080
+DATAVERSE_SERVICE_HOST=localhost
+
+# Postgres settings
+POSTGRES_USER=dataverse
+# POSTGRES_PASSWORD=dvnsecret
+POSTGRES_SERVER=postgres
+POSTGRES_DATABASE=dataverse
+POSTGRES_DB=dataverse
+
+# Domain configuration and init folder
+#hostname=www.yourdataverse.org
+hostname=locahost:8080
+#traefikhost=www.yourdataverse.org
+traefikhost=localhost:8080
+INIT_SCRIPTS_FOLDER=/opt/payara/init.d
+
+# traefik email settings
+useremail=youremail@domain.com
+
+# Webhook configuration to bundle external services 
+WEBHOOK=/opt/payara/triggers/external-services.py
+#CESSDA=True
+#CLARIN=True
+
+# DOI parameters
+# https://guides.dataverse.org/en/latest/installation/config.html#doi-baseurlstring
+#doi_authority=doi_authority
+#doi_provider=doi_provider
+#doi_shoulder=doi_shoulder
+#doi_username=doi_username
+#doi_password=doi_password
+dataciterestapiurlstring=https\\:\/\/api.test.datacite.org
+baseurlstring=https\:\/\/mds.test.datacite.org
+
+# AWS settings
+# https://guides.dataverse.org/en/latest/installation/config.html#id90
+#aws_bucket_name=aws_bucket_name
+#aws_s3_profile=aws_s3_profile
+#aws_endpoint_url=aws_endpoint_url
+
+# Mail relay
+# https://guides.dataverse.org/en/latest/developers/troubleshooting.html
+#system_email=system_email
+#mailhost=mailhost
+#mailuser=mailuser
+#no_reply_email=no_reply_email
+#smtp_password=smtp_password
+#smtp_port=smtp_port
+#socket_port=socket_port
+
+# Federated authentification file
+# https://guides.dataverse.org/en/latest/installation/shibboleth.html
+#federated_json_file=federated_json_file
+
+# MinIO bucket 1
+# https://guides.dataverse.org/en/latest/installation/config.html#id87
+#bucketname_1=bucketname_1
+#minio_label_1=minio_label_1
+#minio_bucket_1=minio_bucket_1
+#minio_profile_1=minio_profile_1
+
+# MinIO bucket 2
+# https://guides.dataverse.org/en/latest/installation/config.html#id87
+#bucketname_2=bucketname_2
+#minio_label_1=minio_label_2
+#minio_bucket_1=minio_bucket_2
+#minio_profile_1=minio_profile_2

pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "deploy"
+version = "0.1.0"
+description = ""
+authors = ["Cian Hughes <cian.hughes@dcu.ie>"]
+readme = "README.md"
+packages = [
+    { include = "deploy", from = "." },
+    # { include = "portainer", from = "src" },
+]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+typer = "^0.9.0"
+portainer = {path = "src/portainer"}
+
+[tool.poetry.group.dev.dependencies]
+snoop = "^0.4.3"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

src/portainer/build.py

@@ -0,0 +1,87 @@
+from typing import Dict, Any
+from pathlib import Path
+import requests  # type: ignore
+import docker  # type: ignore
+
+
+def portainer_login(username: str, password: str) -> Dict[str, Any]:
+    """Logs into portainer and returns a session and the portainer version"""
+    auth_login = {"username": username, "password": password}
+    auth_response = requests.post("http://127.0.0.1:9000/api/auth", json=auth_login)
+    auth_response.raise_for_status()
+    auth_token = auth_response.json()["jwt"]
+    session = requests.Session()
+    session.headers.update({"Authorization": f"Bearer {auth_token}"})
+    portainer_response = session.get("http://127.0.0.1:9000/api/system/version")
+    portainer_response.raise_for_status()
+    portainer_version = portainer_response.json()["ServerVersion"]
+    return {"session": session, "version": portainer_version, "auth_token": auth_token}
+
+
+def fetch_portainer_api_spec(version: str): # -> Dict[str, Any]:
+    """Gets and caches the portainer api spec for a given version"""
+    if not Path(f".cache/portainer_api/{version}/swagger.json").exists():
+        Path(f".cache/portainer_api/{version}").mkdir(parents=True, exist_ok=True)
+        json_request = requests.get(f"https://api.swaggerhub.com/apis/portainer/portainer-ce/{version}/swagger.json")
+        json_request.raise_for_status()
+        with open(f".cache/portainer_api/{version}/swagger.json", "w") as f:
+            f.write(json_request.text)
+    # with open(f".cache/portainer_api/{version}/swagger.json") as f:
+    #     return f.read()
+
+
+# def init_portainer_api(version: str, auth_token: str): # -> openapi.Swagger:
+#     """
+#         Returns a swagger api for portainer which follows the schema outlined here:
+#         https://app.swaggerhub.com/apis/portainer/portainer-ce/2.19.4
+#     """
+#     # spec = get_portainer_api_spec(version)
+#     # api = openapi.loads(spec)
+#     # api.validate()
+#     # return api
+#     app = App.create(f"https://api.swaggerhub.com/apis/portainer/portainer-ce/{version}/swagger.json")
+#     auth = Security(app)
+#     auth.update_with("ApiKeyAuth", auth_token)
+#     client = Client(auth)
+#     return app, client
+
+# def create_portainer_api(version: str, auth_token: str):
+#     spec = get_portainer_api_spec(version)
+#     api = openapi.loads(spec)
+#     api.validate()
+#     return api
+
+def build_portainer_api(version: str):
+    """Builds the portainer api"""
+    # Path("src/portainer_api").mkdir(parents=True, exist_ok=True)
+    docker_client = docker.from_env()
+    docker_client.containers.run(
+        "swaggerapi/swagger-codegen-cli",
+        "generate -i /schemas/swagger.json -l python -o /local -D packageName=portainer",
+        mounts = [
+            docker.types.Mount(
+                target="/local",
+                source=str(Path().resolve()),
+                type="bind"
+            ),
+            docker.types.Mount(
+                target="/schemas",
+                source=str(Path(f".cache/portainer_api/{version}").resolve()),
+                type="bind"
+            ),
+        ],
+        auto_remove=True,
+    )
+
+
+def init_portainer_api(username: str, password: str):
+    portainer_session = portainer_login(username, password)
+    fetch_portainer_api_spec(portainer_session["version"])
+    build_portainer_api(portainer_session["version"])
+    
+def build():
+    init_portainer_api("admin", "%a7DtJkb&^fG%aZKGNvga5V&yBU$#UYBfGjU*pu!v2HS288&kwbR7Gpd@A5MjWr2")
+
+if __name__ == "__main__":
+    build()
+    # import portainer  # noqa: F401

src/portainer/pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "portainer"
+version = "0.1.0"
+description = ""
+authors = ["Cian Hughes <cian.hughes@dcu.ie>"]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+requests = "^2.31.0"
+docker = "^7.0.0"
+
+[tool.poetry.build]
+script = "build.py"
+generate-setup-file = false
+
+[build-system]
+requires = ["poetry-core", "setuptools", "requests", "docker"]
+build-backend = "poetry.core.masonry.api"
+
+[project]
+dynamic = ["dependencies"]
+
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}

src/portainer_backup/build.py

@@ -0,0 +1,87 @@
+from typing import Dict, Any
+from pathlib import Path
+import requests  # type: ignore
+import docker  # type: ignore
+
+
+def portainer_login(username: str, password: str) -> Dict[str, Any]:
+    """Logs into portainer and returns a session and the portainer version"""
+    auth_login = {"username": username, "password": password}
+    auth_response = requests.post("http://127.0.0.1:9000/api/auth", json=auth_login)
+    auth_response.raise_for_status()
+    auth_token = auth_response.json()["jwt"]
+    session = requests.Session()
+    session.headers.update({"Authorization": f"Bearer {auth_token}"})
+    portainer_response = session.get("http://127.0.0.1:9000/api/system/version")
+    portainer_response.raise_for_status()
+    portainer_version = portainer_response.json()["ServerVersion"]
+    return {"session": session, "version": portainer_version, "auth_token": auth_token}
+
+
+def fetch_portainer_api_spec(version: str): # -> Dict[str, Any]:
+    """Gets and caches the portainer api spec for a given version"""
+    if not Path(f".cache/portainer_api/{version}/swagger.json").exists():
+        Path(f".cache/portainer_api/{version}").mkdir(parents=True, exist_ok=True)
+        json_request = requests.get(f"https://api.swaggerhub.com/apis/portainer/portainer-ce/{version}/swagger.json")
+        json_request.raise_for_status()
+        with open(f".cache/portainer_api/{version}/swagger.json", "w") as f:
+            f.write(json_request.text)
+    # with open(f".cache/portainer_api/{version}/swagger.json") as f:
+    #     return f.read()
+
+
+# def init_portainer_api(version: str, auth_token: str): # -> openapi.Swagger:
+#     """
+#         Returns a swagger api for portainer which follows the schema outlined here:
+#         https://app.swaggerhub.com/apis/portainer/portainer-ce/2.19.4
+#     """
+#     # spec = get_portainer_api_spec(version)
+#     # api = openapi.loads(spec)
+#     # api.validate()
+#     # return api
+#     app = App.create(f"https://api.swaggerhub.com/apis/portainer/portainer-ce/{version}/swagger.json")
+#     auth = Security(app)
+#     auth.update_with("ApiKeyAuth", auth_token)
+#     client = Client(auth)
+#     return app, client
+
+# def create_portainer_api(version: str, auth_token: str):
+#     spec = get_portainer_api_spec(version)
+#     api = openapi.loads(spec)
+#     api.validate()
+#     return api
+
+def build_portainer_api(version: str):
+    """Builds the portainer api"""
+    # Path("src/portainer_api").mkdir(parents=True, exist_ok=True)
+    docker_client = docker.from_env()
+    docker_client.containers.run(
+        "swaggerapi/swagger-codegen-cli",
+        "generate -i /schemas/swagger.json -l python -o /local -D packageName=portainer",
+        mounts = [
+            docker.types.Mount(
+                target="/local",
+                source=str(Path().resolve()),
+                type="bind"
+            ),
+            docker.types.Mount(
+                target="/schemas",
+                source=str(Path(f".cache/portainer_api/{version}").resolve()),
+                type="bind"
+            ),
+        ],
+        auto_remove=True,
+    )
+
+
+def init_portainer_api(username: str, password: str):
+    portainer_session = portainer_login(username, password)
+    fetch_portainer_api_spec(portainer_session["version"])
+    build_portainer_api(portainer_session["version"])
+    
+def build():
+    init_portainer_api("admin", "%a7DtJkb&^fG%aZKGNvga5V&yBU$#UYBfGjU*pu!v2HS288&kwbR7Gpd@A5MjWr2")
+
+if __name__ == "__main__":
+    build()
+    # import portainer  # noqa: F401

src/portainer_backup/pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "portainer"
+version = "0.1.0"
+description = ""
+authors = ["Cian Hughes <cian.hughes@dcu.ie>"]
+# packages = [
+#     { include = "src" },
+# ]
+
+[tool.poetry.dependencies]
+python = "^3.11"
+requests = "^2.31.0"
+docker = "^7.0.0"
+six = "^1.16.0"
+
+[tool.poetry.build]
+script = "build.py"
+generate-setup-file = false
+
+[build-system]
+requires = ["poetry-core", "setuptools", "requests", "docker"]
+build-backend = "poetry.core.masonry.api"

stack.yaml

@@ -75,6 +75,11 @@ services:
         condition: on-failure
     networks:
       - i-form_research_server_stack
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.go.rule=Path(`/`)"
+      - "traefik.http.services.go.loadbalancer.server.port=7474"
+      - "traefik.http.services.go.loadbalancer.server.port=7687"
   # # Dataverse requires a postgres database, so we'll add that here
   # postgres:
   #   image: postgres:10.13
@@ -97,6 +102,10 @@ services:
   #     - dataverse_triggers:/triggers
   #   networks:
   #     - i-form_research_server_stack
+  #   labels:
+  #     - "traefik.enable=true"
+  #     - "traefik.http.routers.go.rule=Path(`/`)"
+  #     - "traefik.http.services.go.loadbalancer.server.port=5432"
   # We also want a mysql database for elabftw
   mysql:
     image: mysql/mysql-server:latest
@@ -138,6 +147,10 @@ services:
         condition: on-failure
     networks:
       - i-form_research_server_stack
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.go.rule=Path(`/`)"
+      - "traefik.http.services.go.loadbalancer.server.port=3000"
   # The following service is a simple nginx server that hosts static websites
   nginx:
     image: nginx:latest
@@ -150,6 +163,10 @@ services:
         condition: on-failure
     networks:
       - i-form_research_server_stack
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.go.rule=Path(`/`)"
+      - "traefik.http.services.go.loadbalancer.server.port=80"
   # Then, we add neodash as a service that can be used to visualize the neo4j database
   # This should provide the real AI assisted punching power for this stack
   neodash:
@@ -161,6 +178,10 @@ services:
         condition: on-failure
     networks:
       - i-form_research_server_stack
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.go.rule=Path(`/`)"
+      - "traefik.http.services.go.loadbalancer.server.port=5005"
   # Lastly, we need a LIMS system, but we're not sure which one to use yet
   # As a test, we'll run senaite, with elabftw for lab notebook functionality
   senaite:
@@ -171,6 +192,10 @@ services:
       - senaite:/data
     networks:
       - i-form_research_server_stack
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.go.rule=Path(`/`)"
+      - "traefik.http.services.go.loadbalancer.server.port=8080"
   # We also need to add a service for the elabftw instance and its database
   elabftw:
     image: elabftw/elabimg:latest
@@ -219,6 +244,10 @@ services:
         #- /etc/letsencrypt:/ssl
     networks:
       - i-form_research_server_stack
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.go.rule=Path(`/`)"
+      - "traefik.http.services.go.loadbalancer.server.port=443"
   # # Lastly, we have to add several services to get dataverse to work
   # solr:
   #   image: coronawhy/solr:8.9.0