diff --git a/cli/data/env.template b/cli/data/env.template index 6d2165f..f8746c4 100644 --- a/cli/data/env.template +++ b/cli/data/env.template @@ -28,6 +28,8 @@ INVENIO_CELERY_BROKER_URL=redis://cache:6379/5 # Server settings INVENIO_WSGI_PROXIES=4 +INVENIO_HTTP_PORT=80 +INVENIO_HTTPS_PORT=443 # Invenio-RDM-Records INVENIO_DATACITE_ENABLED=false @@ -60,8 +62,9 @@ INVENIO_GITHUB_APP_CREDENTIALS="" # OAI-PMH INVENIO_OAISERVER_ID_PREFIX=invenio-rdm -# Invenio-Files-REST -INVENIO_FILES_REST_STORAGE_FACTORY=invenio_s3.s3fs_storage_factory +# Storage Backend Selection +# Set to 'false' for local development to use local disk storage instead of S3 +INVENIO_USE_S3=true # Invenio-Search INVENIO_SEARCH_HOSTS=search:9200 diff --git a/cli/deploy.py b/cli/deploy.py index b34ef29..0127161 100644 --- a/cli/deploy.py +++ b/cli/deploy.py @@ -151,7 +151,7 @@ def prod_deploy(): ] & FG docker_compose("up", "-d", "--wait") - setup_cmd = "invenio db init && invenio db create && invenio alembic upgrade heads && invenio collect -v && invenio index init" + setup_cmd = "invenio db init && invenio db create && invenio alembic upgrade heads && (invenio roles create iform_authenticated -d 'Allows uploading research data' || true) && invenio collect -v && invenio index init" docker_compose("exec", "worker", "bash", "-c", setup_cmd) logger.success("Production deployment complete!") @@ -195,7 +195,7 @@ def prod_update( # Restart nginx so it picks up the new container IPs for web-ui and web-api docker_compose("restart", "frontend") - update_cmd = "invenio alembic upgrade heads && invenio collect -v" + update_cmd = "invenio alembic upgrade heads && (invenio roles create iform_authenticated -d 'Allows uploading research data' || true) && invenio collect -v" docker_compose("exec", "worker", "bash", "-c", update_cmd) try: diff --git a/cli/dev.py b/cli/dev.py index a528c79..1a8b0db 100644 --- a/cli/dev.py +++ b/cli/dev.py @@ -96,20 +96,34 @@ def test_local(): repo_dir = get_repo_dir() with local.cwd(repo_dir): - docker_compose("down") - docker( + from plumbum import FG + + docker_compose("down") & FG + docker[ "build", "-t", config.docker_image_name, + "--network", + "host", "--no-cache", "--build-arg", "INSTALL_LOCAL_WHEELS=true", ".", - ) - docker_compose("up", "-d", "--wait") + ] & FG + docker_compose("up", "-d", "--wait") & FG - setup_cmd = "invenio db init || true; invenio db create || true; invenio alembic upgrade || true; invenio index init || true" - docker_compose("exec", "worker", "bash", "-c", setup_cmd) + setup_cmd = "invenio db init || true; invenio db create || true; invenio alembic upgrade || true; invenio index init || true; invenio roles create iform_authenticated -d 'Allows uploading research data' || true" + docker_compose("exec", "worker", "bash", "-c", setup_cmd) & FG + + logger.info( + "Running webpack buildall inside web-ui to update persistent static volume..." + ) + docker_compose( + "exec", "web-ui", "bash", "-c", "uv run invenio webpack buildall" + ) & FG + + logger.info("Restarting frontend proxy to pick up new container IPs...") + docker_compose("restart", "frontend") & FG try: curl("-skI", "https://127.0.0.1:8443/") diff --git a/cli/utils.py b/cli/utils.py index 086768f..da5d166 100644 --- a/cli/utils.py +++ b/cli/utils.py @@ -74,10 +74,26 @@ def docker_compose(*args): raise typer.Exit(1) if _deploy_env_vars is None: - _deploy_env_vars = get_dynamic_s3_credentials() + use_s3 = True + with open(env_file) as f: + for line in f: + if line.startswith("INVENIO_USE_S3"): + val = line.split("=")[1].strip().lower() + use_s3 = val == "true" + + if use_s3: + _deploy_env_vars = get_dynamic_s3_credentials() + else: + logger.info( + "INVENIO_USE_S3 is false. Using dummy S3 credentials to prevent production database corruption." + ) + _deploy_env_vars = { + "INVENIO_S3_ACCESS_KEY_ID": "CHANGE_ME", + "INVENIO_S3_SECRET_ACCESS_KEY": "CHANGE_ME", + } compose = docker[ "compose", "-f", config.docker_compose_file, "--env-file", str(env_file) ] with local.env(**_deploy_env_vars): - return compose(*args) + return compose[*args] diff --git a/i-form-data-repository/docker-compose.full.yml b/i-form-data-repository/docker-compose.full.yml index b3f8076..81a7664 100644 --- a/i-form-data-repository/docker-compose.full.yml +++ b/i-form-data-repository/docker-compose.full.yml @@ -68,9 +68,6 @@ services: depends_on: - web-ui - web-api - ports: - - "0.0.0.0:80:80" - - "0.0.0.0:443:443" # UI Application web-ui: extends: diff --git a/i-form-data-repository/docker-services.yml b/i-form-data-repository/docker-services.yml index e25128c..d151bbd 100644 --- a/i-form-data-repository/docker-services.yml +++ b/i-form-data-repository/docker-services.yml @@ -40,8 +40,8 @@ services: env_file: ../.env restart: "unless-stopped" ports: - - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:8080:80" - - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:8443:443" + - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:${INVENIO_HTTP_PORT:-80}:80" + - "${DOCKER_SERVICES_IP_BIND:-127.0.0.1}:${INVENIO_HTTPS_PORT:-443}:443" logging: driver: "json-file" options: @@ -72,6 +72,7 @@ services: - "POSTGRES_USER=${POSTGRES_USER:-i-form-data-repository}" - "POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-i-form-data-repository}" - "POSTGRES_DB=${POSTGRES_DB:-i-form-data-repository}" + - "ENABLE_WALG=${INVENIO_USE_S3:-true}" - "WALG_S3_PREFIX=s3://${INVENIO_S3_BUCKET_NAME}/walg" - "AWS_ACCESS_KEY_ID=${INVENIO_S3_ACCESS_KEY_ID}" - "AWS_SECRET_ACCESS_KEY=${INVENIO_S3_SECRET_ACCESS_KEY}" diff --git a/i-form-data-repository/docker/nginx/Dockerfile b/i-form-data-repository/docker/nginx/Dockerfile index 4c4acb0..73232bd 100644 --- a/i-form-data-repository/docker/nginx/Dockerfile +++ b/i-form-data-repository/docker/nginx/Dockerfile @@ -1,8 +1,8 @@ FROM docker.io/library/nginx COPY nginx.conf /etc/nginx/nginx.conf COPY conf.d/* /etc/nginx/conf.d/ -COPY test.key /etc/ssl/private/test.key -COPY test.crt /etc/ssl/certs/test.crt +COPY test.key /etc/ssl/private/invenio.key +COPY test.crt /etc/ssl/certs/invenio.crt COPY entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] CMD ["nginx", "-g", "daemon off;"] \ No newline at end of file diff --git a/i-form-data-repository/docker/postgres/setup-walg.sh b/i-form-data-repository/docker/postgres/setup-walg.sh index 9c0eca1..d3ff45b 100644 --- a/i-form-data-repository/docker/postgres/setup-walg.sh +++ b/i-form-data-repository/docker/postgres/setup-walg.sh @@ -1,7 +1,10 @@ #!/bin/bash set -e -# Configure postgres to archive WAL logs via wal-g -echo "archive_mode = on" >> "$PGDATA/postgresql.conf" -echo "archive_command = 'wal-g wal-push %p'" >> "$PGDATA/postgresql.conf" -echo "archive_timeout = 60" >> "$PGDATA/postgresql.conf" +if [ "$ENABLE_WALG" = "true" ]; then + echo "archive_mode = on" >> "$PGDATA/postgresql.conf" + echo "archive_command = 'wal-g wal-push %p'" >> "$PGDATA/postgresql.conf" + echo "archive_timeout = 60" >> "$PGDATA/postgresql.conf" +else + echo "archive_mode = off" >> "$PGDATA/postgresql.conf" +fi diff --git a/i-form-data-repository/invenio.cfg b/i-form-data-repository/invenio.cfg index 49d80e5..6df9599 100644 --- a/i-form-data-repository/invenio.cfg +++ b/i-form-data-repository/invenio.cfg @@ -165,20 +165,21 @@ APP_RDM_DEPOSIT_FORM_AUTOCOMPLETE_NAMES = "search" # "search_only" or "off" # Enable new Uppy-based UI for depositing large files (Invenio RDM v13+) APP_RDM_DEPOSIT_NG_FILES_UI_ENABLED = True -# Invenio-Files-Rest +import os + +use_s3 = os.environ.get("INVENIO_USE_S3", "true").lower() == "true" + +# Invenio-Files-Rest & S3 # ================== -FILES_REST_STORAGE_FACTORY = "invenio_s3.s3fs_storage_factory" +if use_s3: + FILES_REST_STORAGE_FACTORY = "invenio_s3.s3fs_storage_factory" + S3_ENDPOINT_URL = os.environ.get("INVENIO_S3_ENDPOINT_URL", "http://localhost:9000/") + S3_ACCESS_KEY_ID = os.environ.get("INVENIO_S3_ACCESS_KEY_ID", "CHANGE_ME") + S3_SECRET_ACCESS_KEY = os.environ.get("INVENIO_S3_SECRET_ACCESS_KEY", "CHANGE_ME") -# Invenio-S3 -# ========== -S3_ENDPOINT_URL = "http://localhost:9000/" -S3_ACCESS_KEY_ID = "CHANGE_ME" -S3_SECRET_ACCESS_KEY = "CHANGE_ME" - -# Allow S3 endpoint in the CSP rules -APP_DEFAULT_SECURE_HEADERS["content_security_policy"]["default-src"].append( - S3_ENDPOINT_URL -) + APP_DEFAULT_SECURE_HEADERS["content_security_policy"]["default-src"].append( + S3_ENDPOINT_URL + ) # Invenio-Records-Resources # =========================