Attempted to switch backend to invenio-rdm-starter

.gitignore

@@ -44,3 +44,6 @@ yarn-error.log*
 .svelte-kit
 
 versions
+
+secrets.env
+.env

am-d-model.env

@@ -0,0 +1,4 @@
+INVENIO_APP_ALLOWED_HOSTS=["invenio.am-d-model.eu"]
+INVENIO_THEME_FRONTPAGE_TITLE="AM-D-Model Data Repository"
+INVENIO_S3_REGION_NAME="eu-west-1"
+INVENIO_S3_BUCKET_NAME="am-d-model-data-repository"

caddy/Caddyfile

@@ -1,4 +1,4 @@
-import repo.caddyfile {
+{
 	admin off
 	debug
 	log {
@@ -6,35 +6,19 @@ import repo.caddyfile {
 	}
 }
 
-(site) {
-	@landing {
-		path / /contact* /about* /favicon.png
-	}
-	handle /_app/* {
-		reverse_proxy site:3000
-	}
-	handle @landing {
+am-d-model.eu {
 	reverse_proxy site:3000 {
 		health_uri /health
 		health_interval 30s
 		health_timeout 10s
 		health_status 200
 	}
-	}
-	handle /repo* {
-		uri strip_prefix /repo
-		import repo_site
-	}
 }
 
-:80 {
-	import site
-}
-
-http://am-d-model.eu {
-	import site
-}
-
-am-d-model.eu {
-	import site
+invenio.am-d-model.eu {
+	root /static/* /opt/invenio/var/instance/static
+	try_files {path} {path}/ /index.html
+	reverse_proxy invenio-rdm:5000
+	file_server
+	encode gzip
 }

caddy/repo.caddyfile

@@ -1,65 +0,0 @@
-(repo_cors) {
-	header {
-		Access-Control-Allow-Origin *
-		Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS"
-		Access-Control-Allow-Headers "*"
-		Access-Control-Allow-Credentials true
-		Strict-Transport-Security "max-age=15768000"
-		# Request ID tracing
-		+X-Request-ID {uuid}
-	}
-}
-
-(repo_common_proxy) {
-	# Set headers similar to uwsgi_param
-	header_up Host {host}
-	header_up X-Request-ID {uuid}
-	# Remove sensitive headers
-	header_down -X-Session-ID
-	header_down -X-User-ID
-}
-
-(repo_site) {
-	import repo_cors
-
-	# UI Server routes
-	handle / {
-		request_body {
-			max_size 100MB
-		}
-		reverse_proxy repo-web-ui:5000 {
-			import repo_common_proxy
-		}
-	}
-
-	# API routes
-	handle /api/* {
-		request_body {
-			max_size 100MB
-		}
-		reverse_proxy repo-web-api:5000 {
-			import repo_common_proxy
-		}
-	}
-
-	# Large file upload API routes
-	handle_path /api/records/*/draft/files/*/content {
-		request_body {
-			max_size 250GB
-		}
-		encode gzip
-		reverse_proxy repo-web-api:5000 {
-			import repo_common_proxy
-		}
-	}
-
-	handle /static/* {
-		root * /opt/invenio/var/instance/static
-		file_server
-	}
-
-	handle /robots.txt {
-		root * /opt/invenio/var/instance/static
-		file_server
-	}
-}

devenv.nix

@@ -6,8 +6,10 @@
   ...
 }: {
   packages = with pkgs; [
+    awscli2
     bun
     git
+    jq
   ];
 
   languages.javascript.bun = {

docker-compose.yaml

@@ -27,69 +27,181 @@ services:
         restart: unless-stopped
         labels:
             - "io.containers.autoupdate=registry"
-    repo-cache:
-        extends:
-            file: repo/docker-compose.full.yml
-            service: cache
-        networks:
-            - am-d-model-network
-    repo-db:
-        extends:
-            file: repo/docker-compose.full.yml
-            service: db
-        networks:
-            - am-d-model-network
-    repo-mq:
-        extends:
-            file: repo/docker-compose.full.yml
-            service: mq
-        networks:
-            - am-d-model-network
-    repo-search:
-        extends:
-            file: repo/docker-compose.full.yml
-            service: search
-        networks:
-            - am-d-model-network
-    repo-s3:
-        extends:
-            file: repo/docker-compose.full.yml
-            service: s3
-        networks:
-            - am-d-model-network
-    # UI Application
-    repo-web-ui:
-        extends:
-            file: repo/docker-compose.full.yml
-            service: web-ui
-        build:
-            context: ./repo/
-        networks:
-            - am-d-model-network
-    # API Rest Application
-    repo-web-api:
-        extends:
-            file: repo/docker-compose.full.yml
-            service: web-api
-        build:
-            context: ./repo/
-        networks:
-            - am-d-model-network
-    # Worker
-    repo-worker:
-        extends:
-            file: repo/docker-compose.full.yml
-            service: worker
-        build:
-            context: ./repo/
-        networks:
-            - am-d-model-network
+    invenio-rdm:
+      image: ghcr.io/front-matter/invenio-rdm-starter:latest
+      pull_policy: if_not_present
+      volumes:
+        - uploaded_data:/opt/invenio/var/instance/data
+        - archived_data:/opt/invenio/var/instance/archive
+      environment:
+        # Flask
+        - INVENIO_APP_ALLOWED_HOSTS=${INVENIO_APP_ALLOWED_HOSTS:-['0.0.0.0', 'localhost', '127.0.0.1']}
+
+        # Flask-SQLAlchemy
+        - INVENIO_SQLALCHEMY_DATABASE_URI=${INVENIO_SQLALCHEMY_DATABASE_URI:-postgresql+psycopg2://inveniordm:inveniordm@db/inveniordm}
+
+        # Flask-Babel
+        - INVENIO_BABEL_DEFAULT_LOCALE=${INVENIO_BABEL_DEFAULT_LOCALE:-en}
+        - INVENIO_BABEL_DEFAULT_TIMEZONE=${INVENIO_BABEL_DEFAULT_TIMEZONE:-UTC}
+
+        # Invenio-App
+        - INVENIO_CACHE_TYPE=${INVENIO_CACHE_TYPE:-redis}
+        - INVENIO_CACHE_REDIS_URL=${INVENIO_CACHE_REDIS_URL:-redis://cache:6379/0}
+        - INVENIO_ACCOUNTS_SESSION_REDIS_URL=${INVENIO_ACCOUNTS_SESSION_REDIS_URL:-redis://cache:6379/1}
+        - INVENIO_CELERY_RESULT_BACKEND=${INVENIO_CELERY_RESULT_BACKEND:-redis://cache:6379/2}
+        - INVENIO_RATELIMIT_STORAGE_URL=${INVENIO_RATELIMIT_STORAGE_URL:-redis://cache:6379/3}
+        - INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL=${INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL:-redis://cache:6379/4}
+        - INVENIO_BROKER_URL=${INVENIO_BROKER_URL:-redis://cache:6379/5}
+        - INVENIO_CELERY_BROKER_URL=${INVENIO_BROKER_URL:-redis://cache:6379/5}
+
+        - INVENIO_WSGI_PROXIES=4
+        - INVENIO_SECRET_KEY=${INVENIO_SECRET_KEY:-changeme}
+
+        # Invenio-I18N
+        # TODO: fix tuple parsing
+        # - INVENIO_I18N_LANGUAGES=${INVENIO_I18N_LANGUAGES:-'[("de", _("German"))]'}
+
+        # Invenio-Theme
+        - INVENIO_THEME_LOGO=${INVENIO_THEME_LOGO:-images/invenio-rdm-white.svg}
+        - INVENIO_THEME_SITENAME=${INVENIO_THEME_SITENAME:-InvenioRDM Starter}
+        - INVENIO_THEME_FRONTPAGE_TITLE=${INVENIO_THEME_FRONTPAGE_TITLE:-InvenioRDM Starter}
+        - INVENIO_THEME_FRONTPAGE_SUBTITLE=${INVENIO_THEME_FRONTPAGE_SUBTITLE:-A starter project for the turn-key research data management repository.}
+        - INVENIO_THEME_SHOW_FRONTPAGE_INTRO_SECTION=False
+
+        # Invenio-Records-Resources
+        - INVENIO_SITE_UI_URL=${INVENIO_SITE_UI_URL:-https://localhost}
+        - INVENIO_SITE_API_URL=${INVENIO_SITE_API_URL:-https://localhost/api}
+
+        # Invenio-RDM-Records
+        - INVENIO_DATACITE_ENABLED=${INVENIO_DATACITE_ENABLED:-False}
+        - INVENIO_DATACITE_USERNAME=${INVENIO_DATACITE_USERNAME:-}
+        - INVENIO_DATACITE_PASSWORD=${INVENIO_DATACITE_PASSWORD:-}
+        - INVENIO_DATACITE_PREFIX=${INVENIO_DATACITE_PREFIX:-}
+        - INVENIO_DATACITE_TEST_MODE=${INVENIO_DATACITE_TEST_MODE:-True}
+        - INVENIO_DATACITE_DATACENTER_SYMBOL=${INVENIO_DATACITE_DATACENTER_SYMBOL:-}
+
+        - INVENIO_RDM_ALLOW_METADATA_ONLY_RECORDS=${INVENIO_RDM_ALLOW_METADATA_ONLY_RECORDS:-True}
+        - INVENIO_RDM_ALLOW_RESTRICTED_RECORDS=${INVENIO_RDM_ALLOW_RESTRICTED_RECORDS:-True}
+        - INVENIO_RDM_ALLOW_EXTERNAL_DOI_VERSIONING=${INVENIO_RDM_ALLOW_EXTERNAL_DOI_VERSIONING:-True}
+
+        - INVENIO_RDM_CITATION_STYLES_DEFAULT=${INVENIO_RDM_CITATION_STYLES_DEFAULT:-apa}
+        - INVENIO_SECURITY_EMAIL_SENDER=${INVENIO_SECURITY_EMAIL_SENDER:-}
+        # TODO: fix tuple parsing
+        # - INVENIO_RDM_CITATION_STYLES=${INVENIO_RDM_CITATION_STYLES:-"[('apa', 'APA'), ('chicago-note-bibliography', 'Chicago'), ('harvard-cite-them-right', 'Harvard')]"}
+
+        # Invenio-Accounts
+        - INVENIO_ACCOUNTS_LOCAL_LOGIN_ENABLED=${INVENIO_ACCOUNTS_LOCAL_LOGIN_ENABLED:-True}
+        - INVENIO_GITHUB_APP_CREDENTIALS=${INVENIO_GITHUB_APP_CREDENTIALS:-}
+
+        # OAI-PMH
+        - INVENIO_OAISERVER_ID_PREFIX=${OAISERVER_ID_PREFIX:-invenio-rdm}
+
+        # Invenio-Files-REST
+        - INVENIO_FILES_REST_STORAGE_FACTORY=invenio_s3.s3fs_storage_factory
+
+        # Invenio-S3
+        - INVENIO_S3_ENDPOINT_URL=${INVENIO_S3_ENDPOINT_URL:-}
+        - INVENIO_S3_ACCESS_KEY_ID=${INVENIO_S3_ACCESS_KEY_ID:-}
+        - INVENIO_S3_SECRET_ACCESS_KEY=${INVENIO_S3_SECRET_ACCESS_KEY:-}
+        - INVENIO_S3_BUCKET_NAME=${INVENIO_S3_BUCKET_NAME:-}
+
+        # Invenio-Search
+        - INVENIO_SEARCH_HOSTS=${INVENIO_SEARCH_HOSTS:-['search:9200']}
+        - INVENIO_SEARCH_INDEX_PREFIX=${INVENIO_SEARCH_INDEX_PREFIX:-invenio-rdm-}
+
+        # Logging
+        - INVENIO_LOGGING_CONSOLE_LEVEL=${INVENIO_LOGGING_CONSOLE_LEVEL:-WARNING}
+      depends_on:
+        search:
+          condition: service_started
+        cache:
+          condition: service_started
+        # db:
+        #   condition: service_started
+    worker:
+      command: "celery -A invenio_app.celery worker --beat --events --loglevel=WARNING"
+      image: ghcr.io/front-matter/invenio-rdm-starter:latest
+      pull_policy: if_not_present
+      volumes:
+        - uploaded_data:/opt/invenio/var/instance/data
+      environment:
+        - INVENIO_SQLALCHEMY_DATABASE_URI=${INVENIO_SQLALCHEMY_DATABASE_URI:-postgresql+psycopg2://inveniordm:inveniordm@db/inveniordm}
+        - INVENIO_SEARCH_HOSTS=${INVENIO_SEARCH_HOSTS:-['search:9200']}
+        - INVENIO_SEARCH_INDEX_PREFIX=${INVENIO_SEARCH_INDEX_PREFIX:-invenio-rdm-}
+
+        - INVENIO_CACHE_TYPE=${INVENIO_CACHE_TYPE:-redis}
+        - INVENIO_CACHE_REDIS_URL=${INVENIO_CACHE_REDIS_URL:-redis://cache:6379/0}
+        - INVENIO_ACCOUNTS_SESSION_REDIS_URL=${INVENIO_ACCOUNTS_SESSION_REDIS_URL:-redis://cache:6379/1}
+        - INVENIO_CELERY_RESULT_BACKEND=${INVENIO_CELERY_RESULT_BACKEND:-redis://cache:6379/2}
+        - INVENIO_RATELIMIT_STORAGE_URL=${INVENIO_RATELIMIT_STORAGE_URL:-redis://cache:6379/3}
+        - INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL=${INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL:-redis://cache:6379/4}
+        - INVENIO_BROKER_URL=${INVENIO_BROKER_URL:-redis://cache:6379/5}
+        - INVENIO_CELERY_BROKER_URL=${INVENIO_BROKER_URL:-redis://cache:6379/5}
+
+        - INVENIO_MAIL_SUPPRESS_SEND=${INVENIO_MAIL_SUPPRESS_SEND:-True}
+        - INVENIO_MAIL_SERVER=${INVENIO_MAIL_SERVER:-smtp.gmail.com}
+        - INVENIO_MAIL_PORT=${INVENIO_MAIL_PORT:-465}
+        - INVENIO_MAIL_USERNAME=${INVENIO_MAIL_USERNAME:-info}
+        - INVENIO_MAIL_PASSWORD=${INVENIO_MAIL_PASSWORD:-changeme}
+        - INVENIO_MAIL_USE_SSL=${INVENIO_MAIL_USE_SSL:-True}
+
+        # Invenio-Files-REST
+        - INVENIO_FILES_REST_STORAGE_FACTORY=${INVENIO_FILES_REST_STORAGE_FACTORY:-invenio_s3.s3fs_storage_factory}
+
+        # Invenio-S3
+        - INVENIO_S3_ENDPOINT_URL=${INVENIO_S3_ENDPOINT_URL:-}
+        - INVENIO_S3_ACCESS_KEY_ID=${INVENIO_S3_ACCESS_KEY_ID:-}
+        - INVENIO_S3_SECRET_ACCESS_KEY=${INVENIO_S3_SECRET_ACCESS_KEY:-}
+        - INVENIO_S3_BUCKET_NAME=${INVENIO_S3_BUCKET_NAME:-}
+      depends_on:
+        search:
+          condition: service_started
+        cache:
+          condition: service_started
+        # db:
+        #   condition: service_started
+    cache:
+      image: valkey/valkey:7.2.5-bookworm
+      restart: "unless-stopped"
+      ports:
+        - "6379:6379"
+    db:
+      image: postgres:16.4-bookworm
+      restart: "unless-stopped"
+      environment:
+        - POSTGRES_USER=${POSTGRES_USER:-inveniordm}
+        - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-inveniordm}
+        - POSTGRES_DB=${POSTGRES_DB:-inveniordm}
+      ports:
+        - 5432:5432
+    search:
+      image: opensearchproject/opensearch:2.12.0
+      restart: "unless-stopped"
+      environment:
+        - bootstrap.memory_lock=true
+        - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m
+        - DISABLE_INSTALL_DEMO_CONFIG=true
+        - DISABLE_SECURITY_PLUGIN=true
+        - discovery.type=single-node
+      ulimits:
+        memlock:
+          soft: -1
+          hard: -1
+        nofile:
+          soft: 65536
+          hard: 65536
+      mem_limit: 2g
+      ports:
+        - "9200:9200"
+        - "9600:9600"
 
 networks:
     am-d-model-network:
         name: am-d-model-network
 
 volumes:
+  app_data:
+  uploaded_data:
+  archived_data:
   caddy_data:
-    static_data:
-    redis_data:
+  caddy_config:

prepare-env.sh

@@ -1,3 +1,4 @@
 #!/usr/bin/env bash
 aws secretsmanager get-secret-value --secret-id Invenio | \
-  jq -r '.SecretString | fromjson | to_entries | .[] | .key + "=" + .value' > .env
+  jq -r '.SecretString | fromjson | to_entries | .[] | .key + "=" + .value' > secrets.env
+cat am-d-model.env secrets.env > .env