diff --git a/.gitignore b/.gitignore index 2910b4e..4c0de8c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ yarn-error.log* .svelte-kit versions + +secrets.env +.env diff --git a/am-d-model.env b/am-d-model.env new file mode 100644 index 0000000..3dc50c1 --- /dev/null +++ b/am-d-model.env @@ -0,0 +1,4 @@ +INVENIO_APP_ALLOWED_HOSTS=["invenio.am-d-model.eu"] +INVENIO_THEME_FRONTPAGE_TITLE="AM-D-Model Data Repository" +INVENIO_S3_REGION_NAME="eu-west-1" +INVENIO_S3_BUCKET_NAME="am-d-model-data-repository" diff --git a/caddy/Caddyfile b/caddy/Caddyfile index d9f6cd3..d422a8d 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -1,4 +1,4 @@ -import repo.caddyfile { +{ admin off debug log { @@ -6,35 +6,19 @@ import repo.caddyfile { } } -(site) { - @landing { - path / /contact* /about* /favicon.png - } - handle /_app/* { - reverse_proxy site:3000 - } - handle @landing { - reverse_proxy site:3000 { - health_uri /health - health_interval 30s - health_timeout 10s - health_status 200 - } - } - handle /repo* { - uri strip_prefix /repo - import repo_site - } -} - -:80 { - import site -} - -http://am-d-model.eu { - import site -} - am-d-model.eu { - import site + reverse_proxy site:3000 { + health_uri /health + health_interval 30s + health_timeout 10s + health_status 200 + } +} + +invenio.am-d-model.eu { + root /static/* /opt/invenio/var/instance/static + try_files {path} {path}/ /index.html + reverse_proxy invenio-rdm:5000 + file_server + encode gzip } diff --git a/caddy/repo.caddyfile b/caddy/repo.caddyfile deleted file mode 100644 index 20a23e1..0000000 --- a/caddy/repo.caddyfile +++ /dev/null @@ -1,65 +0,0 @@ -(repo_cors) { - header { - Access-Control-Allow-Origin * - Access-Control-Allow-Methods "GET, POST, PUT, PATCH, DELETE, OPTIONS" - Access-Control-Allow-Headers "*" - Access-Control-Allow-Credentials true - Strict-Transport-Security "max-age=15768000" - # Request ID tracing - +X-Request-ID {uuid} - } -} - -(repo_common_proxy) { - # Set headers similar to uwsgi_param - header_up Host {host} - header_up X-Request-ID {uuid} - # Remove sensitive headers - header_down -X-Session-ID - header_down -X-User-ID -} - -(repo_site) { - import repo_cors - - # UI Server routes - handle / { - request_body { - max_size 100MB - } - reverse_proxy repo-web-ui:5000 { - import repo_common_proxy - } - } - - # API routes - handle /api/* { - request_body { - max_size 100MB - } - reverse_proxy repo-web-api:5000 { - import repo_common_proxy - } - } - - # Large file upload API routes - handle_path /api/records/*/draft/files/*/content { - request_body { - max_size 250GB - } - encode gzip - reverse_proxy repo-web-api:5000 { - import repo_common_proxy - } - } - - handle /static/* { - root * /opt/invenio/var/instance/static - file_server - } - - handle /robots.txt { - root * /opt/invenio/var/instance/static - file_server - } -} diff --git a/devenv.nix b/devenv.nix index 19c0b07..a3f28dc 100644 --- a/devenv.nix +++ b/devenv.nix @@ -6,8 +6,10 @@ ... }: { packages = with pkgs; [ + awscli2 bun git + jq ]; languages.javascript.bun = { diff --git a/docker-compose.yaml b/docker-compose.yaml index af21778..6c0b45e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -27,69 +27,181 @@ services: restart: unless-stopped labels: - "io.containers.autoupdate=registry" - repo-cache: - extends: - file: repo/docker-compose.full.yml - service: cache - networks: - - am-d-model-network - repo-db: - extends: - file: repo/docker-compose.full.yml - service: db - networks: - - am-d-model-network - repo-mq: - extends: - file: repo/docker-compose.full.yml - service: mq - networks: - - am-d-model-network - repo-search: - extends: - file: repo/docker-compose.full.yml - service: search - networks: - - am-d-model-network - repo-s3: - extends: - file: repo/docker-compose.full.yml - service: s3 - networks: - - am-d-model-network - # UI Application - repo-web-ui: - extends: - file: repo/docker-compose.full.yml - service: web-ui - build: - context: ./repo/ - networks: - - am-d-model-network - # API Rest Application - repo-web-api: - extends: - file: repo/docker-compose.full.yml - service: web-api - build: - context: ./repo/ - networks: - - am-d-model-network - # Worker - repo-worker: - extends: - file: repo/docker-compose.full.yml - service: worker - build: - context: ./repo/ - networks: - - am-d-model-network + invenio-rdm: + image: ghcr.io/front-matter/invenio-rdm-starter:latest + pull_policy: if_not_present + volumes: + - uploaded_data:/opt/invenio/var/instance/data + - archived_data:/opt/invenio/var/instance/archive + environment: + # Flask + - INVENIO_APP_ALLOWED_HOSTS=${INVENIO_APP_ALLOWED_HOSTS:-['0.0.0.0', 'localhost', '127.0.0.1']} + + # Flask-SQLAlchemy + - INVENIO_SQLALCHEMY_DATABASE_URI=${INVENIO_SQLALCHEMY_DATABASE_URI:-postgresql+psycopg2://inveniordm:inveniordm@db/inveniordm} + + # Flask-Babel + - INVENIO_BABEL_DEFAULT_LOCALE=${INVENIO_BABEL_DEFAULT_LOCALE:-en} + - INVENIO_BABEL_DEFAULT_TIMEZONE=${INVENIO_BABEL_DEFAULT_TIMEZONE:-UTC} + + # Invenio-App + - INVENIO_CACHE_TYPE=${INVENIO_CACHE_TYPE:-redis} + - INVENIO_CACHE_REDIS_URL=${INVENIO_CACHE_REDIS_URL:-redis://cache:6379/0} + - INVENIO_ACCOUNTS_SESSION_REDIS_URL=${INVENIO_ACCOUNTS_SESSION_REDIS_URL:-redis://cache:6379/1} + - INVENIO_CELERY_RESULT_BACKEND=${INVENIO_CELERY_RESULT_BACKEND:-redis://cache:6379/2} + - INVENIO_RATELIMIT_STORAGE_URL=${INVENIO_RATELIMIT_STORAGE_URL:-redis://cache:6379/3} + - INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL=${INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL:-redis://cache:6379/4} + - INVENIO_BROKER_URL=${INVENIO_BROKER_URL:-redis://cache:6379/5} + - INVENIO_CELERY_BROKER_URL=${INVENIO_BROKER_URL:-redis://cache:6379/5} + + - INVENIO_WSGI_PROXIES=4 + - INVENIO_SECRET_KEY=${INVENIO_SECRET_KEY:-changeme} + + # Invenio-I18N + # TODO: fix tuple parsing + # - INVENIO_I18N_LANGUAGES=${INVENIO_I18N_LANGUAGES:-'[("de", _("German"))]'} + + # Invenio-Theme + - INVENIO_THEME_LOGO=${INVENIO_THEME_LOGO:-images/invenio-rdm-white.svg} + - INVENIO_THEME_SITENAME=${INVENIO_THEME_SITENAME:-InvenioRDM Starter} + - INVENIO_THEME_FRONTPAGE_TITLE=${INVENIO_THEME_FRONTPAGE_TITLE:-InvenioRDM Starter} + - INVENIO_THEME_FRONTPAGE_SUBTITLE=${INVENIO_THEME_FRONTPAGE_SUBTITLE:-A starter project for the turn-key research data management repository.} + - INVENIO_THEME_SHOW_FRONTPAGE_INTRO_SECTION=False + + # Invenio-Records-Resources + - INVENIO_SITE_UI_URL=${INVENIO_SITE_UI_URL:-https://localhost} + - INVENIO_SITE_API_URL=${INVENIO_SITE_API_URL:-https://localhost/api} + + # Invenio-RDM-Records + - INVENIO_DATACITE_ENABLED=${INVENIO_DATACITE_ENABLED:-False} + - INVENIO_DATACITE_USERNAME=${INVENIO_DATACITE_USERNAME:-} + - INVENIO_DATACITE_PASSWORD=${INVENIO_DATACITE_PASSWORD:-} + - INVENIO_DATACITE_PREFIX=${INVENIO_DATACITE_PREFIX:-} + - INVENIO_DATACITE_TEST_MODE=${INVENIO_DATACITE_TEST_MODE:-True} + - INVENIO_DATACITE_DATACENTER_SYMBOL=${INVENIO_DATACITE_DATACENTER_SYMBOL:-} + + - INVENIO_RDM_ALLOW_METADATA_ONLY_RECORDS=${INVENIO_RDM_ALLOW_METADATA_ONLY_RECORDS:-True} + - INVENIO_RDM_ALLOW_RESTRICTED_RECORDS=${INVENIO_RDM_ALLOW_RESTRICTED_RECORDS:-True} + - INVENIO_RDM_ALLOW_EXTERNAL_DOI_VERSIONING=${INVENIO_RDM_ALLOW_EXTERNAL_DOI_VERSIONING:-True} + + - INVENIO_RDM_CITATION_STYLES_DEFAULT=${INVENIO_RDM_CITATION_STYLES_DEFAULT:-apa} + - INVENIO_SECURITY_EMAIL_SENDER=${INVENIO_SECURITY_EMAIL_SENDER:-} + # TODO: fix tuple parsing + # - INVENIO_RDM_CITATION_STYLES=${INVENIO_RDM_CITATION_STYLES:-"[('apa', 'APA'), ('chicago-note-bibliography', 'Chicago'), ('harvard-cite-them-right', 'Harvard')]"} + + # Invenio-Accounts + - INVENIO_ACCOUNTS_LOCAL_LOGIN_ENABLED=${INVENIO_ACCOUNTS_LOCAL_LOGIN_ENABLED:-True} + - INVENIO_GITHUB_APP_CREDENTIALS=${INVENIO_GITHUB_APP_CREDENTIALS:-} + + # OAI-PMH + - INVENIO_OAISERVER_ID_PREFIX=${OAISERVER_ID_PREFIX:-invenio-rdm} + + # Invenio-Files-REST + - INVENIO_FILES_REST_STORAGE_FACTORY=invenio_s3.s3fs_storage_factory + + # Invenio-S3 + - INVENIO_S3_ENDPOINT_URL=${INVENIO_S3_ENDPOINT_URL:-} + - INVENIO_S3_ACCESS_KEY_ID=${INVENIO_S3_ACCESS_KEY_ID:-} + - INVENIO_S3_SECRET_ACCESS_KEY=${INVENIO_S3_SECRET_ACCESS_KEY:-} + - INVENIO_S3_BUCKET_NAME=${INVENIO_S3_BUCKET_NAME:-} + + # Invenio-Search + - INVENIO_SEARCH_HOSTS=${INVENIO_SEARCH_HOSTS:-['search:9200']} + - INVENIO_SEARCH_INDEX_PREFIX=${INVENIO_SEARCH_INDEX_PREFIX:-invenio-rdm-} + + # Logging + - INVENIO_LOGGING_CONSOLE_LEVEL=${INVENIO_LOGGING_CONSOLE_LEVEL:-WARNING} + depends_on: + search: + condition: service_started + cache: + condition: service_started + # db: + # condition: service_started + worker: + command: "celery -A invenio_app.celery worker --beat --events --loglevel=WARNING" + image: ghcr.io/front-matter/invenio-rdm-starter:latest + pull_policy: if_not_present + volumes: + - uploaded_data:/opt/invenio/var/instance/data + environment: + - INVENIO_SQLALCHEMY_DATABASE_URI=${INVENIO_SQLALCHEMY_DATABASE_URI:-postgresql+psycopg2://inveniordm:inveniordm@db/inveniordm} + - INVENIO_SEARCH_HOSTS=${INVENIO_SEARCH_HOSTS:-['search:9200']} + - INVENIO_SEARCH_INDEX_PREFIX=${INVENIO_SEARCH_INDEX_PREFIX:-invenio-rdm-} + + - INVENIO_CACHE_TYPE=${INVENIO_CACHE_TYPE:-redis} + - INVENIO_CACHE_REDIS_URL=${INVENIO_CACHE_REDIS_URL:-redis://cache:6379/0} + - INVENIO_ACCOUNTS_SESSION_REDIS_URL=${INVENIO_ACCOUNTS_SESSION_REDIS_URL:-redis://cache:6379/1} + - INVENIO_CELERY_RESULT_BACKEND=${INVENIO_CELERY_RESULT_BACKEND:-redis://cache:6379/2} + - INVENIO_RATELIMIT_STORAGE_URL=${INVENIO_RATELIMIT_STORAGE_URL:-redis://cache:6379/3} + - INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL=${INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL:-redis://cache:6379/4} + - INVENIO_BROKER_URL=${INVENIO_BROKER_URL:-redis://cache:6379/5} + - INVENIO_CELERY_BROKER_URL=${INVENIO_BROKER_URL:-redis://cache:6379/5} + + - INVENIO_MAIL_SUPPRESS_SEND=${INVENIO_MAIL_SUPPRESS_SEND:-True} + - INVENIO_MAIL_SERVER=${INVENIO_MAIL_SERVER:-smtp.gmail.com} + - INVENIO_MAIL_PORT=${INVENIO_MAIL_PORT:-465} + - INVENIO_MAIL_USERNAME=${INVENIO_MAIL_USERNAME:-info} + - INVENIO_MAIL_PASSWORD=${INVENIO_MAIL_PASSWORD:-changeme} + - INVENIO_MAIL_USE_SSL=${INVENIO_MAIL_USE_SSL:-True} + + # Invenio-Files-REST + - INVENIO_FILES_REST_STORAGE_FACTORY=${INVENIO_FILES_REST_STORAGE_FACTORY:-invenio_s3.s3fs_storage_factory} + + # Invenio-S3 + - INVENIO_S3_ENDPOINT_URL=${INVENIO_S3_ENDPOINT_URL:-} + - INVENIO_S3_ACCESS_KEY_ID=${INVENIO_S3_ACCESS_KEY_ID:-} + - INVENIO_S3_SECRET_ACCESS_KEY=${INVENIO_S3_SECRET_ACCESS_KEY:-} + - INVENIO_S3_BUCKET_NAME=${INVENIO_S3_BUCKET_NAME:-} + depends_on: + search: + condition: service_started + cache: + condition: service_started + # db: + # condition: service_started + cache: + image: valkey/valkey:7.2.5-bookworm + restart: "unless-stopped" + ports: + - "6379:6379" + db: + image: postgres:16.4-bookworm + restart: "unless-stopped" + environment: + - POSTGRES_USER=${POSTGRES_USER:-inveniordm} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-inveniordm} + - POSTGRES_DB=${POSTGRES_DB:-inveniordm} + ports: + - 5432:5432 + search: + image: opensearchproject/opensearch:2.12.0 + restart: "unless-stopped" + environment: + - bootstrap.memory_lock=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m + - DISABLE_INSTALL_DEMO_CONFIG=true + - DISABLE_SECURITY_PLUGIN=true + - discovery.type=single-node + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + mem_limit: 2g + ports: + - "9200:9200" + - "9600:9600" networks: am-d-model-network: name: am-d-model-network volumes: - caddy_data: - static_data: - redis_data: + app_data: + uploaded_data: + archived_data: + caddy_data: + caddy_config: diff --git a/prepare-env.sh b/prepare-env.sh index f35180f..25c11ee 100755 --- a/prepare-env.sh +++ b/prepare-env.sh @@ -1,3 +1,4 @@ #!/usr/bin/env bash aws secretsmanager get-secret-value --secret-id Invenio | \ - jq -r '.SecretString | fromjson | to_entries | .[] | .key + "=" + .value' > .env + jq -r '.SecretString | fromjson | to_entries | .[] | .key + "=" + .value' > secrets.env +cat am-d-model.env secrets.env > .env