Set up site for production config behind proxy

2025-12-22 22:11:56 +00:00 · 2025-01-29 12:02:37 +00:00
parent 607f72659d
commit c0c191bc7c
9 changed files with 190 additions and 81 deletions
--- a/.env.example
+++ b/.env.example
@@ -0,0 +1,5 @@
+POSTGRES_PASSWORD=
+MINIO_ROOT_PASSWORD=
+INVENIO_SECRET_KEY=
+S3_SECRET_ACCESS_KEY=
+S3_ACCESS_KEY_ID=
--- a/.gitignore
+++ b/.gitignore
@@ -74,5 +74,5 @@ data/.minio.sys
 # Celery
 celerybeat-schedule

-# Configuration secrets
-secrets.toml
+# Configuration environment
+.env
--- a/2
+++ b/2
@@ -13,7 +13,7 @@ uwsgi = ">=2.0"
 uwsgitop = ">=0.11"
 uwsgi-tools = ">=1.1.1"
 flask-admin = "==1.6.1"
-tomli = "*"
+py-dotenv-safe = "*"

 [requires]
 python_version = "3.9"
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "39ee6b7027666473c34a239e7b6d6c8825f2718b7c21ff5d5a9fb25f98eb691c"
+            "sha256": "90c92776f7fd131f8dbad9aa57898068fc9ae8a4c0a1f4663790503f7138381d"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -114,11 +114,11 @@
        },
        "beautifulsoup4": {
            "hashes": [
-                "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051",
-                "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"
+                "sha256:237484d61be5d1e82b5aedd8568eea763b76191ee146597b1d405e28dbd9f3d9",
+                "sha256:4970105b2620a2fa530de34c76a9063c8f22d393f639d718d939f0750cc4473d"
            ],
            "markers": "python_full_version >= '3.6.0'",
-            "version": "==4.12.3"
+            "version": "==4.13.0b3"
        },
        "bibtexparser": {
            "hashes": [
@@ -155,19 +155,19 @@
        },
        "boto3": {
            "hashes": [
-                "sha256:6d473f0f340d02b4e9ad5b8e68786a09728101a8b950231b89ebdaf72b6dca21",
-                "sha256:b36feae061dc0793cf311468956a0a9e99215ce38bc99a1a4e55a5b105f16297"
+                "sha256:7f61c9d0ea64f484a17c1e3115fdf90fd7b17ab6771e07cb4549f42b9fd28fb9",
+                "sha256:ac47215d320b0c2534340db58d6d5284cb1860b7bff172b4dd6eee2dee1d5779"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==1.36.6"
+            "version": "==1.36.8"
        },
        "botocore": {
            "hashes": [
-                "sha256:4864c53d638da191a34daf3ede3ff1371a3719d952cc0c6bd24ce2836a38dd77",
-                "sha256:f77bbbb03fb420e260174650fb5c0cc142ec20a96967734eed2b0ef24334ef34"
+                "sha256:59d3fdfbae6d916b046e973bebcbeb70a102f9e570ca86d5ba512f1854b78fc2",
+                "sha256:81c88e5566cf018e1411a68304dc1fb9e4156ca2b50a3a0f0befc274299e67fa"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==1.36.6"
+            "version": "==1.36.8"
        },
        "cachelib": {
            "hashes": [
@@ -529,11 +529,11 @@
        },
        "defusedxml": {
            "hashes": [
-                "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69",
-                "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"
+                "sha256:138c7d540a78775182206c7c97fe65b246a2f40b29471e1a2f1b0da76e7a3942",
+                "sha256:1c812964311154c3bf4aaf3bc1443b31ee13530b7f255eaaa062c0553c76103d"
            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==0.7.1"
+            "markers": "python_version >= '3.6'",
+            "version": "==0.8.0rc2"
        },
        "deprecated": {
            "hashes": [
@@ -1537,11 +1537,11 @@
        },
        "kombu": {
            "hashes": [
-                "sha256:14212f5ccf022fc0a70453bb025a1dcc32782a588c49ea866884047d66e14763",
-                "sha256:eef572dd2fd9fc614b37580e3caeafdd5af46c1eff31e7fba89138cdb406f2cf"
+                "sha256:213dc124de2a9dada467aa3387c638d8594e91a9dff2dcf6206cd9c6bcf84a5d",
+                "sha256:f581f3b2945a46d5de540a8fde920e87725308cfed6bdeed6983fa4124879cd0"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==5.4.2"
+            "version": "==5.5.0rc2"
        },
        "limits": {
            "hashes": [
@@ -1925,11 +1925,11 @@
        },
        "mistune": {
            "hashes": [
-                "sha256:b05198cf6d671b3deba6c87ec6cf0d4eb7b72c524636eddb6dbf13823b52cee1",
-                "sha256:dbcac2f78292b9dc066cd03b7a3a26b62d85f8159f2ea5fd28e55df79908d667"
+                "sha256:02106ac2aa4f66e769debbfa028509a275069dcffce0dfa578edd7b991ee700a",
+                "sha256:e0740d635f515119f7d1feb6f9b192ee60f0cc649f80a8f944f905706a21654c"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==3.1.0"
+            "version": "==3.1.1"
        },
        "mkdocs": {
            "hashes": [
@@ -2057,11 +2057,11 @@
        },
        "nbconvert": {
            "hashes": [
-                "sha256:c83467bb5777fdfaac5ebbb8e864f300b277f68692ecc04d6dab72f2d8442344",
-                "sha256:e12eac052d6fd03040af4166c563d76e7aeead2e9aadf5356db552a1784bd547"
+                "sha256:1375a7b67e0c2883678c48e506dc320febb57685e5ee67faa51b18a90f3a712b",
+                "sha256:576a7e37c6480da7b8465eefa66c17844243816ce1ccc372633c6b71c3c0f582"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==7.16.5"
+            "version": "==7.16.6"
        },
        "nbformat": {
            "hashes": [
@@ -2353,6 +2353,15 @@
            ],
            "version": "==0.2.3"
        },
+        "py-dotenv-safe": {
+            "hashes": [
+                "sha256:1ca96558bd3195d706db1ed73fb8bc2cc1f14140ecc371c7022b9b512c4dc9b3",
+                "sha256:f2de7b8fdce6aad4cfa574f54efc113c768e9bf5ac094f118d1b58f3ba445b00"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.6'",
+            "version": "==1.0.1"
+        },
        "pycountry": {
            "hashes": [
                "sha256:b2163a246c585894d808f18783e19137cb70a0c18fb36748dc01fc6f109c1646"
@@ -2389,19 +2398,19 @@
                "crypto"
            ],
            "hashes": [
-                "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953",
-                "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"
+                "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850",
+                "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"
            ],
-            "markers": "python_version >= '3.9'",
-            "version": "==2.10.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.9.0"
        },
        "pymdown-extensions": {
            "hashes": [
-                "sha256:637951cbfbe9874ba28134fb3ce4b8bcadd6aca89ac4998ec29dcbafd554ae08",
-                "sha256:b65801996a0cd4f42a3110810c306c45b7313c09b0610a6f773730f2a9e3c96b"
+                "sha256:7a77b8116dc04193f2c01143760a43387bd9dc4aa05efacb7d838885a7791253",
+                "sha256:f45bc5892410e54fd738ab8ccd736098b7ff0cb27fdb4bf24d0a0c6584bc90e1"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==10.14.1"
+            "version": "==10.14.2"
        },
        "pymysql": {
            "hashes": [
@@ -2435,6 +2444,14 @@
            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
            "version": "==2.9.0.post0"
        },
+        "python-dotenv": {
+            "hashes": [
+                "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca",
+                "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.0.1"
+        },
        "python-geoip": {
            "hashes": [
                "sha256:b7b11dab42bffba56943b3199e3441f41cea145244d215844ecb6de3d5fb2df5",
@@ -2649,11 +2666,11 @@
        },
        "redis": {
            "hashes": [
-                "sha256:16f2e22dff21d5125e8481515e386711a34cbec50f0e44413dd7d9c060a54e0f",
-                "sha256:ee7e1056b9aea0f04c6c2ed59452947f34c4940ee025f5dd83e6a6418b6989e4"
+                "sha256:d05d634b6f75a971ab3481f00c051990ee8ae5c6eb9a9e993aec0d740905f3ed",
+                "sha256:e8be754fdb61a95e4e7c43c4ad9fb94b1c4b407623a3bfaaf6c5f53ffb5a46cc"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==5.2.1"
+            "version": "==5.3.0b4"
        },
        "referencing": {
            "hashes": [
@@ -3384,11 +3401,11 @@
        },
        "tzdata": {
            "hashes": [
-                "sha256:24894909e88cdb28bd1636c6887801df64cb485bd593f2fd83ef29075a81d694",
-                "sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639"
+                "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc",
+                "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"
            ],
            "markers": "python_version >= '2'",
-            "version": "==2025.1"
+            "version": "==2024.2"
        },
        "tzlocal": {
            "hashes": [
@@ -3408,10 +3425,10 @@
        },
        "ua-parser-builtins": {
            "hashes": [
-                "sha256:eb4f93504040c3a990a6b0742a2afd540d87d7f9f05fd66e94c101db1564674d"
+                "sha256:1be9716c3c5994d560ea3b71261985f766ab0dfe90bdec74490304c87f1df3aa"
            ],
            "markers": "python_version >= '3.9'",
-            "version": "==0.18.0.post1"
+            "version": "==0.19.0.dev30"
        },
        "uritemplate": {
            "hashes": [
--- a/docker-compose.full.yml
+++ b/docker-compose.full.yml
@@ -24,10 +24,14 @@ services:
    extends:
      file: docker-services.yml
      service: cache
+    networks:
+      - am-d-model-repo-network
  db:
    extends:
      file: docker-services.yml
      service: db
+    networks:
+      - am-d-model-repo-network
  mq:
    extends:
      file: docker-services.yml
@@ -36,10 +40,14 @@ services:
    extends:
      file: docker-services.yml
      service: search
+    networks:
+      - am-d-model-repo-network
  s3:
    extends:
      file: docker-services.yml
      service: s3
+    networks:
+      - am-d-model-repo-network
  # Frontend
  frontend:
    extends:
@@ -50,9 +58,11 @@ services:
    depends_on:
      - web-ui
      - web-api
-    ports:
-      - "80:80"
-      - "443:443"
+    # ports:
+    #   - "80:80"
+    #   - "443:443"
+    networks:
+      - am-d-model-repo-network
  # UI Application
  web-ui:
    extends:
@@ -64,6 +74,8 @@ services:
      - "5000"
    volumes:
      - static_data:/opt/invenio/var/instance/static
+    networks:
+      - am-d-model-repo-network

  # API Rest Application
  web-api:
@@ -74,6 +86,8 @@ services:
    image: am-d-model-data-repository:latest
    ports:
      - "5000"
+    networks:
+      - am-d-model-repo-network

  # Worker
  worker:
@@ -91,6 +105,12 @@ services:
        condition: service_started
      mq:
        condition: service_started
+    networks:
+      - am-d-model-repo-network
+networks:
+    am-d-model-repo-network:
+        name: am-d-model-repo-network
 volumes:
    static_data:
    data:
+    redis_data:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -14,27 +14,40 @@
 # - OpenSearch (exposed ports: 9200, 9600)
 # - Kibana (view ES indexes) (exposed ports: 5601)
 #
-version: '2.2'
 services:
  cache:
    extends:
      file: docker-services.yml
      service: cache
+    networks:
+      - am-d-model-repo-network
  db:
    extends:
      file: docker-services.yml
      service: db
+    networks:
+      - am-d-model-repo-network
  mq:
    extends:
      file: docker-services.yml
      service: mq
+    networks:
+      - am-d-model-repo-network
  search:
    extends:
      file: docker-services.yml
      service: search
+    networks:
+      - am-d-model-repo-network
  s3:
    extends:
      file: docker-services.yml
      service: s3
+    networks:
+      - am-d-model-repo-network
+networks:
+    am-d-model-repo-network:
+        name: am-d-model-repo-network
 volumes:
    data:
+    redis_data:
--- a/docker-services.yml
+++ b/docker-services.yml
@@ -7,6 +7,8 @@ services:
        - ENVIRONMENT=DEV
    image: am-d-model-data-repository
    restart: "unless-stopped"
+    env_file:
+      - .env
    environment:
      - "INVENIO_ACCOUNTS_SESSION_REDIS_URL=redis://cache:6379/1"
      - "INVENIO_BROKER_URL=amqp://guest:guest@mq:5672/"
@@ -16,10 +18,12 @@ services:
      - "INVENIO_CELERY_RESULT_BACKEND=redis://cache:6379/2"
      - "INVENIO_COMMUNITIES_IDENTITIES_CACHE_REDIS_URL=redis://cache:6379/4"
      - "INVENIO_SEARCH_HOSTS=['search:9200']"
-      - "INVENIO_SECRET_KEY=CHANGE_ME"
+      # - "INVENIO_SECRET_KEY=${aws secretsmanager get-secret-value --secret-id Invenio | jq '.SecretString | fromjson | .INVENIO_SECRET_KEY'}"
      - "INVENIO_SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://am-d-model-data-repository:am-d-model-data-repository@db/am-d-model-data-repository"
      - "INVENIO_WSGI_PROXIES=2"
      - "INVENIO_RATELIMIT_STORAGE_URL=redis://cache:6379/3"
+    networks:
+      - am-d-model-repo-network
  frontend:
    build: ./docker/nginx/
    image: am-d-model-data-repository-frontend
@@ -27,47 +31,62 @@ services:
    ports:
      - "80"
      - "443"
+    networks:
+      - am-d-model-repo-network
  cache:
    image: redis:7
    restart: "unless-stopped"
-    read_only: true
+    # read_only: true
+    command: redis-server --appendonly yes
+    volumes:
+      - redis_data:/data
    ports:
      - "6379:6379"
+    networks:
+      - am-d-model-repo-network
  db:
    image: postgres:14.13
    restart: "unless-stopped"
+    env_file:
+      - .env
    environment:
      - "POSTGRES_USER=am-d-model-data-repository"
-      - "POSTGRES_PASSWORD=am-d-model-data-repository"
+      # - "POSTGRES_PASSWORD=am-d-model-data-repository"
      - "POSTGRES_DB=am-d-model-data-repository"
    ports:
-      - "5432:5432"
+      - "5432"
+    networks:
+      - am-d-model-repo-network
  pgadmin:
    image: dpage/pgadmin4:6
    restart: "unless-stopped"
    ports:
-      - "5050:80"
+      - "5050"
    environment:
      PGADMIN_DEFAULT_EMAIL: "repo@am-d-model.eu"
      PGADMIN_DEFAULT_PASSWORD: "am-d-model-data-repository"
    volumes:
      - ./docker/pgadmin/servers.json:/pgadmin4/servers.json
+    networks:
+      - am-d-model-repo-network
  mq:
    image: rabbitmq:3-management
    restart: "unless-stopped"
    ports:
-      - "15672:15672"
-      - "5672:5672"
+      - "15672"
+      - "5672"
+    networks:
+      - am-d-model-repo-network
  search:
    image: opensearchproject/opensearch:2.17.1
    restart: "unless-stopped"
-    environment:
-      # settings only for development. DO NOT use in production!
-      - bootstrap.memory_lock=true
-      - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
-      - "DISABLE_INSTALL_DEMO_CONFIG=true"
-      - "DISABLE_SECURITY_PLUGIN=true"
-      - "discovery.type=single-node"
+    # environment:
+    #   # settings only for development. DO NOT use in production!
+    #   - bootstrap.memory_lock=true
+    #   - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m"
+    #   - "DISABLE_INSTALL_DEMO_CONFIG=true"
+    #   - "DISABLE_SECURITY_PLUGIN=true"
+    #   - "discovery.type=single-node"
    ulimits:
      memlock:
        soft: -1
@@ -77,27 +96,33 @@ services:
        hard: 65536
    mem_limit: 2g
    ports:
-      - "9200:9200"
-      - "9600:9600"
+      - "9200"
+      - "9600"
+    networks:
+      - am-d-model-repo-network
  opensearch-dashboards:
    image: opensearchproject/opensearch-dashboards:2.17.1
    ports:
      - "5601:5601"
    expose:
      - "5601"
-    environment:
-      # settings only for development. DO NOT use in production!
-      - 'OPENSEARCH_HOSTS=["http://search:9200"]'
-      - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true"
+    # environment:
+    #   # settings only for development. DO NOT use in production!
+    #   - 'OPENSEARCH_HOSTS=["http://search:9200"]'
+    #   - "DISABLE_SECURITY_DASHBOARDS_PLUGIN=true"
+    networks:
+      - am-d-model-repo-network
  s3:
    image: minio/minio:RELEASE.2022-10-24T18-35-07Z
    restart: "unless-stopped"
    ports:
-      - "9000:9000"
-      - "9001:9001"
+      - "9000"
+      - "9001"
+    env_file:
+      - .env
    environment:
-      MINIO_ROOT_USER: CHANGE_ME
-      MINIO_ROOT_PASSWORD: CHANGE_ME
+      MINIO_ROOT_USER: am-d-model-data-repository
+      # MINIO_ROOT_PASSWORD: CHANGE_ME
    volumes:
      - ./data:/data
    command: server /data --console-address :9001
@@ -106,3 +131,12 @@ services:
      interval: 30s
      timeout: 20s
      retries: 3
+    networks:
+      - am-d-model-repo-network
+
+volumes:
+    redis_data:
+
+networks:
+    am-d-model-repo-network:
+        name: am-d-model-repo-network
--- a/invenio.cfg
+++ b/invenio.cfg
@@ -14,13 +14,30 @@ from invenio_i18n import lazy_gettext as _
 def _(x):  # needed to avoid start time failure with lazy strings
    return x

-# Custom function and constant to manage secrets more easily
-def read_secrets():
-    import tomli
-    with open("secrets.toml", "rb") as f:
-        return tomli.load(f)
+# Custom functions and to get env variables more safely
+def init_env():
+    from py_dotenv_safe import config

-SECRETS = read_secrets()
+    options = {
+        "dotenvPath": ".env",
+        "examplePath": ".env.example",
+        "allowEmptyValues": False,
+    }
+
+    config(options)
+    print("Environment variables loaded successfully.")
+
+init_env()
+
+def get_env_variable(key):
+    import os
+
+    x = os.getenv(key)
+
+    if x is None:
+        raise EnvironmentError(f"Environment variable {key} not found")
+
+    return x

 # Flask
 # =====
@@ -35,7 +52,7 @@ SEND_FILE_MAX_AGE_DEFAULT = 300
 # SECURITY WARNING: keep the secret key used in production secret!
 # Do not commit it to a source code repository.
 # TODO: Set
-SECRET_KEY=SECRETS["SECRET_KEY"]
+SECRET_KEY=get_env_variable("INVENIO_SECRET_KEY")

 # Since HAProxy and Nginx route all requests no matter the host header
 # provided, the allowed hosts variable is set to localhost. In production it
@@ -154,8 +171,8 @@ FILES_REST_STORAGE_FACTORY='invenio_s3.s3fs_storage_factory'
 # Invenio-S3
 # ==========
 S3_ENDPOINT_URL='http://localhost:9000/'
-S3_ACCESS_KEY_ID='CHANGE_ME'
-S3_SECRET_ACCESS_KEY='CHANGE_ME'
+S3_ACCESS_KEY_ID=get_env_variable("S3_ACCESS_KEY_ID")
+S3_SECRET_ACCESS_KEY=get_env_variable("S3_SECRET_ACCESS_KEY")

 # Allow S3 endpoint in the CSP rules
 APP_DEFAULT_SECURE_HEADERS['content_security_policy']['default-src'].append(
--- a/prepare-env.sh
+++ b/prepare-env.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+aws secretsmanager get-secret-value --secret-id Invenio | \
+  jq -r '.SecretString | fromjson | to_entries | .[] | .key + "=" + .value' > .env