docker-compose.integration.yaml

# Full end-to-end stack for DataPusher+ integration testing.
#
# Brings up Postgres + Solr + Redis + CKAN + Prefect server + Prefect worker
# (with qsv baked in via Dockerfile.worker). The worker mounts the repo
# editable so changes to DP+ code are picked up after `docker compose
# restart prefect-worker`.
#
# Usage:
#   # 1. Bring up the stack
#   docker compose -f docker-compose.integration.yaml up -d --build
#
#   # 2. Wait for health checks
#   docker compose -f docker-compose.integration.yaml ps
#
#   # 3. Register the DP+ deployment inside the CKAN container
#   docker compose -f docker-compose.integration.yaml exec ckan \
#       ckan -c /etc/ckan/default/ckan.ini datapusher_plus prefect-deploy
#
#   # 4. Run the integration tests
#   INTEGRATION=1 pytest tests/integration/
#
#   # 5. Tear down
#   docker compose -f docker-compose.integration.yaml down -v
#
# Service URLs (defaults — override via env in tests/integration/conftest.py):
#   CKAN          http://localhost:5000
#   Prefect UI    http://localhost:4200
#   Postgres      localhost:5432   (user=postgres pass=postgres)
#   Solr          http://localhost:8983
#   Redis         localhost:6379

services:
  postgres:
    image: postgres:15
    environment:
      POSTGRES_USER: postgres
      POSTGRES_PASSWORD: postgres
      POSTGRES_DB: postgres
    ports:
      - "${POSTGRES_HOST_PORT:-5432}:5432"
    volumes:
      - integration-postgres:/var/lib/postgresql/data
      - ./tests/integration/init-postgres.sh:/docker-entrypoint-initdb.d/init-postgres.sh:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U postgres"]
      interval: 5s
      retries: 10

  solr:
    image: ckan/ckan-solr:2.11-solr9
    ports:
      - "${SOLR_HOST_PORT:-8983}:8983"
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://localhost:8983/solr/ckan/admin/ping || exit 1"]
      interval: 10s
      retries: 10

  redis:
    image: redis:7
    ports:
      - "${REDIS_HOST_PORT:-6379}:6379"
    healthcheck:
      test: ["CMD-SHELL", "redis-cli ping"]
      interval: 5s
      retries: 10

  prefect-server:
    image: prefecthq/prefect:3-latest
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      PREFECT_API_DATABASE_CONNECTION_URL: postgresql+asyncpg://postgres:postgres@postgres:5432/prefect
      PREFECT_SERVER_API_HOST: 0.0.0.0
      # Tracks ${PREFECT_HOST_PORT} below — the UI is served from the
      # browser's perspective, so the API URL it calls must match
      # whatever host port the API is published on. Hard-coding :4200
      # would break the UI as soon as a user remaps PREFECT_HOST_PORT
      # to avoid a conflict.
      PREFECT_SERVER_UI_API_URL: http://localhost:${PREFECT_HOST_PORT:-4200}/api
    command: prefect server start
    ports:
      - "${PREFECT_HOST_PORT:-4200}:4200"
    healthcheck:
      test: ["CMD", "python", "-c", "import urllib.request as u; u.urlopen('http://localhost:4200/api/health', timeout=1)"]
      interval: 10s
      retries: 10
      start_period: 60s

  prefect-worker:
    build:
      context: .
      dockerfile: Dockerfile.worker
    # Override ckan-dev's default user (``ckan``) → root so the on-start
    # bash command can ``pip install -e`` and write to the editable
    # install. Same reason the ``ckan`` service has ``user: root``.
    user: root
    depends_on:
      prefect-server:
        condition: service_healthy
    environment:
      PREFECT_API_URL: http://prefect-server:4200/api
      CKAN_INI: /etc/ckan/default/ckan.ini
    volumes:
      # Match the CKAN container's bind path (``/srv/app/src/datapusher-plus``).
      # ``ckan datapusher_plus prefect-deploy`` resolves the flow source
      # via ``dp_pkg.__file__`` inside CKAN and bakes that absolute path
      # into the Prefect deployment, so the worker has to be able to
      # ``chdir`` to the same path or the ``set_working_directory`` step
      # crashes and flow runs stay ``Pending`` forever.
      - ./:/srv/app/src/datapusher-plus
      - ./tests/integration/ckan.ini:/etc/ckan/default/ckan.ini:ro
    command: >
      bash -c "
        pip install -e /srv/app/src/datapusher-plus &&
        prefect worker start --pool datapusher-plus
      "
    restart: on-failure

  ckan:
    image: ckan/ckan-dev:2.11
    # Override default user (``ckan``) → root so the on-start command
    # can ``apt-get install postgresql-client`` (needed for piping
    # ``datastore set-permissions`` into psql). The CI workflow does
    # the equivalent via ``options: --user root``.
    user: root
    depends_on:
      postgres:
        condition: service_healthy
      solr:
        condition: service_healthy
      redis:
        condition: service_healthy
      prefect-server:
        condition: service_healthy
    environment:
      CKAN_SQLALCHEMY_URL: postgresql://ckan:ckan@postgres/ckan
      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:datastore@postgres/datastore
      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:datastore@postgres/datastore
      CKAN_SOLR_URL: http://solr:8983/solr/ckan
      CKAN_REDIS_URL: redis://redis:6379/1
      # Docker-network hostname (not ``localhost``): the prefect-worker
      # downloads resources by fetching this URL from its OWN network
      # context, where ``localhost`` would resolve to the worker
      # container itself. ``ckan`` is reachable from any service in
      # this compose stack. Host-side tests reach CKAN via the
      # separate ``CKAN_URL=http://localhost:5050`` env var.
      CKAN_SITE_URL: http://ckan:5000
      PREFECT_API_URL: http://prefect-server:4200/api
    volumes:
      - ./:/srv/app/src/datapusher-plus
      # Bind-mounted RW because the bootstrap's ``ckan config-tool``
      # appends the freshly-minted DP+ service-account token to this
      # file, and the prefect-worker bind-mounts the SAME file (also
      # at /etc/ckan/default/ckan.ini) — the worker's Download stage
      # reads ``ckanext.datapusher_plus.api_token`` to authenticate
      # the resource fetch back to CKAN. A copy-from-source pattern
      # would isolate writes but desync the two containers.
      #
      # Side effect: this file shows as modified after every stack-up
      # while the stack is running. ``scripts/integration-down``
      # resets it to its committed state so the JWT never accidentally
      # gets committed. Don't ``git add`` it manually while the stack
      # is up.
      - ./tests/integration/ckan.ini:/etc/ckan/default/ckan.ini
    ports:
      - "${CKAN_HOST_PORT:-5000}:5000"
    # psql is piped from ``datastore set-permissions`` to grant the
    # datastore_read role its SELECT privileges; the base
    # ``ckan/ckan-dev:2.11`` image doesn't carry the postgresql-client
    # binaries, so install them on first start. Cheap (~1s) on
    # subsequent restarts because the apt cache layer is already
    # populated.
    # End-to-end first-start dance:
    # 1. apt-install psql so the set-permissions pipe works.
    # 2. pip-install DP+ in editable mode (pulls in fiona / shapely /
    #    prefect via the dynamic-dependencies block in pyproject.toml).
    # 3. db init / datastore set-permissions / db upgrade.
    # 4. Mint the integration sysadmin + an API token, then inject
    #    that token as the DP+ service-account token in the ini via
    #    ``ckan config-tool`` — without this the resource_create /
    #    auto-submit path 500s with "No API token found. API token
    #    required for DP+ service account." (see utils.py:89). Mirrors
    #    main.yml:516-528.
    # 5. Stash the same token to ``/tmp/integration_admin_token`` so
    #    the integration tests on the host can grab it via
    #    ``docker exec ... cat``.
    # 6. ``ckan run`` foreground (this stays alive).
    # Outer ``bash -c`` uses SINGLE quotes so the inner script can use
    # double quotes freely. YAML folded scalar (``>``) collapses newlines
    # to spaces, which bash treats as statement separators, so the
    # multi-line shell still parses as one command.
    command: >
      bash -c '
        # ``-o pipefail`` so a failure in ``ckan datastore set-permissions``
        # (left of the psql pipe below) surfaces — without it psql''s exit
        # status would mask it and the stack would come up half-bootstrapped.
        set -euo pipefail
        apt-get update -qq && apt-get install -y --no-install-recommends postgresql-client >/dev/null
        pip install -e /srv/app/src/datapusher-plus
        ckan -c /etc/ckan/default/ckan.ini db init
        ckan -c /etc/ckan/default/ckan.ini datastore set-permissions | psql postgresql://postgres:postgres@postgres/postgres
        ckan -c /etc/ckan/default/ckan.ini db upgrade -p datapusher_plus
        if ! ckan -c /etc/ckan/default/ckan.ini user show admin 2>/dev/null | grep -q "name=admin"; then
          echo y | ckan -c /etc/ckan/default/ckan.ini sysadmin add admin password=adminadmin email=admin@test.local
        fi
        # Mint the API token only on first start. Re-running on every
        # container restart was creating duplicate token rows AND
        # silently rotating ``api_token`` in the ini under any test
        # process still holding the previous value (review on PR #300).
        # ``/tmp/integration_admin_token`` is on the container writable
        # layer: persists across ``docker compose restart`` but wiped on
        # ``rm``/``up -d`` after ``down -v``, which is exactly the
        # behaviour we want.
        if [ ! -s /tmp/integration_admin_token ]; then
          # Parse the JWT directly by regex rather than positional grep
          # so a wording change in ``user token add`` does not silently
          # leave $$TOKEN empty. Do NOT swallow stderr — a real
          # failure should surface. ($$ escapes compose env-var
          # substitution so the literal $ reaches bash.)
          TOKEN=$$(ckan -c /etc/ckan/default/ckan.ini user token add admin integration \
                   | grep -Eo "[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+" \
                   | head -1)
          if [ -z "$$TOKEN" ]; then
            echo "FATAL: could not extract API token from ckan user token add output" >&2
            exit 1
          fi
          ckan -c /etc/ckan/default/ckan.ini config-tool /etc/ckan/default/ckan.ini "ckanext.datapusher_plus.api_token=$$TOKEN"
          printf "%s" "$$TOKEN" > /tmp/integration_admin_token
          echo "--- Integration admin token written to /tmp/integration_admin_token (grab via docker exec) ---"
        else
          echo "--- Reusing existing token from /tmp/integration_admin_token ---"
        fi
        exec ckan -c /etc/ckan/default/ckan.ini run --host 0.0.0.0 --port 5000
      '

volumes:
  integration-postgres: