118 lines
3.8 KiB
YAML
118 lines
3.8 KiB
YAML
---
|
|
|
|
services:
|
|
# This service runs the postgres DB used by dagster for run storage, schedule storage,
|
|
# and event log storage. Depending on the hardware you run this Compose on, you may be able
|
|
# to reduce the interval and timeout in the healthcheck to speed up your `docker-compose up` times.
|
|
dagster_postgresql:
|
|
image: postgres:17
|
|
container_name: dagster_postgresql
|
|
environment:
|
|
POSTGRES_USER: ${POSTGRES_USER}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
|
POSTGRES_DB: ${POSTGRES_DB}
|
|
env_file:
|
|
- .env
|
|
volumes:
|
|
- ./postgres_data:/var/lib/postgresql/data
|
|
networks:
|
|
- dagster
|
|
healthcheck:
|
|
test: ['CMD-SHELL', 'pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}']
|
|
interval: 10s
|
|
timeout: 8s
|
|
retries: 5
|
|
|
|
# This service runs the gRPC server that loads your user code, in both dagster-webserver
|
|
# and dagster-daemon. By setting DAGSTER_CURRENT_IMAGE to its own image, we tell the
|
|
# run launcher to use this same image when launching runs in a new container as well.
|
|
# Multiple containers like this can be deployed separately - each just needs to run on
|
|
# its own port, and have its own entry in the workspace.yaml file that's loaded by the
|
|
# webserver.
|
|
dagster_user_code_gtfs:
|
|
build:
|
|
context: .
|
|
dockerfile: ./Dockerfile_user_code_gtfs
|
|
container_name: dagster_user_code_gtfs
|
|
image: dagster_user_code_gtfs
|
|
restart: always
|
|
environment:
|
|
DAGSTER_POSTGRES_USER: ${POSTGRES_USER}
|
|
DAGSTER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
|
DAGSTER_POSTGRES_DB: ${POSTGRES_DB}
|
|
DAGSTER_CURRENT_IMAGE: 'dagster_user_code_gtfs'
|
|
MOBILITY_DB_REFRESH_TOKEN: ${MOBILITY_DB_REFRESH_TOKEN}
|
|
env_file:
|
|
- .env
|
|
volumes:
|
|
- ./data:/opt/dagster/app/data
|
|
networks:
|
|
- dagster
|
|
|
|
# This service runs dagster-webserver, which loads your user code from the user code container.
|
|
# Since our instance uses the QueuedRunCoordinator, any runs submitted from the webserver will be put on
|
|
# a queue and later dequeued and launched by dagster-daemon.
|
|
dagster_webserver:
|
|
build:
|
|
context: .
|
|
dockerfile: ./Dockerfile_dagster
|
|
entrypoint:
|
|
- dagster-webserver
|
|
- -h
|
|
- '0.0.0.0'
|
|
- -p
|
|
- '3000'
|
|
- -w
|
|
- workspace.yaml
|
|
container_name: dagster_webserver
|
|
ports:
|
|
- 3001:3000
|
|
environment:
|
|
DAGSTER_POSTGRES_USER: ${POSTGRES_USER}
|
|
DAGSTER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
|
DAGSTER_POSTGRES_DB: ${POSTGRES_DB}
|
|
env_file:
|
|
- .env
|
|
volumes: # Make docker client accessible so we can terminate containers from the webserver
|
|
- /var/run/docker.sock:/var/run/docker.sock
|
|
- /tmp/io_manager_storage:/tmp/io_manager_storage
|
|
networks:
|
|
- dagster
|
|
depends_on:
|
|
dagster_postgresql:
|
|
condition: service_healthy
|
|
dagster_user_code_gtfs:
|
|
condition: service_started
|
|
|
|
# This service runs the dagster-daemon process, which is responsible for taking runs
|
|
# off of the queue and launching them, as well as creating runs from schedules or sensors.
|
|
dagster_daemon:
|
|
build:
|
|
context: .
|
|
dockerfile: ./Dockerfile_dagster
|
|
entrypoint:
|
|
- dagster-daemon
|
|
- run
|
|
container_name: dagster_daemon
|
|
restart: on-failure
|
|
environment:
|
|
DAGSTER_POSTGRES_USER: ${POSTGRES_USER}
|
|
DAGSTER_POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
|
DAGSTER_POSTGRES_DB: ${POSTGRES_DB}
|
|
env_file:
|
|
- .env
|
|
volumes: # Make docker client accessible so we can launch containers using host docker
|
|
- /var/run/docker.sock:/var/run/docker.sock
|
|
- /tmp/io_manager_storage:/tmp/io_manager_storage
|
|
networks:
|
|
- dagster
|
|
depends_on:
|
|
dagster_postgresql:
|
|
condition: service_healthy
|
|
dagster_user_code_gtfs:
|
|
condition: service_started
|
|
|
|
networks:
|
|
dagster:
|
|
driver: bridge
|
|
name: dagster
|