diff --git a/dagster.yaml b/dagster.yaml index a9b563c..4f4718e 100644 --- a/dagster.yaml +++ b/dagster.yaml @@ -8,10 +8,27 @@ run_coordinator: class: QueuedRunCoordinator config: max_concurrent_runs: 5 + tag_concurrency_limits: + - key: "operation" + value: "example" + limit: 5 run_launcher: - module: dagster.core.launcher - class: DefaultRunLauncher + module: dagster_docker + class: DockerRunLauncher + config: + env_vars: + - DAGSTER_POSTGRES_USER + - DAGSTER_POSTGRES_PASSWORD + - DAGSTER_POSTGRES_DB + - MOBILITY_DB_REFRESH_TOKEN + network: dagster + container_kwargs: + volumes_from: + - dagster_code + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/io_manager_storage:/tmp/io_manager_storage run_storage: module: dagster_postgres.run_storage diff --git a/docker-compose.yaml b/docker-compose.yaml index 28065cc..c128a99 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -14,7 +14,7 @@ services: env_file: - .env volumes: - - ${POSTGRES_DIRECTORY}:/var/lib/postgresql/data + - ./postgres_data:/var/lib/postgresql/data networks: - dagster healthcheck: @@ -44,7 +44,7 @@ services: MOBILITY_DB_REFRESH_TOKEN: ${MOBILITY_DB_REFRESH_TOKEN} env_file: - .env - volumes: + volumes: # these volumes are also passed on to the containers that are spawned to run the assets. - ${DATA_DIRECTORY}:/opt/dagster/app/data - ${CONFIG_DIRECTORY}:/opt/dagster/app/config networks: @@ -74,10 +74,9 @@ services: DAGSTER_POSTGRES_DB: ${POSTGRES_DB} env_file: - .env -## Don't need for DefaultRunLauncher, uncomment for DockerRunLauncher -# volumes: -# - /var/run/docker.sock:/var/run/docker.sock -# - /tmp/io_manager_storage:/tmp/io_manager_storage + volumes: # Make docker client accessible so we can terminate containers from the webserver + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/io_manager_storage:/tmp/io_manager_storage networks: - dagster depends_on: @@ -103,10 +102,9 @@ services: DAGSTER_POSTGRES_DB: ${POSTGRES_DB} env_file: - .env -## Don't need for DefaultRunLauncher, uncomment for DockerRunLauncher -# volumes: -# - /var/run/docker.sock:/var/run/docker.sock -# - /tmp/io_manager_storage:/tmp/io_manager_storage + volumes: # Make docker client accessible so we can launch containers using host docker + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/io_manager_storage:/tmp/io_manager_storage networks: - dagster depends_on: