diff --git a/dagster.yaml b/dagster.yaml index a9b563c..b5ac980 100644 --- a/dagster.yaml +++ b/dagster.yaml @@ -8,10 +8,27 @@ run_coordinator: class: QueuedRunCoordinator config: max_concurrent_runs: 5 + tag_concurrency_limits: + - key: "operation" + value: "example" + limit: 5 run_launcher: - module: dagster.core.launcher - class: DefaultRunLauncher + module: dagster_docker + class: DockerRunLauncher + config: + env_vars: + - DAGSTER_POSTGRES_USER + - DAGSTER_POSTGRES_PASSWORD + - DAGSTER_POSTGRES_DB + - MOBILITY_DB_REFRESH_TOKEN + network: dagster + container_kwargs: + volumes: # Make docker client accessible to any launched containers as well + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/io_manager_storage:/tmp/io_manager_storage + - /home/ben/code/gtfs-dagster/data:/opt/dagster/app/data + - /home/ben/code/gtfs-dagster/config:/opt/dagster/app/config run_storage: module: dagster_postgres.run_storage diff --git a/docker-compose.yaml b/docker-compose.yaml index 28065cc..cdb1af9 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -14,7 +14,7 @@ services: env_file: - .env volumes: - - ${POSTGRES_DIRECTORY}:/var/lib/postgresql/data + - ./postgres_data:/var/lib/postgresql/data networks: - dagster healthcheck: @@ -45,8 +45,8 @@ services: env_file: - .env volumes: - - ${DATA_DIRECTORY}:/opt/dagster/app/data - - ${CONFIG_DIRECTORY}:/opt/dagster/app/config + - ./data:/opt/dagster/app/data + - ./config:/opt/dagster/app/config networks: - dagster @@ -74,10 +74,9 @@ services: DAGSTER_POSTGRES_DB: ${POSTGRES_DB} env_file: - .env -## Don't need for DefaultRunLauncher, uncomment for DockerRunLauncher -# volumes: -# - /var/run/docker.sock:/var/run/docker.sock -# - /tmp/io_manager_storage:/tmp/io_manager_storage + volumes: # Make docker client accessible so we can terminate containers from the webserver + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/io_manager_storage:/tmp/io_manager_storage networks: - dagster depends_on: @@ -103,10 +102,9 @@ services: DAGSTER_POSTGRES_DB: ${POSTGRES_DB} env_file: - .env -## Don't need for DefaultRunLauncher, uncomment for DockerRunLauncher -# volumes: -# - /var/run/docker.sock:/var/run/docker.sock -# - /tmp/io_manager_storage:/tmp/io_manager_storage + volumes: # Make docker client accessible so we can launch containers using host docker + - /var/run/docker.sock:/var/run/docker.sock + - /tmp/io_manager_storage:/tmp/io_manager_storage networks: - dagster depends_on: