rearranged directory structure, automaterialize to automationCondition

This commit is contained in:
Ben Varick 2025-11-07 07:40:46 -08:00
parent dcff5c78e7
commit 11003a8bda
Signed by: ben
SSH key fingerprint: SHA256:jWnpFDAcacYM5aPFpYRqlsamlDyKNpSj3jj+k4ojtUo
11 changed files with 38 additions and 31 deletions

View file

@ -1,2 +1,2 @@
# user_code/__init__.py
# This file makes user_code a package
from . import assets, resources

View file

@ -4,19 +4,21 @@ from dagster import (
AssetExecutionContext,
Output,
MetadataValue,
AutoMaterializePolicy
AutomationCondition
)
from dagster_duckdb import DuckDBResource
from resources import MobilityDatabaseAPI # Direct import instead of relative
from resources import MobilityDatabaseAPI
import json
@asset
@asset(
group_name="gtfs_metadata",
)
def agency_list(duckdb: DuckDBResource) -> None:
"""Load agency list from CSV into DuckDB."""
# Read the CSV (path is relative to container working directory)
df = pd.read_csv('data/gtfs/agency_list.csv')
df = pd.read_csv('config/agency_list.csv')
# Write to DuckDB
with duckdb.get_connection() as conn:
@ -29,7 +31,7 @@ def agency_list(duckdb: DuckDBResource) -> None:
@asset(
deps=["agency_list"],
group_name="gtfs_metadata",
auto_materialize_policy=AutoMaterializePolicy.eager()
automation_condition=AutomationCondition.eager()
)
def gtfs_feed_metadata(
context: AssetExecutionContext,

View file

@ -1,4 +1,10 @@
from dagster import Definitions, load_assets_from_modules, EnvVar
from dagster import (
Definitions,
load_assets_from_modules,
EnvVar,
DefaultSensorStatus,
AutomationConditionSensorDefinition
)
from dagster_duckdb import DuckDBResource
import assets
@ -8,9 +14,16 @@ all_assets = load_assets_from_modules([assets])
defs = Definitions(
assets=all_assets,
sensors=[
AutomationConditionSensorDefinition(
"asset_automation_sensor",
target="*",
default_status=DefaultSensorStatus.RUNNING,
)
],
resources={
"duckdb": DuckDBResource(
database="data/gtfs/gtfs.duckdb"
database="data/duckdb/gtfs.duckdb"
),
"mobility_db": MobilityDatabaseAPI(
refresh_token=EnvVar("MOBILITY_DB_REFRESH_TOKEN"),

View file

@ -1,2 +0,0 @@
# user_code/gtfs/__init__.py
from . import assets, resources