rearranged directory structure, automaterialize to automationCondition
This commit is contained in:
parent
dcff5c78e7
commit
11003a8bda
11 changed files with 38 additions and 31 deletions
|
|
@ -1,2 +1,2 @@
|
|||
# user_code/__init__.py
|
||||
# This file makes user_code a package
|
||||
from . import assets, resources
|
||||
|
|
|
|||
|
|
@ -4,19 +4,21 @@ from dagster import (
|
|||
AssetExecutionContext,
|
||||
Output,
|
||||
MetadataValue,
|
||||
AutoMaterializePolicy
|
||||
AutomationCondition
|
||||
)
|
||||
from dagster_duckdb import DuckDBResource
|
||||
from resources import MobilityDatabaseAPI # Direct import instead of relative
|
||||
from resources import MobilityDatabaseAPI
|
||||
import json
|
||||
|
||||
|
||||
@asset
|
||||
@asset(
|
||||
group_name="gtfs_metadata",
|
||||
)
|
||||
def agency_list(duckdb: DuckDBResource) -> None:
|
||||
"""Load agency list from CSV into DuckDB."""
|
||||
|
||||
# Read the CSV (path is relative to container working directory)
|
||||
df = pd.read_csv('data/gtfs/agency_list.csv')
|
||||
df = pd.read_csv('config/agency_list.csv')
|
||||
|
||||
# Write to DuckDB
|
||||
with duckdb.get_connection() as conn:
|
||||
|
|
@ -29,7 +31,7 @@ def agency_list(duckdb: DuckDBResource) -> None:
|
|||
@asset(
|
||||
deps=["agency_list"],
|
||||
group_name="gtfs_metadata",
|
||||
auto_materialize_policy=AutoMaterializePolicy.eager()
|
||||
automation_condition=AutomationCondition.eager()
|
||||
)
|
||||
def gtfs_feed_metadata(
|
||||
context: AssetExecutionContext,
|
||||
|
|
@ -1,4 +1,10 @@
|
|||
from dagster import Definitions, load_assets_from_modules, EnvVar
|
||||
from dagster import (
|
||||
Definitions,
|
||||
load_assets_from_modules,
|
||||
EnvVar,
|
||||
DefaultSensorStatus,
|
||||
AutomationConditionSensorDefinition
|
||||
)
|
||||
from dagster_duckdb import DuckDBResource
|
||||
|
||||
import assets
|
||||
|
|
@ -8,9 +14,16 @@ all_assets = load_assets_from_modules([assets])
|
|||
|
||||
defs = Definitions(
|
||||
assets=all_assets,
|
||||
sensors=[
|
||||
AutomationConditionSensorDefinition(
|
||||
"asset_automation_sensor",
|
||||
target="*",
|
||||
default_status=DefaultSensorStatus.RUNNING,
|
||||
)
|
||||
],
|
||||
resources={
|
||||
"duckdb": DuckDBResource(
|
||||
database="data/gtfs/gtfs.duckdb"
|
||||
database="data/duckdb/gtfs.duckdb"
|
||||
),
|
||||
"mobility_db": MobilityDatabaseAPI(
|
||||
refresh_token=EnvVar("MOBILITY_DB_REFRESH_TOKEN"),
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
# user_code/gtfs/__init__.py
|
||||
from . import assets, resources
|
||||
Loading…
Add table
Add a link
Reference in a new issue