added asset to read agency_list.csv and add it to table in gtfs.duckdb
This commit is contained in:
parent
7791d034ae
commit
2b47a45b8f
9 changed files with 44 additions and 4 deletions
2
user_code/gtfs/__init__.py
Normal file
2
user_code/gtfs/__init__.py
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
# user_code/gtfs/__init__.py
|
||||
from .assets import *
|
||||
18
user_code/gtfs/assets.py
Normal file
18
user_code/gtfs/assets.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import pandas as pd
|
||||
from dagster import asset
|
||||
from dagster_duckdb import DuckDBResource
|
||||
|
||||
|
||||
@asset
|
||||
def agency_list(duckdb: DuckDBResource) -> None:
|
||||
"""Load agency list from CSV into DuckDB."""
|
||||
|
||||
# Read the CSV (path is relative to container working directory)
|
||||
df = pd.read_csv('data/gtfs/agency_list.csv')
|
||||
|
||||
# Write to DuckDB
|
||||
with duckdb.get_connection() as conn:
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS agency_list AS
|
||||
SELECT * FROM df
|
||||
""")
|
||||
12
user_code/gtfs/definitions.py
Normal file
12
user_code/gtfs/definitions.py
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
from dagster import Definitions
|
||||
from dagster_duckdb import DuckDBResource
|
||||
from assets import agency_list
|
||||
|
||||
defs = Definitions(
|
||||
assets=[agency_list],
|
||||
resources={
|
||||
"duckdb": DuckDBResource(
|
||||
database="data/gtfs/gtfs.duckdb"
|
||||
)
|
||||
}
|
||||
)
|
||||
Loading…
Add table
Add a link
Reference in a new issue