mirror of
https://git.turbo-data.com/nprasad2077/Streamlit_NBA.git
synced 2026-11-04 18:22:51 +00:00
43 lines
1.3 KiB
Python
Executable File
43 lines
1.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Extract source tables from Postgres into DuckDB using the DuckDB Python library.
|
|
Reads SQL commands from scripts/extract.sql with environment variable interpolation.
|
|
"""
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
import duckdb
|
|
|
|
def main():
|
|
repo_root = Path(__file__).resolve().parent.parent
|
|
db_path = repo_root / "data" / "DB" / "dbt_nba.duckdb"
|
|
sql_path = repo_root / "scripts" / "extract.sql"
|
|
|
|
postgres_url = os.getenv("POSTGRES_URL")
|
|
if not postgres_url:
|
|
print("ERROR: POSTGRES_URL environment variable is not set", file=sys.stderr)
|
|
print(f"Please export POSTGRES_URL or configure it in {repo_root}/.env", file=sys.stderr)
|
|
sys.exit(1)
|
|
|
|
db_path.parent.mkdir(parents=True, exist_ok=True)
|
|
if db_path.exists():
|
|
db_path.unlink()
|
|
|
|
with open(sql_path, "r", encoding="utf-8") as f:
|
|
sql = f.read()
|
|
|
|
# Replace variable
|
|
sql = sql.replace("${POSTGRES_URL}", postgres_url)
|
|
|
|
print(f"==> Connecting to DuckDB at {db_path}...")
|
|
con = duckdb.connect(str(db_path))
|
|
try:
|
|
print("==> Executing extraction queries...")
|
|
con.execute(sql)
|
|
print("==> Extraction complete successfully!")
|
|
finally:
|
|
con.close()
|
|
|
|
if __name__ == "__main__":
|
|
main()
|