diff --git a/moped-database/metadata/databases/default/tables/public_combined_project_funding_view.yaml b/moped-database/metadata/databases/default/tables/public_combined_project_funding_view.yaml new file mode 100644 index 0000000000..d7824323d5 --- /dev/null +++ b/moped-database/metadata/databases/default/tables/public_combined_project_funding_view.yaml @@ -0,0 +1,61 @@ +table: + name: combined_project_funding_view + schema: public +select_permissions: + - role: moped-admin + permission: + columns: + - amount + - created_at + - description + - ecapris_subproject_id + - fao_id + - fdu + - id + - is_synced_from_ecapris + - original_id + - program_name + - project_id + - source_name + - status_name + - updated_at + filter: {} + comment: "" + - role: moped-editor + permission: + columns: + - amount + - created_at + - description + - ecapris_subproject_id + - fao_id + - fdu + - id + - is_synced_from_ecapris + - original_id + - program_name + - project_id + - source_name + - status_name + - updated_at + filter: {} + comment: "" + - role: moped-viewer + permission: + columns: + - amount + - created_at + - description + - ecapris_subproject_id + - fao_id + - fdu + - id + - is_synced_from_ecapris + - original_id + - program_name + - project_id + - source_name + - status_name + - updated_at + filter: {} + comment: "" diff --git a/moped-database/metadata/databases/default/tables/public_ecapris_subproject_funding.yaml b/moped-database/metadata/databases/default/tables/public_ecapris_subproject_funding.yaml new file mode 100644 index 0000000000..2b1d03c62b --- /dev/null +++ b/moped-database/metadata/databases/default/tables/public_ecapris_subproject_funding.yaml @@ -0,0 +1,3 @@ +table: + name: ecapris_subproject_funding + schema: public diff --git a/moped-database/metadata/databases/default/tables/public_moped_proj_funding.yaml b/moped-database/metadata/databases/default/tables/public_moped_proj_funding.yaml index 3454deab65..4d9fab7482 100644 --- a/moped-database/metadata/databases/default/tables/public_moped_proj_funding.yaml +++ b/moped-database/metadata/databases/default/tables/public_moped_proj_funding.yaml @@ -17,6 +17,7 @@ insert_permissions: updated_by_user_id: x-hasura-user-db-id columns: - dept_unit + - ecapris_funding_id - fdu - fund - funding_amount @@ -36,6 +37,7 @@ insert_permissions: updated_by_user_id: x-hasura-user-db-id columns: - dept_unit + - ecapris_funding_id - fdu - fund - funding_amount @@ -65,7 +67,6 @@ select_permissions: - funding_source_id - funding_status_id - is_deleted - - is_editable - is_legacy_funding_record - proj_funding_id - project_id @@ -91,7 +92,6 @@ select_permissions: - funding_source_id - funding_status_id - is_deleted - - is_editable - is_legacy_funding_record - proj_funding_id - project_id @@ -117,7 +117,6 @@ select_permissions: - funding_source_id - funding_status_id - is_deleted - - is_editable - is_legacy_funding_record - proj_funding_id - project_id @@ -131,6 +130,7 @@ update_permissions: permission: columns: - dept_unit + - ecapris_funding_id - fdu - fund - funding_amount @@ -150,6 +150,7 @@ update_permissions: permission: columns: - dept_unit + - ecapris_funding_id - fdu - fund - funding_amount diff --git a/moped-database/metadata/databases/default/tables/public_moped_project.yaml b/moped-database/metadata/databases/default/tables/public_moped_project.yaml index ad864857da..b627f8daff 100644 --- a/moped-database/metadata/databases/default/tables/public_moped_project.yaml +++ b/moped-database/metadata/databases/default/tables/public_moped_project.yaml @@ -309,6 +309,7 @@ event_triggers: - project_lead_id - public_process_status_id - should_sync_ecapris_statuses + - should_sync_ecapris_funding - project_name - project_description - ecapris_subproject_id diff --git a/moped-database/metadata/databases/default/tables/tables.yaml b/moped-database/metadata/databases/default/tables/tables.yaml index 8b3d7ff041..ee50ea3c8c 100644 --- a/moped-database/metadata/databases/default/tables/tables.yaml +++ b/moped-database/metadata/databases/default/tables/tables.yaml @@ -1,8 +1,10 @@ - "!include deprecated_moped_project_types.yaml" - "!include deprecated_moped_types.yaml" +- "!include public_combined_project_funding_view.yaml" - "!include public_combined_project_notes_view.yaml" - "!include public_component_arcgis_online_view.yaml" - "!include public_current_phase_view.yaml" +- "!include public_ecapris_subproject_funding.yaml" - "!include public_ecapris_subproject_statuses.yaml" - "!include public_exploded_component_arcgis_online_view.yaml" - "!include public_feature_drawn_lines.yaml" diff --git a/moped-database/migrations/default/1761074353404_ecapris_funding/down.sql b/moped-database/migrations/default/1761074353404_ecapris_funding/down.sql deleted file mode 100644 index 15bac2f238..0000000000 --- a/moped-database/migrations/default/1761074353404_ecapris_funding/down.sql +++ /dev/null @@ -1,32 +0,0 @@ --- Remove funding sync flag from projects table -ALTER TABLE moped_project -DROP COLUMN should_sync_ecapris_funding; - --- Remove added columns from moped_proj_funding table -ALTER TABLE moped_proj_funding -DROP COLUMN ecapris_funding_id, -DROP COLUMN is_legacy_funding_record, -DROP COLUMN is_editable, -DROP COLUMN fdu, -DROP COLUMN unit_long_name; - --- Restore previous view to use fund_dept_unit column -DROP VIEW IF EXISTS project_funding_view; - -CREATE OR REPLACE VIEW project_funding_view AS SELECT - mp.project_id, - mpf.proj_funding_id, - mpf.funding_amount, - mpf.funding_description, - mpf.fund_dept_unit, - mpf.created_at, - mpf.updated_at, - mfs.funding_source_name, - mfp.funding_program_name, - mfst.funding_status_name -FROM moped_project AS mp -LEFT JOIN moped_proj_funding AS mpf ON mp.project_id = mpf.project_id -LEFT JOIN moped_fund_sources AS mfs ON mpf.funding_source_id = mfs.funding_source_id -LEFT JOIN moped_fund_programs AS mfp ON mpf.funding_program_id = mfp.funding_program_id -LEFT JOIN moped_fund_status AS mfst ON mpf.funding_status_id = mfst.funding_status_id -WHERE true AND mp.is_deleted = false AND mpf.is_deleted = false; diff --git a/moped-database/migrations/default/1761074353404_ecapris_funding/up.sql b/moped-database/migrations/default/1761074353404_ecapris_funding/up.sql deleted file mode 100644 index 3f86891269..0000000000 --- a/moped-database/migrations/default/1761074353404_ecapris_funding/up.sql +++ /dev/null @@ -1,68 +0,0 @@ --- Add funding sync tracking to projects table -ALTER TABLE moped_project -ADD COLUMN should_sync_ecapris_funding BOOLEAN DEFAULT FALSE NOT NULL; - -COMMENT ON COLUMN moped_project.should_sync_ecapris_funding IS 'Indicates if project funding should be synced from eCAPRIS'; --- Fix eCAPRIS name in existing comment -COMMENT ON COLUMN moped_project.should_sync_ecapris_statuses IS 'Indicates if project statuses should be synced from eCAPRIS'; - --- Update moped_proj_funding table -ALTER TABLE moped_proj_funding -ADD COLUMN ecapris_funding_id INTEGER, -ADD COLUMN is_legacy_funding_record BOOLEAN DEFAULT FALSE NOT NULL, -ADD COLUMN is_editable BOOLEAN GENERATED ALWAYS AS (ecapris_funding_id IS NULL) STORED, -ADD COLUMN fdu TEXT DEFAULT NULL, -ADD COLUMN unit_long_name TEXT DEFAULT NULL; - -COMMENT ON COLUMN moped_proj_funding.ecapris_funding_id IS 'References the eCAPRIS FDU unique fao_id if applicable'; -COMMENT ON COLUMN moped_proj_funding.is_legacy_funding_record IS 'Indicates if the funding record was created before eCAPRIS sync integration (Nov 2025)'; - --- Mark all existing funding records as legacy before eCAPRIS sync integration launches -UPDATE moped_proj_funding -SET is_legacy_funding_record = TRUE; - --- Add comments on other existing moped_proj_funding columns -COMMENT ON COLUMN moped_proj_funding.proj_funding_id IS 'Primary key for the project funding record'; -COMMENT ON COLUMN moped_proj_funding.created_by_user_id IS 'ID of the user who last created the record'; -COMMENT ON COLUMN moped_proj_funding.created_at IS 'Timestamp when the record was last created'; -COMMENT ON COLUMN moped_proj_funding.project_id IS 'References the project this funding record is associated with'; -COMMENT ON COLUMN moped_proj_funding.funding_source_id IS 'References the funding source for this funding record'; -COMMENT ON COLUMN moped_proj_funding.funding_program_id IS 'References the funding program for this funding record'; -COMMENT ON COLUMN moped_proj_funding.funding_amount IS 'The amount of funding allocated from this funding source'; -COMMENT ON COLUMN moped_proj_funding.funding_description IS 'A description of the funding source'; -COMMENT ON COLUMN moped_proj_funding.funding_status_id IS 'References the current status of this funding record'; -COMMENT ON COLUMN moped_proj_funding.fund IS 'Legacy JSONB object containing additional fund details from eCAPRIS (Socrata jega-nqf6)'; -COMMENT ON COLUMN moped_proj_funding.dept_unit IS 'Legacy JSONB object containing additional department/unit details from eCAPRIS (Socrata bgrt-2m2z)'; -COMMENT ON COLUMN moped_proj_funding.is_editable IS 'Indicates if the funding record is editable (false if linked to an eCAPRIS funding record)'; -COMMENT ON COLUMN moped_proj_funding.fdu IS 'The FDU (Fund-Dept-Unit) code associated with this funding record'; -COMMENT ON COLUMN moped_proj_funding.unit_long_name IS 'The long name of the unit associated with this funding record'; - --- Populate new fdu column based on existing fund_dept_unit data if available and --- populate unit_long_name from dept_unit JSONB --- Note: fund_dept_unit is a generated column that is null if fund or dept_unit is null -UPDATE moped_proj_funding -SET - fdu = fund_dept_unit, - unit_long_name = (dept_unit ->> 'unit_long_name') -WHERE fund_dept_unit IS NOT NULL; - --- Drop and recreate view to use new fdu column instead of fund_dept_unit -DROP VIEW IF EXISTS project_funding_view; - -CREATE OR REPLACE VIEW project_funding_view AS SELECT - mp.project_id, - mpf.proj_funding_id, - mpf.funding_amount, - mpf.funding_description, - mpf.fdu AS fund_dept_unit, - mpf.created_at, - mpf.updated_at, - mfs.funding_source_name, - mfp.funding_program_name, - mfst.funding_status_name -FROM moped_project AS mp -LEFT JOIN moped_proj_funding AS mpf ON mp.project_id = mpf.project_id -LEFT JOIN moped_fund_sources AS mfs ON mpf.funding_source_id = mfs.funding_source_id -LEFT JOIN moped_fund_programs AS mfp ON mpf.funding_program_id = mfp.funding_program_id -LEFT JOIN moped_fund_status AS mfst ON mpf.funding_status_id = mfst.funding_status_id -WHERE TRUE AND mp.is_deleted = FALSE AND mpf.is_deleted = FALSE; diff --git a/moped-database/migrations/default/1763481385375_ecapris_funding/down.sql b/moped-database/migrations/default/1763481385375_ecapris_funding/down.sql new file mode 100644 index 0000000000..e2c1824c6d --- /dev/null +++ b/moped-database/migrations/default/1763481385375_ecapris_funding/down.sql @@ -0,0 +1,23 @@ +-- Drop combined_project_funding_view +DROP VIEW IF EXISTS combined_project_funding_view; + +-- Remove funding sync flag from projects table +ALTER TABLE moped_project +DROP COLUMN should_sync_ecapris_funding; + +-- Remove added columns from moped_proj_funding table +ALTER TABLE moped_proj_funding +DROP COLUMN ecapris_funding_id, +DROP COLUMN is_legacy_funding_record, +DROP COLUMN fdu, +DROP COLUMN unit_long_name; + +-- Drop added indexes +DROP INDEX IF EXISTS idx_moped_proj_funding_fdu_not_deleted; +DROP INDEX IF EXISTS idx_moped_proj_funding_project_id; +DROP INDEX IF EXISTS idx_moped_proj_funding_status_id; +DROP INDEX IF EXISTS idx_moped_proj_funding_source_id; +DROP INDEX IF EXISTS idx_moped_proj_funding_program_id; + +-- Drop ecapris_subproject_funding table +DROP TABLE IF EXISTS public.ecapris_subproject_funding; diff --git a/moped-database/migrations/default/1763481385375_ecapris_funding/up.sql b/moped-database/migrations/default/1763481385375_ecapris_funding/up.sql new file mode 100644 index 0000000000..5f9402a564 --- /dev/null +++ b/moped-database/migrations/default/1763481385375_ecapris_funding/up.sql @@ -0,0 +1,137 @@ +-- Add funding sync tracking to projects table +ALTER TABLE moped_project +ADD COLUMN should_sync_ecapris_funding BOOLEAN DEFAULT FALSE NOT NULL; + +COMMENT ON COLUMN moped_project.should_sync_ecapris_funding IS 'Indicates if project funding should be synced from eCAPRIS'; +-- Fix eCAPRIS name in existing comment +COMMENT ON COLUMN moped_project.should_sync_ecapris_statuses IS 'Indicates if project statuses should be synced from eCAPRIS'; + +-- Update moped_proj_funding table to include information available new eCAPRIS data source and to track legacy records +ALTER TABLE moped_proj_funding +ADD COLUMN ecapris_funding_id INTEGER, +ADD COLUMN is_legacy_funding_record BOOLEAN DEFAULT FALSE NOT NULL, +ADD COLUMN fdu TEXT DEFAULT NULL, +ADD COLUMN unit_long_name TEXT DEFAULT NULL; + +-- Index fdu since we'll be querying by it to avoid duplicates in the combined view (using NOT EXISTS) +CREATE INDEX idx_moped_proj_funding_fdu_not_deleted +ON moped_proj_funding (fdu) +WHERE is_deleted = FALSE AND fdu IS NOT NULL; + +-- Index project_id since we are going to be joining this in the project_list_view +CREATE INDEX idx_moped_proj_funding_project_id +ON moped_proj_funding (project_id) +WHERE is_deleted = FALSE; + +-- Index foreign keys of source, program, and status since we join in the combined view +CREATE INDEX idx_moped_proj_funding_status_id +ON moped_proj_funding (funding_status_id); + +CREATE INDEX idx_moped_proj_funding_source_id +ON moped_proj_funding (funding_source_id); + +CREATE INDEX idx_moped_proj_funding_program_id +ON moped_proj_funding (funding_program_id); + +COMMENT ON COLUMN moped_proj_funding.ecapris_funding_id IS 'References the eCAPRIS FDU unique fao_id of imported eCAPRIS funding records'; +COMMENT ON COLUMN moped_proj_funding.is_legacy_funding_record IS 'Indicates if the funding record was created before eCAPRIS sync integration (Dec 2025)'; + +-- Add comments on other existing moped_proj_funding columns +COMMENT ON COLUMN moped_proj_funding.proj_funding_id IS 'Primary key for the project funding record'; +COMMENT ON COLUMN moped_proj_funding.created_by_user_id IS 'ID of the user who last created the record'; +COMMENT ON COLUMN moped_proj_funding.created_at IS 'Timestamp when the record was last created'; +COMMENT ON COLUMN moped_proj_funding.project_id IS 'References the project this funding record is associated with'; +COMMENT ON COLUMN moped_proj_funding.funding_source_id IS 'References the funding source for this funding record'; +COMMENT ON COLUMN moped_proj_funding.funding_program_id IS 'References the funding program for this funding record'; +COMMENT ON COLUMN moped_proj_funding.funding_amount IS 'The amount of funding allocated from this funding source'; +COMMENT ON COLUMN moped_proj_funding.funding_description IS 'A description of the funding source'; +COMMENT ON COLUMN moped_proj_funding.funding_status_id IS 'References the current status of this funding record'; +COMMENT ON COLUMN moped_proj_funding.fund IS 'Legacy JSONB object containing additional fund details from eCAPRIS (Socrata jega-nqf6)'; +COMMENT ON COLUMN moped_proj_funding.dept_unit IS 'Legacy JSONB object containing additional department/unit details from eCAPRIS (Socrata bgrt-2m2z)'; +COMMENT ON COLUMN moped_proj_funding.fdu IS 'The FDU (Fund-Dept-Unit) code associated with this funding record from eCAPRIS'; +COMMENT ON COLUMN moped_proj_funding.unit_long_name IS 'The long name of the unit associated with this funding record from eCAPRIS'; + +-- Create ecapris_subproject_funding table with column comments +CREATE TABLE public.ecapris_subproject_funding ( + id SERIAL PRIMARY KEY, + ecapris_subproject_id TEXT NOT NULL, + fao_id INTEGER NOT NULL UNIQUE, + fdu TEXT NOT NULL, + app INT4 NOT NULL, + unit_long_name TEXT NOT NULL, + subprogram TEXT DEFAULT NULL, + program TEXT DEFAULT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL, + updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL, + created_by_user_id INTEGER REFERENCES moped_users (user_id) ON DELETE RESTRICT ON UPDATE CASCADE, + updated_by_user_id INTEGER REFERENCES moped_users (user_id) ON DELETE RESTRICT ON UPDATE CASCADE +); + +-- Index fdu since we'll be querying by it to avoid duplicates in the combined view (using NOT EXISTS) +CREATE INDEX idx_ecapris_subproject_funding_fdu +ON ecapris_subproject_funding (fdu); + +-- Index ecapris_subproject_id for our future GraphQL query for the project funding table +CREATE INDEX idx_ecapris_subproject_funding_subproject_id +ON ecapris_subproject_funding (ecapris_subproject_id); + +COMMENT ON TABLE public.ecapris_subproject_funding IS 'Stores eCAPRIS subproject fund records synced from the FSD Data Warehouse to supplement the moped_proj_funding table records.'; +COMMENT ON COLUMN public.ecapris_subproject_funding.id IS 'Primary key for the table'; +COMMENT ON COLUMN public.ecapris_subproject_funding.ecapris_subproject_id IS 'eCapris subproject ID number'; +COMMENT ON COLUMN public.ecapris_subproject_funding.fao_id IS 'Unique ID for the FDU (Fund-Dept-Unit) from eCAPRIS'; +COMMENT ON COLUMN public.ecapris_subproject_funding.fdu IS 'The FDU (Fund-Dept-Unit) code associated with this funding record from eCAPRIS'; +COMMENT ON COLUMN public.ecapris_subproject_funding.app IS 'The appropriated amount associated with this funding record from eCAPRIS'; +COMMENT ON COLUMN public.ecapris_subproject_funding.unit_long_name IS 'The long name of the unit associated with this funding record from eCAPRIS'; +COMMENT ON COLUMN public.ecapris_subproject_statuses.created_at IS 'Timestamp when the record was created'; +COMMENT ON COLUMN public.ecapris_subproject_statuses.updated_at IS 'Timestamp when the record was last updated'; +COMMENT ON COLUMN public.ecapris_subproject_statuses.created_by_user_id IS 'ID of the user who created the record'; +COMMENT ON COLUMN public.ecapris_subproject_statuses.updated_by_user_id IS 'ID of the user who updated the record'; + +-- Create a combined_project_funding_view for the project funding UI to consume. This view combines +-- both moped_proj_funding and ecapris_subproject_funding data and removes duplicates based on FDU. +CREATE OR REPLACE VIEW combined_project_funding_view AS +SELECT + ('moped_' || moped_proj_funding.proj_funding_id) AS id, + moped_proj_funding.proj_funding_id AS original_id, + moped_proj_funding.created_at, + moped_proj_funding.updated_at, + moped_proj_funding.project_id, + moped_proj_funding.fdu AS fdu, + moped_proj_funding.funding_amount AS amount, + moped_proj_funding.funding_description AS description, + moped_fund_sources.funding_source_name AS source_name, + moped_fund_status.funding_status_name AS status_name, + moped_fund_programs.funding_program_name AS program_name, + NULL AS fao_id, + NULL AS ecapris_subproject_id, + FALSE AS is_synced_from_ecapris +FROM + moped_proj_funding +LEFT JOIN moped_fund_status ON moped_proj_funding.funding_status_id = moped_fund_status.funding_status_id +LEFT JOIN moped_fund_sources ON moped_proj_funding.funding_source_id = moped_fund_sources.funding_source_id +LEFT JOIN moped_fund_programs ON moped_proj_funding.funding_program_id = moped_fund_programs.funding_program_id +WHERE moped_proj_funding.is_deleted = FALSE +UNION ALL +SELECT + ('ecapris_' || ecapris_subproject_funding.id) AS id, + ecapris_subproject_funding.id AS original_id, + ecapris_subproject_funding.created_at, + ecapris_subproject_funding.updated_at, + NULL AS project_id, + ecapris_subproject_funding.fdu AS fdu, + ecapris_subproject_funding.app AS amount, + 'Synced from eCAPRIS' AS description, + NULL AS source_name, + 'Set up' AS status_name, + NULL AS program_name, + ecapris_subproject_funding.fao_id, + ecapris_subproject_funding.ecapris_subproject_id, + TRUE AS is_synced_from_ecapris +FROM + ecapris_subproject_funding +WHERE NOT EXISTS ( + SELECT 1 + FROM moped_proj_funding + WHERE moped_proj_funding.fdu = ecapris_subproject_funding.fdu + AND moped_proj_funding.is_deleted = FALSE + ); diff --git a/moped-database/views/combined_project_funding_view.sql b/moped-database/views/combined_project_funding_view.sql new file mode 100644 index 0000000000..7fe2f273b3 --- /dev/null +++ b/moped-database/views/combined_project_funding_view.sql @@ -0,0 +1,44 @@ +-- Most recent migration: moped-database/migrations/default/1763481385375_ecapris_funding/up.sql + +CREATE OR REPLACE VIEW combined_project_funding_view AS SELECT + 'moped_'::text || moped_proj_funding.proj_funding_id AS id, + moped_proj_funding.proj_funding_id AS original_id, + moped_proj_funding.created_at, + moped_proj_funding.updated_at, + moped_proj_funding.project_id, + moped_proj_funding.fdu, + moped_proj_funding.funding_amount AS amount, + moped_proj_funding.funding_description AS description, + moped_fund_sources.funding_source_name AS source_name, + moped_fund_status.funding_status_name AS status_name, + moped_fund_programs.funding_program_name AS program_name, + NULL::integer AS fao_id, + NULL::text AS ecapris_subproject_id, + FALSE AS is_synced_from_ecapris +FROM moped_proj_funding +LEFT JOIN moped_fund_status ON moped_proj_funding.funding_status_id = moped_fund_status.funding_status_id +LEFT JOIN moped_fund_sources ON moped_proj_funding.funding_source_id = moped_fund_sources.funding_source_id +LEFT JOIN moped_fund_programs ON moped_proj_funding.funding_program_id = moped_fund_programs.funding_program_id +WHERE moped_proj_funding.is_deleted = FALSE +UNION ALL +SELECT + 'ecapris_'::text || ecapris_subproject_funding.id AS id, + ecapris_subproject_funding.id AS original_id, + ecapris_subproject_funding.created_at, + ecapris_subproject_funding.updated_at, + NULL::integer AS project_id, + ecapris_subproject_funding.fdu, + ecapris_subproject_funding.app AS amount, + 'Synced from eCAPRIS'::text AS description, + NULL::text AS source_name, + 'Set up'::text AS status_name, + NULL::text AS program_name, + ecapris_subproject_funding.fao_id, + ecapris_subproject_funding.ecapris_subproject_id, + TRUE AS is_synced_from_ecapris +FROM ecapris_subproject_funding +WHERE NOT (EXISTS ( + SELECT 1 + FROM moped_proj_funding + WHERE moped_proj_funding.fdu = ecapris_subproject_funding.fdu AND moped_proj_funding.is_deleted = FALSE + )); diff --git a/moped-database/views/project_funding_view.sql b/moped-database/views/project_funding_view.sql index 1ed2ea25d7..11b0e69753 100644 --- a/moped-database/views/project_funding_view.sql +++ b/moped-database/views/project_funding_view.sql @@ -1,11 +1,11 @@ --- Most recent migration: moped-database/migrations/default/1761074353404_ecapris_funding/up.sql +-- Most recent migration: moped-database/migrations/default/1736291471358_add_proj_fund_view/up.sql CREATE OR REPLACE VIEW project_funding_view AS SELECT mp.project_id, mpf.proj_funding_id, mpf.funding_amount, mpf.funding_description, - mpf.fdu AS fund_dept_unit, + mpf.fund_dept_unit, mpf.created_at, mpf.updated_at, mfs.funding_source_name, diff --git a/moped-etl/ecapris-funding/.dockerignore b/moped-etl/ecapris-funding/.dockerignore new file mode 100644 index 0000000000..ebfcfbe643 --- /dev/null +++ b/moped-etl/ecapris-funding/.dockerignore @@ -0,0 +1,3 @@ +env_file +.git% +__pycache__ diff --git a/moped-etl/ecapris-funding/Dockerfile b/moped-etl/ecapris-funding/Dockerfile new file mode 100644 index 0000000000..46355cb5db --- /dev/null +++ b/moped-etl/ecapris-funding/Dockerfile @@ -0,0 +1,7 @@ +FROM python:3.14-slim + +WORKDIR /app +COPY . /app + +RUN pip3.14 install --upgrade pip +RUN pip3.14 install -r requirements.txt diff --git a/moped-etl/ecapris-funding/README.md b/moped-etl/ecapris-funding/README.md new file mode 100644 index 0000000000..839437e92c --- /dev/null +++ b/moped-etl/ecapris-funding/README.md @@ -0,0 +1,39 @@ +# eCAPRIS funding → Moped eCAPRIS funding table + +Python script that transfers eCAPRIS funding to the Moped database to provide a cached dataset for the UI and reporting + +## Sync eCAPRIS funding + +The script `ecapris_funding_sync.py` is used to copy eCAPRIS subproject funding records to the Moped database: +- FDUs are synced from a EDP dataset (s4mj-68pg) that is populated nightly by our [atd-finance-data program tagging ETL](https://github.com/cityofaustin/atd-airflow/blob/production/dags/atd_finance_data_fdu_program_tagging.py). +- These records contain program and subprogram information tagging in addition to eCAPRIS data. +- The original FDUs come from an Oracle Data Warehouse and a view set up by FSD called `ATD_FDUS_VW`. +- These FDUs are up-to-date as of end of business of the previous day so nightly schedules are sufficient for providing the latest data. +- The upsert mutation utilizes the unique `fao_id` as a constraint to avoid duplicating FDUs from the EDP dataset and consume updates. + +## Testing the script locally using Docker Compose + +1. Ensure the local Moped stack is running with a current snapshot. +1. Configure an `env_file` according to the `env_template` example. Find the Socrata (ODP) secrets in the secret store entry called `Socrata Key ID, Secret, and Token`. +1. `docker compose build` to build the container. +1. Dry run the script via: + ```bash + docker compose run ecapris-funding -n + ``` +1. Run the script via: + ```bash + docker compose run ecapris-funding + ``` + +## Testing the script locally using Airflow + +1. Ensure the local Moped stack is running with a current snapshot. +1. If you are making code updates, you will want to update the `development` tag Docker image that the Airflow DAG will reference as you makes changes. + ```bash + # See https://www.docker.com/blog/how-to-rapidly-build-multi-architecture-images-with-buildx/ + docker buildx build --push \ + --platform linux/amd64,linux/arm64 \ + --tag atddocker/atd-moped-etl-ecapris-funding:development . + ``` +1. Start your local Airflow instance ([atd-airflow](https://github.com/cityofaustin/atd-airflow)). You will need to make sure you are running the Airflow web server on a port other than 8080 to prevent conflicts with the Moped local stack. 8082 works well. +1. The DAG is setup to use `development` as the Docker image tag in the local development stack which references the local Hasura API for queries and mutations done by the ETL. diff --git a/moped-etl/ecapris-funding/docker-compose.yaml b/moped-etl/ecapris-funding/docker-compose.yaml new file mode 100644 index 0000000000..86891fbfd8 --- /dev/null +++ b/moped-etl/ecapris-funding/docker-compose.yaml @@ -0,0 +1,9 @@ +services: + ecapris-funding: + build: + context: . + volumes: + - .:/app + entrypoint: python3.14 /app/ecapris_funding_sync.py + env_file: + - env_file diff --git a/moped-etl/ecapris-funding/ecapris_funding_sync.py b/moped-etl/ecapris-funding/ecapris_funding_sync.py new file mode 100644 index 0000000000..36817b37a2 --- /dev/null +++ b/moped-etl/ecapris-funding/ecapris_funding_sync.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +""" +Fetch eCAPRIS funding records from the ODP and sync to Moped moped_proj_funding table +""" +import os +import sys +import logging +import sodapy + +from process.args import get_cli_args +from process.logging import get_logger +from process.queries import GRAPHQL_QUERIES +from process.request import make_hasura_request + +SOCRATA_ENDPOINT = os.getenv("SOCRATA_ENDPOINT") +SOCRATA_TOKEN = os.getenv("SOCRATA_TOKEN") +SOCRATA_API_KEY_ID = os.getenv("SOCRATA_API_KEY_ID") +SOCRATA_API_KEY_SECRET = os.getenv("SOCRATA_API_KEY_SECRET") +FUNDING_DATASET_IDENTIFIER = os.getenv("FUNDING_DATASET_IDENTIFIER") +CHUNK_SIZE = os.getenv("CHUNK_SIZE", 500) + + +def get_socrata_client(): + return sodapy.Socrata( + SOCRATA_ENDPOINT, + SOCRATA_TOKEN, + username=SOCRATA_API_KEY_ID, + password=SOCRATA_API_KEY_SECRET, + timeout=60, + ) + + +def main(args): + # Query ODP for all funding records and make list of records to upsert into Moped DB + socrata_client = get_socrata_client() + + # Fetch all funding records from ODP + logger.info(f"Fetching all eCAPRIS funding records from ODP...") + ecapris_funding_records = socrata_client.get( + FUNDING_DATASET_IDENTIFIER, limit=100000 + ) + + logger.info( + f"Fetched {len(ecapris_funding_records)} total eCAPRIS funding records from ODP." + ) + + # Upsert into the ecapris_subproject_funding table + funding_records_to_upsert = [] + + for record in ecapris_funding_records: + funding_records_to_upsert.append( + { + "ecapris_subproject_id": record.get("sp_number"), + "fao_id": record.get("fao_id"), + "fdu": record.get("fdu"), + "app": int(float(record.get("app", 0))), + "unit_long_name": record.get("unit_long_name"), + "subprogram": record.get("subprogram"), + "program": record.get("program"), + "created_by_user_id": 1, + "updated_by_user_id": 1, + } + ) + + logger.info(f"Built {len(funding_records_to_upsert)} funding records to upsert.") + + # Upsert records into Moped DB + for chunk in range(0, len(funding_records_to_upsert), CHUNK_SIZE): + chunk_payload = funding_records_to_upsert[chunk : chunk + CHUNK_SIZE] + + if args.dry_run: + logger.info( + f"[DRY RUN] Would upsert chunk of {len(chunk_payload)} funding records into Moped DB..." + ) + else: + logger.info( + f"Upserting chunk of {len(chunk_payload)} funding records into Moped DB..." + ) + results = make_hasura_request( + query=GRAPHQL_QUERIES["project_funding_upsert"], + variables={"objects": chunk_payload}, + ) + + +if __name__ == "__main__": + log_level = logging.DEBUG + logger = get_logger(name="moped-ecapris-funding-sync", level=log_level) + logger.info( + f"Starting sync. Transferring eCapris funding records from ODP to Moped DB." + ) + + args = get_cli_args() + + main(args) diff --git a/moped-etl/ecapris-funding/env_template b/moped-etl/ecapris-funding/env_template new file mode 100644 index 0000000000..a65832c485 --- /dev/null +++ b/moped-etl/ecapris-funding/env_template @@ -0,0 +1,7 @@ +SOCRATA_ENDPOINT= +SOCRATA_TOKEN= +SOCRATA_API_KEY_ID= +SOCRATA_API_KEY_SECRET= +FUNDING_DATASET_IDENTIFIER=s4mj-68pg +HASURA_ENDPOINT=http://host.docker.internal:8080/v1/graphql +HASURA_ADMIN_SECRET=hasurapassword diff --git a/moped-etl/ecapris-funding/process/args.py b/moped-etl/ecapris-funding/process/args.py new file mode 100644 index 0000000000..693a5edf2a --- /dev/null +++ b/moped-etl/ecapris-funding/process/args.py @@ -0,0 +1,20 @@ +import argparse +from datetime import datetime, timezone, timedelta + + +def get_cli_args(): + """Create the CLI and parse args + + Returns: + argparse.Namespace: The CLI namespace + """ + parser = argparse.ArgumentParser() + + parser.add_argument( + "-n", + "--dry-run", + action="store_true", + help="Log what changes would be made without executing them", + ) + + return parser.parse_args() diff --git a/moped-etl/ecapris-funding/process/logging.py b/moped-etl/ecapris-funding/process/logging.py new file mode 100644 index 0000000000..28ebae76c6 --- /dev/null +++ b/moped-etl/ecapris-funding/process/logging.py @@ -0,0 +1,13 @@ +import sys +import logging + + +def get_logger(name, level=logging.DEBUG): + """Return a module logger that streams to stdout""" + logger = logging.getLogger(name) + handler = logging.StreamHandler(stream=sys.stdout) + formatter = logging.Formatter(fmt=" %(name)s.%(levelname)s: %(message)s") + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(level) + return logger diff --git a/moped-etl/ecapris-funding/process/queries.py b/moped-etl/ecapris-funding/process/queries.py new file mode 100644 index 0000000000..e3fbcb9184 --- /dev/null +++ b/moped-etl/ecapris-funding/process/queries.py @@ -0,0 +1,27 @@ +""" +Queries for Moped DB + +The upsert query is used to insert or update the eCapris funding records into the Moped DB. +eCapris funding records can be edited so we need to handle updates. +""" + +GRAPHQL_QUERIES = { + "subprojects_to_query_for_funding": """ + query MopedProjects { + moped_project(where: {ecapris_subproject_id: {_is_null: false}, is_deleted: {_eq: false}}, distinct_on: ecapris_subproject_id) { + ecapris_subproject_id + } + } + """, + "project_funding_upsert": """ + mutation UpsertEcaprisFunding($objects: [ecapris_subproject_funding_insert_input!]!) { + insert_ecapris_subproject_funding(objects: $objects, on_conflict: {constraint: ecapris_subproject_funding_fao_id_key, + update_columns: [app, unit_long_name, subprogram, program]}) { + returning { + fao_id + ecapris_subproject_id + } + } + } + """, +} diff --git a/moped-etl/ecapris-funding/process/request.py b/moped-etl/ecapris-funding/process/request.py new file mode 100644 index 0000000000..5a2f14cf17 --- /dev/null +++ b/moped-etl/ecapris-funding/process/request.py @@ -0,0 +1,36 @@ +# +# Request Helper - Makes requests to a Hasura/GraphQL endpoint +# + +import os +import requests + +HASURA_ENDPOINT = os.getenv("HASURA_ENDPOINT") +HASURA_ADMIN_SECRET = os.getenv("HASURA_ADMIN_SECRET") + + +def make_hasura_request(*, query, variables=None): + """Fetch data from hasura + + Args: + query (str): the hasura query + variables (dict, optional): the query variables + + Raises: + ValueError: If no data is returned + + Returns: + dict: Hasura JSON response data + """ + headers = { + "X-Hasura-Admin-Secret": HASURA_ADMIN_SECRET, + "content-type": "application/json", + } + payload = {"query": query, "variables": variables} + res = requests.post(HASURA_ENDPOINT, json=payload, headers=headers) + res.raise_for_status() + data = res.json() + try: + return data["data"] + except KeyError: + raise ValueError(data) diff --git a/moped-etl/ecapris-funding/requirements.txt b/moped-etl/ecapris-funding/requirements.txt new file mode 100644 index 0000000000..4c2e0168e4 --- /dev/null +++ b/moped-etl/ecapris-funding/requirements.txt @@ -0,0 +1,2 @@ +requests==2.* +sodapy==2.1.*