diff --git a/.github/scripts/test_alembic_v2_lifecycle.sh b/.github/scripts/test_alembic_v2_lifecycle.sh index b122419df..7b6293df5 100755 --- a/.github/scripts/test_alembic_v2_lifecycle.sh +++ b/.github/scripts/test_alembic_v2_lifecycle.sh @@ -11,4 +11,5 @@ uv run --frozen --extra dev pytest -q \ tests/integration/test_alembic_v2_saved_item_keys.py \ tests/integration/test_alembic_v2_user_types.py \ tests/integration/test_alembic_v2_withheld_area_codes.py \ - tests/integration/test_alembic_v2_shared_report_keys.py + tests/integration/test_alembic_v2_shared_report_keys.py \ + tests/integration/test_alembic_v2_cliff_impacts.py diff --git a/changelog.d/simulation-run-cliff-impacts.added.md b/changelog.d/simulation-run-cliff-impacts.added.md new file mode 100644 index 000000000..f858abb2b --- /dev/null +++ b/changelog.d/simulation-run-cliff-impacts.added.md @@ -0,0 +1 @@ +The v2 database can store one cliff result per simulation run, its cliff gap and cliff share, with the state of its calculation. The simulation service may start each calculation, its Modal workers may write the result, and the Public API may read it. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 157f8aecf..9e0f86351 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -255,8 +255,8 @@ and a revision runs once per database, so it can't repair drift. | `policyengine_v2_runtime` | The Public API on Cloud Run | | `policyengine_v2_catalog_writer` | `scripts/publish_v2_metadata_catalog.py`, which also runs `ANALYZE` on seven catalog tables after every publication, with `MAINTAIN` (PostgreSQL 17) | | `policyengine_stage12_` | The simulation API and its Modal worker, on the temporary comparison-run tables | -| `policyengine_v2_simulation_service` | The simulation API on Cloud Run: creates simulations, reports, their runs, budget-window runs and household jobs, and reads the policy, dynamic, household and catalog rows a run needs. It updates only the status, Modal call ID, error and completion columns of the runs and household jobs whose Modal calls it starts, and the status, error and completion columns of a budget-window run, which it fails at once when Modal refuses the calls of the window's first years | -| `policyengine_v2_modal_worker` | The simulation API's Modal workers: write their runs' and household jobs' status and results, and budget-window runs' status, and read the rows a run needs. It updates only a run's status, Modal call ID, error, time and result columns, and a budget-window run's status, error and time columns: each year's coordinator starts the window's next unstarted year when its own year finishes, the last year to finish marks the window succeeded, and a failed year fails it. The workers' code, not the database, keeps a worker to its own run | +| `policyengine_v2_simulation_service` | The simulation API on Cloud Run: creates simulations, reports, their runs, budget-window runs, household jobs and simulation runs' cliff calculations, and reads the policy, dynamic, household and catalog rows a run needs. It updates only the status, Modal call ID, error and completion columns of the runs, household jobs and cliff calculations whose Modal calls it starts, and the status, error and completion columns of a budget-window run, which it fails at once when Modal refuses the calls of the window's first years | +| `policyengine_v2_modal_worker` | The simulation API's Modal workers: write their runs', household jobs' and cliff calculations' status and results, and budget-window runs' status, and read the rows a run needs. It updates only a run's or cliff calculation's status, Modal call ID, error, time and result columns, and a budget-window run's status, error and time columns: each year's coordinator starts the window's next unstarted year when its own year finishes, the last year to finish marks the window succeeded, and a failed year fails it. The workers' code, not the database, keeps a worker to its own run | Every v2 table has an entry for every role, so "no access" is a decision, not an omission. A privilege is declared on the whole table, or with @@ -456,6 +456,8 @@ it publishes that version, with nothing to restore it. Downgrading past the revision is therefore safe only while no version is pruned (no `tax_benefit_model_versions` row has `catalog_values_pruned_at` set). From the head, such a downgrade first runs the later revisions' downgrades: +`bc334f9877b7`'s drops `cliff_impacts` with every simulation run's cliff +results (see "API v2 simulation-run cliff impacts"), `f519a77ba122`'s drops `user_invitations` and needs no recorded change with source `invitation` (see "API v2 user invitations"), `94dc43079186`'s drops every user's report-run record, the simulation and @@ -1012,3 +1014,74 @@ change's source is `invitation`. `test_user_invitations_revision_adds_invitations_and_dismissals_only` and `test_user_invitations_revision_records_invitation_changes_until_downgraded` in `tests/integration/test_alembic_v2_user_types.py` check both. + +### API v2 simulation-run cliff impacts + +Revision `bc334f9877b7` (parent `f519a77ba122`) was autogenerated, with its +ID given by `--rev-id`, after the `CliffImpact` model was added. Each +simulation run can have one cliff result, linked to the run and not to any +report, which every report naming the run shares: cliffs need marginal tax +rates, which recalculate each household with extra earnings, so a shared run +never stores the cliff columns (US national runs keep their region-group +split), and a request that asks for cliffs starts a separate calculation of +the run on its dataset, policy and dynamic. + +It creates `cliff_impacts`, one row per cliff calculation of a simulation run: + +- `simulation_run_id`, a foreign key to `simulation_runs` with `CASCADE`, as + `poverty` and `inequality` have: the result is part of its run, and nothing + references it. A run can be deleted only once no report run names it. +- The calculation's state: `status` (`pending`, `running`, `succeeded` or + `failed`, in the `v2_simulation_status` type simulation runs use), + `modal_call_id`, `error_message` (`code: message`, as runs record theirs), + `started_at` and `completed_at`, with the run tables' check that a + succeeded or failed row has `completed_at` + (`ck_cliff_impacts_finished_completion`). +- The result: `cliff_gap`, the weighted total of the net income people would + lose from a small rise in their earnings, and `cliff_share`, the weighted + number of people on a cliff divided by the weighted number of adults, both + double precision. A succeeded row has both, and no other row has either + (`ck_cliff_impacts_result_when_succeeded`). Neither is range-checked: .py + doesn't keep the share to at most 1, and a filtered simulation with no + adults gives NaN. +- `uq_cliff_impacts_unfailed`, a partial unique index on `simulation_run_id` + where `status <> 'failed'` (also declared for SQLite): a run has at most one + calculation that hasn't failed, and so at most one result. The simulation + service is to start one with `INSERT ... ON CONFLICT (simulation_run_id) + WHERE status <> 'failed' DO NOTHING`, so that two requests at once start + one calculation and the second joins it. A failed calculation never counts: + it stays as a record, and the next request inserts another. +- An index on `(status, started_at)`, as `simulation_runs` has, for finding + unfinished calculations by age, as the simulation API's stuck-run sweep + does for runs: a calculation whose worker ended without recording its + outcome stays unfinished, and keeps later requests from starting another, + until something fails it. + +A result has no versions of its own: it inherits its run's catalog and +package versions, so it is calculated only for a run on the versions the +simulation API serves. The simulation service may read and insert rows and update their +status, Modal call ID, error and completion columns; the Modal worker may read +them and update their state and result columns, never insert them; the +runtime role may read them; neither simulation role may change the run a row +names or delete a row. The reconcile step after each schema upgrade grants +them. Until the simulation API writes the table, no run has a cliff +calculation, and its cliffs stay refused. `report_runs`' four cliff columns +stay until a later revision drops them, once the Public API reads cliffs from +here. + +It has one post-generation correction, commented in the revision: `status` +reuses the baseline's `v2_simulation_status` type +(`postgresql.ENUM(..., create_type=False)`), as `1356f658347d` does for +`simulation_runs.status`; the generated `create_table` would create the type +again and fail. So the downgrade, which drops only the table, leaves the type +to `simulation_runs`. + +The revision writes no data and changes no other table, and the previous +Public API revision, which doesn't map the table, keeps working at it. The +downgrade drops the table with every calculation and result; a later +calculation of the same runs on the same versions gives the same numbers. +`tests/integration/test_alembic_v2_cliff_impacts.py` checks that nothing +else changes, that the downgrade leaves the type and an upgrade works again, +and the index, checks and delete rule; +`tests/integration/test_v2_simulation_role_privileges.py` runs the writes +and reads as the roles that do them. diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index d93eec83a..13f8475d8 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -770,7 +770,9 @@ admin; `scripts/set_v2_user_type.py` makes another admin afterwards. `users.primary_country`, so it fails while legacy-only rows, any simulation or report associations, or any unauthenticated user exist. Removing them first deletes users' saved work. From the head, the later revisions' - downgrades run first: the user invitations revision (`f519a77ba122`) drops + downgrades run first: the cliff impacts revision (`bc334f9877b7`) drops + every simulation run's cliff results, the user invitations revision + (`f519a77ba122`) drops every invitation and dismissal and needs no change with source `invitation`, the shared reports revision (`94dc43079186`) drops every user's report-run record, every simulation and report key, the runs' diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 785d4a49a..431a5a148 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -835,8 +835,10 @@ pruning"). A schema downgrade past revision `334859ef4fec` drops `catalog_last_published_at` and `catalog_values_pruned_at`. From the head, -it first runs the later revisions' downgrades: `f519a77ba122`'s drops -`user_invitations` and needs no recorded change with source `invitation`, +it first runs the later revisions' downgrades: `bc334f9877b7`'s drops +`cliff_impacts` with every simulation run's cliff results, `f519a77ba122`'s +drops `user_invitations` and needs no recorded change with source +`invitation`, `94dc43079186`'s drops `user_report_run_associations` with every user's report-run record, the simulation and report keys, the runs' `forced` markers, the window runs' .py diff --git a/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py b/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py new file mode 100644 index 000000000..b9bd3badd --- /dev/null +++ b/migrations/v2/versions/bc334f9877b7_record_each_simulation_run_s_cliff_.py @@ -0,0 +1,86 @@ +"""record each simulation run's cliff calculations + +Revision ID: bc334f9877b7 +Revises: f519a77ba122 +Create Date: 2026-11-17 17:38:09.806260 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate --rev-id bc334f9877b7 + +One post-generation correction, commented where it is made: +cliff_impacts.status reuses the existing v2_simulation_status type instead of +creating it again. + +Each simulation run can have one cliff result: its cliff gap and cliff share, +linked to the run and not to any report. It adds cliff_impacts, one row per +cliff calculation of a simulation run, holding the calculation's state +(status, Modal call, error and times) and, once it succeeds, the two numbers. +A partial unique index allows one calculation per run that hasn't failed, so +identical requests join it; a failed calculation never counts. Deleting a run +deletes its calculations. A result has no versions of its own: it inherits its +run's. + +It writes no data and changes no other table, and the previous Public API +revision, which doesn't map the table, keeps working at it. The downgrade +drops the table with every calculation and result, and leaves the type, +which simulation_runs still uses. +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import postgresql + + +revision: str = 'bc334f9877b7' +down_revision: Union[str, None] = 'f519a77ba122' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('cliff_impacts', + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('simulation_run_id', sa.Uuid(), nullable=False), + sa.Column( + 'status', + # Dialect correction: a cliff calculation has a simulation run's + # states, so the column reuses the baseline's v2_simulation_status + # type instead of creating it again, which would fail. + postgresql.ENUM( + 'pending', + 'running', + 'succeeded', + 'failed', + name='v2_simulation_status', + create_type=False, + ), + nullable=False, + ), + sa.Column('modal_call_id', sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + sa.Column('error_message', sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('cliff_gap', sa.Float(), nullable=True), + sa.Column('cliff_share', sa.Float(), nullable=True), + sa.CheckConstraint("(cliff_gap IS NULL) = (cliff_share IS NULL) AND (status = 'succeeded') = (cliff_gap IS NOT NULL)", name=op.f('ck_cliff_impacts_result_when_succeeded')), + sa.CheckConstraint("status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", name=op.f('ck_cliff_impacts_finished_completion')), + sa.ForeignKeyConstraint(['simulation_run_id'], ['simulation_runs.id'], name=op.f('fk_cliff_impacts_simulation_run_id_simulation_runs'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_cliff_impacts')) + ) + op.create_index(op.f('ix_cliff_impacts_simulation_run_id'), 'cliff_impacts', ['simulation_run_id'], unique=False) + op.create_index('ix_cliff_impacts_status_started_at', 'cliff_impacts', ['status', 'started_at'], unique=False) + op.create_index('uq_cliff_impacts_unfailed', 'cliff_impacts', ['simulation_run_id'], unique=True, postgresql_where=sa.text("status <> 'failed'"), sqlite_where=sa.text("status <> 'failed'")) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index('uq_cliff_impacts_unfailed', table_name='cliff_impacts', postgresql_where=sa.text("status <> 'failed'"), sqlite_where=sa.text("status <> 'failed'")) + op.drop_index('ix_cliff_impacts_status_started_at', table_name='cliff_impacts') + op.drop_index(op.f('ix_cliff_impacts_simulation_run_id'), table_name='cliff_impacts') + op.drop_table('cliff_impacts') + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index b51fd461c..21c2a2a5c 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "f519a77ba122" +V2_ALEMBIC_HEAD_REVISION = "bc334f9877b7" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 973eb9c11..dfa9c0bdf 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -78,6 +78,7 @@ AggregateType, BudgetSummary, ChangeAggregate, + CliffImpact, CongressionalDistrictImpact, ConstituencyImpact, DecileImpact, @@ -124,6 +125,7 @@ "CatalogEntity", "CatalogEnumValue", "ChangeAggregate", + "CliffImpact", "CongressionalDistrictImpact", "ConstituencyImpact", "Country", diff --git a/policyengine_api/data/v2/models/run_results.py b/policyengine_api/data/v2/models/run_results.py index 2e691bd2f..3451515bf 100644 --- a/policyengine_api/data/v2/models/run_results.py +++ b/policyengine_api/data/v2/models/run_results.py @@ -2,10 +2,11 @@ Results that compare a report run's baseline and reform belong to the report run and name its two simulation runs; each such row is unique within its report -run. Poverty and inequality, which describe one simulation, belong to a -simulation run. +run. Poverty, inequality and cliff impacts, which describe one simulation, +belong to a simulation run. """ +from datetime import datetime from enum import StrEnum from typing import Any from uuid import UUID @@ -13,9 +14,16 @@ import sqlalchemy as sa from sqlmodel import Field, Relationship -from policyengine_api.data.v2.models.base import IdentifiedModel, enum_type +from policyengine_api.data.v2.models.base import ( + IdentifiedModel, + TimestampedModel, + enum_type, +) from policyengine_api.data.v2.models.reports import ReportRun -from policyengine_api.data.v2.models.simulations import SimulationRun +from policyengine_api.data.v2.models.simulations import ( + SimulationRun, + SimulationStatus, +) class OutputStatus(StrEnum): @@ -401,6 +409,78 @@ class Poverty(IdentifiedModel, table=True): simulation_run: SimulationRun = Relationship(back_populates="poverty_results") +# The cliff calculations a new request may join, of which a simulation run has +# at most one: a failed calculation never counts. +UNFAILED_CLIFF_CALCULATIONS = "status <> 'failed'" + + +class CliffImpact(TimestampedModel, table=True): + """One cliff calculation of a simulation run, and its result once it succeeds. + + ``cliff_gap`` is the weighted total of the net income people would lose + from a small rise in their earnings, and ``cliff_share`` the weighted + number of people on a cliff divided by the weighted number of adults. + They need marginal tax rates, which recalculate each household with extra + earnings, so a run's cliffs are calculated apart from the run, on its + dataset, policy and dynamic, only when a request asks for them, and only + for a run on the current versions. The result has no versions of its own: + it inherits its run's, and every report naming the run shares it. + + The simulation service inserts a pending calculation and records its + Modal call, or fails it when Modal refuses the call; the Modal worker + claims it under its call, then writes the two numbers with ``succeeded``, + or ``failed`` with its ``code: message`` error. A run has at most one + calculation that hasn't failed, which identical requests join; a failed + calculation never counts, and the next request starts another. Deleting + the run deletes its calculations. + """ + + __tablename__ = "cliff_impacts" + __table_args__ = ( + sa.CheckConstraint( + "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", + name="ck_cliff_impacts_finished_completion", + ), + sa.CheckConstraint( + "(cliff_gap IS NULL) = (cliff_share IS NULL) " + "AND (status = 'succeeded') = (cliff_gap IS NOT NULL)", + name="ck_cliff_impacts_result_when_succeeded", + ), + sa.Index("ix_cliff_impacts_status_started_at", "status", "started_at"), + sa.Index( + "uq_cliff_impacts_unfailed", + "simulation_run_id", + unique=True, + postgresql_where=sa.text(UNFAILED_CLIFF_CALCULATIONS), + sqlite_where=sa.text(UNFAILED_CLIFF_CALCULATIONS), + ), + ) + + simulation_run_id: UUID = Field( + foreign_key="simulation_runs.id", + ondelete="CASCADE", + index=True, + ) + status: SimulationStatus = Field( + default=SimulationStatus.PENDING, + sa_type=enum_type(SimulationStatus, "v2_simulation_status"), + ) + modal_call_id: str | None = Field(default=None, max_length=255) + error_message: str | None = None + started_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + completed_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + cliff_gap: float | None = None + cliff_share: float | None = None + + simulation_run: SimulationRun = Relationship(back_populates="cliff_impacts") + + class ProgramStatistics(IdentifiedModel, table=True): """Baseline and reform totals, counts, winners and losers for one program. diff --git a/policyengine_api/data/v2/models/simulations.py b/policyengine_api/data/v2/models/simulations.py index ca213d3fb..28ba6b052 100644 --- a/policyengine_api/data/v2/models/simulations.py +++ b/policyengine_api/data/v2/models/simulations.py @@ -24,7 +24,11 @@ if TYPE_CHECKING: from policyengine_api.data.v2.models.reports import Report - from policyengine_api.data.v2.models.run_results import Inequality, Poverty + from policyengine_api.data.v2.models.run_results import ( + CliffImpact, + Inequality, + Poverty, + ) class SimulationStatus(StrEnum): @@ -168,7 +172,9 @@ class SimulationRun(TimestampedModel, table=True): dataset can't be deleted while a run names them, and deleting the region clears the reference. A household simulation's run stores its household in ``household_result``; an economy run's output file is ``output_uri``, with - its SHA-256. Each poverty and inequality result belongs to one run. + its SHA-256. Each poverty and inequality result belongs to one run, and so + does each calculation of its cliff impact (``CliffImpact``), which a + request asking for cliffs starts apart from the run. A simulation has at most one run that hasn't failed for each year, catalog version and pair of package versions, which identical requests join; a @@ -268,3 +274,7 @@ class SimulationRun(TimestampedModel, table=True): back_populates="simulation_run", cascade_delete=True, ) + cliff_impacts: list["CliffImpact"] = Relationship( + back_populates="simulation_run", + cascade_delete=True, + ) diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index ba116de22..5a27d1d6f 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -29,20 +29,22 @@ analyzes afterwards; its code isn't organized as connector modules, so the deploy check, not the connector analysis, covers it. - ``policyengine_v2_simulation_service``: the simulation API on Cloud Run. It - creates simulations, reports, their runs, budget-window runs and household - jobs, and reads the policy, dynamic, household and catalog rows a run needs. - It starts the Modal calls of runs and household jobs, so it may update their + creates simulations, reports, their runs, budget-window runs, household + jobs and simulation runs' cliff calculations, and reads the policy, + dynamic, household and catalog rows a run needs. It starts the Modal calls + of runs, household jobs and cliff calculations, so it may update their status, Modal call ID, error and completion columns: it records a call's ID once Modal accepts the call, and fails the run when Modal refuses it. It starts a budget-window run's first years at submit, so it may update the window run's status, error and completion columns, to fail it at once when Modal refuses those years' calls. - ``policyengine_v2_modal_worker``: the simulation API's Modal workers. They - write the status of the runs and household jobs they were given, and their - results, and read the rows a run needs. They also write a budget-window - run's status, error and times: each year's coordinator starts the window's - next unstarted year when its own year finishes, the last year to finish - marks the window succeeded, and a failed year fails it. + write the status of the runs, household jobs and cliff calculations they + were given, and their results, and read the rows a run needs. They also + write a budget-window run's status, error and times: each year's + coordinator starts the window's next unstarted year when its own year + finishes, the last year to finish marks the window succeeded, and a failed + year fails it. Both simulation API roles update only the columns ``update_columns`` names, so the database refuses any change to what a run is (its simulation or report, @@ -267,6 +269,11 @@ def _run_table( "reports": replace(_SIMULATION_DEFINITION, catalog_writer=READ), "simulation_runs": _run_table("output_uri", "output_sha256", "household_result"), "simulations": replace(_SIMULATION_DEFINITION, catalog_writer=READ), + # A simulation run's cliff calculations, started and finished as a run is: + # the simulation service inserts one and records or fails its Modal call, + # and the Modal worker claims it and writes its cliff gap and share. Neither + # may change the run a calculation names. + "cliff_impacts": _run_table("cliff_gap", "cliff_share"), # Results of simulation runs and report runs. "labor_supply_decile_responses": _RESULT, "labor_supply_responses": _RESULT, diff --git a/tests/integration/test_alembic_v2_cliff_impacts.py b/tests/integration/test_alembic_v2_cliff_impacts.py new file mode 100644 index 000000000..56f9b3013 --- /dev/null +++ b/tests/integration/test_alembic_v2_cliff_impacts.py @@ -0,0 +1,369 @@ +"""Check the cliff impacts revision against disposable Postgres. + +Revision bc334f9877b7, on the user invitations revision f519a77ba122, adds +``cliff_impacts``: one row per cliff calculation of a simulation run, with its +state and, once it succeeds, the run's cliff gap and cliff share. Its status +reuses the ``v2_simulation_status`` type, and it changes no other table. The +downgrade drops the table with its rows and leaves the type, which +``simulation_runs`` still uses. +""" + +from collections.abc import Iterator +from typing import Any +from uuid import UUID, uuid4 + +import pytest +from alembic import command +from sqlalchemy import Connection, Engine, create_engine, inspect, text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.pool import NullPool + +from tests.fixtures.integration.alembic_v2 import ( + alembic_v2_config, + disposable_v2_url, +) +from tests.integration.simulation_rows import ( + RUN_VERSIONS, + insert_public_api_rows, + insert_row, + refused, +) + +USER_INVITATIONS_REVISION = "f519a77ba122" +CLIFF_IMPACTS_REVISION = "bc334f9877b7" +# What a request starting a run's cliff calculation inserts: the conflict +# target names the partial unique index's whole condition. +START_CLIFF_CALCULATION = ( + "INSERT INTO cliff_impacts (id, simulation_run_id, status) " + "VALUES (:id, :simulation_run_id, 'pending') " + "ON CONFLICT (simulation_run_id) WHERE status <> 'failed' DO NOTHING " + "RETURNING id" +) + + +def _schema_shape(engine: Engine) -> dict[str, dict[str, Any]]: + """Reflect every table's columns, keys, indexes and checks.""" + inspector = inspect(engine) + return { + table_name: { + "columns": [ + ( + column["name"], + str(column["type"]), + column["nullable"], + column.get("default"), + ) + for column in inspector.get_columns(table_name) + ], + "primary_key": inspector.get_pk_constraint(table_name), + "foreign_keys": sorted( + ( + key["name"], + key["constrained_columns"], + key["referred_table"], + key["options"].get("ondelete"), + ) + for key in inspector.get_foreign_keys(table_name) + ), + "unique_constraints": sorted( + (unique["name"], unique["column_names"]) + for unique in inspector.get_unique_constraints(table_name) + ), + "indexes": sorted( + ( + index["name"], + index["column_names"], + index["unique"], + index.get("dialect_options", {}).get("postgresql_where"), + ) + for index in inspector.get_indexes(table_name) + ), + "checks": { + check["name"]: check["sqltext"] + for check in inspector.get_check_constraints(table_name) + }, + } + for table_name in inspector.get_table_names(schema="public") + if table_name != "alembic_version" + } + + +def _enum_labels(engine: Engine) -> dict[str, list[str]]: + """Return each enum type in the public schema with its labels in order.""" + with engine.connect() as connection: + rows = connection.execute( + text( + "SELECT type.typname, enum.enumlabel FROM pg_type AS type " + "JOIN pg_enum AS enum ON enum.enumtypid = type.oid " + "JOIN pg_namespace AS namespace ON namespace.oid = type.typnamespace " + "WHERE namespace.nspname = 'public' " + "ORDER BY type.typname, enum.enumsortorder" + ) + ).all() + labels: dict[str, list[str]] = {} + for type_name, label in rows: + labels.setdefault(type_name, []).append(label) + return labels + + +@pytest.fixture +def engine() -> Iterator[Engine]: + """Yield an engine on the disposable database, left at the v2 head.""" + database_engine = create_engine(disposable_v2_url(), poolclass=NullPool) + try: + yield database_engine + finally: + command.upgrade(alembic_v2_config(), "head") + database_engine.dispose() + + +def test_cliff_impacts_revision_adds_only_its_table(engine: Engine) -> None: + """The revision adds one table, with no new enum type, and changes nothing else.""" + config = alembic_v2_config() + + command.upgrade(config, "head") + command.downgrade(config, USER_INVITATIONS_REVISION) + parent_shape = _schema_shape(engine) + parent_enums = _enum_labels(engine) + command.upgrade(config, CLIFF_IMPACTS_REVISION) + revision_shape = _schema_shape(engine) + + assert set(revision_shape) - set(parent_shape) == {"cliff_impacts"} + assert { + table_name: shape + for table_name, shape in revision_shape.items() + if table_name != "cliff_impacts" + } == parent_shape + assert _enum_labels(engine) == parent_enums + cliff_impacts = revision_shape["cliff_impacts"] + assert cliff_impacts["primary_key"]["constrained_columns"] == ["id"] + assert [ + (columns, referred_table, ondelete) + for _name, columns, referred_table, ondelete in cliff_impacts["foreign_keys"] + ] == [(["simulation_run_id"], "simulation_runs", "CASCADE")] + assert cliff_impacts["unique_constraints"] == [] + assert cliff_impacts["indexes"] == [ + ("ix_cliff_impacts_simulation_run_id", ["simulation_run_id"], False, None), + ("ix_cliff_impacts_status_started_at", ["status", "started_at"], False, None), + ( + "uq_cliff_impacts_unfailed", + ["simulation_run_id"], + True, + "(status <> 'failed'::v2_simulation_status)", + ), + ] + assert set(cliff_impacts["checks"]) == { + "ck_cliff_impacts_finished_completion", + "ck_cliff_impacts_result_when_succeeded", + } + with engine.connect() as connection: + column_types = dict( + connection.execute( + text( + "SELECT column_name, udt_name || ':' || is_nullable " + "FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = 'cliff_impacts'" + ) + ).all() + ) + assert column_types == { + "id": "uuid:NO", + "created_at": "timestamptz:NO", + "updated_at": "timestamptz:NO", + "simulation_run_id": "uuid:NO", + "status": "v2_simulation_status:NO", + "modal_call_id": "varchar:YES", + "error_message": "varchar:YES", + "started_at": "timestamptz:YES", + "completed_at": "timestamptz:YES", + "cliff_gap": "float8:YES", + "cliff_share": "float8:YES", + } + + +def test_cliff_impacts_downgrade_drops_the_table_and_keeps_the_status_type( + engine: Engine, +) -> None: + """Downgrading drops only the table, so upgrading again works.""" + config = alembic_v2_config() + + command.upgrade(config, "head") + command.downgrade(config, USER_INVITATIONS_REVISION) + + assert "cliff_impacts" not in inspect(engine).get_table_names() + # ``simulation_runs.status`` still uses the type. + assert _enum_labels(engine)["v2_simulation_status"] == [ + "pending", + "running", + "succeeded", + "failed", + ] + + command.upgrade(config, CLIFF_IMPACTS_REVISION) + + assert "cliff_impacts" in inspect(engine).get_table_names() + + +def _insert_simulation_run(connection: Connection) -> UUID: + """Store an economy simulation and one pending run of it.""" + public_api_rows = insert_public_api_rows(connection) + simulation_id = uuid4() + run_id = uuid4() + insert_row( + connection, + "simulations", + id=simulation_id, + simulation_type="economy", + dataset_id=public_api_rows.dataset_id, + tax_benefit_model_version_id=public_api_rows.version_id, + year=2026, + ) + insert_row( + connection, + "simulation_runs", + id=run_id, + simulation_id=simulation_id, + tax_benefit_model_version_id=public_api_rows.version_id, + year=2026, + status="pending", + **RUN_VERSIONS, + ) + return run_id + + +def _start_cliff_calculation(connection: Connection, run_id: UUID) -> UUID | None: + """Insert a pending calculation unless one that hasn't failed exists.""" + return connection.execute( + text(START_CLIFF_CALCULATION), {"id": uuid4(), "simulation_run_id": run_id} + ).scalar_one_or_none() + + +def _fail_cliff_calculation(connection: Connection, calculation_id: UUID) -> None: + connection.execute( + text( + "UPDATE cliff_impacts SET status = 'failed', " + "error_message = 'modal_call_ended: the call ended', " + "completed_at = now() WHERE id = :id" + ), + {"id": calculation_id}, + ) + + +def _cliff_statuses(connection: Connection, run_id: UUID) -> list[str]: + return list( + connection.execute( + text( + "SELECT status::text FROM cliff_impacts " + "WHERE simulation_run_id = :run_id ORDER BY created_at, status" + ), + {"run_id": run_id}, + ).scalars() + ) + + +def test_a_run_starts_one_cliff_calculation_until_it_fails(engine: Engine) -> None: + """Two requests at once start one calculation; a failed one never counts.""" + command.upgrade(alembic_v2_config(), "head") + + with engine.connect() as connection: + transaction = connection.begin() + try: + run_id = _insert_simulation_run(connection) + first_calculation = _start_cliff_calculation(connection, run_id) + joined_calculation = _start_cliff_calculation(connection, run_id) + + assert first_calculation is not None + # The second request joins the pending calculation. + assert joined_calculation is None + with refused(connection, IntegrityError, match="uq_cliff_impacts_unfailed"): + insert_row( + connection, + "cliff_impacts", + id=uuid4(), + simulation_run_id=run_id, + status="running", + ) + + _fail_cliff_calculation(connection, first_calculation) + retried_calculation = _start_cliff_calculation(connection, run_id) + + assert retried_calculation is not None + connection.execute( + text( + "UPDATE cliff_impacts SET status = 'succeeded', " + "cliff_gap = 1.5e9, cliff_share = 0.04, completed_at = now() " + "WHERE id = :id" + ), + {"id": retried_calculation}, + ) + assert _start_cliff_calculation(connection, run_id) is None + assert sorted(_cliff_statuses(connection, run_id)) == [ + "failed", + "succeeded", + ] + finally: + transaction.rollback() + + +@pytest.mark.parametrize( + ("assignments", "check_name"), + [ + ("status = 'succeeded', completed_at = now()", "result_when_succeeded"), + ( + "status = 'succeeded', cliff_gap = 1.0, completed_at = now()", + "result_when_succeeded", + ), + ("cliff_gap = 1.0, cliff_share = 0.1", "result_when_succeeded"), + ( + "status = 'failed', cliff_gap = 1.0, cliff_share = 0.1, " + "completed_at = now()", + "result_when_succeeded", + ), + ("status = 'failed'", "finished_completion"), + ( + "status = 'succeeded', cliff_gap = 1.0, cliff_share = 0.1", + "finished_completion", + ), + ], +) +def test_only_a_finished_calculation_ends_and_only_a_succeeded_one_has_numbers( + engine: Engine, assignments: str, check_name: str +) -> None: + command.upgrade(alembic_v2_config(), "head") + + with engine.connect() as connection: + transaction = connection.begin() + try: + run_id = _insert_simulation_run(connection) + calculation_id = _start_cliff_calculation(connection, run_id) + + with refused( + connection, IntegrityError, match=f"ck_cliff_impacts_{check_name}" + ): + connection.execute( + text(f"UPDATE cliff_impacts SET {assignments} WHERE id = :id"), + {"id": calculation_id}, + ) + finally: + transaction.rollback() + + +def test_a_runs_cliff_calculations_are_deleted_with_it(engine: Engine) -> None: + command.upgrade(alembic_v2_config(), "head") + + with engine.connect() as connection: + transaction = connection.begin() + try: + run_id = _insert_simulation_run(connection) + failed_calculation = _start_cliff_calculation(connection, run_id) + assert failed_calculation is not None + _fail_cliff_calculation(connection, failed_calculation) + _start_cliff_calculation(connection, run_id) + + connection.execute( + text("DELETE FROM simulation_runs WHERE id = :id"), {"id": run_id} + ) + + assert _cliff_statuses(connection, run_id) == [] + finally: + transaction.rollback() diff --git a/tests/integration/test_alembic_v2_simulation_runs.py b/tests/integration/test_alembic_v2_simulation_runs.py index 7d2ddf406..f70d2fe22 100644 --- a/tests/integration/test_alembic_v2_simulation_runs.py +++ b/tests/integration/test_alembic_v2_simulation_runs.py @@ -72,11 +72,12 @@ } ) # The tables the simulation service and its Modal workers write; the Public API -# writes every other table. withheld_area_codes came in a later revision. +# writes every other table. withheld_area_codes and cliff_impacts came in later +# revisions. SIMULATION_SIDE_TABLES = ( CHANGED_TABLES | NEW_TABLES - | {"aggregates", "change_aggregates", "withheld_area_codes"} + | {"aggregates", "change_aggregates", "withheld_area_codes", "cliff_impacts"} ) # The foreign keys from the simulation service's tables to the Public API's # rows, with their delete rules. diff --git a/tests/integration/test_v2_simulation_role_privileges.py b/tests/integration/test_v2_simulation_role_privileges.py index 2bc48db88..944aeeb9e 100644 --- a/tests/integration/test_v2_simulation_role_privileges.py +++ b/tests/integration/test_v2_simulation_role_privileges.py @@ -1,8 +1,9 @@ """Run the simulation tables' writes as the roles that do them in production. The simulation service creates simulations, reports, their runs, budget-window -runs and household jobs; its Modal workers write their runs' status and -results and their budget-window runs' status; the Public API reads them. Each +runs, household jobs and simulation runs' cliff calculations; its Modal +workers write their runs' and cliff calculations' status and results and +their budget-window runs' status; the Public API reads them. Each role holds only its declared privileges (``policyengine_api/data/v2/privileges.py``), so a statement outside them fails here as it would in staging and production. @@ -561,6 +562,7 @@ def test_each_simulation_role_updates_exactly_its_declared_columns( "report_runs", "household_jobs", "budget_window_runs", + "cliff_impacts", ): updatable = { column @@ -577,3 +579,156 @@ def test_each_simulation_role_updates_exactly_its_declared_columns( connection.execute(text(statement)) else: _refused(connection, statement) + + +# What the simulation service inserts to start a run's cliff calculation, +# unless one that hasn't failed exists. Naming the partial unique index's +# conflict target needs SELECT as well as INSERT. +START_CLIFF_CALCULATION = ( + "INSERT INTO cliff_impacts (id, simulation_run_id, status) " + "VALUES (:id, :simulation_run_id, 'pending') " + "ON CONFLICT (simulation_run_id) WHERE status <> 'failed' DO NOTHING " + "RETURNING id" +) + + +def _start_cliff_calculation(connection: Connection, run_id: object) -> object: + return connection.execute( + text(START_CLIFF_CALCULATION), {"id": uuid4(), "simulation_run_id": run_id} + ).scalar_one_or_none() + + +def test_the_simulation_service_starts_cliff_calculations_and_records_calls( + connection: Connection, managed_v2_roles, public_api_rows: PublicApiRows +) -> None: + """The service inserts a run's calculation once, and records or fails its call. + + It never claims a calculation, writes its numbers, changes the run it + calculates or deletes it. + """ + _act_as(connection, managed_v2_roles.simulation_service) + ids = _create_simulation_rows(connection, public_api_rows) + + calculation_id = _start_cliff_calculation(connection, ids["run"]) + joined_calculation = _start_cliff_calculation(connection, ids["run"]) + recorded = connection.execute( + text( + "UPDATE cliff_impacts SET modal_call_id = 'fc-cliffs', " + "updated_at = now() WHERE id = :id AND status = 'pending' " + "AND modal_call_id IS NULL RETURNING id" + ), + {"id": calculation_id}, + ).scalar_one() + failed = connection.execute( + text( + "UPDATE cliff_impacts SET status = 'failed', " + "error_message = 'modal_refused: refused', completed_at = now(), " + "updated_at = now() WHERE id = :id AND status = 'pending' RETURNING id" + ), + {"id": calculation_id}, + ).scalar_one() + + assert calculation_id is not None + assert joined_calculation is None + assert recorded == failed == calculation_id + for statement in ( + "UPDATE cliff_impacts SET started_at = now() WHERE id = :id", + "UPDATE cliff_impacts SET cliff_gap = 1.0 WHERE id = :id", + "UPDATE cliff_impacts SET simulation_run_id = simulation_run_id WHERE id = :id", + "DELETE FROM cliff_impacts WHERE id = :id", + ): + _refused(connection, statement, id=calculation_id) + + +def test_modal_workers_claim_cliff_calculations_and_write_their_results( + connection: Connection, managed_v2_roles, public_api_rows: PublicApiRows +) -> None: + """A worker claims a pending calculation under its call and finishes it. + + It never inserts a calculation, changes the run it calculates or deletes + one. + """ + _act_as(connection, managed_v2_roles.simulation_service) + ids = _create_simulation_rows(connection, public_api_rows) + calculation_id = _start_cliff_calculation(connection, ids["run"]) + _act_as(connection, managed_v2_roles.modal_worker) + + claimed = connection.execute( + text( + "UPDATE cliff_impacts SET status = 'running', started_at = now(), " + "modal_call_id = 'fc-cliffs', updated_at = now() " + "WHERE id = :id AND status = 'pending' RETURNING id" + ), + {"id": calculation_id}, + ).scalar_one() + succeeded = connection.execute( + text( + "UPDATE cliff_impacts SET status = 'succeeded', cliff_gap = 1.5e9, " + "cliff_share = 0.04, completed_at = now(), updated_at = now() " + "WHERE id = :id AND status = 'running' RETURNING id" + ), + {"id": calculation_id}, + ).scalar_one() + + assert claimed == succeeded == calculation_id + _refused( + connection, + "INSERT INTO cliff_impacts (id, simulation_run_id, status) " + "VALUES (:id, :run_id, 'pending')", + id=uuid4(), + run_id=ids["run"], + ) + for statement in ( + "UPDATE cliff_impacts SET simulation_run_id = simulation_run_id WHERE id = :id", + "DELETE FROM cliff_impacts WHERE id = :id", + ): + _refused(connection, statement, id=calculation_id) + + +def test_the_public_api_reads_cliff_results_and_other_roles_cannot( + connection: Connection, managed_v2_roles, public_api_rows: PublicApiRows +) -> None: + """The Public API reads a report run's two cliff results and writes none. + + The catalog writer and the Stage 12 role can't read them. + """ + _act_as(connection, managed_v2_roles.simulation_service) + ids = _create_simulation_rows(connection, public_api_rows) + calculation_id = _start_cliff_calculation(connection, ids["run"]) + _act_as(connection, managed_v2_roles.modal_worker) + connection.execute( + text( + "UPDATE cliff_impacts SET status = 'succeeded', cliff_gap = 1.5e9, " + "cliff_share = 0.04, completed_at = now() WHERE id = :id" + ), + {"id": calculation_id}, + ) + _act_as(connection, managed_v2_roles.runtime) + + results = connection.execute( + text( + "SELECT cliff_impacts.cliff_gap, cliff_impacts.cliff_share " + "FROM report_runs JOIN cliff_impacts ON cliff_impacts.simulation_run_id " + "IN (report_runs.baseline_simulation_run_id, " + "report_runs.reform_simulation_run_id) " + "WHERE report_runs.id = :id AND cliff_impacts.status = 'succeeded'" + ), + {"id": ids["report_run"]}, + ).all() + + assert [tuple(result) for result in results] == [(1.5e9, 0.04)] + _refused( + connection, + "INSERT INTO cliff_impacts (id, simulation_run_id, status) " + "VALUES (:id, :run_id, 'pending')", + id=uuid4(), + run_id=ids["run"], + ) + _refused( + connection, + "UPDATE cliff_impacts SET cliff_gap = 0 WHERE id = :id", + id=calculation_id, + ) + for role in (managed_v2_roles.stage12, CATALOG_WRITER_ROLE): + _act_as(connection, role) + _refused(connection, "SELECT count(*) FROM cliff_impacts") diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 73b063d32..7676d8a93 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -214,6 +214,7 @@ def test_reusable_v2_alembic_check_uses_only_disposable_postgres(): assert "test_alembic_v2_user_types.py" in lifecycle_script assert "test_alembic_v2_withheld_area_codes.py" in lifecycle_script assert "test_alembic_v2_shared_report_keys.py" in lifecycle_script + assert "test_alembic_v2_cliff_impacts.py" in lifecycle_script assert "redis:7.2-alpine" not in workflow assert "RUNTIME_CACHE_TEST_URL" not in workflow assert "test_v2_catalog_installed.py" not in workflow diff --git a/tests/unit/v2/test_alembic_v2_revisions.py b/tests/unit/v2/test_alembic_v2_revisions.py index 2c484b86d..326a811f7 100644 --- a/tests/unit/v2/test_alembic_v2_revisions.py +++ b/tests/unit/v2/test_alembic_v2_revisions.py @@ -21,6 +21,7 @@ POST_BASELINE_TABLE_NAMES = frozenset( { "budget_window_runs", + "cliff_impacts", "countries", "legacy_household_mappings", "legacy_policy_mappings", @@ -66,16 +67,21 @@ # The shared reports revision, on the withheld area codes revision, and parent # of the user invitations revision. SHARED_REPORTS_REVISION = "94dc43079186" -# The user invitations revision, on the shared reports revision. +# The user invitations revision, on the shared reports revision, and parent of +# the cliff impacts revision. USER_INVITATIONS_REVISION = "f519a77ba122" +# The cliff impacts revision, on the user invitations revision. +CLIFF_IMPACTS_REVISION = "bc334f9877b7" def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> None: config = Config(str(REPO / "alembic-v2.ini")) script = ScriptDirectory.from_config(config) assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] + assert V2_ALEMBIC_HEAD_REVISION == CLIFF_IMPACTS_REVISION assert [revision.revision for revision in script.walk_revisions()] == [ - V2_ALEMBIC_HEAD_REVISION, + CLIFF_IMPACTS_REVISION, + USER_INVITATIONS_REVISION, SHARED_REPORTS_REVISION, WITHHELD_AREA_CODES_REVISION, SPM_AVAILABILITY_REVISION, @@ -1132,3 +1138,56 @@ def test_user_invitations_revision_adds_its_table_and_the_dismissal_column() -> ) assert "ck_user_type_changes_source_allowed" in upgrade assert "source IN ('admin_route', 'script', 'invitation')" in upgrade + + +def test_cliff_impacts_revision_creates_only_its_table_with_one_correction() -> None: + """The revision after user invitations adds ``cliff_impacts``. + + It is generated, creates the table with its two indexes and its partial + unique index, and writes no data. Its one post-generation correction makes + ``status`` reuse the ``v2_simulation_status`` type the baseline created, + which the generated ``create_table`` would create again and fail on; so + the downgrade, which drops only the table, leaves the type to + ``simulation_runs``. + """ + (path,) = (REPO / "migrations/v2/versions").glob(f"{CLIFF_IMPACTS_REVISION}_*.py") + revision = path.read_text(encoding="utf-8") + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert re.search( + rf"""down_revision: Union\[str, None\] = ["']{USER_INVITATIONS_REVISION}["']""", + revision, + ) + assert revision.count("Dialect correction:") == 1 + assert re.search( + r"""name=["']v2_simulation_status["'],\s*create_type=False""", upgrade + ) + assert re.findall(r"""op\.create_table\(\s*["'](\w+)["']""", upgrade) == [ + "cliff_impacts" + ] + assert sorted( + re.findall(r"""op\.create_index\(\s*(?:op\.f\()?["'](\w+)["']""", upgrade) + ) == [ + "ix_cliff_impacts_simulation_run_id", + "ix_cliff_impacts_status_started_at", + "uq_cliff_impacts_unfailed", + ] + assert re.search( + r"""postgresql_where=sa\.text\(["']status <> 'failed'["']\)""", upgrade + ) + assert re.findall(r"""op\.drop_table\(\s*["'](\w+)["']""", downgrade) == [ + "cliff_impacts" + ] + for body in (upgrade, downgrade): + assert "op.execute(" not in body + assert "op.bulk_insert(" not in body + assert ".drop(op.get_bind())" not in body + assert "op.alter_column(" not in upgrade + assert "op.add_column(" not in upgrade diff --git a/tests/unit/v2/test_cliff_impact_models.py b/tests/unit/v2/test_cliff_impact_models.py new file mode 100644 index 000000000..c2009d4bd --- /dev/null +++ b/tests/unit/v2/test_cliff_impact_models.py @@ -0,0 +1,224 @@ +"""Each simulation run can have one cliff result (C1 ยท 24's answer 4b). + +A cliff calculation of a simulation run records its state and, once it +succeeds, the run's cliff gap and cliff share. A run has at most one +calculation that hasn't failed, so two requests asking for its cliffs at once +start one calculation; a failed calculation never counts, and the next request +starts another. The result inherits its run's versions, and is linked to the +run, not to any report. +""" + +from collections.abc import Iterator +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import pytest +import sqlalchemy as sa +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from policyengine_api.data.v2.models import ( + V2_METADATA, + CliffImpact, + SimulationRun, + SimulationStatus, +) + +CLIFF_IMPACT_COLUMNS = frozenset( + { + "id", + "created_at", + "updated_at", + "simulation_run_id", + "status", + "modal_call_id", + "error_message", + "started_at", + "completed_at", + "cliff_gap", + "cliff_share", + } +) +UNFAILED_CALCULATIONS = "status <> 'failed'" +FINISHED_STATUSES = frozenset({SimulationStatus.SUCCEEDED, SimulationStatus.FAILED}) + + +def _cliff_impacts() -> sa.Table: + return V2_METADATA.tables["cliff_impacts"] + + +def _checks() -> dict[str, str]: + return { + str(constraint.name): str(constraint.sqltext) + for constraint in _cliff_impacts().constraints + if isinstance(constraint, sa.CheckConstraint) + } + + +def _indexes() -> dict[str, sa.Index]: + return {str(index.name): index for index in _cliff_impacts().indexes} + + +def test_cliff_impacts_hold_one_calculation_of_a_simulation_run() -> None: + """The run it calculates, the calculation's state and the two numbers.""" + cliff_impacts = _cliff_impacts() + + assert set(cliff_impacts.c.keys()) == CLIFF_IMPACT_COLUMNS + assert not cliff_impacts.c.simulation_run_id.nullable + (run_key,) = cliff_impacts.c.simulation_run_id.foreign_keys + assert run_key.target_fullname == "simulation_runs.id" + # The result is part of its run, as poverty and inequality rows are. + assert run_key.ondelete == "CASCADE" + status = cliff_impacts.c.status + assert not status.nullable + assert isinstance(status.type, sa.Enum) + # A cliff calculation has a simulation run's states, in the same type. + assert status.type.name == "v2_simulation_status" + assert status.type.enums == ["pending", "running", "succeeded", "failed"] + assert cliff_impacts.c.modal_call_id.type.length == 255 + for timestamp in ("started_at", "completed_at"): + assert cliff_impacts.c[timestamp].nullable + assert cliff_impacts.c[timestamp].type.timezone + for number in ("cliff_gap", "cliff_share"): + assert cliff_impacts.c[number].nullable + assert isinstance(cliff_impacts.c[number].type, sa.Float) + # No version of its own: it inherits its run's. + assert { + "tax_benefit_model_version_id", + "country_package_version", + "policyengine_version", + "report_run_id", + }.isdisjoint(cliff_impacts.c.keys()) + + +def test_only_a_succeeded_cliff_calculation_has_a_result() -> None: + assert _checks() == { + "ck_cliff_impacts_finished_completion": ( + "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL" + ), + "ck_cliff_impacts_result_when_succeeded": ( + "(cliff_gap IS NULL) = (cliff_share IS NULL) " + "AND (status = 'succeeded') = (cliff_gap IS NOT NULL)" + ), + } + + +def test_a_simulation_run_has_one_cliff_calculation_that_has_not_failed() -> None: + """The partial unique index the simulation service names on conflict.""" + indexes = _indexes() + unfailed = indexes["uq_cliff_impacts_unfailed"] + + assert unfailed.unique + assert tuple(column.name for column in unfailed.columns) == ("simulation_run_id",) + assert str(unfailed.dialect_options["postgresql"]["where"]) == ( + UNFAILED_CALCULATIONS + ) + assert str(unfailed.dialect_options["sqlite"]["where"]) == UNFAILED_CALCULATIONS + # The run's key, for its cascade and for reads that include failed rows, + # and the stuck-run sweep's search by state and age. + assert { + name: tuple(column.name for column in index.columns) + for name, index in indexes.items() + if not index.unique + } == { + "ix_cliff_impacts_simulation_run_id": ("simulation_run_id",), + "ix_cliff_impacts_status_started_at": ("status", "started_at"), + } + + +def test_a_simulation_runs_cliff_calculations_go_with_it() -> None: + relationship = sa.inspect(SimulationRun).relationships["cliff_impacts"] + + assert relationship.mapper.local_table is _cliff_impacts() + assert "delete" in relationship.cascade + assert sa.inspect(CliffImpact).relationships["simulation_run"].mapper.class_ is ( + SimulationRun + ) + + +@pytest.fixture +def sqlite_session() -> Iterator[Session]: + engine = sa.create_engine("sqlite://") + V2_METADATA.create_all(engine) + with Session(engine) as session: + yield session + engine.dispose() + + +def _cliff_calculation( + simulation_run_id: UUID, + *, + status: SimulationStatus = SimulationStatus.PENDING, + cliff_gap: float | None = None, + cliff_share: float | None = None, +) -> CliffImpact: + return CliffImpact( + simulation_run_id=simulation_run_id, + status=status, + completed_at=datetime.now(UTC) if status in FINISHED_STATUSES else None, + cliff_gap=cliff_gap, + cliff_share=cliff_share, + ) + + +def test_sqlite_refuses_a_second_unfailed_cliff_calculation_like_postgres( + sqlite_session: Session, +) -> None: + """Failed calculations stay beside the one that hasn't failed.""" + simulation_run_id = uuid4() + sqlite_session.add_all( + [ + _cliff_calculation(simulation_run_id, status=SimulationStatus.FAILED), + _cliff_calculation(simulation_run_id, status=SimulationStatus.FAILED), + _cliff_calculation( + simulation_run_id, + status=SimulationStatus.SUCCEEDED, + cliff_gap=1.5e9, + cliff_share=0.04, + ), + _cliff_calculation(uuid4(), status=SimulationStatus.RUNNING), + ] + ) + sqlite_session.commit() + + sqlite_session.add(_cliff_calculation(simulation_run_id)) + with pytest.raises(IntegrityError, match="UNIQUE constraint failed: cliff_impacts"): + sqlite_session.commit() + + +@pytest.mark.parametrize( + ("status", "cliff_gap", "cliff_share", "check_name"), + [ + (SimulationStatus.SUCCEEDED, None, None, "ck_cliff_impacts_result_when"), + (SimulationStatus.SUCCEEDED, 1.0, None, "ck_cliff_impacts_result_when"), + (SimulationStatus.PENDING, 1.0, 0.1, "ck_cliff_impacts_result_when"), + (SimulationStatus.FAILED, 1.0, 0.1, "ck_cliff_impacts_result_when"), + (SimulationStatus.RUNNING, None, 0.1, "ck_cliff_impacts_result_when"), + ], +) +def test_sqlite_refuses_numbers_on_any_row_but_a_succeeded_one( + sqlite_session: Session, + status: SimulationStatus, + cliff_gap: float | None, + cliff_share: float | None, + check_name: str, +) -> None: + sqlite_session.add( + _cliff_calculation( + uuid4(), status=status, cliff_gap=cliff_gap, cliff_share=cliff_share + ) + ) + + with pytest.raises(IntegrityError, match=check_name): + sqlite_session.commit() + + +def test_sqlite_refuses_a_finished_cliff_calculation_without_its_end( + sqlite_session: Session, +) -> None: + sqlite_session.add( + CliffImpact(simulation_run_id=uuid4(), status=SimulationStatus.FAILED) + ) + + with pytest.raises(IntegrityError, match="ck_cliff_impacts_finished_completion"): + sqlite_session.commit() diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index 9bd2d7389..637e6d406 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -69,7 +69,7 @@ def reject_ddl(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 50 +assert len(V2_METADATA.tables) == 51 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules assert "policyengine_api.data.v2.household_migration_qualification" not in sys.modules diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 0a20c74a3..857872b5b 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -45,7 +45,8 @@ PROOF_FIXTURES = PROJECT_ROOT / "rules/fixtures/api.v2.runtime-privileges-declared" STAGE12_PACKAGES = frozenset({"comparison_runs"}) TEST_ENVIRONMENT = "staging" -# The rows the simulation service creates, and the run tables among them. +# The rows the simulation service creates, and the run tables among them. A +# simulation run's cliff calculation is started and finished like a run. SIMULATION_SERVICE_CREATES = frozenset( { "simulations", @@ -54,9 +55,12 @@ "report_runs", "budget_window_runs", "household_jobs", + "cliff_impacts", } ) -RUN_STATUS_TABLES = frozenset({"simulation_runs", "report_runs", "household_jobs"}) +RUN_STATUS_TABLES = frozenset( + {"simulation_runs", "report_runs", "household_jobs", "cliff_impacts"} +) # The run columns the Modal workers update: a run's state and its results. WORKER_RUN_STATE_COLUMNS = frozenset( { @@ -81,6 +85,7 @@ } ), "household_jobs": frozenset({"result"}), + "cliff_impacts": frozenset({"cliff_gap", "cliff_share"}), } # The run columns the simulation service updates: it records a Modal call's ID # when Modal accepts the call, and fails the run when Modal refuses it. @@ -150,6 +155,8 @@ "recalculate", } ), + # A cliff calculation always calculates the run it was inserted for. + "cliff_impacts": frozenset({"simulation_run_id"}), } # The policy, dynamic, household and catalog rows a run needs. RUN_INPUT_TABLES = frozenset( @@ -480,3 +487,25 @@ def test_the_public_api_records_each_users_report_run_requests() -> None: for role, tables in declared.items(): if role != RUNTIME_ROLE: assert tables["user_report_run_associations"] == NO_ACCESS, role + + +def test_cliff_calculations_are_started_by_the_service_and_finished_by_workers() -> ( + None +): + """The simulation service inserts a run's cliff calculation and records its call. + + The Modal worker claims it, writes the cliff gap and share or the failure, + and never inserts one; the Public API reads the results. The catalog + writer and the Stage 12 role can do neither. + """ + declared = declared_privileges(TEST_ENVIRONMENT) + + assert declared[SIMULATION_SERVICE_ROLE]["cliff_impacts"] == READ_INSERT | { + update_columns(*SERVICE_RUN_STATE_COLUMNS) + } + assert declared[MODAL_WORKER_ROLE]["cliff_impacts"] == READ | { + update_columns(*WORKER_RUN_STATE_COLUMNS, "cliff_gap", "cliff_share") + } + assert declared[RUNTIME_ROLE]["cliff_impacts"] == READ + for role in (CATALOG_WRITER_ROLE, stage12_role(TEST_ENVIRONMENT)): + assert declared[role]["cliff_impacts"] == NO_ACCESS, role diff --git a/tests/unit/v2/test_simulation_run_models.py b/tests/unit/v2/test_simulation_run_models.py index 0abdaca73..d3282dcb6 100644 --- a/tests/unit/v2/test_simulation_run_models.py +++ b/tests/unit/v2/test_simulation_run_models.py @@ -46,7 +46,8 @@ "relative_income_effect", "relative_substitution_effect", ) -PER_SIMULATION_RESULT_TABLES = ("inequality", "poverty") +# A simulation run's cliff calculation is one of its results too. +PER_SIMULATION_RESULT_TABLES = ("inequality", "poverty", "cliff_impacts") COMPARISON_RESULT_TABLES = ( "budget_summary", "congressional_district_impacts",