diff --git a/changelog.d/v2-dynamics-like-policies.added.md b/changelog.d/v2-dynamics-like-policies.added.md new file mode 100644 index 000000000..ce2f7b9f4 --- /dev/null +++ b/changelog.d/v2-dynamics-like-policies.added.md @@ -0,0 +1 @@ +Store dynamics, a simulation's model assumptions, as policies are stored: once per country, catalog version and content hash, with no name or description. `POST /v2/dynamics` finds or creates one (201 or 200) from parameters under `gov.simulation`, and in the US also `simulation`, refusing other parameters and an empty dynamic with 422, and `GET /v2/dynamics/{dynamic_id}` reads one in its country. `POST /v2/policies` refuses those parameters with 422 (code `dynamic_parameters`, naming them). `/v2/simulations` and `/v2/reports` refuse a dynamic the country does not store (404) or one that sets parameters the running catalog lacks (422 `missing_parameters`). The v1-to-v2 policy copy splits a v1 policy's `gov.simulation` and US `simulation` values into a dynamic, rewriting the UK's old `labor_supply_responses` spelling, and maps each v1 policy to its policy and dynamic (`legacy_policy_mappings.dynamic_id`). diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index cd6308cbc..3533d11c3 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -7,8 +7,8 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | Metric | Count | | --- | ---: | | route group count | 9 | -| workflow count | 18 | -| request count | 101 | +| workflow count | 19 | +| request count | 103 | | db entity count | 6 | | sim flow count | 3 | @@ -19,7 +19,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check`, `deployment` | `none` | `none` | | `specification` | `specification` | `none` | `none` | | `metadata` | `metadata`, `countries`, `datasets`, `economy-options`, `model-catalog`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | -| `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | +| `policy` | `policy`, `policies`, `dynamics`, `user-policy`, `user-policies` | `policy` | `none` | | `household` | `household`, `households`, `household-calculations`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | | `simulation` | `simulation`, `simulations`, `user-simulations` | `simulation` | `economy` | @@ -49,6 +49,16 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/policies/{policy_id}?country_id=us` | 200 | `policy` | none | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at`, `result.item.missing_parameters` | | `GET` | `/v2/policies?country_id=us` | 200 | `policy` | none | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | +### `dynamic_resources_v2` + +- Current contract: `typed_v2_resources` +- Future owner: Dynamics stored like policies + +| Method | Path | Status | Route group | Required headers | Stable response fields | +| --- | --- | ---: | --- | --- | --- | +| `POST` | `/v2/dynamics?country_id=us` | 201 | `policy` | none | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at`, `result.item.missing_parameters` | +| `GET` | `/v2/dynamics/{dynamic_id}?country_id=us` | 200 | `policy` | none | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at`, `result.item.missing_parameters` | + ### `user_policy_associations_v2` - Current contract: `typed_v2_resources` diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 157f8aecf..487a17850 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -310,8 +310,9 @@ production, so that its absence fails the deploy again. The declarations include write privileges for code paths production doesn't turn on: the dual-write copies into `legacy_policy_mappings` and -`legacy_household_mappings`, and the writes behind the native `/v2/policies` -and `/v2/households` routes. Narrow them when those tables are removed. +`legacy_household_mappings`, and the writes behind the native `/v2/policies`, +`/v2/dynamics` and `/v2/households` routes. Narrow them when those tables are +removed. When a pull request adds or changes a v2 table, or a connector's use of one: @@ -456,6 +457,8 @@ it publishes that version, with nothing to restore it. Downgrading past the revision is therefore safe only while no version is pruned (no `tax_benefit_model_versions` row has `catalog_values_pruned_at` set). From the head, such a downgrade first runs the later revisions' downgrades: +`9d1d12a7ac58`'s needs `dynamics` empty (see "API v2 dynamics stored like +policies"), `f519a77ba122`'s drops `user_invitations` and needs no recorded change with source `invitation` (see "API v2 user invitations"), `94dc43079186`'s drops every user's report-run record, the simulation and @@ -1012,3 +1015,61 @@ change's source is `invitation`. `test_user_invitations_revision_adds_invitations_and_dismissals_only` and `test_user_invitations_revision_records_invitation_changes_until_downgraded` in `tests/integration/test_alembic_v2_user_types.py` check both. + +### API v2 dynamics stored like policies + +Revision `9d1d12a7ac58` (parent `f519a77ba122`) was autogenerated, with no +post-generation corrections apart from `ruff format`, after these model +changes, so that dynamics are stored as policies are: + +- `dynamics` drops `name` and `description`, which nothing wrote or read: a + user's saved dynamic will hold its name, as `user_policies` holds a + policy's. It gains `policies`' columns: `country_id` (a key to + `countries`), `tax_benefit_model_id` and `tax_benefit_model_version_id` + (indexed), each `RESTRICT`, and `canonicalization_version` and + `content_hash`, unique together + (`uq_dynamics_canonicalization_content_hash`), with the checks policies + have (a positive version, a 64-character hash). `(id, country_id)` is + unique (`uq_dynamics_id_country`), so other tables can name a dynamic of + one country. +- `parameter_values` gains `uq_parameter_values_dynamic_parameter_start_date`, + a partial unique index on `(dynamic_id, parameter_id, start_date)` for rows + with a dynamic: a dynamic sets one value per parameter and start date, as a + policy does. Rows without a dynamic never conflict in a unique key, so the + partial index refuses exactly what a full unique constraint would, but it + holds only dynamics' rows instead of every current-law value. Building it + still reads every row of `parameter_values` (millions of current-law + values), and, as Alembic creates it without `CONCURRENTLY`, inserts into + that table wait until the build finishes. It is also declared for SQLite, + so the unit tests' databases keep the same rule. +- `legacy_policy_mappings` gains the nullable `dynamic_id`, the dynamic the + v1-to-v2 policy copy split from a v1 policy: a key with the mapping's + country to `dynamics` (`fk_legacy_policy_mappings_dynamic_country`, + `RESTRICT`), indexed. A mapping without a dynamic is unaffected. + +A dynamic is hashed by the policies' function (`canonical_policy_document`) +with the same canonicalization version, so identical content in a policy +and a dynamic has the same hash; they are in different tables. The runtime +role may now read and insert `dynamics`; the simulation service and its +Modal workers keep reading them. Which parameters a dynamic holds, and how +the copy splits a v1 policy, is in `policyengine_api/dynamic_parameters.py`. + +Nothing wrote `dynamics` before this revision, and its new columns are +required with no server default, so the upgrade fails, changing nothing, +while `dynamics` has a row. Before deploying it to an environment, count +them read-only: + +```sql +SELECT count(*) FROM dynamics; +``` + +The previous Public API revision maps `name` and `description` but never +reads or writes `dynamics`, and its copy writes mappings without +`dynamic_id`, which stays null, so it keeps working at this revision. The +simulation service selects only `dynamics.id`. The downgrade drops the new +columns, constraints and indexes and the mappings' `dynamic_id` with its +values, and adds `name` back as required, so it fails while `dynamics` has a +row. `tests/integration/test_alembic_v2_dynamics.py` checks that nothing else +changes, the downgrade and upgrade, both refusals, and the keys and unique +rules. + diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 4c5d23ceb..52e8312de 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,10 +1,10 @@ { "metadata": { "db_entity_count": 6, - "request_count": 101, + "request_count": 103, "route_group_count": 9, "sim_flow_count": 3, - "workflow_count": 18 + "workflow_count": 19 }, "route_groups": [ { @@ -51,6 +51,7 @@ "path_segments": [ "policy", "policies", + "dynamics", "user-policy", "user-policies" ], @@ -201,6 +202,49 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "Dynamics stored like policies", + "name": "dynamic_resources_v2", + "requests": [ + { + "expected_status": 201, + "headers": [], + "method": "POST", + "path": "/v2/dynamics?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + "result.item.missing_parameters" + ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/dynamics/{dynamic_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + "result.item.missing_parameters" + ] + } + ] + }, { "current_contract": "typed_v2_resources", "future_owner_pr": "PR 12A: User Login and v1-Anchored User Associations", diff --git a/docs/migration/stage-10-v2-policies.md b/docs/migration/stage-10-v2-policies.md index d83aa2187..ca5f2834e 100644 --- a/docs/migration/stage-10-v2-policies.md +++ b/docs/migration/stage-10-v2-policies.md @@ -134,7 +134,26 @@ the running catalog whatever country package version the v1 row was saved under (its stored `api_version`), the same way the Public API simulates an old policy with the running country package. A policy is copied when every parameter path it sets exists in the running catalog and its periods and values -are valid. A saved-policy mutation commits +are valid. + +The copy splits off the v1 policy's model assumptions: every value under +`gov.simulation.`, and in the US also `simulation.` +(`policyengine_api/dynamic_parameters.py`). It stores them as a v2 dynamic, +hashed and deduplicated as a policy is, and keeps the rest in the v2 policy, +which is empty when the v1 policy set nothing else. Before looking up the +parameters it rewrites the UK's old spelling +`gov.simulation.labor_supply_responses.` to +`gov.simulation.labour_supply_responses.`, the only one the UK catalog has; a +v1 policy that sets one parameter under both spellings with different values +fails as `conflicting_values`. The mapping in `legacy_policy_mappings` names the +policy and the dynamic (`dynamic_id`), or no dynamic when the v1 policy sets no +assumptions. A v1 policy copied before the split existed keeps its copy: its +assumptions stay policy values and its mapping names no dynamic, until the +one-time migration splits it again. Native `POST /v2/policies` refuses such +values with 422 instead (code `dynamic_parameters`), and they are created +through `POST /v2/dynamics`. + +A saved-policy mutation commits its source row, incremented revision, and complete event in one Cloud SQL transaction. The same request then processes that source's pending events in revision order and records `processed_at` only after the corresponding @@ -206,14 +225,16 @@ Monitor structured events with metric names `v1_policy_mirror_operations` and | `failure_category` | `failure_reason` | | --- | --- | | `catalog_or_translation` | `missing_parameters` (the policy sets a parameter the running catalog lacks), `invalid_parameter_values`, `empty_parameter_values`, `invalid_period`, `conflicting_values`, `too_many_values` (more than 1,000 dated values), `catalog_absent` (no catalog for the running PolicyEngine.py version), `catalog_query_failed`, `catalog_version_not_found`, `catalog_mismatch` | -| `integrity` | `mapping_conflict`, `content_hash_collision`, `stored_policy_integrity`, `saved_policy_integrity` (any integrity check of a saved-policy copy) | +| `integrity` | `mapping_conflict`, `content_hash_collision` (of the policy or the dynamic), `stored_policy_integrity`, `stored_dynamic_integrity`, `saved_policy_integrity` (any integrity check of a saved-policy copy) | | `database` | `database_error` | | `configuration` | `configuration` | | `unexpected` | `missing_reform_policy` (a saved policy's v1 reform could not be loaded), `unexpected` | | `missing_mirror_snapshot`, `missing_mirror_revision` | the same value as the category | Events include the legacy integer ID, -destination UUID when committed, attempted and completed database sources, and +destination UUID when committed (a policy copy's record also names its +dynamic's, `destination_dynamic_id`, and whether it created it, +`dynamic_created`), attempted and completed database sources, and duration. They do not include policy JSON, presentation data, database URLs, or credentials. diff --git a/migrations/v2/versions/9d1d12a7ac58_store_dynamics_like_policies_and_map_v1_.py b/migrations/v2/versions/9d1d12a7ac58_store_dynamics_like_policies_and_map_v1_.py new file mode 100644 index 000000000..9ce710982 --- /dev/null +++ b/migrations/v2/versions/9d1d12a7ac58_store_dynamics_like_policies_and_map_v1_.py @@ -0,0 +1,207 @@ +"""store dynamics like policies and map v1 policies to them + +Revision ID: 9d1d12a7ac58 +Revises: f519a77ba122 +Create Date: 2026-11-17 15:29:53.770002 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate + +Stores dynamics as policies are stored. dynamics drops name and description, +which nothing wrote, and gains policies' columns: the country, the +tax-benefit model and catalog version (each RESTRICT), and the +canonicalization version and content hash, unique together. Checks require a +positive canonicalization version and a 64-character hash, and (id, +country_id) is unique so other tables can name a dynamic of one country. +parameter_values gains a partial unique index on (dynamic_id, parameter_id, +start_date) for rows with a dynamic: a dynamic's values are unique per +parameter and start date, as a policy's are, and the index holds only +dynamics' rows. legacy_policy_mappings gains a nullable dynamic_id, the +dynamic the v1-to-v2 copy split from the v1 policy, a foreign key (with the +country) to dynamics, RESTRICT, and indexed. The operations are exactly as +generated, apart from ruff format. + +Nothing wrote dynamics before this revision, and its new columns are +required with no default, so the upgrade fails, changing nothing, while +dynamics has a row. The previous Public API revision maps the columns this +revision drops, but never reads or writes dynamics, and its v1-to-v2 copy +writes mappings without dynamic_id, which stays null. The simulation service +selects only dynamics.id. The downgrade drops the new columns, their +constraints and the mappings' dynamic_id with its values, and adds name back +as required, so it fails while dynamics has a row. +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "9d1d12a7ac58" +down_revision: Union[str, None] = "f519a77ba122" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "dynamics", + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + ) + op.add_column( + "dynamics", sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False) + ) + op.add_column( + "dynamics", sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False) + ) + op.add_column( + "dynamics", sa.Column("canonicalization_version", sa.Integer(), nullable=False) + ) + op.add_column( + "dynamics", + sa.Column( + "content_hash", sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False + ), + ) + op.create_index( + op.f("ix_dynamics_tax_benefit_model_id"), + "dynamics", + ["tax_benefit_model_id"], + unique=False, + ) + op.create_index( + op.f("ix_dynamics_tax_benefit_model_version_id"), + "dynamics", + ["tax_benefit_model_version_id"], + unique=False, + ) + op.create_unique_constraint( + "uq_dynamics_canonicalization_content_hash", + "dynamics", + ["canonicalization_version", "content_hash"], + ) + op.create_unique_constraint( + "uq_dynamics_id_country", "dynamics", ["id", "country_id"] + ) + op.create_foreign_key( + op.f("fk_dynamics_tax_benefit_model_version_id_tax_benefit_model_versions"), + "dynamics", + "tax_benefit_model_versions", + ["tax_benefit_model_version_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + op.f("fk_dynamics_tax_benefit_model_id_tax_benefit_models"), + "dynamics", + "tax_benefit_models", + ["tax_benefit_model_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + op.f("fk_dynamics_country_id_countries"), + "dynamics", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_check_constraint( + op.f("ck_dynamics_canonicalization_version"), + "dynamics", + "canonicalization_version > 0", + ) + op.create_check_constraint( + op.f("ck_dynamics_content_hash_length"), "dynamics", "length(content_hash) = 64" + ) + op.drop_column("dynamics", "description") + op.drop_column("dynamics", "name") + op.add_column( + "legacy_policy_mappings", sa.Column("dynamic_id", sa.Uuid(), nullable=True) + ) + op.create_index( + "ix_legacy_policy_mappings_dynamic", + "legacy_policy_mappings", + ["dynamic_id"], + unique=False, + ) + op.create_foreign_key( + "fk_legacy_policy_mappings_dynamic_country", + "legacy_policy_mappings", + "dynamics", + ["dynamic_id", "country_id"], + ["id", "country_id"], + ondelete="RESTRICT", + ) + op.create_index( + "uq_parameter_values_dynamic_parameter_start_date", + "parameter_values", + ["dynamic_id", "parameter_id", "start_date"], + unique=True, + postgresql_where=sa.text("dynamic_id IS NOT NULL"), + sqlite_where=sa.text("dynamic_id IS NOT NULL"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "uq_parameter_values_dynamic_parameter_start_date", + table_name="parameter_values", + postgresql_where=sa.text("dynamic_id IS NOT NULL"), + sqlite_where=sa.text("dynamic_id IS NOT NULL"), + ) + op.drop_constraint( + "fk_legacy_policy_mappings_dynamic_country", + "legacy_policy_mappings", + type_="foreignkey", + ) + op.drop_index( + "ix_legacy_policy_mappings_dynamic", table_name="legacy_policy_mappings" + ) + op.drop_column("legacy_policy_mappings", "dynamic_id") + op.add_column( + "dynamics", + sa.Column("name", sa.VARCHAR(length=255), autoincrement=False, nullable=False), + ) + op.add_column( + "dynamics", + sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), + ) + op.drop_constraint( + op.f("ck_dynamics_content_hash_length"), "dynamics", type_="check" + ) + op.drop_constraint( + op.f("ck_dynamics_canonicalization_version"), "dynamics", type_="check" + ) + op.drop_constraint( + op.f("fk_dynamics_country_id_countries"), "dynamics", type_="foreignkey" + ) + op.drop_constraint( + op.f("fk_dynamics_tax_benefit_model_id_tax_benefit_models"), + "dynamics", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_dynamics_tax_benefit_model_version_id_tax_benefit_model_versions"), + "dynamics", + type_="foreignkey", + ) + op.drop_constraint("uq_dynamics_id_country", "dynamics", type_="unique") + op.drop_constraint( + "uq_dynamics_canonicalization_content_hash", "dynamics", type_="unique" + ) + op.drop_index( + op.f("ix_dynamics_tax_benefit_model_version_id"), table_name="dynamics" + ) + op.drop_index(op.f("ix_dynamics_tax_benefit_model_id"), table_name="dynamics") + op.drop_column("dynamics", "content_hash") + op.drop_column("dynamics", "canonicalization_version") + op.drop_column("dynamics", "tax_benefit_model_version_id") + op.drop_column("dynamics", "tax_benefit_model_id") + op.drop_column("dynamics", "country_id") + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index b51fd461c..9c4fcbf26 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "f519a77ba122" +V2_ALEMBIC_HEAD_REVISION = "9d1d12a7ac58" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 30115f15a..018bb6edb 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -80,6 +80,7 @@ class TaxBenefitModel(TimestampedModel, table=True): back_populates="tax_benefit_model" ) policies: list["Policy"] = Relationship(back_populates="tax_benefit_model") + dynamics: list["Dynamic"] = Relationship(back_populates="tax_benefit_model") reports: list["Report"] = Relationship(back_populates="tax_benefit_model") @@ -156,6 +157,7 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): datasets: list["Dataset"] = Relationship(back_populates="tax_benefit_model_version") regions: list["Region"] = Relationship(back_populates="tax_benefit_model_version") policies: list["Policy"] = Relationship(back_populates="tax_benefit_model_version") + dynamics: list["Dynamic"] = Relationship(back_populates="tax_benefit_model_version") class Region(TimestampedModel, table=True): @@ -418,8 +420,9 @@ class Parameter(IdentifiedModel, table=True): class ParameterValue(IdentifiedModel, table=True): """A parameter's value from ``start_date`` until ``end_date``, or indefinitely. - A value owned by a policy or dynamic is deleted with its owner. A value with - neither is current law, and a parameter has at most one per start date. + A value owned by a policy or dynamic is deleted with its owner, and each + owner sets at most one per parameter and start date. A value with neither + is current law, and a parameter has at most one per start date. """ __tablename__ = "parameter_values" @@ -438,6 +441,18 @@ class ParameterValue(IdentifiedModel, table=True): "start_date", name="uq_parameter_values_policy_parameter_start_date", ), + # A dynamic's values are unique as a policy's are. The index covers + # only dynamics' rows, so it doesn't index every current-law value; + # rows without a dynamic never conflict in a unique key anyway. + sa.Index( + "uq_parameter_values_dynamic_parameter_start_date", + "dynamic_id", + "parameter_id", + "start_date", + unique=True, + postgresql_where=sa.text("dynamic_id IS NOT NULL"), + sqlite_where=sa.text("dynamic_id IS NOT NULL"), + ), sa.Index( "ix_parameter_values_parameter_period", "parameter_id", diff --git a/policyengine_api/data/v2/models/policies.py b/policyengine_api/data/v2/models/policies.py index f0da6e8cc..ba134ed0f 100644 --- a/policyengine_api/data/v2/models/policies.py +++ b/policyengine_api/data/v2/models/policies.py @@ -96,16 +96,60 @@ class Policy(TimestampedModel, table=True): class Dynamic(TimestampedModel, table=True): - """Named parameter values, separate from any policy, that a simulation can apply. - - Its parameter values are deleted with it. + """Immutable model assumptions, stored once per canonicalization version and hash. + + A dynamic holds parameter values a simulation applies alongside its + policy: every parameter under ``gov.simulation.``, plus the US's + top-level ``simulation.`` (``policyengine_api.dynamic_parameters``). It + is stored and hashed as a policy is, bound to one catalog version, with + no name or description: a user's saved dynamics name it. Its parameter + values are deleted with it. """ __tablename__ = "dynamics" + __table_args__ = ( + sa.UniqueConstraint( + "id", + "country_id", + name="uq_dynamics_id_country", + ), + sa.UniqueConstraint( + "canonicalization_version", + "content_hash", + name="uq_dynamics_canonicalization_content_hash", + ), + sa.CheckConstraint( + "canonicalization_version > 0", + name="ck_dynamics_canonicalization_version", + ), + sa.CheckConstraint( + "length(content_hash) = 64", + name="ck_dynamics_content_hash_length", + ), + ) - name: str = Field(max_length=255) - description: str | None = None + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) + tax_benefit_model_id: UUID = Field( + foreign_key="tax_benefit_models.id", + ondelete="RESTRICT", + index=True, + ) + tax_benefit_model_version_id: UUID = Field( + foreign_key="tax_benefit_model_versions.id", + ondelete="RESTRICT", + index=True, + ) + canonicalization_version: int + content_hash: str = Field(max_length=64) + tax_benefit_model: TaxBenefitModel = Relationship(back_populates="dynamics") + tax_benefit_model_version: "TaxBenefitModelVersion" = Relationship( + back_populates="dynamics" + ) parameter_values: list["ParameterValue"] = Relationship( back_populates="dynamic", cascade_delete=True, diff --git a/policyengine_api/data/v2/models/policy_mappings.py b/policyengine_api/data/v2/models/policy_mappings.py index 7cdbc91b9..64dd79d93 100644 --- a/policyengine_api/data/v2/models/policy_mappings.py +++ b/policyengine_api/data/v2/models/policy_mappings.py @@ -39,7 +39,13 @@ class LegacyUserMapping(SQLModel, table=True): class LegacyPolicyMapping(TimestampedModel, table=True): - """Map one country-scoped v1 policy ID to deduplicated v2 content.""" + """Map one country-scoped v1 policy ID to deduplicated v2 content. + + The copy splits a v1 policy's model assumptions into a dynamic of the same + country (``dynamic_id``), and keeps the rest in the policy. A v1 policy + with no such values, and every v1 policy copied before the split existed, + maps to no dynamic. + """ __tablename__ = "legacy_policy_mappings" __table_args__ = ( @@ -58,15 +64,26 @@ class LegacyPolicyMapping(TimestampedModel, table=True): name="fk_legacy_policy_mappings_policy_country", ondelete="RESTRICT", ), + sa.ForeignKeyConstraint( + ["dynamic_id", "country_id"], + ["dynamics.id", "dynamics.country_id"], + name="fk_legacy_policy_mappings_dynamic_country", + ondelete="RESTRICT", + ), sa.Index( "ix_legacy_policy_mappings_policy", "policy_id", ), + sa.Index( + "ix_legacy_policy_mappings_dynamic", + "dynamic_id", + ), ) country_id: str = Field(max_length=2) legacy_policy_id: int = Field(sa_type=sa.BigInteger) policy_id: UUID + dynamic_id: UUID | None = None source_policy_hash: str = Field(max_length=255) policy: "Policy" = Relationship(back_populates="legacy_mappings") diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index ba116de22..45fef0ec1 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -245,8 +245,10 @@ def _run_table( # Custom totals, unused until they return. "aggregates": _access(), "change_aggregates": _access(), - # A simulation's or household job's dynamic, which a run reads. - "dynamics": _access(**_RUN_INPUT_READS), + # A simulation's or household job's dynamic, which a run reads. The Public + # API creates them, as it does policies, through /v2/dynamics and the + # v1-to-v2 policy copy; a dynamic never changes once stored. + "dynamics": _access(runtime=READ_INSERT, **_RUN_INPUT_READS), # Simulations, reports, their runs and household jobs. The catalog writer # checks that publication leaves the rows of three of them untouched. "budget_window_runs": _access( @@ -299,9 +301,9 @@ def _run_table( "parameter_tree_children": _access( runtime=READ, catalog_writer=READ_INSERT_DELETE | _ANALYZE ), - # The Public API also inserts a v2 policy's own parameter values. Catalog - # pruning deletes an older version's current-law values (those with no - # policy or dynamic), never a policy's or a dynamic's. + # The Public API also inserts a v2 policy's or dynamic's own parameter + # values. Catalog pruning deletes an older version's current-law values + # (those with no policy or dynamic), never a policy's or a dynamic's. "parameter_values": _access( runtime=READ_INSERT, catalog_writer=READ_WRITE | _ANALYZE, **_RUN_INPUT_READS ), diff --git a/policyengine_api/dynamic_parameters.py b/policyengine_api/dynamic_parameters.py new file mode 100644 index 000000000..753e53a84 --- /dev/null +++ b/policyengine_api/dynamic_parameters.py @@ -0,0 +1,136 @@ +"""Which parameters a dynamic holds, and how a v1 policy's values are split. + +A dynamic holds the model's assumptions: every parameter under +``gov.simulation.``, in every country, and the US's top-level +``simulation.`` tree (C1 ยท 35, Q1). These are behavioural responses, model +settings and data assumptions, which a report applies alongside a policy. +Parameters a country package adds later under these prefixes belong to +dynamics with no code change. + +The rule is used in three places: + +- ``POST /v2/policies`` refuses a policy that sets any such parameter, and + ``POST /v2/dynamics`` refuses a dynamic that sets any other. +- The copy of each v1 policy into v2 splits the v1 policy's values with + ``split_legacy_policy_json``: these values become a dynamic and the rest + the policy. +- The one-time migration splits v1 policies, and v2 copies made before the + split existed, with the same functions. + +The UK renamed ``gov.simulation.labor_supply_responses`` to +``gov.simulation.labour_supply_responses`` on 2026-05-01. Its catalog has +only the new names, so a UK v1 policy saved with the old spelling is +rewritten before its parameters are looked up. In the US, +``labor_supply_responses`` is the parameter's own name and is never +rewritten. + +The module imports nothing from the package, so validators, transformations +and command-line code can all import it. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from collections.abc import Mapping + + from policyengine_api.supported_countries import SupportedCountryId + +GOV_SIMULATION_PREFIX: Final = "gov.simulation." +DYNAMIC_PARAMETER_PREFIXES: Final[Mapping[SupportedCountryId, tuple[str, ...]]] = { + "uk": (GOV_SIMULATION_PREFIX,), + "us": (GOV_SIMULATION_PREFIX, "simulation."), +} +UK_OLD_LABOUR_SUPPLY_PREFIX: Final = "gov.simulation.labor_supply_responses." +UK_LABOUR_SUPPLY_PREFIX: Final = "gov.simulation.labour_supply_responses." + + +class ConflictingParameterSpellingsError(ValueError): + """A v1 policy sets one parameter under both UK spellings, and they disagree. + + Both spellings' values are period objects, merged into one; the policy is + refused when either is not a period object or both give one period + different values. + """ + + +@dataclass(frozen=True) +class LegacyPolicySplit: + """A v1 policy's ``{path: {period: value}}`` object, split in two. + + ``dynamic_values`` holds the parameters a dynamic holds, and + ``policy_values`` the rest, each keyed by the parameter's current name. + Either may be empty. + """ + + policy_values: dict[str, object] + dynamic_values: dict[str, object] + + +def is_dynamic_parameter(country_id: SupportedCountryId, parameter_name: str) -> bool: + """Return whether the country's parameter belongs in a dynamic, not a policy.""" + return parameter_name.startswith(DYNAMIC_PARAMETER_PREFIXES[country_id]) + + +def canonical_parameter_name( + country_id: SupportedCountryId, parameter_name: str +) -> str: + """Return the parameter's current name: the UK's old spelling rewritten.""" + if country_id == "uk" and parameter_name.startswith(UK_OLD_LABOUR_SUPPLY_PREFIX): + return UK_LABOUR_SUPPLY_PREFIX + parameter_name.removeprefix( + UK_OLD_LABOUR_SUPPLY_PREFIX + ) + return parameter_name + + +def _merged_period_values( + parameter_name: str, earlier: object, later: object +) -> dict[object, object]: + """Merge two spellings' period objects; raise when they disagree.""" + if type(earlier) is not dict or type(later) is not dict: + msg = f"parameter {parameter_name!r} is set under both spellings" + raise ConflictingParameterSpellingsError(msg) + for period, value in later.items(): + if period in earlier and earlier[period] != value: + msg = ( + f"parameter {parameter_name!r} is set under both spellings " + f"with different values for {period!r}" + ) + raise ConflictingParameterSpellingsError(msg) + return {**earlier, **later} + + +def split_legacy_policy_json( + country_id: SupportedCountryId, policy_json: Mapping[str, object] +) -> LegacyPolicySplit: + """Split a v1 policy's values into the policy's and the dynamic's. + + Each parameter path is first given its current name + (``canonical_parameter_name``). A UK policy that sets one parameter under + both spellings has their period objects merged; raise + ConflictingParameterSpellingsError when they can't be. The values + themselves are left as they are, for the translation to check. + """ + renamed: dict[str, object] = {} + for parameter_name, period_values in policy_json.items(): + current_name = canonical_parameter_name(country_id, parameter_name) + if current_name in renamed: + renamed[current_name] = _merged_period_values( + current_name, renamed[current_name], period_values + ) + else: + renamed[current_name] = period_values + return LegacyPolicySplit( + policy_values={ + name: values + for name, values in renamed.items() + if not is_dynamic_parameter(country_id, name) + }, + dynamic_values={ + name: values + for name, values in renamed.items() + if is_dynamic_parameter(country_id, name) + }, + ) diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index aeb4411e2..e886b9250 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -18,6 +18,7 @@ from policyengine_api.fastapi_routes.resource_service_protocols import ( V2AccountResourceService, V2CountryResourceService, + V2DynamicResourceService, V2HouseholdCalculationResourceService, V2HouseholdResourceService, V2PolicyResourceService, @@ -71,6 +72,7 @@ "SimulationGatewayProbe", "V2AccountResourceService", "V2CountryResourceService", + "V2DynamicResourceService", "V2HouseholdCalculationResourceService", "V2HouseholdResourceService", "V2MetadataResourceReader", @@ -174,6 +176,21 @@ def _default_v2_policy_service_factory() -> V2PolicyResourceService: ) +def _default_v2_dynamic_service_factory() -> V2DynamicResourceService: + from policyengine_api.data.v2.database import ( + get_v2_session_factory, + ) + from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, + ) + from policyengine_api.services.v2.dynamics.services import V2DynamicService + + return V2DynamicService( + DynamicDatabaseSession(get_v2_session_factory()), + running_policyengine_version=_running_policyengine_version(), + ) + + def _default_v2_household_service_factory() -> V2HouseholdResourceService: from policyengine_api.data.v2.database import ( get_v2_session_factory, @@ -234,6 +251,7 @@ def _default_simulation_submissions() -> SimulationServiceClient: countries=_default_v2_country_service_factory(), households=_default_v2_household_service_factory(), policies=_default_v2_policy_service_factory(), + dynamics=_default_v2_dynamic_service_factory(), client_factory=process_simulation_service_client, ) @@ -545,6 +563,7 @@ class NativeRouteDependencies: v2_metadata_reader_factory: Callable[[], V2MetadataResourceReader] | None = None v2_country_service_factory: Callable[[], V2CountryResourceService] | None = None v2_policy_service_factory: Callable[[], V2PolicyResourceService] | None = None + v2_dynamic_service_factory: Callable[[], V2DynamicResourceService] | None = None v2_household_service_factory: Callable[[], V2HouseholdResourceService] | None = None v2_household_calculation_service_factory: ( Callable[[], V2HouseholdCalculationResourceService] | None diff --git a/policyengine_api/fastapi_routes/resource_service_protocols.py b/policyengine_api/fastapi_routes/resource_service_protocols.py index aa2818879..cb710ff6b 100644 --- a/policyengine_api/fastapi_routes/resource_service_protocols.py +++ b/policyengine_api/fastapi_routes/resource_service_protocols.py @@ -10,6 +10,11 @@ from policyengine_api.data.v2.models import UserType from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.dynamics.types import ( + CheckedDynamicRead, + NativeDynamicCreation, + NativeDynamicCreationInput, + ) from policyengine_api.services.v2.household_calculations.types import ( HouseholdCalculationInput, HouseholdCalculationRead, @@ -130,6 +135,20 @@ def list_policies( ... +class V2DynamicResourceService(Protocol): + """Route-independent native dynamic operations, stored as policies are.""" + + def create_dynamic( + self, dynamic_input: NativeDynamicCreationInput + ) -> NativeDynamicCreation: + """Store the dynamic, or return the identical one; ``created`` says which.""" + ... + + def get_dynamic(self, *, country_id: str, dynamic_id: UUID) -> CheckedDynamicRead: + """Return the country's dynamic, or raise ``DynamicNotFoundError``.""" + ... + + class V2HouseholdResourceService(Protocol): """Route-independent native household operations for one request.""" @@ -268,10 +287,10 @@ def create_report( country without household calculations, ``HouseholdNotFoundError`` for a household the country does not store, ``HouseholdCalculationYearError`` for one with no default year, - ``PolicyNotFoundError`` - for a policy the country does not store, - ``PolicyParametersMissingError`` for a policy that sets parameters the - running catalog version lacks, ``SubmissionStorageError`` when the + ``PolicyNotFoundError`` or ``DynamicNotFoundError`` for a policy or + dynamic the country does not store, ``PolicyParametersMissingError`` + (``DynamicParametersMissingError`` for a dynamic) for one setting + parameters the running catalog lacks, ``SubmissionStorageError`` when the database or its catalog cannot be read for those checks, the simulation service's errors, ``ReportRunNotRecordedError`` when the request could not be recorded, and ``CreatedReportUnreadableError`` @@ -338,7 +357,7 @@ def create_simulation( Raise the checks' and the simulation service's errors as ``V2StoredReportResourceService.create_report`` does, for the - simulation's one policy, and ``CreatedSimulationUnreadableError`` when + simulation's policy and dynamic, and ``CreatedSimulationUnreadableError`` when the simulation it created cannot be read back. """ ... diff --git a/policyengine_api/fastapi_routes/v2/dynamics/__init__.py b/policyengine_api/fastapi_routes/v2/dynamics/__init__.py new file mode 100644 index 000000000..657ea9884 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/dynamics/__init__.py @@ -0,0 +1 @@ +"""FastAPI adapters for immutable v2 dynamics.""" diff --git a/policyengine_api/fastapi_routes/v2/dynamics/request_models.py b/policyengine_api/fastapi_routes/v2/dynamics/request_models.py new file mode 100644 index 000000000..310d7e529 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/dynamics/request_models.py @@ -0,0 +1,14 @@ +"""Strict HTTP request models for the native v2 dynamic API.""" + +from policyengine_api.services.v2.dynamics.types import DynamicCreationInput + +MAXIMUM_DYNAMIC_REQUEST_BYTES = 1_048_576 + + +class DynamicCreateRequest(DynamicCreationInput): + """Native body containing immutable dynamic content only. + + A dynamic holds the model's assumptions: parameters under + ``gov.simulation``, and in the US also ``simulation``. Any other + parameter belongs in a policy (``POST /v2/policies``). + """ diff --git a/policyengine_api/fastapi_routes/v2/dynamics/response_models.py b/policyengine_api/fastapi_routes/v2/dynamics/response_models.py new file mode 100644 index 000000000..bb1f29f57 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/dynamics/response_models.py @@ -0,0 +1,111 @@ +"""Strict HTTP response models for the native v2 dynamic API.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING, Annotated, Any, Literal +from uuid import UUID + +from pydantic import Field + +from policyengine_api.fastapi_routes.v2.errors_support import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.policies.response_models import ( + PolicyParameterValueItem, + PolicySuccessResponse, + StrictPolicyAPIModel, +) + +if TYPE_CHECKING: + from policyengine_api.services.v2.dynamics.types import CheckedDynamicRead + + +class DynamicItem(StrictPolicyAPIModel): + """A stored dynamic: the model assumptions a simulation applies with its policy. + + Dynamics are stored as policies are: immutable, bound to one catalog + version and deduplicated, so creating one with identical content returns + the existing dynamic and its ``id``. A dynamic has no name; a user's saved + dynamic names it. + """ + + id: UUID + country_id: str + tax_benefit_model_id: UUID + created_at: datetime + updated_at: datetime + parameter_values: list[PolicyParameterValueItem] + missing_parameters: Annotated[ + list[str] | None, + Field( + description=( + "The parameter paths this dynamic sets that the current " + "tax-benefit model (the catalog of the running PolicyEngine.py " + "version) no longer has, sorted; empty when it has them all. " + "Null when that catalog is not published, so nothing could be " + "checked. A dynamic with missing parameters cannot be run on " + "the current model." + ) + ), + ] + + @classmethod + def from_read(cls, item: CheckedDynamicRead) -> DynamicItem: + """Build the response item by validating the service read record's fields.""" + return cls.model_validate(item) + + +class DynamicDetailResult(StrictPolicyAPIModel): + """Result holding one dynamic in ``item``.""" + + item: DynamicItem + + +class DynamicDetailResponse(PolicySuccessResponse[DynamicDetailResult]): + """Success response holding one dynamic.""" + + +class V2PolicyParametersErrorResponse(V2ErrorResponse): + """A refused dynamic that sets parameters belonging in a policy. + + ``policy_parameters`` names each parameter the dynamic sets that is not + one of the model's assumptions, sorted. + """ + + code: Literal["policy_parameters"] = "policy_parameters" + policy_parameters: list[str] + + +DYNAMIC_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 400: { + "model": V2ErrorResponse, + "description": "The dynamic content or country selection is invalid.", + }, + 404: { + "model": V2ErrorResponse, + "description": "The selected dynamic or catalog does not exist.", + }, + 409: { + "model": V2ErrorResponse, + "description": "Immutable dynamic identity conflicts with stored state.", + }, + 413: { + "model": V2ErrorResponse, + "description": "The dynamic request body exceeds 1 MiB.", + }, + 422: { + "model": V2PolicyParametersErrorResponse | V2ErrorResponse, + "description": ( + "The request does not match the dynamic schema (a dynamic sets at " + "least one value), or the dynamic sets parameters that belong in a " + "policy (code policy_parameters, naming them)." + ), + }, + 500: { + "model": V2ErrorResponse, + "description": "Stored dynamic integrity validation failed.", + }, + 503: { + "model": V2ErrorResponse, + "description": "Supabase dynamic persistence is unavailable.", + }, +} diff --git a/policyengine_api/fastapi_routes/v2/dynamics/routes.py b/policyengine_api/fastapi_routes/v2/dynamics/routes.py new file mode 100644 index 000000000..72d5ff401 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/dynamics/routes.py @@ -0,0 +1,208 @@ +"""Native FastAPI routes for immutable v2 dynamics. + +A dynamic holds the model's assumptions a simulation applies alongside its +policy. It is created and read as a policy is, needs no user credential (a +user's saved dynamics name it), and is never listed, changed or deleted. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, TypeVar +from uuid import UUID + +from fastapi import APIRouter, Depends +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_dynamic_service_factory, +) +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.fastapi_routes.v2.dynamics.request_models import ( + MAXIMUM_DYNAMIC_REQUEST_BYTES, + DynamicCreateRequest, +) +from policyengine_api.fastapi_routes.v2.dynamics.response_models import ( + DYNAMIC_ERROR_RESPONSES, + DynamicDetailResponse, + DynamicDetailResult, + DynamicItem, + V2PolicyParametersErrorResponse, +) +from policyengine_api.fastapi_routes.v2.errors_support import ( + V2ErrorMapping, + run_v2_operation, + v2_error_response, +) +from policyengine_api.fastapi_routes.v2.user_resources_support import ( + request_size_limit, +) +from policyengine_api.query_parameters import DynamicCreateQuery, DynamicDetailQuery +from policyengine_api.services.v2.dynamics.types import NativeDynamicCreationInput +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, + DynamicNotFoundError, + DynamicSetsPolicyParametersError, + DynamicStorageError, +) +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2DynamicResourceService, + ) + + +DynamicCreateQueryDependency = Annotated[ + DynamicCreateQuery, Depends(query_dependency(DynamicCreateQuery)) +] +DynamicDetailQueryDependency = Annotated[ + DynamicDetailQuery, Depends(query_dependency(DynamicDetailQuery)) +] +enforce_dynamic_request_size = request_size_limit( + MAXIMUM_DYNAMIC_REQUEST_BYTES, label="1 MiB" +) + +OperationT = TypeVar("OperationT") + +# Checked in order, so the collision is matched before its integrity base class. +DYNAMIC_ERROR_MAPPINGS = ( + V2ErrorMapping((PolicyCatalogValidationError,), 400), + V2ErrorMapping((MetadataCatalogVersionNotFoundError, DynamicNotFoundError), 404), + V2ErrorMapping( + (DynamicContentHashCollisionError,), + 409, + "Dynamic content hash conflicts with storage", + ), + V2ErrorMapping( + (DynamicCreationIntegrityError,), 500, "Stored dynamic integrity failed" + ), + V2ErrorMapping( + (V2ConfigurationError, MetadataCatalogUnavailableError, DynamicStorageError), + 503, + "V2 dynamic persistence is unavailable", + ), +) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2DynamicResourceService]: + if dependencies.v2_dynamic_service_factory is not None: + return dependencies.v2_dynamic_service_factory + return _default_v2_dynamic_service_factory + + +def _policy_parameters_response( + error: DynamicSetsPolicyParametersError, +) -> JSONResponse: + """Serialize the 422 refusing a dynamic that sets a policy's parameters.""" + refusal = V2PolicyParametersErrorResponse( + message=str(error), policy_parameters=list(error.parameter_names) + ) + return JSONResponse(status_code=422, content=refusal.model_dump(mode="json")) + + +def _dynamic_operation( + operation: Callable[[], OperationT], +) -> OperationT | JSONResponse: + def refused() -> OperationT | JSONResponse: + try: + return operation() + except DynamicSetsPolicyParametersError as error: + return _policy_parameters_response(error) + + return run_v2_operation( + refused, + DYNAMIC_ERROR_MAPPINGS, + fallback_message="V2 dynamic operation failed", + ) + + +def build_v2_dynamic_router(dependencies: NativeRouteDependencies) -> APIRouter: + """Build native dynamic routes without opening a database connection.""" + router = APIRouter(prefix="/v2", responses=DYNAMIC_ERROR_RESPONSES) + service_factory = _service_factory(dependencies) + + @router.post( + "/dynamics", + response_model=DynamicDetailResponse, + status_code=201, + responses={ + 200: { + "model": DynamicDetailResponse, + "description": "Equivalent immutable content already exists.", + } + }, + summary="Create or find an immutable dynamic", + description=( + "Binds the model assumptions to the catalog of policyengine_version, " + "or of the running PolicyEngine.py version when it names none, and " + "stores them once per content hash, as policies are stored. Every " + "parameter must be under gov.simulation, or in the US also " + "simulation; any other answers 422 with code policy_parameters. " + "Answers 201 when it creates the dynamic and 200 when identical " + "content is already stored. There is no list route." + ), + ) + def create_dynamic( + body: DynamicCreateRequest, + query: DynamicCreateQueryDependency, + _size: Annotated[None, Depends(enforce_dynamic_request_size)], + ) -> DynamicDetailResponse | JSONResponse: + if body.country_id != query.country_id: + return v2_error_response(400, "Body country_id must match query country_id") + + def create() -> DynamicDetailResponse | JSONResponse: + result = service_factory().create_dynamic( + NativeDynamicCreationInput( + **body.model_dump(), + policyengine_version=query.policyengine_version, + ) + ) + response = DynamicDetailResponse( + result=DynamicDetailResult(item=DynamicItem.from_read(result.item)) + ) + if result.created: + return response + return JSONResponse( + status_code=200, content=response.model_dump(mode="json") + ) + + return _dynamic_operation(create) + + @router.get( + "/dynamics/{dynamic_id}", + response_model=DynamicDetailResponse, + summary="Read one immutable dynamic", + description=( + "Returns the country's dynamic with its values, and the parameters " + "it sets that the running catalog no longer has. Another country's " + "dynamic answers 404." + ), + ) + def get_dynamic( + dynamic_id: UUID, + query: DynamicDetailQueryDependency, + ) -> DynamicDetailResponse | JSONResponse: + def read() -> DynamicDetailResponse: + item = service_factory().get_dynamic( + country_id=query.country_id, dynamic_id=dynamic_id + ) + return DynamicDetailResponse( + result=DynamicDetailResult(item=DynamicItem.from_read(item)) + ) + + return _dynamic_operation(read) + + return router diff --git a/policyengine_api/fastapi_routes/v2/errors_support.py b/policyengine_api/fastapi_routes/v2/errors_support.py index 632442309..d809815a0 100644 --- a/policyengine_api/fastapi_routes/v2/errors_support.py +++ b/policyengine_api/fastapi_routes/v2/errors_support.py @@ -57,11 +57,11 @@ class V2CodedErrorResponse(V2ErrorResponse): class V2MissingParametersErrorResponse(V2ErrorResponse): - """A refused calculation whose policy sets parameters the catalog lacks. + """A refused calculation whose policy or dynamic sets parameters the catalog lacks. - ``missing_parameters`` names each parameter the policy sets that the - running catalog version does not have, sorted; the calculation would - otherwise run without those values. + ``missing_parameters`` names each parameter the policy or dynamic sets + that the running catalog version does not have, sorted; the calculation + would otherwise run without those values. """ code: Literal["missing_parameters"] = "missing_parameters" @@ -102,7 +102,7 @@ def v2_coded_error_response( def v2_missing_parameters_response( message: str, missing_parameters: Sequence[str] ) -> JSONResponse: - """Serialize a 422 refusing a policy whose parameters the catalog lacks.""" + """Serialize a 422 refusing a policy or dynamic the catalog lacks parameters of.""" error = V2MissingParametersErrorResponse( message=message, missing_parameters=list(missing_parameters) ) diff --git a/policyengine_api/fastapi_routes/v2/policies/response_models.py b/policyengine_api/fastapi_routes/v2/policies/response_models.py index a73c24ce8..45b88a983 100644 --- a/policyengine_api/fastapi_routes/v2/policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/policies/response_models.py @@ -123,6 +123,18 @@ class PolicyPageResponse(PolicySuccessResponse[PolicyPageResult]): """Success response holding one page of policies.""" +class V2DynamicParametersErrorResponse(V2ErrorResponse): + """A refused policy that sets parameters only a dynamic may hold. + + ``dynamic_parameters`` names each of the model's assumptions the policy + sets, sorted: parameters under ``gov.simulation``, and in the US also + ``simulation``. Send them as a dynamic through ``POST /v2/dynamics``. + """ + + code: Literal["dynamic_parameters"] = "dynamic_parameters" + dynamic_parameters: list[str] + + POLICY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": V2ErrorResponse, @@ -141,8 +153,12 @@ class PolicyPageResponse(PolicySuccessResponse[PolicyPageResult]): "description": "The policy request body exceeds 1 MiB.", }, 422: { - "model": V2ErrorResponse, - "description": "The request does not match the policy schema.", + "model": V2DynamicParametersErrorResponse | V2ErrorResponse, + "description": ( + "The request does not match the policy schema, or the policy sets " + "parameters only a dynamic may hold (code dynamic_parameters, " + "naming them)." + ), }, 500: { "model": V2ErrorResponse, diff --git a/policyengine_api/fastapi_routes/v2/policies/routes.py b/policyengine_api/fastapi_routes/v2/policies/routes.py index 8b07feeaa..14d57f665 100644 --- a/policyengine_api/fastapi_routes/v2/policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/policies/routes.py @@ -35,6 +35,7 @@ PolicyItem, PolicyPageResponse, PolicyPageResult, + V2DynamicParametersErrorResponse, ) from policyengine_api.query_parameters import ( PolicyCollectionQuery, @@ -47,6 +48,7 @@ PolicyContentHashCollisionError, PolicyCreationIntegrityError, PolicyNotFoundError, + PolicySetsDynamicParametersError, ) if TYPE_CHECKING: @@ -116,11 +118,27 @@ def _service_factory( ) +def _dynamic_parameters_response( + error: PolicySetsDynamicParametersError, +) -> JSONResponse: + """Serialize the 422 refusing a policy that sets a dynamic's parameters.""" + refusal = V2DynamicParametersErrorResponse( + message=str(error), dynamic_parameters=list(error.parameter_names) + ) + return JSONResponse(status_code=422, content=refusal.model_dump(mode="json")) + + def _policy_operation( operation: Callable[[], OperationT], ) -> OperationT | JSONResponse: + def refused() -> OperationT | JSONResponse: + try: + return operation() + except PolicySetsDynamicParametersError as error: + return _dynamic_parameters_response(error) + return run_v2_operation( - operation, + refused, POLICY_ERROR_MAPPINGS, fallback_message="V2 policy operation failed", ) @@ -144,6 +162,16 @@ def build_v2_policy_router( } }, summary="Create or find an immutable policy", + description=( + "Binds the policy to the catalog of policyengine_version, or of the " + "running PolicyEngine.py version when it names none, and stores it " + "once per content hash. A policy may not set the model's " + "assumptions, the parameters under gov.simulation (and in the US " + "simulation): those answer 422 with code dynamic_parameters, and " + "go in a dynamic through POST /v2/dynamics. Answers 201 when it " + "creates the policy and 200 when identical content is already " + "stored." + ), ) def create_policy( body: PolicyCreateRequest, diff --git a/policyengine_api/fastapi_routes/v2/reports/response_models.py b/policyengine_api/fastapi_routes/v2/reports/response_models.py index 3b9e3c760..c21f0188c 100644 --- a/policyengine_api/fastapi_routes/v2/reports/response_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/response_models.py @@ -485,8 +485,8 @@ class ReportRunNotRecordedResponse(V2ErrorResponse): 401: error_response(UNAUTHENTICATED_REQUESTER), 403: error_response(MACHINE_REQUESTER), 404: error_response( - "A policy the report names, or a household report's household, is not " - "stored in the country." + "A policy or dynamic the report names, or a household report's " + "household, is not stored in the country." ), 413: error_response("The request body exceeds 16 KiB."), 422: { @@ -497,11 +497,12 @@ class ReportRunNotRecordedResponse(V2ErrorResponse): "country has no economy reports (such as a household-only country), " "for an economy report or budget window, or no household " "calculations, for a household report; a household " - "report's household has no default year; a policy the report names " - "sets parameters the running catalog version does not have " - "(code missing_parameters, which names them; the baseline's policy is " - "checked first); or the simulation service cannot run the report: its " - "422, or its 400 for a region, dataset or dynamic it does not have. The " + "report's household has no default year; a policy or dynamic the " + "report names sets parameters the running catalog version does not " + "have (code missing_parameters, which names them; the policies are " + "checked before the dynamics, the baseline's first); or the " + "simulation service cannot run the report: its " + "422, or its 400 for a region or dataset it does not have. The " "message says which; for the simulation service's refusal, it gives " "the service's code, when it gave one, and its reason, but not its " "status." diff --git a/policyengine_api/fastapi_routes/v2/reports/routes.py b/policyengine_api/fastapi_routes/v2/reports/routes.py index 2628f3053..b78448dd4 100644 --- a/policyengine_api/fastapi_routes/v2/reports/routes.py +++ b/policyengine_api/fastapi_routes/v2/reports/routes.py @@ -465,8 +465,8 @@ def build_v2_report_router(dependencies: NativeRouteDependencies) -> APIRouter: summary="Create an economy report, a household report or a budget window", description=( "Checks that the country has economy reports and stores each named " - "policy, and that the running catalog version has every parameter " - "those policies set, then has the simulation service create the " + "policy and dynamic, and that the running catalog version has every " + "parameter they set, then has the simulation service create the " "report, its two " "simulations and their runs on its default worker's PolicyEngine.py " "version, and start the run. With data_source local_area_dataset, " diff --git a/policyengine_api/fastapi_routes/v2/reports/run_models.py b/policyengine_api/fastapi_routes/v2/reports/run_models.py index 4680fa289..8b00b970c 100644 --- a/policyengine_api/fastapi_routes/v2/reports/run_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/run_models.py @@ -182,7 +182,8 @@ class RerunReportUnreadableResponse(V2ErrorResponse): "reports, or no household calculations for a household report; a " "policy of the report's simulations sets " "parameters the running catalog version does not have (code " - "missing_parameters, which names them); or the simulation service " + "missing_parameters, which names them), where a dynamic that does is " + "the simulation service's 422 instead; or the simulation service " "cannot run the report as stored: its 422, or its 400 for a region, " "dataset or dynamic the report names that the current catalog version " "does not have, with its code and reason." diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 51e96ee25..589bcc73d 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -16,6 +16,9 @@ from policyengine_api.fastapi_routes.v2.countries.routes import ( build_v2_country_router, ) +from policyengine_api.fastapi_routes.v2.dynamics.routes import ( + build_v2_dynamic_router, +) from policyengine_api.fastapi_routes.v2.errors_support import V2ErrorResponse from policyengine_api.fastapi_routes.v2.household_calculations.routes import ( build_v2_household_calculation_router, @@ -79,6 +82,7 @@ def build_v2_router( router.include_router(build_v2_household_router(dependencies)) router.include_router(build_v2_household_calculation_router(dependencies)) router.include_router(build_v2_policy_router(dependencies)) + router.include_router(build_v2_dynamic_router(dependencies)) router.include_router(build_v2_report_router(dependencies)) router.include_router(build_v2_simulation_router(dependencies)) router.include_router(build_v2_country_router(dependencies)) diff --git a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py index fbb15814e..36e517939 100644 --- a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py +++ b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py @@ -22,6 +22,7 @@ run_v2_operation, v2_missing_parameters_response, ) +from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError from policyengine_api.services.v2.household_calculations.validators import ( HouseholdCalculationCountryError, HouseholdCalculationYearError, @@ -52,6 +53,7 @@ # not logged. SUBMISSION_REFUSALS: tuple[type[Exception], ...] = ( PolicyNotFoundError, + DynamicNotFoundError, HouseholdNotFoundError, EconomyCalculationsUnsupportedError, HouseholdCalculationCountryError, @@ -69,7 +71,9 @@ # the record it created and failed is still a 503, and each create route # answers it with the record's ID (``unavailable_after_creation_answer``). SUBMISSION_ERROR_MAPPINGS = ( - V2ErrorMapping((PolicyNotFoundError, HouseholdNotFoundError), 404), + V2ErrorMapping( + (PolicyNotFoundError, DynamicNotFoundError, HouseholdNotFoundError), 404 + ), V2ErrorMapping( ( EconomyCalculationsUnsupportedError, @@ -154,10 +158,10 @@ def simulation_resource_operation( ) -> OperationT | JSONResponse: """Run one service call and answer its failure with a typed v2 error. - A policy that sets parameters the running catalog version lacks answers - 422 with code ``missing_parameters``, naming them, whatever the route. A - failure of a type in ``refusals`` is the caller's to act on and is not - logged, nor is that 422; any other is logged under ``event``. + A policy or dynamic that sets parameters the running catalog version + lacks answers 422 with code ``missing_parameters``, naming them, whatever + the route. A failure of a type in ``refusals`` is the caller's to act on + and is not logged, nor is that 422; any other is logged under ``event``. """ def logged() -> OperationT | JSONResponse: diff --git a/policyengine_api/fastapi_routes/v2/simulations/response_models.py b/policyengine_api/fastapi_routes/v2/simulations/response_models.py index 7b95a579e..f3279ab03 100644 --- a/policyengine_api/fastapi_routes/v2/simulations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/simulations/response_models.py @@ -126,17 +126,18 @@ class UnavailableAfterSimulationCreatedResponse(V2ErrorResponse): CREATE_RESPONSES: dict[int | str, dict[str, Any]] = { 400: error_response("The body and query name different countries."), 404: error_response( - "The policy the simulation names is not stored in the country." + "The policy or dynamic the simulation names is not stored in the country." ), 413: error_response("The request body exceeds 16 KiB."), 422: { "model": V2MissingParametersErrorResponse | V2ErrorResponse, "description": ( "The request does not match the schema; the country has no economy " - "calculations; the policy sets parameters the running catalog version " - "does not have (code missing_parameters, which names them); or the " + "calculations; the policy or dynamic sets parameters the running " + "catalog version does not have (code missing_parameters, which names " + "them; the policy is checked first); or the " "simulation service cannot run the simulation: its 422, or its 400 " - "for a region, dataset or dynamic it does not have. The message says " + "for a region or dataset it does not have. The message says " "which; for the simulation service's refusal, it gives the service's " "code, when it gave one, and its reason, but not its status." ), diff --git a/policyengine_api/fastapi_routes/v2/simulations/routes.py b/policyengine_api/fastapi_routes/v2/simulations/routes.py index 043f61402..0486307cf 100644 --- a/policyengine_api/fastapi_routes/v2/simulations/routes.py +++ b/policyengine_api/fastapi_routes/v2/simulations/routes.py @@ -170,8 +170,8 @@ def build_v2_simulation_router(dependencies: NativeRouteDependencies) -> APIRout summary="Create an economy simulation", description=( "Checks that the country has economy calculations and stores the " - "named policy, and that the running catalog version has every " - "parameter it sets, then has the simulation service create the " + "named policy and dynamic, and that the running catalog version has " + "every parameter they set, then has the simulation service create the " "simulation and its first run on its default worker's " "PolicyEngine.py version, and start the run. Answers the simulation " "as stored, with its pending run. A created simulation that cannot " diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index 93c5874b5..6c7e808cb 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -56,7 +56,13 @@ class RouteGroupConfig: ), RouteGroupConfig( name="policy", - path_segments=("policy", "policies", "user-policy", "user-policies"), + path_segments=( + "policy", + "policies", + "dynamics", + "user-policy", + "user-policies", + ), db_entity="policy", ), RouteGroupConfig( diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index c39281438..a2a0c5278 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -356,6 +356,14 @@ class PolicyDetailQuery(CountryQuery): """Query contract for country-scoped policy detail reads.""" +class DynamicCreateQuery(CatalogQuery): + """Query contract for native immutable dynamic creation.""" + + +class DynamicDetailQuery(CountryQuery): + """Query contract for country-scoped dynamic detail reads.""" + + class PolicyCollectionQuery(CountryQuery, PaginationQuery): """Query contract for an exact-filtered policy collection.""" diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py index b46299a6f..c26844aad 100644 --- a/policyengine_api/services/policy_mirroring.py +++ b/policyengine_api/services/policy_mirroring.py @@ -17,6 +17,10 @@ ) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.gcp_logging import logger +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, +) from policyengine_api.services.v2.policies.validators import ( LegacyPolicyMappingIntegrityError, LegacyPolicyTranslationError, @@ -91,6 +95,7 @@ class PolicyCopyFailureReason(StrEnum): MAPPING_CONFLICT = "mapping_conflict" CONTENT_HASH_COLLISION = "content_hash_collision" STORED_POLICY_INTEGRITY = "stored_policy_integrity" + STORED_DYNAMIC_INTEGRITY = "stored_dynamic_integrity" SAVED_POLICY_INTEGRITY = "saved_policy_integrity" MISSING_REFORM_POLICY = "missing_reform_policy" MISSING_MIRROR_SNAPSHOT = "missing_mirror_snapshot" @@ -121,7 +126,7 @@ class PolicyCopyFailure: ) -# Checked in order, so the hash collision is matched before its integrity base +# Checked in order, so each hash collision is matched before its integrity base # class. _FAILURES_BY_ERROR_TYPE: tuple[tuple[type[Exception], PolicyCopyFailure], ...] = ( ( @@ -166,6 +171,20 @@ class PolicyCopyFailure: PolicyCopyFailureReason.STORED_POLICY_INTEGRITY, ), ), + ( + DynamicContentHashCollisionError, + PolicyCopyFailure( + PolicyCopyFailureCategory.INTEGRITY, + PolicyCopyFailureReason.CONTENT_HASH_COLLISION, + ), + ), + ( + DynamicCreationIntegrityError, + PolicyCopyFailure( + PolicyCopyFailureCategory.INTEGRITY, + PolicyCopyFailureReason.STORED_DYNAMIC_INTEGRITY, + ), + ), ( SQLAlchemyError, PolicyCopyFailure( @@ -207,8 +226,10 @@ class _MirrorOutcome: outcome: str actual_write_sources: list[str] destination_policy_id: object | None = None + destination_dynamic_id: object | None = None failure: PolicyCopyFailure | None = None policy_created: bool | None = None + dynamic_created: bool | None = None mapping_created: bool | None = None @@ -219,6 +240,7 @@ def _log_mirror_event( mirror_outcome: _MirrorOutcome, ) -> None: destination_policy_id = mirror_outcome.destination_policy_id + destination_dynamic_id = mirror_outcome.destination_dynamic_id failure = mirror_outcome.failure # Metric logging must not change the mirror outcome. with contextlib.suppress(Exception): @@ -237,10 +259,16 @@ def _log_mirror_event( if destination_policy_id is not None else None ), + "destination_dynamic_id": ( + str(destination_dynamic_id) + if destination_dynamic_id is not None + else None + ), "outcome": mirror_outcome.outcome, "failure_category": failure.category.value if failure else None, "failure_reason": failure.reason.value if failure else None, "policy_created": mirror_outcome.policy_created, + "dynamic_created": mirror_outcome.dynamic_created, "mapping_created": mirror_outcome.mapping_created, "duration_ms": round((time.perf_counter() - started_at) * 1000, 3), }, @@ -253,7 +281,7 @@ def mirror_policy_after_commit( *, mirror_factory: Callable[[], LegacyPolicyMirror] | None = None, ) -> LegacyPolicyPersistenceResult: - """Copy a committed v1 policy and its mapping into v2 in one Supabase transaction. + """Copy a committed v1 policy, its dynamic and its mapping into v2 at once. Log the outcome. On failure, raise PolicyMirrorUnavailableError after logging it; the v1 route still returns its normal response. @@ -283,7 +311,9 @@ def mirror_policy_after_commit( outcome="ok", actual_write_sources=["cloud_sql", "supabase"], destination_policy_id=result.policy_id, + destination_dynamic_id=result.dynamic_id, policy_created=result.policy_created, + dynamic_created=result.dynamic_created, mapping_created=result.mapping_created, ), ) diff --git a/policyengine_api/services/v2/dynamics/__init__.py b/policyengine_api/services/v2/dynamics/__init__.py new file mode 100644 index 000000000..b7311d76a --- /dev/null +++ b/policyengine_api/services/v2/dynamics/__init__.py @@ -0,0 +1 @@ +"""Application services for immutable v2 dynamics, stored as policies are.""" diff --git a/policyengine_api/services/v2/dynamics/database_connectors/__init__.py b/policyengine_api/services/v2/dynamics/database_connectors/__init__.py new file mode 100644 index 000000000..e9b9241cb --- /dev/null +++ b/policyengine_api/services/v2/dynamics/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for v2 dynamic services.""" diff --git a/policyengine_api/services/v2/dynamics/database_connectors/creates.py b/policyengine_api/services/v2/dynamics/database_connectors/creates.py new file mode 100644 index 000000000..142e79d06 --- /dev/null +++ b/policyengine_api/services/v2/dynamics/database_connectors/creates.py @@ -0,0 +1,69 @@ +"""Database inserts used by immutable v2 dynamic operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, col + +from policyengine_api.data.v2.models import Dynamic, ParameterValue + +if TYPE_CHECKING: + from policyengine_api.services.v2.dynamics.types import ( + ResolvedDynamicCreationInput, + ) + + +def create_dynamic( + session: Session, + dynamic_input: ResolvedDynamicCreationInput, + *, + canonicalization_version: int, + content_hash: str, +) -> UUID | None: + """Insert a dynamic unless one with the same canonical version and hash exists. + + Return the new id, or None when the unique content-hash constraint made the + insert a no-op; the caller then reads the existing row. Parameter values are + inserted separately by create_dynamic_parameter_values. + """ + dynamic_id = uuid4() + statement = ( + insert(Dynamic) + .values( + id=dynamic_id, + country_id=dynamic_input.country_id, + tax_benefit_model_id=dynamic_input.tax_benefit_model_id, + tax_benefit_model_version_id=dynamic_input.tax_benefit_model_version_id, + canonicalization_version=canonicalization_version, + content_hash=content_hash, + ) + .on_conflict_do_nothing(constraint="uq_dynamics_canonicalization_content_hash") + .returning(col(Dynamic.id)) + ) + return session.execute(statement).scalar_one_or_none() + + +def create_dynamic_parameter_values( + session: Session, + *, + dynamic_id: UUID, + dynamic_input: ResolvedDynamicCreationInput, +) -> None: + """Insert a new dynamic's parameter values and flush them for later reads.""" + session.add_all( + [ + ParameterValue( + policy_id=None, + dynamic_id=dynamic_id, + parameter_id=value.parameter_id, + value_json=value.value, + start_date=value.start_date, + end_date=value.end_date, + ) + for value in dynamic_input.parameter_values + ] + ) + session.flush() diff --git a/policyengine_api/services/v2/dynamics/database_connectors/reads.py b/policyengine_api/services/v2/dynamics/database_connectors/reads.py new file mode 100644 index 000000000..003ae1a8a --- /dev/null +++ b/policyengine_api/services/v2/dynamics/database_connectors/reads.py @@ -0,0 +1,66 @@ +"""Database selections used by immutable v2 dynamic operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Dynamic, Parameter, ParameterValue + +if TYPE_CHECKING: + from uuid import UUID + + +def read_dynamic_by_content_identity( + session: Session, *, canonicalization_version: int, content_hash: str +) -> Dynamic | None: + """Return the dynamic with this canonical version and content hash, or None.""" + return session.exec( + select(Dynamic).where( + Dynamic.canonicalization_version == canonicalization_version, + Dynamic.content_hash == content_hash, + ) + ).one_or_none() + + +def read_dynamic_parameter_values( + session: Session, dynamic_id: UUID +) -> list[ParameterValue]: + """Return every parameter value stored for one dynamic, in no particular order.""" + return list( + session.exec( + select(ParameterValue).where(ParameterValue.dynamic_id == dynamic_id) + ).all() + ) + + +def read_dynamic_row( + session: Session, *, country_id: str, dynamic_id: UUID +) -> Dynamic | None: + """Return the dynamic with this id in this country, or None.""" + return session.exec( + select(Dynamic).where( + Dynamic.id == dynamic_id, Dynamic.country_id == country_id + ) + ).one_or_none() + + +def read_dynamic_parameter_values_with_names( + session: Session, dynamic_ids: list[UUID] +) -> list[tuple[ParameterValue, str]]: + """Return the dynamics' values with parameter names, by name then start date.""" + if not dynamic_ids: + return [] + return list( + session.exec( + select(ParameterValue, Parameter.name) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where(col(ParameterValue.dynamic_id).in_(dynamic_ids)) + .order_by( + col(Parameter.name), + col(ParameterValue.start_date), + col(ParameterValue.id), + ) + ).all() + ) diff --git a/policyengine_api/services/v2/dynamics/database_session.py b/policyengine_api/services/v2/dynamics/database_session.py new file mode 100644 index 000000000..192433c2d --- /dev/null +++ b/policyengine_api/services/v2/dynamics/database_session.py @@ -0,0 +1,32 @@ +"""Database session lifetime management for v2 dynamic services.""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.orm import sessionmaker + from sqlmodel import Session + + +class DynamicDatabaseSession: + """Open read sessions and transaction-scoped sessions for dynamic services.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + """Use ``session_factory`` to open every session.""" + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + """Yield a session for reads; nothing is committed.""" + with self._session_factory() as session: + yield session + + @contextmanager + def transaction(self) -> Iterator[Session]: + """Yield a session whose transaction commits on exit and rolls back on error.""" + with self._session_factory.begin() as session: + yield session diff --git a/policyengine_api/services/v2/dynamics/services.py b/policyengine_api/services/v2/dynamics/services.py new file mode 100644 index 000000000..28d8c7d81 --- /dev/null +++ b/policyengine_api/services/v2/dynamics/services.py @@ -0,0 +1,291 @@ +"""Router-facing services for v2 dynamics, stored as policies are. + +A dynamic is bound to a catalog version and deduplicated by the policies' +canonical content hash, with the policies' catalog reads. This module never +imports ``policies.services``: the v1-to-v2 policy copy there creates +dynamics through ``create_resolved_dynamic``. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogNotPublishedError, +) +from policyengine_api.services.v2.dynamics.database_connectors.creates import ( + create_dynamic, + create_dynamic_parameter_values, +) +from policyengine_api.services.v2.dynamics.database_connectors.reads import ( + read_dynamic_by_content_identity, + read_dynamic_parameter_values, + read_dynamic_parameter_values_with_names, + read_dynamic_row, +) +from policyengine_api.services.v2.dynamics.transformations import ( + checked_dynamic_read, + dynamic_creation_input, + dynamic_parameter_values_by_dynamic, + dynamic_read, +) +from policyengine_api.services.v2.dynamics.types import ( + CheckedDynamicRead, + DynamicCreationInput, + DynamicCreationResult, + DynamicRead, + NativeDynamicCreation, + NativeDynamicCreationInput, + ResolvedDynamicCreationInput, +) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, + DynamicNotFoundError, + DynamicStorageError, + require_dynamic_parameters, +) +from policyengine_api.services.v2.policies.database_connectors.reads import ( + read_model_version, + read_parameters_by_name, + read_policy_catalog, + read_version_parameter_names, +) +from policyengine_api.services.v2.policies.transformations import ( + canonical_policy_document, + canonicalize_policy, + policy_missing_parameters, + stored_policy_creation_input, +) +from policyengine_api.services.v2.policies.validators import validate_policy_catalog + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from uuid import UUID + + from sqlmodel import Session + + from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, + ) + from policyengine_api.services.v2.policies.types import ( + CanonicalPolicyContent, + ResolvedPolicyCreationInput, + ) + + +@contextmanager +def _storage_failures() -> Iterator[None]: + """Raise DynamicStorageError for a database read or write that failed.""" + try: + yield + except SQLAlchemyError as error: + msg = "the v2 database could not be read or written for a dynamic" + raise DynamicStorageError(msg) from error + + +def resolve_dynamic_creation_input( + session: Session, + dynamic_input: DynamicCreationInput, + *, + policyengine_version: str | None = None, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> ResolvedDynamicCreationInput: + """Bind dynamic content to the selected catalog's model version. + + Raise PolicyCatalogValidationError if the model or any parameter id is not + part of the selected catalog, and DynamicSetsPolicyParametersError if any + parameter is not one of the model's assumptions. + """ + selected = read_policy_catalog( + session, + dynamic_input.country_id, + policyengine_version=policyengine_version, + running_policyengine_version=running_policyengine_version, + ) + parameter_names = read_version_parameter_names( + session, + model_version_id=selected.model_version.id, + requested_ids={value.parameter_id for value in dynamic_input.parameter_values}, + ) + resolved = validate_policy_catalog( + dynamic_input, + selected=selected, + resolved_parameter_ids=set(parameter_names), + ) + require_dynamic_parameters(dynamic_input.country_id, parameter_names.values()) + return dynamic_creation_input(resolved) + + +def create_resolved_dynamic( + session: Session, + dynamic_input: ResolvedDynamicCreationInput, + *, + canonicalizer: Callable[ + [ResolvedPolicyCreationInput], CanonicalPolicyContent + ] = canonicalize_policy, +) -> DynamicCreationResult: + """Store a resolved dynamic, or return the existing one with identical content. + + Raise DynamicCreationIntegrityError if the hash conflict finds no usable + row, and DynamicContentHashCollisionError if the stored row's canonical + bytes differ. + """ + content = canonicalizer(dynamic_input) + created_dynamic_id = create_dynamic( + session, + dynamic_input, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if created_dynamic_id is not None: + create_dynamic_parameter_values( + session, dynamic_id=created_dynamic_id, dynamic_input=dynamic_input + ) + return DynamicCreationResult(dynamic_id=created_dynamic_id, created=True) + + existing = read_dynamic_by_content_identity( + session, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if existing is None: + msg = "dynamic hash conflict did not resolve to a stored dynamic" + raise DynamicCreationIntegrityError(msg) + model_version = read_model_version(session, existing.tax_benefit_model_version_id) + if model_version is None: + msg = "stored dynamic references an absent model version" + raise DynamicCreationIntegrityError(msg) + stored_input = stored_policy_creation_input( + existing, + model_version, + read_dynamic_parameter_values(session, existing.id), + ) + if canonical_policy_document(stored_input) != content.document: + msg = "stored dynamic content differs for the same canonical version and hash" + raise DynamicContentHashCollisionError(msg) + return DynamicCreationResult(dynamic_id=existing.id, created=False) + + +def read_complete_dynamic( + session: Session, *, country_id: str, dynamic_id: UUID +) -> DynamicRead: + """Return one dynamic of this country, or raise DynamicNotFoundError.""" + dynamic = read_dynamic_row(session, country_id=country_id, dynamic_id=dynamic_id) + if dynamic is None: + msg = f"dynamic {dynamic_id} was not found" + raise DynamicNotFoundError(msg) + values = dynamic_parameter_values_by_dynamic( + [dynamic.id], read_dynamic_parameter_values_with_names(session, [dynamic.id]) + ) + return dynamic_read(dynamic, values) + + +def with_missing_dynamic_parameters( + session: Session, + dynamic: DynamicRead, + *, + country_id: str, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> CheckedDynamicRead: + """Return the dynamic with the parameters it sets that the running catalog lacks. + + As for a policy, ``missing_parameters`` is None when the running + PolicyEngine.py version has no published catalog for the country. A + catalog that cannot be queried raises, as any other read would. + """ + try: + selected = read_policy_catalog( + session, + country_id, + running_policyengine_version=running_policyengine_version, + ) + except MetadataCatalogNotPublishedError: + return checked_dynamic_read(dynamic, None) + running_model_version_id = selected.model_version.id + names_to_check = ( + set() + if dynamic.tax_benefit_model_version_id == running_model_version_id + else {value.parameter_name for value in dynamic.parameter_values} + ) + running_parameters = read_parameters_by_name( + session, model_version_id=running_model_version_id, names=names_to_check + ) + return checked_dynamic_read( + dynamic, + policy_missing_parameters( + dynamic, + running_model_version_id=running_model_version_id, + running_parameter_names=running_parameters, + ), + ) + + +class V2DynamicService: + """Sequence dynamic operations while delegating database lifetime management.""" + + def __init__( + self, + database_session: DynamicDatabaseSession, + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + ) -> None: + """Open sessions through ``database_session``. + + ``running_policyengine_version`` selects the catalog when a request names none. + """ + self._database_session = database_session + self._running_policyengine_version = running_policyengine_version + + def create_dynamic( + self, dynamic_input: NativeDynamicCreationInput + ) -> NativeDynamicCreation: + """Store native dynamic content, deduplicated by content hash. + + Return the stored dynamic and whether this call created it. Raise + DynamicStorageError when the database cannot be read or written. + """ + content = DynamicCreationInput.model_validate( + dynamic_input.model_dump(exclude={"policyengine_version"}) + ) + with _storage_failures(), self._database_session.transaction() as session: + resolved = resolve_dynamic_creation_input( + session, + content, + policyengine_version=dynamic_input.policyengine_version, + running_policyengine_version=self._running_policyengine_version, + ) + persisted = create_resolved_dynamic(session, resolved) + item = read_complete_dynamic( + session, + country_id=dynamic_input.country_id, + dynamic_id=persisted.dynamic_id, + ) + checked = with_missing_dynamic_parameters( + session, + item, + country_id=dynamic_input.country_id, + running_policyengine_version=self._running_policyengine_version, + ) + return NativeDynamicCreation(item=checked, created=persisted.created) + + def get_dynamic(self, *, country_id: str, dynamic_id: UUID) -> CheckedDynamicRead: + """Return one dynamic of this country, or raise DynamicNotFoundError. + + Its ``missing_parameters`` are checked against the running catalog. + Raise DynamicStorageError when the database cannot be read. + """ + with _storage_failures(), self._database_session.read() as session: + item = read_complete_dynamic( + session, country_id=country_id, dynamic_id=dynamic_id + ) + return with_missing_dynamic_parameters( + session, + item, + country_id=country_id, + running_policyengine_version=self._running_policyengine_version, + ) diff --git a/policyengine_api/services/v2/dynamics/transformations.py b/policyengine_api/services/v2/dynamics/transformations.py new file mode 100644 index 000000000..11aa3f42a --- /dev/null +++ b/policyengine_api/services/v2/dynamics/transformations.py @@ -0,0 +1,86 @@ +"""Pure representation transformations for v2 dynamic operations. + +A dynamic's canonical bytes and content hash are a policy's +(``policies.transformations.canonicalize_policy``), so a dynamic needs no +canonicalization of its own. +""" + +from __future__ import annotations + +from dataclasses import fields as dataclass_fields +from typing import TYPE_CHECKING + +from policyengine_api.services.v2.dynamics.types import ( + CheckedDynamicRead, + DynamicRead, + ResolvedDynamicCreationInput, +) +from policyengine_api.services.v2.policies.types import PolicyParameterValueRead + +if TYPE_CHECKING: + from uuid import UUID + + from policyengine_api.data.v2.models import Dynamic, ParameterValue + from policyengine_api.services.v2.policies.types import ( + ResolvedPolicyCreationInput, + ) + + +def dynamic_creation_input( + content: ResolvedPolicyCreationInput, +) -> ResolvedDynamicCreationInput: + """Return resolved content as a dynamic's; it must set at least one value.""" + return ResolvedDynamicCreationInput.model_validate(dict(content)) + + +def dynamic_parameter_values_by_dynamic( + dynamic_ids: list[UUID], + rows: list[tuple[ParameterValue, str]], +) -> dict[UUID, tuple[PolicyParameterValueRead, ...]]: + """Group named value rows by dynamic id; every requested dynamic gets an entry.""" + grouped: dict[UUID, list[PolicyParameterValueRead]] = { + dynamic_id: [] for dynamic_id in dynamic_ids + } + for value, parameter_name in rows: + if value.dynamic_id is None: + continue + grouped[value.dynamic_id].append( + PolicyParameterValueRead( + id=value.id, + parameter_id=value.parameter_id, + parameter_name=parameter_name, + value=value.value_json, + start_date=value.start_date, + end_date=value.end_date, + ) + ) + return {dynamic_id: tuple(values) for dynamic_id, values in grouped.items()} + + +def dynamic_read( + dynamic: Dynamic, + values: dict[UUID, tuple[PolicyParameterValueRead, ...]], +) -> DynamicRead: + """Convert a stored dynamic and its grouped values to the service's read model.""" + return DynamicRead( + id=dynamic.id, + country_id=dynamic.country_id, + tax_benefit_model_id=dynamic.tax_benefit_model_id, + tax_benefit_model_version_id=dynamic.tax_benefit_model_version_id, + created_at=dynamic.created_at, + updated_at=dynamic.updated_at, + parameter_values=values.get(dynamic.id, ()), + ) + + +def checked_dynamic_read( + dynamic: DynamicRead, missing_parameters: tuple[str, ...] | None +) -> CheckedDynamicRead: + """Return the dynamic read with the result of its missing-parameter check.""" + return CheckedDynamicRead( + **{ + field.name: getattr(dynamic, field.name) + for field in dataclass_fields(DynamicRead) + }, + missing_parameters=missing_parameters, + ) diff --git a/policyengine_api/services/v2/dynamics/types.py b/policyengine_api/services/v2/dynamics/types.py new file mode 100644 index 000000000..aa173ef7c --- /dev/null +++ b/policyengine_api/services/v2/dynamics/types.py @@ -0,0 +1,95 @@ +"""Framework-independent data exchanged by v2 dynamic layers. + +A dynamic's content has a policy's fields, is bound to one catalog version +as a policy is, and is hashed by the same canonical document +(``policies.transformations.canonical_policy_document``). It differs in two +ways: it sets at least one value, since "no assumptions beyond the model's +defaults" is ``dynamic_id`` null, and it holds only the model's assumptions +(``policyengine_api.dynamic_parameters``), which ``dynamics.validators`` +checks once the catalog has named the parameters. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Annotated + +from pydantic import Field + +from policyengine_api.query_parameters import PolicyEngineVersion +from policyengine_api.services.v2.policies.types import ( + ParameterValueInputs, + PolicyCreationInput, + PolicyRead, + ResolvedPolicyCreationInput, +) + +if TYPE_CHECKING: + from uuid import UUID + +# A dynamic's values: at least one, and at most as many as a policy's. +DynamicParameterValueInputs = Annotated[ + ParameterValueInputs, + Field( + min_length=1, + description=( + "The model assumptions the dynamic sets, by catalog parameter ID: " + "parameters under gov.simulation, and in the US also simulation. " + "At least one: for none, send dynamic_id null." + ), + ), +] + + +class DynamicCreationInput(PolicyCreationInput): + """Complete immutable dynamic content accepted from any source.""" + + parameter_values: DynamicParameterValueInputs + + +class NativeDynamicCreationInput(DynamicCreationInput): + """Native dynamic content plus an optional catalog-version selection.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class ResolvedDynamicCreationInput(ResolvedPolicyCreationInput): + """Dynamic content bound to one exact initialized catalog version.""" + + parameter_values: DynamicParameterValueInputs + + +@dataclass(frozen=True) +class DynamicCreationResult: + """New or deduplicated immutable dynamic identity.""" + + dynamic_id: UUID + created: bool + + +@dataclass(frozen=True) +class DynamicRead(PolicyRead): + """A stored v2 dynamic with its parameter values, as returned to routes. + + It has a policy's fields, read from ``dynamics`` instead of ``policies``. + """ + + +@dataclass(frozen=True) +class CheckedDynamicRead(DynamicRead): + """A stored dynamic checked against the running catalog, as routes return it. + + ``missing_parameters`` lists the parameter names the dynamic sets that the + running catalog lacks, sorted; it is None only when the running + PolicyEngine.py version has no published catalog for the country. + """ + + missing_parameters: tuple[str, ...] | None + + +@dataclass(frozen=True) +class NativeDynamicCreation: + """Complete dynamic read plus whether this request inserted it.""" + + item: CheckedDynamicRead + created: bool diff --git a/policyengine_api/services/v2/dynamics/validators.py b/policyengine_api/services/v2/dynamics/validators.py new file mode 100644 index 000000000..b6557f298 --- /dev/null +++ b/policyengine_api/services/v2/dynamics/validators.py @@ -0,0 +1,106 @@ +"""Database-independent validation for v2 dynamic operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.dynamic_parameters import is_dynamic_parameter +from policyengine_api.services.v2.policies.validators import ( + PolicyParametersMissingError, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + from uuid import UUID + + from policyengine_api.supported_countries import SupportedCountryId + + +class DynamicNotFoundError(LookupError): + """Raised when a dynamic UUID is absent from the selected country.""" + + +class DynamicStorageError(RuntimeError): + """Raised when the v2 database could not be read or written for a dynamic.""" + + +class DynamicCreationIntegrityError(RuntimeError): + """Raised when stored dynamic content cannot support safe deduplication.""" + + +class DynamicContentHashCollisionError(DynamicCreationIntegrityError): + """Raised when equal version/hash keys identify different canonical bytes.""" + + +class DynamicSetsPolicyParametersError(ValueError): + """Raised when a dynamic sets parameters that are not model assumptions. + + A dynamic holds only parameters under ``gov.simulation.``, plus the US's + ``simulation.`` (``policyengine_api.dynamic_parameters``); the others + belong in a policy. ``parameter_names`` lists them, sorted. + """ + + def __init__(self, parameter_names: tuple[str, ...]) -> None: + """Name each parameter the dynamic may not set.""" + self.parameter_names = parameter_names + super().__init__(parameter_names) + + def __str__(self) -> str: + """Say which parameters to send as a policy, and where.""" + return ( + "A dynamic holds only the model's assumptions, the parameters under " + "gov.simulation (and, in the US, simulation): send the others as a " + "policy through POST /v2/policies. This dynamic sets: " + + ", ".join(self.parameter_names) + ) + + +class DynamicParametersMissingError(PolicyParametersMissingError): + """Raised when a dynamic sets parameters the running catalog version lacks. + + A calculation refuses it as it refuses such a policy, with the same 422 + and ``missing_parameters``; ``dynamic_id`` names the dynamic (also as + ``policy_id``, the base error's field). + """ + + @property + def dynamic_id(self) -> UUID: + """The dynamic that sets the missing parameters.""" + return self.policy_id + + def __str__(self) -> str: + """Name the dynamic and each missing parameter.""" + return ( + f"Dynamic {self.policy_id} sets parameters the running catalog " + f"version does not have: {', '.join(self.missing_parameters)}" + ) + + +def require_dynamic_parameters( + country_id: SupportedCountryId, parameter_names: Iterable[str] +) -> None: + """Raise DynamicSetsPolicyParametersError unless each parameter is an assumption.""" + policy_parameter_names = tuple( + sorted( + { + name + for name in parameter_names + if not is_dynamic_parameter(country_id, name) + } + ) + ) + if policy_parameter_names: + raise DynamicSetsPolicyParametersError(policy_parameter_names) + + +def require_dynamic_parameters_present( + dynamic_id: UUID, missing_parameters: tuple[str, ...] | None +) -> None: + """Raise DynamicParametersMissingError when the dynamic sets missing parameters. + + ``missing_parameters`` is a checked dynamic's (``CheckedDynamicRead``): + None when the running PolicyEngine.py version has no published catalog + for the country, so nothing could be checked, which passes. + """ + if missing_parameters: + raise DynamicParametersMissingError(dynamic_id, missing_parameters) diff --git a/policyengine_api/services/v2/policies/database_connectors/creates.py b/policyengine_api/services/v2/policies/database_connectors/creates.py index 18e81b853..8627aaf55 100644 --- a/policyengine_api/services/v2/policies/database_connectors/creates.py +++ b/policyengine_api/services/v2/policies/database_connectors/creates.py @@ -11,7 +11,10 @@ from policyengine_api.data.v2.models import LegacyPolicyMapping, ParameterValue, Policy if TYPE_CHECKING: - from policyengine_api.services.v2.policies.types import ResolvedPolicyCreationInput + from policyengine_api.services.v2.policies.types import ( + LegacyPolicyMappingInput, + ResolvedPolicyCreationInput, + ) def create_policy( @@ -68,14 +71,9 @@ def create_parameter_values( def create_legacy_policy_mapping( - session: Session, - *, - country_id: str, - legacy_policy_id: int, - source_policy_hash: str, - policy_id: UUID, + session: Session, mapping: LegacyPolicyMappingInput ) -> UUID | None: - """Insert the mapping from a v1 policy id to its v2 policy. + """Insert the mapping from a v1 policy id to its v2 policy and dynamic. Return the new mapping id, or None when a mapping for this country and v1 id already exists (for example, a concurrent copy won the race). @@ -83,10 +81,11 @@ def create_legacy_policy_mapping( return session.execute( insert(LegacyPolicyMapping) .values( - country_id=country_id, - legacy_policy_id=legacy_policy_id, - policy_id=policy_id, - source_policy_hash=source_policy_hash, + country_id=mapping.country_id, + legacy_policy_id=mapping.legacy_policy_id, + policy_id=mapping.policy_id, + dynamic_id=mapping.dynamic_id, + source_policy_hash=mapping.source_policy_hash, ) .on_conflict_do_nothing(constraint="uq_legacy_policy_mappings_country_legacy") .returning(col(LegacyPolicyMapping.id)) diff --git a/policyengine_api/services/v2/policies/database_connectors/reads.py b/policyengine_api/services/v2/policies/database_connectors/reads.py index 1eba1685b..d26e33dfc 100644 --- a/policyengine_api/services/v2/policies/database_connectors/reads.py +++ b/policyengine_api/services/v2/policies/database_connectors/reads.py @@ -46,15 +46,15 @@ def read_policy_catalog( ) -def read_version_parameter_ids( +def read_version_parameter_names( session: Session, *, model_version_id: UUID, requested_ids: set[UUID] -) -> set[UUID]: - """Return the requested parameter ids that exist in this model version.""" +) -> dict[UUID, str]: + """Return the names of the requested parameters this model version has, by id.""" if not requested_ids: - return set() - return set( + return {} + return dict( session.exec( - select(Parameter.id).where( + select(Parameter.id, Parameter.name).where( Parameter.tax_benefit_model_version_id == model_version_id, col(Parameter.id).in_(requested_ids), ) diff --git a/policyengine_api/services/v2/policies/services.py b/policyengine_api/services/v2/policies/services.py index 6829d826b..9327d7b85 100644 --- a/policyengine_api/services/v2/policies/services.py +++ b/policyengine_api/services/v2/policies/services.py @@ -8,6 +8,9 @@ from policyengine_api.data.v2.catalog.catalog_selection_validation import ( MetadataCatalogNotPublishedError, ) +from policyengine_api.services.v2.dynamics.transformations import ( + dynamic_creation_input, +) from policyengine_api.services.v2.policies.database_connectors.creates import ( create_legacy_policy_mapping, create_parameter_values, @@ -23,7 +26,7 @@ read_policy_catalog, read_policy_row, read_policy_rows, - read_version_parameter_ids, + read_version_parameter_names, ) from policyengine_api.services.v2.policies.transformations import ( canonical_policy_document, @@ -33,6 +36,7 @@ policy_page, policy_parameter_values_by_policy, policy_read, + split_legacy_policy, stored_policy_creation_input, translate_legacy_policy, ) @@ -40,6 +44,7 @@ CanonicalPolicyContent, CheckedPolicyPage, CheckedPolicyRead, + LegacyPolicyMappingInput, LegacyPolicyPersistenceResult, LegacyPolicySnapshot, NativePolicyCreation, @@ -55,6 +60,7 @@ PolicyContentHashCollisionError, PolicyCreationIntegrityError, PolicyNotFoundError, + refuse_dynamic_parameters, validate_policy_catalog, verify_legacy_policy_mapping, ) @@ -80,7 +86,8 @@ def resolve_policy_creation_input( """Bind policy content to the selected catalog's model version. Raise PolicyCatalogValidationError if the model or any parameter id is not - part of the selected catalog. + part of the selected catalog, and PolicySetsDynamicParametersError if any + parameter is one only a dynamic may hold. """ selected = read_policy_catalog( session, @@ -91,16 +98,18 @@ def resolve_policy_creation_input( requested_parameter_ids = { value.parameter_id for value in policy_input.parameter_values } - resolved_parameter_ids = read_version_parameter_ids( + parameter_names = read_version_parameter_names( session, model_version_id=selected.model_version.id, requested_ids=requested_parameter_ids, ) - return validate_policy_catalog( + resolved = validate_policy_catalog( policy_input, selected=selected, - resolved_parameter_ids=resolved_parameter_ids, + resolved_parameter_ids=set(parameter_names), ) + refuse_dynamic_parameters(policy_input.country_id, parameter_names.values()) + return resolved def create_resolved_policy( @@ -245,12 +254,22 @@ def mirror_legacy_policy_in_session( """Copy one committed v1 policy into v2 and map its v1 id to the result. The policy is translated against the running catalog, whichever country - package version it was saved under. A v1 policy that already has a v2 copy - returns that copy without being translated again, so writing it again after - a PolicyEngine.py or country package upgrade still succeeds. Raise + package version it was saved under. Its model assumptions (every + ``gov.simulation.`` value, and the US's ``simulation.``) are copied into + a dynamic, with the UK's old labour-supply spelling rewritten, and the + rest into the policy, which is empty when nothing else remains; the + mapping names both. A v1 policy that sets no assumptions maps to no + dynamic. A v1 policy that already has a v2 copy returns that copy without + being translated again, so writing it again after a PolicyEngine.py or + country package upgrade still succeeds; a copy made before the split + existed keeps its assumptions in its policy and maps to no dynamic. Raise LegacyPolicyMappingIntegrityError if the v1 id is already mapped with a different source hash. """ + from policyengine_api.services.v2.dynamics.services import ( + create_resolved_dynamic, + ) + existing = read_legacy_policy_mapping( session, country_id=snapshot.country_id, @@ -267,6 +286,7 @@ def mirror_legacy_policy_in_session( policy_id=existing.policy_id, policy_created=False, mapping_created=False, + dynamic_id=existing.dynamic_id, ) selected = read_policy_catalog( @@ -274,33 +294,43 @@ def mirror_legacy_policy_in_session( snapshot.country_id, running_policyengine_version=running_policyengine_version, ) - policy_json = snapshot.policy_json - if not isinstance(policy_json, dict): - msg = "legacy policy_json must be an object" - raise TypeError(msg) + split = split_legacy_policy(snapshot) parameters = read_parameters_by_name( session, model_version_id=selected.model_version.id, - names=set(policy_json), + names={*split.policy_values, *split.dynamic_values}, ) - policy_input = translate_legacy_policy( + translation = translate_legacy_policy( snapshot, selected=selected, parameters=parameters, ) - policy_result = create_resolved_policy(session, policy_input) + policy_result = create_resolved_policy(session, translation.policy) + dynamic_result = ( + None + if translation.dynamic is None + else create_resolved_dynamic( + session, dynamic_creation_input(translation.dynamic) + ) + ) + dynamic_id = None if dynamic_result is None else dynamic_result.dynamic_id mapping_id = create_legacy_policy_mapping( session, - country_id=snapshot.country_id, - legacy_policy_id=snapshot.legacy_policy_id, - source_policy_hash=snapshot.source_policy_hash, - policy_id=policy_result.policy_id, + LegacyPolicyMappingInput( + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + source_policy_hash=snapshot.source_policy_hash, + policy_id=policy_result.policy_id, + dynamic_id=dynamic_id, + ), ) if mapping_id is not None: return LegacyPolicyPersistenceResult( policy_id=policy_result.policy_id, policy_created=policy_result.created, mapping_created=True, + dynamic_id=dynamic_id, + dynamic_created=dynamic_result is not None and dynamic_result.created, ) concurrent = read_legacy_policy_mapping( session, @@ -314,9 +344,9 @@ def mirror_legacy_policy_in_session( # A concurrent copy mapped the policy first. As with a mapping found at the # start, the same source hash makes its v2 policy the copy, even when it was # translated with another catalog version than this one. If this copy's - # translation differed, the v2 policy it inserted commits without a mapping, - # and the result's policy_created is false because it reports the mapped - # policy, not that one. + # translation differed, the v2 policy and dynamic it inserted commit + # without a mapping, and the result's policy_created and dynamic_created + # are false because it reports the mapped ones, not those. verify_legacy_policy_mapping( concurrent, source_policy_hash=snapshot.source_policy_hash ) @@ -324,6 +354,7 @@ def mirror_legacy_policy_in_session( policy_id=concurrent.policy_id, policy_created=False, mapping_created=False, + dynamic_id=concurrent.dynamic_id, ) diff --git a/policyengine_api/services/v2/policies/transformations.py b/policyengine_api/services/v2/policies/transformations.py index 799fac982..2259f334b 100644 --- a/policyengine_api/services/v2/policies/transformations.py +++ b/policyengine_api/services/v2/policies/transformations.py @@ -13,10 +13,16 @@ period as parse_policyengine_period, ) +from policyengine_api.dynamic_parameters import ( + ConflictingParameterSpellingsError, + LegacyPolicySplit, + split_legacy_policy_json, +) from policyengine_api.services.v2.policies.types import ( CanonicalPolicyContent, CheckedPolicyRead, LegacyPolicySnapshot, + LegacyPolicyTranslation, LegacyPolicyTranslationReason, PolicyCreationInput, PolicyPage, @@ -40,11 +46,13 @@ SelectedCatalog, ) from policyengine_api.data.v2.models import ( + Dynamic, Parameter, ParameterValue, Policy, TaxBenefitModelVersion, ) + from policyengine_api.supported_countries import SupportedCountryId POLICY_CANONICALIZATION_VERSION = 1 # A v1 date-range period key is "start.end": two dates joined by one dot. @@ -139,11 +147,11 @@ def canonicalize_policy( def stored_policy_creation_input( - policy: Policy, + policy: Policy | Dynamic, model_version: TaxBenefitModelVersion, values: list[ParameterValue], ) -> ResolvedPolicyCreationInput: - """Rebuild a stored policy's input so its canonical bytes can be compared.""" + """Rebuild a stored policy's or dynamic's input to compare its canonical bytes.""" return ResolvedPolicyCreationInput.model_validate( { "country_id": policy.country_id, @@ -296,36 +304,43 @@ def parse_legacy_period(value: str) -> tuple[datetime, datetime]: return start_date, end_date -def translate_legacy_policy( - snapshot: LegacyPolicySnapshot, - *, - selected: SelectedCatalog, - parameters: Mapping[str, Parameter], -) -> ResolvedPolicyCreationInput: - """Translate a v1 policy into policy content bound to the running catalog. - - The country package version the v1 policy was saved under is not compared: - the Public API simulates an old policy with the running package, so the copy - binds it to the running catalog the same way. +def split_legacy_policy(snapshot: LegacyPolicySnapshot) -> LegacyPolicySplit: + """Split a v1 policy's values into the policy's and its model assumptions. - ``parameters`` holds the running catalog's parameters with the policy's - parameter paths, keyed by path. Raise LegacyPolicyTranslationError, with a - fixed reason, if the v1 policy sets a parameter the catalog lacks, sets no - value for a parameter, or has invalid or conflicting periods. + The split is ``split_legacy_policy_json``'s, which also rewrites the UK's + old labour-supply spelling. Raise LegacyPolicyTranslationError, with + reason ``conflicting_values``, when a UK policy sets one parameter under + both spellings and they disagree. """ policy_json = snapshot.policy_json if not isinstance(policy_json, dict): msg = "legacy policy_json must be an object" raise TypeError(msg) - if parameters_missing_from_model(policy_json, parameters): - msg = "every legacy parameter path must exist in the running catalog" + try: + return split_legacy_policy_json(snapshot.country_id, policy_json) + except ConflictingParameterSpellingsError as error: + msg = "legacy parameter spellings conflict" raise LegacyPolicyTranslationError( - msg, reason=LegacyPolicyTranslationReason.MISSING_PARAMETERS - ) + msg, reason=LegacyPolicyTranslationReason.CONFLICTING_VALUES + ) from error + + +def _translate_legacy_values( + country_id: SupportedCountryId, + values_by_name: Mapping[str, object], + *, + selected: SelectedCatalog, + parameters: Mapping[str, Parameter], +) -> ResolvedPolicyCreationInput: + """Translate one part of a v1 policy's values into content bound to the catalog. + Every name in ``values_by_name`` is in ``parameters``. Raise + LegacyPolicyTranslationError for values that are not period objects, + invalid or conflicting periods, or more values than v2 content may hold. + """ parameter_values: list[PolicyParameterValueInput] = [] - for parameter_name in sorted(policy_json): - period_values = policy_json[parameter_name] + for parameter_name in sorted(values_by_name): + period_values = values_by_name[parameter_name] if type(period_values) is not dict: msg = ( f"legacy parameter {parameter_name!r} must contain period/value entries" @@ -354,13 +369,13 @@ def translate_legacy_policy( ) ) if len(parameter_values) > MAXIMUM_POLICY_PARAMETER_VALUES: - msg = "legacy policy has more values than a v2 policy may hold" + msg = "legacy policy has more values than a v2 policy or dynamic may hold" raise LegacyPolicyTranslationError( msg, reason=LegacyPolicyTranslationReason.TOO_MANY_VALUES ) try: - policy_input = PolicyCreationInput( - country_id=snapshot.country_id, + content = PolicyCreationInput( + country_id=country_id, tax_benefit_model_id=selected.model.id, parameter_values=parameter_values, ) @@ -372,7 +387,56 @@ def translate_legacy_policy( msg, reason=LegacyPolicyTranslationReason.CONFLICTING_VALUES ) from error return validate_policy_catalog( - policy_input, + content, + selected=selected, + resolved_parameter_ids={parameters[name].id for name in values_by_name}, + ) + + +def translate_legacy_policy( + snapshot: LegacyPolicySnapshot, + *, + selected: SelectedCatalog, + parameters: Mapping[str, Parameter], +) -> LegacyPolicyTranslation: + """Translate a v1 policy into a policy and a dynamic bound to the running catalog. + + The v1 policy's model assumptions (``policyengine_api.dynamic_parameters``) + become the dynamic's content, None when it sets none, and the rest the + policy's, which may be empty. The UK's old labour-supply spelling is + rewritten first (``split_legacy_policy``). + + The country package version the v1 policy was saved under is not compared: + the Public API simulates an old policy with the running package, so the copy + binds it to the running catalog the same way. + + ``parameters`` holds the running catalog's parameters with the split's + parameter names, keyed by name. Raise LegacyPolicyTranslationError, with a + fixed reason, if the v1 policy sets a parameter the catalog lacks, sets no + value for a parameter, or has invalid or conflicting periods or spellings. + """ + split = split_legacy_policy(snapshot) + if parameters_missing_from_model( + [*split.policy_values, *split.dynamic_values], parameters + ): + msg = "every legacy parameter path must exist in the running catalog" + raise LegacyPolicyTranslationError( + msg, reason=LegacyPolicyTranslationReason.MISSING_PARAMETERS + ) + policy = _translate_legacy_values( + snapshot.country_id, + split.policy_values, selected=selected, - resolved_parameter_ids={parameter.id for parameter in parameters.values()}, + parameters=parameters, + ) + dynamic = ( + _translate_legacy_values( + snapshot.country_id, + split.dynamic_values, + selected=selected, + parameters=parameters, + ) + if split.dynamic_values + else None ) + return LegacyPolicyTranslation(policy=policy, dynamic=dynamic) diff --git a/policyengine_api/services/v2/policies/types.py b/policyengine_api/services/v2/policies/types.py index d5ef827d1..80d9aa8c6 100644 --- a/policyengine_api/services/v2/policies/types.py +++ b/policyengine_api/services/v2/policies/types.py @@ -66,15 +66,22 @@ def validate_period(self) -> PolicyParameterValueInput: return self +# A policy's or a dynamic's values, at most MAXIMUM_POLICY_PARAMETER_VALUES. +ParameterValueInputs = Annotated[ + list[PolicyParameterValueInput], + Field(max_length=MAXIMUM_POLICY_PARAMETER_VALUES), +] + + class PolicyCreationInput(StrictPolicyInput): - """Complete immutable policy content accepted from any source.""" + """Complete immutable policy content accepted from any source. + + A dynamic's content has the same fields (``dynamics.types``). + """ country_id: CountryId tax_benefit_model_id: UUID - parameter_values: Annotated[ - list[PolicyParameterValueInput], - Field(max_length=MAXIMUM_POLICY_PARAMETER_VALUES), - ] + parameter_values: ParameterValueInputs @field_validator("parameter_values") @classmethod @@ -143,13 +150,44 @@ class PolicyCreationResult: created: bool +@dataclass(frozen=True) +class LegacyPolicyTranslation: + """A v1 policy translated against the running catalog, split in two. + + ``policy`` holds the values that stay in the policy, possibly none. + ``dynamic`` holds the model's assumptions, the content of the dynamic + the copy stores, or is None when the v1 policy sets none. + """ + + policy: ResolvedPolicyCreationInput + dynamic: ResolvedPolicyCreationInput | None + + +@dataclass(frozen=True) +class LegacyPolicyMappingInput: + """The mapping a copy records from a v1 policy to its v2 policy and dynamic.""" + + country_id: CountryId + legacy_policy_id: int + source_policy_hash: str + policy_id: UUID + dynamic_id: UUID | None + + @dataclass(frozen=True) class LegacyPolicyPersistenceResult: - """Destination identity and insertion outcomes for one mirror attempt.""" + """Destination identity and insertion outcomes for one mirror attempt. + + ``dynamic_id`` is the dynamic the v1 policy's model assumptions were + copied into, None when it sets none or was copied before the split + existed; ``dynamic_created`` says whether this attempt inserted it. + """ policy_id: UUID policy_created: bool mapping_created: bool + dynamic_id: UUID | None = None + dynamic_created: bool = False @dataclass(frozen=True) diff --git a/policyengine_api/services/v2/policies/validators.py b/policyengine_api/services/v2/policies/validators.py index 05786fdb7..cb1145832 100644 --- a/policyengine_api/services/v2/policies/validators.py +++ b/policyengine_api/services/v2/policies/validators.py @@ -6,6 +6,8 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Any +from policyengine_api.dynamic_parameters import is_dynamic_parameter + if TYPE_CHECKING: from collections.abc import Container, Iterable from uuid import UUID @@ -19,6 +21,7 @@ PolicyCreationInput, ResolvedPolicyCreationInput, ) + from policyengine_api.supported_countries import SupportedCountryId MAXIMUM_POLICY_PARAMETER_VALUES = 1_000 @@ -58,6 +61,43 @@ def __init__(self, message: str, *, reason: LegacyPolicyTranslationReason) -> No self.reason = reason +class PolicySetsDynamicParametersError(ValueError): + """Raised when a policy sets parameters only a dynamic may hold. + + These are the model's assumptions, every parameter under + ``gov.simulation.`` plus the US's ``simulation.`` + (``policyengine_api.dynamic_parameters``), sent as a dynamic through + ``POST /v2/dynamics``. ``parameter_names`` lists them, sorted. + """ + + def __init__(self, parameter_names: tuple[str, ...]) -> None: + """Name each parameter the policy may not set.""" + self.parameter_names = parameter_names + super().__init__(parameter_names) + + def __str__(self) -> str: + """Say which parameters to send as a dynamic, and where.""" + return ( + "A policy cannot set the model's assumptions, the parameters under " + "gov.simulation (and, in the US, simulation): send them as a " + "dynamic through POST /v2/dynamics. This policy sets: " + + ", ".join(self.parameter_names) + ) + + +def refuse_dynamic_parameters( + country_id: SupportedCountryId, parameter_names: Iterable[str] +) -> None: + """Raise PolicySetsDynamicParametersError if any parameter belongs in a dynamic.""" + dynamic_parameter_names = tuple( + sorted( + {name for name in parameter_names if is_dynamic_parameter(country_id, name)} + ) + ) + if dynamic_parameter_names: + raise PolicySetsDynamicParametersError(dynamic_parameter_names) + + class PolicyParametersMissingError(ValueError): """Raised when a policy sets parameters the running catalog version lacks. diff --git a/policyengine_api/services/v2/simulation_service/services.py b/policyengine_api/services/v2/simulation_service/services.py index 6b1540bb6..f363d6d00 100644 --- a/policyengine_api/services/v2/simulation_service/services.py +++ b/policyengine_api/services/v2/simulation_service/services.py @@ -8,10 +8,14 @@ country does not), that each policy the submission names is stored in the country, through the same service as ``/v2/policies`` (current law names none), and that none of those policies sets a parameter the running catalog -version lacks (``require_runnable_policies``, which re-runs call too). -Only then is the simulation service asked. The dynamics a submission names -and its region are checked by the simulation service, which reads them, and -the region's dataset, in the catalog version its default worker runs. +version lacks (``require_runnable_policies``, which re-runs call too). An +economy simulation, economy report or budget window then has each dynamic it +names checked the same way, through the same service as ``/v2/dynamics`` +(none means no assumptions beyond the model's defaults): +``require_runnable_dynamics``. +Only then is the simulation service asked. The submission's region is +checked by the simulation service, which reads it, and the region's dataset, +in the catalog version its default worker runs. A household report is checked the same way, except that its country's row must allow household calculations instead (a household-only country passes), @@ -22,7 +26,9 @@ Running a stored report again (``POST /reports/{report_id}/runs``) checks its country as a new report of its kind is checked, then the policies its -simulations name. +simulations name. Its dynamics are not checked again: each was checked to be +the country's when the report was created, and the simulation service refuses +a re-run whose dynamic sets parameters its catalog version lacks. """ from __future__ import annotations @@ -35,6 +41,10 @@ from policyengine_api.data.v2.catalog.catalog_selection_validation import ( MetadataCatalogUnavailableError, ) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicStorageError, + require_dynamic_parameters_present, +) from policyengine_api.services.v2.household_calculations.validators import ( require_default_year, require_household_country, @@ -58,6 +68,7 @@ from policyengine_api.services.v2.simulation_service.types import ( BudgetWindowReportCreationInput, CountrySource, + DynamicSource, EconomyReportCreationInput, EconomySimulationCreationInput, HouseholdReportCreationInput, @@ -79,12 +90,17 @@ def _storage_failures() -> Iterator[None]: """Raise SubmissionStorageError for a read that failed. - That includes the policy check's catalog query, which the policy service - reports as MetadataCatalogUnavailableError. + That includes the policy and dynamic checks' catalog query, which their + services report as MetadataCatalogUnavailableError, and the dynamic + service's DynamicStorageError. """ try: yield - except (SQLAlchemyError, MetadataCatalogUnavailableError) as error: + except ( + SQLAlchemyError, + MetadataCatalogUnavailableError, + DynamicStorageError, + ) as error: msg = "V2 storage is unavailable to check the submission" raise SubmissionStorageError(msg) from error @@ -113,6 +129,28 @@ def require_runnable_policies( require_policy_parameters_present(policy.id, policy.missing_parameters) +def require_runnable_dynamics( + dynamics: DynamicSource, *, country_id: str, dynamic_ids: Iterable[UUID | None] +) -> None: + """Raise unless each named dynamic is the country's and its parameters all exist. + + As for policies, every dynamic is read before any is checked, so + DynamicNotFoundError for a dynamic the country does not store comes + before DynamicParametersMissingError for one that sets parameters the + running catalog version lacks. None names no dynamic, and a dynamic the + running catalog could not check passes. Raise SubmissionStorageError when + the database or its catalog cannot be read. + """ + with _storage_failures(): + stored_dynamics = [ + dynamics.get_dynamic(country_id=country_id, dynamic_id=dynamic_id) + for dynamic_id in dict.fromkeys(dynamic_ids) + if dynamic_id is not None + ] + for dynamic in stored_dynamics: + require_dynamic_parameters_present(dynamic.id, dynamic.missing_parameters) + + class SimulationSubmissionService: """Check submissions, then send them to the simulation service.""" @@ -122,33 +160,37 @@ def __init__( countries: CountrySource, households: HouseholdSource, policies: PolicySource, + dynamics: DynamicSource, client_factory: Callable[[], SimulationServiceClient], ) -> None: """Check through these sources; send through ``client_factory``'s client. - ``households`` is read only for a household report. The client is - built only once a submission passes the checks, so a refused - submission gets its own answer even when the simulation service is - not configured. + ``households`` is read only for a household report, and ``dynamics`` + only for a submission that names a dynamic. The client is built only + once a submission passes the checks, so a refused submission gets its + own answer even when the simulation service is not configured. """ self._countries = countries self._households = households self._policies = policies + self._dynamics = dynamics self._client_factory = client_factory def _check( self, country_id: str, policy_ids: Iterable[UUID | None], + dynamic_ids: Iterable[UUID | None] = (), *, require_country: CountryCheck = require_economy_country, ) -> None: - """Raise unless the country passes ``require_country`` and the policies can run. + """Raise unless the country passes ``require_country`` and all can run. By default the country must have economy calculations. Raise the country check's error (EconomyCalculationsUnsupportedError by default), then PolicyNotFoundError, then PolicyParametersMissingError, - and SubmissionStorageError when the database or its catalog cannot be + then DynamicNotFoundError, then DynamicParametersMissingError, and + SubmissionStorageError when the database or its catalog cannot be read. """ with _storage_failures(): @@ -156,6 +198,9 @@ def _check( require_runnable_policies( self._policies, country_id=country_id, policy_ids=policy_ids ) + require_runnable_dynamics( + self._dynamics, country_id=country_id, dynamic_ids=dynamic_ids + ) def submit_simulation( self, simulation: EconomySimulationCreationInput @@ -164,18 +209,22 @@ def submit_simulation( Raise the checks' errors, then the simulation service's. """ - self._check(simulation.country_id, (simulation.policy_id,)) + self._check( + simulation.country_id, (simulation.policy_id,), (simulation.dynamic_id,) + ) return self._client_factory().submit_simulation(simulation) def submit_report(self, report: EconomyReportCreationInput) -> SubmittedReport: """Check the report, then have the simulation service create and start it. Both the baseline's and the reform's policies are checked, the - baseline's first. Raise the checks' errors, then the simulation - service's. + baseline's first, then their dynamics in the same order. Raise the + checks' errors, then the simulation service's. """ self._check( - report.country_id, (report.baseline.policy_id, report.reform.policy_id) + report.country_id, + (report.baseline.policy_id, report.reform.policy_id), + (report.baseline.dynamic_id, report.reform.dynamic_id), ) return self._client_factory().submit_report(report) @@ -218,7 +267,9 @@ def submit_budget_window( simulation service's. """ self._check( - report.country_id, (report.baseline.policy_id, report.reform.policy_id) + report.country_id, + (report.baseline.policy_id, report.reform.policy_id), + (report.baseline.dynamic_id, report.reform.dynamic_id), ) return self._client_factory().submit_budget_window(report) diff --git a/policyengine_api/services/v2/simulation_service/types.py b/policyengine_api/services/v2/simulation_service/types.py index 4691ba88e..6d7f674bb 100644 --- a/policyengine_api/services/v2/simulation_service/types.py +++ b/policyengine_api/services/v2/simulation_service/types.py @@ -42,6 +42,7 @@ from uuid import UUID from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.dynamics.types import CheckedDynamicRead from policyengine_api.services.v2.policies.types import CheckedPolicyRead # The bounds the simulation service and the tables accept. @@ -470,7 +471,8 @@ class SimulationServiceClient(Protocol): it through a client, also raises EconomyCalculationsUnsupportedError, HouseholdCalculationCountryError, HouseholdNotFoundError, HouseholdCalculationYearError, PolicyNotFoundError, - PolicyParametersMissingError and + PolicyParametersMissingError, DynamicNotFoundError, + DynamicParametersMissingError and SubmissionStorageError from its checks. """ @@ -527,3 +529,11 @@ class PolicySource(Protocol): def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicyRead: """Return the country's policy, or raise ``PolicyNotFoundError``.""" ... + + +class DynamicSource(Protocol): + """The stored dynamics a submission names.""" + + def get_dynamic(self, *, country_id: str, dynamic_id: UUID) -> CheckedDynamicRead: + """Return the country's dynamic, or raise ``DynamicNotFoundError``.""" + ... diff --git a/policyengine_api/supported_countries.py b/policyengine_api/supported_countries.py index 70dcce58a..2efe6d226 100644 --- a/policyengine_api/supported_countries.py +++ b/policyengine_api/supported_countries.py @@ -24,6 +24,10 @@ v1 ``household_mirror_events`` table, which the v1 household insert writes in its own transaction. +A new country also needs its dynamic parameter prefixes, the parameters +only a dynamic may hold (``DYNAMIC_PARAMETER_PREFIXES`` in +``policyengine_api.dynamic_parameters``); a unit test fails until it has them. + Catalog publication also needs the country's checked-in region reference file (``policyengine_api/data/v2/catalog/region_reference_files``) and its region scoping rules (``REGION_SCOPING_RULES`` in diff --git a/pyproject.toml b/pyproject.toml index e55f43fc8..038b79298 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -402,6 +402,8 @@ banned-module-level-imports = [ "policyengine_api.libs.simulation_reports", "policyengine_api.services.v2.countries.database_session", "policyengine_api.services.v2.countries.services", + "policyengine_api.services.v2.dynamics.database_session", + "policyengine_api.services.v2.dynamics.services", "policyengine_api.services.v2.household_calculations.database_session", "policyengine_api.services.v2.household_calculations.services", "policyengine_api.services.v2.households.database_session", diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 75b3220fa..0bf798576 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -211,6 +211,47 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="dynamic_resources_v2", + current_contract="typed_v2_resources", + future_owner_pr="Dynamics stored like policies", + requests=( + ContractRequest( + method="POST", + path="/v2/dynamics?country_id=us", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + "result.item.missing_parameters", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/v2/dynamics/{dynamic_id}?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + "result.item.missing_parameters", + ), + route_group="policy", + ), + ), + ), WorkflowContract( name="user_policy_associations_v2", current_contract="typed_v2_resources", diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py index 8642f0542..cfe8c896d 100644 --- a/tests/contract/test_app_v2_workflow_contracts.py +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -10,6 +10,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): assert {workflow.name for workflow in APP_V2_WORKFLOW_CONTRACTS} == { "policy_save_search", "policy_resources_v2", + "dynamic_resources_v2", "user_policy_associations_v2", "household_save_read", "household_resources_v2", @@ -37,6 +38,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "detailed_save_reports_v2", "metadata_resources_v2_preview", "policy_resources_v2", + "dynamic_resources_v2", "user_policy_associations_v2", "household_resources_v2", "household_calculations_v2", diff --git a/tests/integration/test_alembic_v2_dynamics.py b/tests/integration/test_alembic_v2_dynamics.py new file mode 100644 index 000000000..3da55d7cf --- /dev/null +++ b/tests/integration/test_alembic_v2_dynamics.py @@ -0,0 +1,339 @@ +"""Check the revision that stores dynamics like policies against disposable Postgres. + +The revision gives ``dynamics`` policies' columns and drops ``name`` and +``description``, adds a partial unique index on a dynamic's values, and adds +``legacy_policy_mappings.dynamic_id``, a key to a dynamic of the mapping's +country. Nothing else changes. Its new required columns have no default, so +the upgrade fails while ``dynamics`` has a row, and the downgrade, which adds +``name`` back as required, fails the same way. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import pytest +from alembic import command +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.exc import IntegrityError + +from tests.fixtures.integration.alembic_v2 import alembic_v2_config, disposable_v2_url +from tests.integration.simulation_rows import ( + PublicApiRows, + delete_public_api_rows, + insert_public_api_rows, + insert_row, + refused, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Connection, Engine + +PREVIOUS_REVISION = "f519a77ba122" +DYNAMICS_REVISION = "9d1d12a7ac58" +CHANGED_TABLES = frozenset({"dynamics", "legacy_policy_mappings", "parameter_values"}) + + +@pytest.fixture +def engine() -> Iterator[Engine]: + """Yield an engine on a database at head, and leave it at head.""" + command.upgrade(alembic_v2_config(), "head") + database_engine = create_engine(disposable_v2_url()) + try: + yield database_engine + finally: + command.upgrade(alembic_v2_config(), "head") + database_engine.dispose() + + +@pytest.fixture +def connection(engine: Engine) -> Iterator[Connection]: + """Yield a connection at head whose writes are rolled back after the test.""" + with engine.connect() as database_connection: + transaction = database_connection.begin() + try: + yield database_connection + finally: + transaction.rollback() + + +def _schema(engine: Engine) -> dict[str, dict[str, Any]]: + """Return each table's columns, keys, checks and indexes, comparably.""" + inspector = inspect(engine) + return { + table_name: { + "columns": sorted( + (column["name"], str(column["type"]), column["nullable"]) + for column in inspector.get_columns(table_name) + ), + "foreign_keys": sorted( + ( + str(key["name"]), + tuple(key["constrained_columns"]), + key["referred_table"], + tuple(key["referred_columns"]), + key["options"].get("ondelete"), + ) + for key in inspector.get_foreign_keys(table_name) + ), + "unique": sorted( + (str(constraint["name"]), tuple(constraint["column_names"])) + for constraint in inspector.get_unique_constraints(table_name) + ), + "checks": sorted( + (str(check["name"]), check["sqltext"]) + for check in inspector.get_check_constraints(table_name) + ), + "indexes": sorted( + ( + str(index["name"]), + tuple(str(column) for column in index["column_names"]), + bool(index["unique"]), + str(index.get("dialect_options", {}).get("postgresql_where")), + ) + for index in inspector.get_indexes(table_name) + ), + } + for table_name in inspector.get_table_names() + } + + +def _current_revision(engine: Engine) -> str: + with engine.connect() as connection: + return str( + connection.execute( + text("SELECT version_num FROM alembic_version") + ).scalar_one() + ) + + +def _insert_dynamic( + connection: Connection, + rows: PublicApiRows, + *, + content_hash: str = "d" * 64, + country_id: str = "us", +) -> object: + dynamic_id = uuid4() + insert_row( + connection, + "dynamics", + id=dynamic_id, + country_id=country_id, + tax_benefit_model_id=rows.model_id, + tax_benefit_model_version_id=rows.version_id, + canonicalization_version=1, + content_hash=content_hash, + ) + return dynamic_id + + +def _insert_parameter(connection: Connection, rows: PublicApiRows) -> object: + parameter_id = uuid4() + insert_row( + connection, + "parameters", + id=parameter_id, + name="gov.simulation.labor_supply_responses.elasticities.income", + tax_benefit_model_version_id=rows.version_id, + ) + return parameter_id + + +def _insert_policy( + connection: Connection, rows: PublicApiRows, *, country_id: str +) -> object: + policy_id = uuid4() + insert_row( + connection, + "policies", + id=policy_id, + country_id=country_id, + tax_benefit_model_id=rows.model_id, + tax_benefit_model_version_id=rows.version_id, + canonicalization_version=1, + content_hash=uuid4().hex * 2, + ) + return policy_id + + +def test_the_revision_changes_only_dynamics_their_values_and_mappings( + engine: Engine, +) -> None: + config = alembic_v2_config() + command.downgrade(config, PREVIOUS_REVISION) + before = _schema(engine) + command.upgrade(config, DYNAMICS_REVISION) + after = _schema(engine) + + assert {name: before[name] for name in before if name not in CHANGED_TABLES} == { + name: after[name] for name in after if name not in CHANGED_TABLES + } + assert {column[0] for column in after["dynamics"]["columns"]} == { + column[0] for column in after["policies"]["columns"] + } + assert {column[0] for column in before["dynamics"]["columns"]} == { + "id", + "created_at", + "updated_at", + "name", + "description", + } + assert set(after["legacy_policy_mappings"]["columns"]) - set( + before["legacy_policy_mappings"]["columns"] + ) == {("dynamic_id", "UUID", True)} + assert set(after["legacy_policy_mappings"]["foreign_keys"]) - set( + before["legacy_policy_mappings"]["foreign_keys"] + ) == { + ( + "fk_legacy_policy_mappings_dynamic_country", + ("dynamic_id", "country_id"), + "dynamics", + ("id", "country_id"), + "RESTRICT", + ) + } + assert set(after["parameter_values"]["indexes"]) - set( + before["parameter_values"]["indexes"] + ) == { + ( + "uq_parameter_values_dynamic_parameter_start_date", + ("dynamic_id", "parameter_id", "start_date"), + True, + "(dynamic_id IS NOT NULL)", + ) + } + assert { + key: value + for key, value in after["parameter_values"].items() + if key != "indexes" + } == { + key: value + for key, value in before["parameter_values"].items() + if key != "indexes" + } + + command.downgrade(config, PREVIOUS_REVISION) + assert _schema(engine) == before + command.upgrade(config, "head") + command.check(config) + + +def test_a_dynamic_is_stored_once_and_sets_one_value_per_parameter_and_date( + connection: Connection, +) -> None: + rows = insert_public_api_rows(connection) + parameter_id = _insert_parameter(connection, rows) + dynamic_id = _insert_dynamic(connection, rows) + + with refused(connection, match="uq_dynamics_canonicalization_content_hash"): + _insert_dynamic(connection, rows) + with refused(connection, match="ck_dynamics_content_hash_length"): + _insert_dynamic(connection, rows, content_hash="short") + + value = { + "parameter_id": parameter_id, + "dynamic_id": dynamic_id, + "value_json": "0.1", + "start_date": "2026-01-01T00:00:00Z", + } + insert_row(connection, "parameter_values", id=uuid4(), **value) + with refused(connection, match="uq_parameter_values_dynamic_parameter_start_date"): + insert_row(connection, "parameter_values", id=uuid4(), **value) + # Another dynamic may set the same parameter from the same date. + other_dynamic_id = _insert_dynamic(connection, rows, content_hash="e" * 64) + insert_row( + connection, + "parameter_values", + id=uuid4(), + **{**value, "dynamic_id": other_dynamic_id}, + ) + + +def test_a_mapping_names_a_dynamic_of_its_country_which_stays_while_named( + connection: Connection, +) -> None: + rows = insert_public_api_rows(connection) + us_dynamic_id = _insert_dynamic(connection, rows) + us_policy_id = _insert_policy(connection, rows, country_id="us") + uk_policy_id = _insert_policy(connection, rows, country_id="uk") + + with refused(connection, match="fk_legacy_policy_mappings_dynamic_country"): + insert_row( + connection, + "legacy_policy_mappings", + id=uuid4(), + country_id="uk", + legacy_policy_id=1, + policy_id=uk_policy_id, + dynamic_id=us_dynamic_id, + source_policy_hash="v1-hash", + ) + insert_row( + connection, + "legacy_policy_mappings", + id=uuid4(), + country_id="us", + legacy_policy_id=1, + policy_id=us_policy_id, + dynamic_id=us_dynamic_id, + source_policy_hash="v1-hash", + ) + # A mapping without a dynamic is unaffected. + insert_row( + connection, + "legacy_policy_mappings", + id=uuid4(), + country_id="us", + legacy_policy_id=2, + policy_id=us_policy_id, + source_policy_hash="v1-hash-2", + ) + with refused(connection, match="fk_legacy_policy_mappings_dynamic_country"): + connection.execute( + text("DELETE FROM dynamics WHERE id = :id"), {"id": us_dynamic_id} + ) + + +def test_the_upgrade_fails_changing_nothing_while_a_dynamic_is_stored( + engine: Engine, +) -> None: + config = alembic_v2_config() + command.downgrade(config, PREVIOUS_REVISION) + dynamic_id = uuid4() + with engine.begin() as connection: + insert_row(connection, "dynamics", id=dynamic_id, name="stored before") + try: + with pytest.raises(IntegrityError, match="country_id"): + command.upgrade(config, DYNAMICS_REVISION) + assert _current_revision(engine) == PREVIOUS_REVISION + dynamics_columns = inspect(engine).get_columns("dynamics") + assert "name" in {column["name"] for column in dynamics_columns} + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM dynamics WHERE id = :id"), {"id": dynamic_id} + ) + + +def test_the_downgrade_fails_changing_nothing_while_a_dynamic_is_stored( + engine: Engine, +) -> None: + config = alembic_v2_config() + command.upgrade(config, DYNAMICS_REVISION) + with engine.begin() as connection: + rows = insert_public_api_rows(connection) + dynamic_id = _insert_dynamic(connection, rows) + try: + with pytest.raises(IntegrityError, match="name"): + command.downgrade(config, PREVIOUS_REVISION) + assert _current_revision(engine) == DYNAMICS_REVISION + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM dynamics WHERE id = :id"), {"id": dynamic_id} + ) + delete_public_api_rows(connection, rows) diff --git a/tests/integration/test_v2_budget_window_report_creation.py b/tests/integration/test_v2_budget_window_report_creation.py index f1913b40f..25f8d1909 100644 --- a/tests/integration/test_v2_budget_window_report_creation.py +++ b/tests/integration/test_v2_budget_window_report_creation.py @@ -35,6 +35,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -157,6 +161,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) dependencies = NativeRouteDependencies( diff --git a/tests/integration/test_v2_catalog_publication_pruning.py b/tests/integration/test_v2_catalog_publication_pruning.py index 21f192827..075ebe715 100644 --- a/tests/integration/test_v2_catalog_publication_pruning.py +++ b/tests/integration/test_v2_catalog_publication_pruning.py @@ -75,7 +75,6 @@ US_MODEL = "policyengine-us" UK_MODEL = "policyengine-uk" MODEL_NAMES = (US_MODEL, UK_MODEL) -PROBE_DYNAMIC_NAME = "catalog-pruning-probe" # Each fixture catalog version holds, per country, two current-law values, # three parameter tree rows and one search file. FIXTURE_PRUNED_ROWS = {"parameter_values": 2, "tree_rows": 3, "search_files": 1} @@ -99,12 +98,9 @@ def _disposable_url() -> str: def _clear_catalog(engine: Engine) -> None: + # Policies and dynamics name a model, so the truncation removes them too. with engine.begin() as connection: connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) - connection.execute( - text("DELETE FROM dynamics WHERE name = :name"), - {"name": PROBE_DYNAMIC_NAME}, - ) @pytest.fixture @@ -267,7 +263,13 @@ def _add_saved_reform_and_dynamic(engine: Engine, version: str) -> None: canonicalization_version=1, content_hash=uuid4().hex * 2, ) - dynamic = Dynamic(name=PROBE_DYNAMIC_NAME) + dynamic = Dynamic( + country_id="us", + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=model_version.id, + canonicalization_version=1, + content_hash=uuid4().hex * 2, + ) session.add_all([policy, dynamic]) session.flush() session.add_all( diff --git a/tests/integration/test_v2_dynamic_persistence.py b/tests/integration/test_v2_dynamic_persistence.py new file mode 100644 index 000000000..34d4ce312 --- /dev/null +++ b/tests/integration/test_v2_dynamic_persistence.py @@ -0,0 +1,397 @@ +"""PostgreSQL tests for dynamics, created by ``/v2/dynamics`` and the v1-to-v2 copy. + +The services run as the Public API's runtime role, which proves its declared +privileges cover them (rule ``api.v2.runtime-privileges-tested-as-runtime-role``); +the catalog is written and everything removed as the superuser. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import create_engine, delete, or_, select +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + Dynamic, + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService +from policyengine_api.services.v2.dynamics.types import NativeDynamicCreationInput +from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError +from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, +) +from policyengine_api.services.v2.policies.services import V2PolicyService +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot + +if TYPE_CHECKING: + from collections.abc import Iterator + +RUNNING_VERSION = "5.2.0" +US_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" +US_MTR_DELTA = "simulation.marginal_tax_rate_delta" +US_TAX_RATE = "gov.irs.income.rate" +UK_INCOME_ELASTICITY = "gov.simulation.labour_supply_responses.income_elasticity" +UK_OLD_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.income_elasticity" +UK_TAX_RATE = "gov.hmrc.income_tax.rate" + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get(V2_ALEMBIC_DISPOSABLE_TEST, ""), + } + ) + if not settings.disposable_test: + pytest.fail("dynamic persistence tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +@dataclass(frozen=True) +class StoredCatalog: + """A country's running catalog version and its parameters' IDs by name.""" + + model_id: UUID + version_id: UUID + parameter_ids: dict[str, UUID] + + +def _store_catalog( + session: Session, country_id: str, names: list[str] +) -> StoredCatalog: + model = TaxBenefitModel(name=f"policyengine-{country_id}") + version = TaxBenefitModelVersion( + model=model, + version=RUNNING_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + parameters = [ + Parameter(name=name, tax_benefit_model_version=version) for name in names + ] + session.add_all(parameters) + session.flush() + return StoredCatalog( + model_id=model.id, + version_id=version.id, + parameter_ids={parameter.name: parameter.id for parameter in parameters}, + ) + + +def _remove_catalog(engine, catalog: StoredCatalog) -> None: + with engine.begin() as connection: + dynamic_ids = select(Dynamic.id).where( + Dynamic.tax_benefit_model_id == catalog.model_id + ) + policy_ids = select(Policy.id).where( + Policy.tax_benefit_model_id == catalog.model_id + ) + connection.execute( + delete(LegacyPolicyMapping).where( + or_( + LegacyPolicyMapping.policy_id.in_(policy_ids), + LegacyPolicyMapping.dynamic_id.in_(dynamic_ids), + ) + ) + ) + connection.execute( + delete(ParameterValue).where( + or_( + ParameterValue.policy_id.in_(policy_ids), + ParameterValue.dynamic_id.in_(dynamic_ids), + ) + ) + ) + connection.execute( + delete(Dynamic).where(Dynamic.tax_benefit_model_id == catalog.model_id) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == catalog.model_id) + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id == catalog.version_id + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.id == catalog.version_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == catalog.model_id) + ) + + +@pytest.fixture +def superuser_engine() -> Iterator[object]: + engine = create_engine(_disposable_url()) + try: + yield engine + finally: + engine.dispose() + + +@pytest.fixture +def runtime_sessions( + managed_v2_roles, v2_engine_as_role +) -> Iterator[sessionmaker[Session]]: + """Open sessions as the Public API's runtime role.""" + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime) + try: + yield sessionmaker(runtime_engine, class_=Session, expire_on_commit=False) + finally: + runtime_engine.dispose() + + +@pytest.fixture +def us_catalog(superuser_engine) -> Iterator[StoredCatalog]: + with Session(superuser_engine) as session, session.begin(): + catalog = _store_catalog( + session, "us", [US_INCOME_ELASTICITY, US_MTR_DELTA, US_TAX_RATE] + ) + try: + yield catalog + finally: + _remove_catalog(superuser_engine, catalog) + + +@pytest.fixture +def uk_catalog(superuser_engine) -> Iterator[StoredCatalog]: + with Session(superuser_engine) as session, session.begin(): + catalog = _store_catalog(session, "uk", [UK_INCOME_ELASTICITY, UK_TAX_RATE]) + try: + yield catalog + finally: + _remove_catalog(superuser_engine, catalog) + + +def _dynamic_service(sessions: sessionmaker[Session]) -> V2DynamicService: + return V2DynamicService( + DynamicDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ) + + +def _policy_service(sessions: sessionmaker[Session]) -> V2PolicyService: + return V2PolicyService( + PolicyDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ) + + +def _dynamic_input(catalog: StoredCatalog, value: float) -> NativeDynamicCreationInput: + return NativeDynamicCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model_id, + parameter_values=[ + { + "parameter_id": catalog.parameter_ids[US_INCOME_ELASTICITY], + "value": value, + "start_date": "2026-01-01T00:00:00Z", + }, + { + "parameter_id": catalog.parameter_ids[US_MTR_DELTA], + "value": 500, + "start_date": "2026-01-01T00:00:00Z", + }, + ], + ) + + +def _snapshot( + country_id: str, legacy_policy_id: int, policy_json: dict[str, object] +) -> LegacyPolicySnapshot: + return LegacyPolicySnapshot( + country_id=country_id, + legacy_policy_id=legacy_policy_id, + label=None, + api_version="1.0.0", + policy_json=policy_json, + source_policy_hash=f"v1-hash-{legacy_policy_id}", + ) + + +def _stored_names(session: Session, *, policy_id=None, dynamic_id=None) -> set[str]: + owner = ( + ParameterValue.policy_id == policy_id + if policy_id is not None + else ParameterValue.dynamic_id == dynamic_id + ) + return set( + session.scalars( + select(Parameter.name) + .join(ParameterValue, ParameterValue.parameter_id == Parameter.id) + .where(owner) + ) + ) + + +def test_identical_dynamic_content_is_stored_once_and_read_back( + runtime_sessions, us_catalog, superuser_engine +) -> None: + service = _dynamic_service(runtime_sessions) + + created = service.create_dynamic(_dynamic_input(us_catalog, 0.25)) + found = service.create_dynamic(_dynamic_input(us_catalog, 0.25)) + other = service.create_dynamic(_dynamic_input(us_catalog, 0.5)) + read = service.get_dynamic(country_id="us", dynamic_id=created.item.id) + + assert created.created is True + assert found.created is False + assert found.item.id == created.item.id + assert other.created is True + assert other.item.id != created.item.id + assert read.tax_benefit_model_version_id == us_catalog.version_id + assert read.missing_parameters == () + assert {(value.parameter_name, value.value) for value in read.parameter_values} == { + (US_INCOME_ELASTICITY, 0.25), + (US_MTR_DELTA, 500), + } + with Session(superuser_engine) as session: + stored = session.scalars( + select(Dynamic).where(Dynamic.tax_benefit_model_id == us_catalog.model_id) + ).all() + assert len(stored) == 2 + assert _stored_names(session, dynamic_id=created.item.id) == { + US_INCOME_ELASTICITY, + US_MTR_DELTA, + } + + +def test_a_dynamic_is_read_only_in_its_own_country( + runtime_sessions, us_catalog +) -> None: + service = _dynamic_service(runtime_sessions) + created = service.create_dynamic(_dynamic_input(us_catalog, 0.25)) + + with pytest.raises(DynamicNotFoundError): + service.get_dynamic(country_id="uk", dynamic_id=created.item.id) + with pytest.raises(DynamicNotFoundError): + service.get_dynamic(country_id="us", dynamic_id=uuid4()) + + +def test_the_copy_splits_a_us_policy_into_a_policy_and_a_dynamic( + runtime_sessions, us_catalog, superuser_engine +) -> None: + service = _policy_service(runtime_sessions) + snapshot = _snapshot( + "us", + 9101, + { + US_TAX_RATE: {"2026": 0.2}, + US_INCOME_ELASTICITY: {"2026": 0.25}, + US_MTR_DELTA: {"2026": 500}, + }, + ) + + first = service.mirror_legacy_policy(snapshot) + retry = service.mirror_legacy_policy(snapshot) + + assert first.policy_created is True + assert first.dynamic_created is True + assert first.dynamic_id is not None + assert retry.policy_id == first.policy_id + assert retry.dynamic_id == first.dynamic_id + assert retry.mapping_created is False + with Session(superuser_engine) as session: + assert _stored_names(session, policy_id=first.policy_id) == {US_TAX_RATE} + assert _stored_names(session, dynamic_id=first.dynamic_id) == { + US_INCOME_ELASTICITY, + US_MTR_DELTA, + } + mapping = session.scalars( + select(LegacyPolicyMapping).where( + LegacyPolicyMapping.country_id == "us", + LegacyPolicyMapping.legacy_policy_id == 9101, + ) + ).one() + assert (mapping.policy_id, mapping.dynamic_id) == ( + first.policy_id, + first.dynamic_id, + ) + + +def test_the_copy_rewrites_the_uk_old_spelling_into_the_dynamic( + runtime_sessions, uk_catalog, superuser_engine +) -> None: + service = _policy_service(runtime_sessions) + dynamic_values_only = service.mirror_legacy_policy( + _snapshot("uk", 9201, {UK_OLD_INCOME_ELASTICITY: {"2026": 0.3}}) + ) + same_values_new_spelling = service.mirror_legacy_policy( + _snapshot("uk", 9202, {UK_INCOME_ELASTICITY: {"2026": 0.3}}) + ) + no_dynamic_values = service.mirror_legacy_policy( + _snapshot("uk", 9203, {UK_TAX_RATE: {"2026": 0.4}}) + ) + + assert dynamic_values_only.dynamic_id is not None + assert same_values_new_spelling.dynamic_id == dynamic_values_only.dynamic_id + assert same_values_new_spelling.dynamic_created is False + assert no_dynamic_values.dynamic_id is None + assert no_dynamic_values.dynamic_created is False + with Session(superuser_engine) as session: + # A v1 policy holding only dynamic values maps to an empty policy. + assert _stored_names(session, policy_id=dynamic_values_only.policy_id) == set() + assert _stored_names(session, dynamic_id=dynamic_values_only.dynamic_id) == { + UK_INCOME_ELASTICITY + } + dynamic = session.get(Dynamic, dynamic_values_only.dynamic_id) + assert dynamic is not None + assert dynamic.country_id == "uk" + assert dynamic.tax_benefit_model_version_id == uk_catalog.version_id + + +def test_a_copy_made_before_the_split_keeps_its_mapping_without_a_dynamic( + runtime_sessions, us_catalog, superuser_engine +) -> None: + snapshot = _snapshot("us", 9301, {US_INCOME_ELASTICITY: {"2026": 0.25}}) + with Session(superuser_engine) as session, session.begin(): + earlier_copy = Policy( + country_id="us", + tax_benefit_model_id=us_catalog.model_id, + tax_benefit_model_version_id=us_catalog.version_id, + canonicalization_version=1, + content_hash=uuid4().hex * 2, + ) + session.add(earlier_copy) + session.flush() + earlier_copy_id = earlier_copy.id + session.add( + LegacyPolicyMapping( + country_id="us", + legacy_policy_id=9301, + policy_id=earlier_copy_id, + source_policy_hash=snapshot.source_policy_hash, + ) + ) + + result = _policy_service(runtime_sessions).mirror_legacy_policy(snapshot) + + assert result.policy_id == earlier_copy_id + assert result.dynamic_id is None + assert result.mapping_created is False diff --git a/tests/integration/test_v2_household_reports.py b/tests/integration/test_v2_household_reports.py index 4d72c15c6..a64d971e8 100644 --- a/tests/integration/test_v2_household_reports.py +++ b/tests/integration/test_v2_household_reports.py @@ -39,6 +39,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -343,6 +347,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) dependencies = NativeRouteDependencies( diff --git a/tests/integration/test_v2_report_creation.py b/tests/integration/test_v2_report_creation.py index 7ce1d32b7..98ef13935 100644 --- a/tests/integration/test_v2_report_creation.py +++ b/tests/integration/test_v2_report_creation.py @@ -33,6 +33,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -182,6 +186,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) dependencies = NativeRouteDependencies( diff --git a/tests/integration/test_v2_report_runs.py b/tests/integration/test_v2_report_runs.py index ca4612001..96e17b9b0 100644 --- a/tests/integration/test_v2_report_runs.py +++ b/tests/integration/test_v2_report_runs.py @@ -35,6 +35,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -287,6 +291,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) signed_in = {} if requester is None else {"optional_user": lambda: requester} diff --git a/tests/integration/user_report_run_support.py b/tests/integration/user_report_run_support.py index 9aac21c8a..4ffeb2c9b 100644 --- a/tests/integration/user_report_run_support.py +++ b/tests/integration/user_report_run_support.py @@ -29,6 +29,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -239,6 +243,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) signed_in = {} if requester is None else {"optional_user": lambda: requester} diff --git a/tests/unit/services/test_mirror_log_payloads.py b/tests/unit/services/test_mirror_log_payloads.py index 20cf5e693..27b95233c 100644 --- a/tests/unit/services/test_mirror_log_payloads.py +++ b/tests/unit/services/test_mirror_log_payloads.py @@ -45,6 +45,7 @@ ) DESTINATION_ID = UUID("00000000-0000-0000-0000-000000000010") +DESTINATION_DYNAMIC_ID = UUID("00000000-0000-0000-0000-000000000011") ASSOCIATION_ID = UUID("00000000-0000-0000-0000-000000000060") DATABASE_ERROR = OperationalError("statement", {}, Exception("database")) @@ -324,6 +325,8 @@ def test_policy_mirror_success_logs_one_complete_record() -> None: policy_id=DESTINATION_ID, policy_created=False, mapping_created=True, + dynamic_id=DESTINATION_DYNAMIC_ID, + dynamic_created=True, ) with patch("policyengine_api.services.policy_mirroring.logger") as logger: @@ -340,10 +343,12 @@ def test_policy_mirror_success_logs_one_complete_record() -> None: "country_id": "uk", "legacy_policy_id": 5, "destination_policy_id": str(DESTINATION_ID), + "destination_dynamic_id": str(DESTINATION_DYNAMIC_ID), "outcome": "ok", "failure_category": None, "failure_reason": None, "policy_created": False, + "dynamic_created": True, "mapping_created": True, } assert list(payload) == list(expected) @@ -372,10 +377,12 @@ def test_policy_mirror_failure_logs_one_complete_record() -> None: "country_id": "uk", "legacy_policy_id": 5, "destination_policy_id": None, + "destination_dynamic_id": None, "outcome": "error", "failure_category": "database", "failure_reason": "database_error", "policy_created": None, + "dynamic_created": None, "mapping_created": None, } assert list(payload) == list(expected) diff --git a/tests/unit/services/test_policy_mirroring.py b/tests/unit/services/test_policy_mirroring.py index de695a56d..3e34a0d14 100644 --- a/tests/unit/services/test_policy_mirroring.py +++ b/tests/unit/services/test_policy_mirroring.py @@ -22,6 +22,10 @@ classify_policy_copy_failure, mirror_policy_after_commit, ) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, +) from policyengine_api.services.v2.policies.types import ( LegacyPolicyPersistenceResult, LegacyPolicySnapshot, @@ -214,6 +218,18 @@ def _translation_failure( "stored_policy_integrity", id="stored-policy-integrity", ), + pytest.param( + DynamicContentHashCollisionError("secret"), + "integrity", + "content_hash_collision", + id="dynamic-content-hash-collision", + ), + pytest.param( + DynamicCreationIntegrityError("secret"), + "integrity", + "stored_dynamic_integrity", + id="stored-dynamic-integrity", + ), pytest.param( PoolTimeoutError("pool timeout secret"), "database", diff --git a/tests/unit/test_dynamic_parameters.py b/tests/unit/test_dynamic_parameters.py new file mode 100644 index 000000000..5e3935cdf --- /dev/null +++ b/tests/unit/test_dynamic_parameters.py @@ -0,0 +1,147 @@ +"""Which parameters a dynamic holds, and how a v1 policy's values are split.""" + +from typing import get_args + +import pytest + +from policyengine_api.dynamic_parameters import ( + DYNAMIC_PARAMETER_PREFIXES, + ConflictingParameterSpellingsError, + canonical_parameter_name, + is_dynamic_parameter, + split_legacy_policy_json, +) +from policyengine_api.supported_countries import SupportedCountryId + +UK_OLD_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.income_elasticity" +UK_INCOME_ELASTICITY = "gov.simulation.labour_supply_responses.income_elasticity" +US_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" + + +def test_every_supported_country_lists_its_prefixes() -> None: + assert set(DYNAMIC_PARAMETER_PREFIXES) == set(get_args(SupportedCountryId)) + assert DYNAMIC_PARAMETER_PREFIXES["us"] == ("gov.simulation.", "simulation.") + assert DYNAMIC_PARAMETER_PREFIXES["uk"] == ("gov.simulation.",) + + +@pytest.mark.parametrize( + ("country_id", "parameter_name"), + [ + ("us", US_INCOME_ELASTICITY), + ("us", "gov.simulation.branch_to_determine_itemization"), + ("us", "simulation.marginal_tax_rate_delta"), + ("uk", UK_INCOME_ELASTICITY), + ("uk", "gov.simulation.marginal_tax_rate_delta"), + ], +) +def test_parameters_under_the_country_s_prefixes_are_dynamic( + country_id: SupportedCountryId, parameter_name: str +) -> None: + assert is_dynamic_parameter(country_id, parameter_name) + + +@pytest.mark.parametrize( + ("country_id", "parameter_name"), + [ + ("us", "gov.irs.credits.ctc.amount.base[0].amount"), + # A name that only starts with the same letters is not under the prefix. + ("us", "gov.simulations_count"), + ("us", "simulationX.rate"), + # The UK has no top-level simulation tree. + ("uk", "simulation.marginal_tax_rate_delta"), + ("uk", "gov.hmrc.income_tax.allowances.personal_allowance.amount"), + ], +) +def test_other_parameters_are_not_dynamic( + country_id: SupportedCountryId, parameter_name: str +) -> None: + assert not is_dynamic_parameter(country_id, parameter_name) + + +def test_only_the_uk_old_labour_supply_spelling_is_rewritten() -> None: + assert canonical_parameter_name("uk", UK_OLD_INCOME_ELASTICITY) == ( + UK_INCOME_ELASTICITY + ) + assert canonical_parameter_name("uk", UK_INCOME_ELASTICITY) == ( + UK_INCOME_ELASTICITY + ) + # In the US, labor_supply_responses is the parameter's own name. + assert canonical_parameter_name("us", US_INCOME_ELASTICITY) == ( + US_INCOME_ELASTICITY + ) + assert canonical_parameter_name("uk", "gov.hmrc.rate") == "gov.hmrc.rate" + + +def test_a_us_policy_is_split_into_its_policy_and_dynamic_values() -> None: + policy_json = { + "gov.irs.rate": {"2026-01-01.2100-12-31": 0.2}, + US_INCOME_ELASTICITY: {"2026": 0.1}, + "simulation.marginal_tax_rate_delta": {"2026": 500}, + } + + split = split_legacy_policy_json("us", policy_json) + + assert split.policy_values == {"gov.irs.rate": {"2026-01-01.2100-12-31": 0.2}} + assert split.dynamic_values == { + US_INCOME_ELASTICITY: {"2026": 0.1}, + "simulation.marginal_tax_rate_delta": {"2026": 500}, + } + + +def test_a_uk_policy_s_old_spelling_is_rewritten_into_its_dynamic() -> None: + split = split_legacy_policy_json( + "uk", + { + "gov.hmrc.rate": {"2026": 0.2}, + UK_OLD_INCOME_ELASTICITY: {"2026": 0.3}, + }, + ) + + assert split.policy_values == {"gov.hmrc.rate": {"2026": 0.2}} + assert split.dynamic_values == {UK_INCOME_ELASTICITY: {"2026": 0.3}} + + +def test_both_uk_spellings_of_one_parameter_merge_their_periods() -> None: + split = split_legacy_policy_json( + "uk", + { + UK_OLD_INCOME_ELASTICITY: {"2026": 0.3, "2027": 0.4}, + UK_INCOME_ELASTICITY: {"2027": 0.4, "2028": 0.5}, + }, + ) + + assert split.policy_values == {} + assert split.dynamic_values == { + UK_INCOME_ELASTICITY: {"2026": 0.3, "2027": 0.4, "2028": 0.5} + } + + +@pytest.mark.parametrize( + ("old_spelling_values", "new_spelling_values"), + [ + ({"2026": 0.3}, {"2026": 0.4}), + ({"2026": 0.3}, 0.4), + ("0.3", {"2027": 0.4}), + ], +) +def test_both_uk_spellings_that_disagree_are_refused( + old_spelling_values: object, new_spelling_values: object +) -> None: + with pytest.raises(ConflictingParameterSpellingsError): + split_legacy_policy_json( + "uk", + { + UK_OLD_INCOME_ELASTICITY: old_spelling_values, + UK_INCOME_ELASTICITY: new_spelling_values, + }, + ) + + +def test_a_policy_with_no_dynamic_values_is_left_whole() -> None: + policy_json = {"gov.irs.rate": {"2026": 0.2}} + + split = split_legacy_policy_json("us", policy_json) + + assert split.policy_values == policy_json + assert split.dynamic_values == {} + assert split_legacy_policy_json("us", {}) == split_legacy_policy_json("uk", {}) diff --git a/tests/unit/v2/test_alembic_v2_revisions.py b/tests/unit/v2/test_alembic_v2_revisions.py index 2c484b86d..da9be4836 100644 --- a/tests/unit/v2/test_alembic_v2_revisions.py +++ b/tests/unit/v2/test_alembic_v2_revisions.py @@ -66,8 +66,11 @@ # The shared reports revision, on the withheld area codes revision, and parent # of the user invitations revision. SHARED_REPORTS_REVISION = "94dc43079186" -# The user invitations revision, on the shared reports revision. +# The user invitations revision, on the shared reports revision, and parent +# of the dynamics revision. USER_INVITATIONS_REVISION = "f519a77ba122" +# The dynamics revision, on the user invitations revision. +DYNAMICS_REVISION = "9d1d12a7ac58" def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> None: @@ -76,6 +79,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + USER_INVITATIONS_REVISION, SHARED_REPORTS_REVISION, WITHHELD_AREA_CODES_REVISION, SPM_AVAILABILITY_REVISION, @@ -1132,3 +1136,76 @@ def test_user_invitations_revision_adds_its_table_and_the_dismissal_column() -> ) assert "ck_user_type_changes_source_allowed" in upgrade assert "source IN ('admin_route', 'script', 'invitation')" in upgrade + + +def test_dynamics_revision_stores_dynamics_like_policies_and_maps_them() -> None: + """The revision after user invitations stores dynamics as policies are stored. + + It is generated with no corrections: ``dynamics`` gains policies' five + columns, their two indexes, two unique constraints, three foreign keys + and two checks, and loses ``name`` and ``description``; + ``legacy_policy_mappings`` gains ``dynamic_id`` with its index and key; + ``parameter_values`` gains the partial unique index on dynamics' values. + The downgrade undoes each, and neither writes data. + """ + (path,) = (REPO / "migrations/v2/versions").glob(f"{DYNAMICS_REVISION}_*.py") + revision = path.read_text(encoding="utf-8") + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert V2_ALEMBIC_HEAD_REVISION == DYNAMICS_REVISION + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert ( + f'down_revision: Union[str, None] = "{USER_INVITATIONS_REVISION}"' in revision + ) + assert "correction" not in revision.lower() + assert sorted(re.findall(r"op\.\w+", upgrade)) == sorted( + [ + *["op.add_column"] * 6, + *["op.create_index"] * 4, + *["op.create_unique_constraint"] * 2, + *["op.create_foreign_key"] * 4, + *["op.create_check_constraint"] * 2, + *["op.drop_column"] * 2, + # Two generated index names, three key names and two check names. + *["op.f"] * 7, + ] + ) + assert sorted(re.findall(r"op\.\w+", downgrade)) == sorted( + [ + *["op.drop_index"] * 4, + *["op.drop_constraint"] * 8, + *["op.drop_column"] * 6, + *["op.add_column"] * 2, + *["op.f"] * 7, + ] + ) + for column_name in ( + "country_id", + "tax_benefit_model_id", + "tax_benefit_model_version_id", + "canonicalization_version", + "content_hash", + ): + assert re.search( + rf'op\.add_column\(\s*"dynamics",\s*sa\.Column\(\s*"{column_name}",' + r"[^)]*\)?[^)]*nullable=False", + upgrade, + ) + assert re.search(r'op\.drop_column\("dynamics", "name"\)', upgrade) + assert re.search(r'op\.drop_column\("dynamics", "description"\)', upgrade) + assert re.search( + r'op\.add_column\(\s*"legacy_policy_mappings",\s*' + r'sa\.Column\("dynamic_id", sa\.Uuid\(\), nullable=True\)', + upgrade, + ) + assert "fk_legacy_policy_mappings_dynamic_country" in upgrade + assert "uq_parameter_values_dynamic_parameter_start_date" in upgrade + assert 'postgresql_where=sa.text("dynamic_id IS NOT NULL")' in upgrade + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision diff --git a/tests/unit/v2/test_budget_window_submissions.py b/tests/unit/v2/test_budget_window_submissions.py index b85336f75..91be8837a 100644 --- a/tests/unit/v2/test_budget_window_submissions.py +++ b/tests/unit/v2/test_budget_window_submissions.py @@ -16,6 +16,7 @@ from pydantic import ValidationError from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError from policyengine_api.services.v2.policies.types import CheckedPolicyRead from policyengine_api.services.v2.policies.validators import ( PolicyNotFoundError, @@ -31,7 +32,12 @@ from policyengine_api.services.v2.simulation_service.validators import ( EconomyCalculationsUnsupportedError, ) -from tests.unit.v2.test_simulation_submissions import UnreadHouseholds +from tests.unit.v2.test_simulation_submissions import ( + STORED_DYNAMIC_ID, + UK_DYNAMIC_ID, + StoredDynamics, + UnreadHouseholds, +) STORED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c101") OUTDATED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c102") @@ -111,6 +117,7 @@ def submissions( countries=StoredCountries(), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=lambda: client, ) @@ -188,3 +195,21 @@ def test_a_window_may_end_in_2200() -> None: window = window_input(year=2191, budget_window_size=10) assert (window.year, window.budget_window_size) == (2191, 10) + + +def test_a_budget_window_s_dynamics_are_checked_in_its_country() -> None: + client = RecordingSimulationService() + stored_dynamic = { + "policy_id": str(STORED_POLICY_ID), + "dynamic_id": str(STORED_DYNAMIC_ID), + } + + submissions(client, StoredPolicies()).submit_budget_window( + window_input(baseline=stored_dynamic, reform=stored_dynamic) + ) + with pytest.raises(DynamicNotFoundError, match=str(UK_DYNAMIC_ID)): + submissions(client, StoredPolicies()).submit_budget_window( + window_input(reform={"dynamic_id": str(UK_DYNAMIC_ID)}) + ) + + assert len(client.windows) == 1 diff --git a/tests/unit/v2/test_dynamic_catalog.py b/tests/unit/v2/test_dynamic_catalog.py new file mode 100644 index 000000000..b53267f43 --- /dev/null +++ b/tests/unit/v2/test_dynamic_catalog.py @@ -0,0 +1,282 @@ +"""Binding dynamics and policies to a catalog, and keeping their parameters apart. + +A dynamic holds only parameters under ``gov.simulation.`` (plus the US's +``simulation.``), and a policy holds none of them; each is refused with the +parameter names it may not set. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from uuid import UUID + +import pytest +import sqlalchemy as sa +from pydantic import ValidationError +from sqlalchemy.exc import OperationalError, SQLAlchemyError +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, +) +from policyengine_api.data.v2.models import ( + V2_METADATA, + Parameter, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import ( + V2DynamicService, + resolve_dynamic_creation_input, +) +from policyengine_api.services.v2.dynamics.types import ( + DynamicCreationInput, + NativeDynamicCreationInput, + ResolvedDynamicCreationInput, +) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicSetsPolicyParametersError, + DynamicStorageError, +) +from policyengine_api.services.v2.policies.services import ( + resolve_policy_creation_input, +) +from policyengine_api.services.v2.policies.types import PolicyCreationInput +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, + PolicySetsDynamicParametersError, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + +INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" +MARGINAL_TAX_RATE_DELTA = "simulation.marginal_tax_rate_delta" +TAX_RATE = "gov.irs.income.rate" +RUNNING_VERSION = "5.2.0" + + +class Catalog: + """A US catalog version with two dynamic parameters and one policy parameter.""" + + def __init__(self, session: Session) -> None: + self.model = TaxBenefitModel(name="policyengine-us") + self.version = TaxBenefitModelVersion( + model=self.model, + version=RUNNING_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + self.parameter_ids = {} + for name in (INCOME_ELASTICITY, MARGINAL_TAX_RATE_DELTA, TAX_RATE): + parameter = Parameter(name=name, tax_benefit_model_version=self.version) + session.add(parameter) + self.parameter_ids[name] = parameter.id + session.commit() + + +@pytest.fixture +def session() -> Iterator[Session]: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + database_session = Session(engine) + try: + yield database_session + finally: + database_session.close() + engine.dispose() + + +@pytest.fixture +def catalog(session: Session) -> Catalog: + return Catalog(session) + + +def _values(catalog: Catalog, *names: str) -> list[dict[str, object]]: + return [ + { + "parameter_id": catalog.parameter_ids[name], + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + } + for name in names + ] + + +def _dynamic_input(catalog: Catalog, *names: str) -> DynamicCreationInput: + return DynamicCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model.id, + parameter_values=_values(catalog, *names), + ) + + +def test_a_dynamic_is_bound_to_the_running_catalog_version( + session: Session, catalog: Catalog +) -> None: + resolved = resolve_dynamic_creation_input( + session, + _dynamic_input(catalog, INCOME_ELASTICITY, MARGINAL_TAX_RATE_DELTA), + running_policyengine_version=RUNNING_VERSION, + ) + + assert isinstance(resolved, ResolvedDynamicCreationInput) + assert resolved.tax_benefit_model_version_id == catalog.version.id + assert resolved.policyengine_version == RUNNING_VERSION + assert {value.parameter_id for value in resolved.parameter_values} == { + catalog.parameter_ids[INCOME_ELASTICITY], + catalog.parameter_ids[MARGINAL_TAX_RATE_DELTA], + } + + +def test_a_dynamic_setting_policy_parameters_is_refused_naming_them( + session: Session, catalog: Catalog +) -> None: + with pytest.raises(DynamicSetsPolicyParametersError) as refusal: + resolve_dynamic_creation_input( + session, + _dynamic_input(catalog, INCOME_ELASTICITY, TAX_RATE), + running_policyengine_version=RUNNING_VERSION, + ) + + assert refusal.value.parameter_names == (TAX_RATE,) + assert TAX_RATE in str(refusal.value) + assert "/v2/policies" in str(refusal.value) + + +def test_a_dynamic_with_a_parameter_of_another_catalog_is_refused( + session: Session, catalog: Catalog +) -> None: + dynamic_input = DynamicCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model.id, + parameter_values=[ + { + "parameter_id": UUID(int=7), + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + } + ], + ) + + with pytest.raises(PolicyCatalogValidationError, match="every parameter_id"): + resolve_dynamic_creation_input( + session, dynamic_input, running_policyengine_version=RUNNING_VERSION + ) + + +@pytest.mark.parametrize( + "dynamic_type", [DynamicCreationInput, ResolvedDynamicCreationInput] +) +def test_a_dynamic_with_no_values_is_refused(dynamic_type: type) -> None: + fields: dict[str, object] = { + "country_id": "us", + "tax_benefit_model_id": UUID(int=1), + "parameter_values": [], + } + if dynamic_type is ResolvedDynamicCreationInput: + fields |= { + "tax_benefit_model_version_id": UUID(int=2), + "policyengine_version": RUNNING_VERSION, + } + + with pytest.raises(ValidationError, match="at least 1 item"): + dynamic_type(**fields) + + +def test_a_policy_setting_dynamic_parameters_is_refused_naming_them( + session: Session, catalog: Catalog +) -> None: + policy_input = PolicyCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model.id, + parameter_values=_values( + catalog, TAX_RATE, MARGINAL_TAX_RATE_DELTA, INCOME_ELASTICITY + ), + ) + + with pytest.raises(PolicySetsDynamicParametersError) as refusal: + resolve_policy_creation_input( + session, policy_input, running_policyengine_version=RUNNING_VERSION + ) + + assert refusal.value.parameter_names == ( + INCOME_ELASTICITY, + MARGINAL_TAX_RATE_DELTA, + ) + assert "/v2/dynamics" in str(refusal.value) + + +def test_a_policy_without_dynamic_parameters_is_still_bound( + session: Session, catalog: Catalog +) -> None: + resolved = resolve_policy_creation_input( + session, + PolicyCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model.id, + parameter_values=_values(catalog, TAX_RATE), + ), + running_policyengine_version=RUNNING_VERSION, + ) + + assert resolved.tax_benefit_model_version_id == catalog.version.id + assert [value.parameter_id for value in resolved.parameter_values] == [ + catalog.parameter_ids[TAX_RATE] + ] + + +def test_a_dynamic_storage_that_cannot_be_read_is_a_storage_error() -> None: + # A database without the v2 tables fails every read with OperationalError. + # A create's first read is the catalog query, which reports its own error. + engine = create_engine("sqlite://") + service = V2DynamicService( + DynamicDatabaseSession(sessionmaker(engine, class_=Session)), + running_policyengine_version=RUNNING_VERSION, + ) + try: + with pytest.raises(DynamicStorageError) as failure: + service.get_dynamic(country_id="us", dynamic_id=UUID(int=3)) + with pytest.raises(MetadataCatalogUnavailableError): + service.create_dynamic( + NativeDynamicCreationInput( + country_id="us", + tax_benefit_model_id=UUID(int=1), + parameter_values=[ + { + "parameter_id": UUID(int=2), + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + } + ], + ) + ) + finally: + engine.dispose() + + assert isinstance(failure.value.__cause__, OperationalError) + + +def test_a_dynamic_that_cannot_be_written_is_a_storage_error( + session: Session, catalog: Catalog +) -> None: + engine = session.get_bind() + with engine.begin() as connection: + connection.execute(sa.text("DROP TABLE dynamics")) + service = V2DynamicService( + DynamicDatabaseSession(sessionmaker(engine, class_=Session)), + running_policyengine_version=RUNNING_VERSION, + ) + + with pytest.raises(DynamicStorageError) as failure: + service.create_dynamic( + NativeDynamicCreationInput( + **_dynamic_input(catalog, INCOME_ELASTICITY).model_dump() + ) + ) + + assert isinstance(failure.value.__cause__, SQLAlchemyError) diff --git a/tests/unit/v2/test_dynamic_routes.py b/tests/unit/v2/test_dynamic_routes.py new file mode 100644 index 000000000..048305261 --- /dev/null +++ b/tests/unit/v2/test_dynamic_routes.py @@ -0,0 +1,327 @@ +"""Native FastAPI contract tests for v2 dynamics, stored as policies are.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from uuid import UUID, uuid4 + +import pytest +from fastapi.testclient import TestClient +from flask import Flask, jsonify + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.dynamics.types import ( + CheckedDynamicRead, + NativeDynamicCreation, + NativeDynamicCreationInput, +) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, + DynamicNotFoundError, + DynamicSetsPolicyParametersError, + DynamicStorageError, +) +from policyengine_api.services.v2.policies.types import PolicyParameterValueRead +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, +) + +DYNAMIC_ID = UUID("00000000-0000-0000-0000-000000000011") +MODEL_ID = UUID("00000000-0000-0000-0000-000000000020") +MODEL_VERSION_ID = UUID("00000000-0000-0000-0000-000000000030") +PARAMETER_ID = UUID("00000000-0000-0000-0000-000000000040") +VALUE_ID = UUID("00000000-0000-0000-0000-000000000050") +CREATED_AT = datetime(2026, 1, 1, tzinfo=UTC) +INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" + + +def _dynamic_read( + *, dynamic_id: UUID = DYNAMIC_ID, missing_parameters: tuple[str, ...] | None = () +) -> CheckedDynamicRead: + return CheckedDynamicRead( + id=dynamic_id, + country_id="us", + tax_benefit_model_id=MODEL_ID, + tax_benefit_model_version_id=MODEL_VERSION_ID, + created_at=CREATED_AT, + updated_at=CREATED_AT + timedelta(seconds=1), + parameter_values=( + PolicyParameterValueRead( + id=VALUE_ID, + parameter_id=PARAMETER_ID, + parameter_name=INCOME_ELASTICITY, + value=0.25, + start_date=CREATED_AT, + end_date=None, + ), + ), + missing_parameters=missing_parameters, + ) + + +class FakeDynamicService: + """Records each call; raises ``error`` when set.""" + + def __init__(self) -> None: + self.created = True + self.error: Exception | None = None + self.missing_parameters: tuple[str, ...] | None = () + self.calls: list[tuple[str, object]] = [] + + def create_dynamic( + self, dynamic_input: NativeDynamicCreationInput + ) -> NativeDynamicCreation: + self.calls.append(("create_dynamic", dynamic_input)) + if self.error is not None: + raise self.error + return NativeDynamicCreation(item=_dynamic_read(), created=self.created) + + def get_dynamic(self, **filters: object) -> CheckedDynamicRead: + self.calls.append(("get_dynamic", filters)) + if self.error is not None: + raise self.error + dynamic_id = filters["dynamic_id"] + assert isinstance(dynamic_id, UUID) + return _dynamic_read( + dynamic_id=dynamic_id, missing_parameters=self.missing_parameters + ) + + +def _client(service: FakeDynamicService) -> tuple[TestClient, dict[str, int]]: + flask_calls = {"count": 0} + flask_app = Flask(__name__) + + @flask_app.route("/", methods=["GET", "POST", "PATCH", "DELETE"]) + def fallback(resource: str): + flask_calls["count"] += 1 + return jsonify({"source": "flask", "resource": resource}) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + v2_dynamic_service_factory=lambda: service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app(flask_app, dependencies=dependencies, route_settings=settings), + raise_server_exceptions=False, + ) + return client, flask_calls + + +def _body(**changes: object) -> dict[str, object]: + body: dict[str, object] = { + "country_id": "us", + "tax_benefit_model_id": str(MODEL_ID), + "parameter_values": [ + { + "parameter_id": str(PARAMETER_ID), + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + } + ], + } + body.update(changes) + return body + + +def test_create_answers_201_for_a_new_dynamic_and_200_for_stored_content() -> None: + service = FakeDynamicService() + client, flask_calls = _client(service) + + created = client.post( + "/v2/dynamics?country_id=us&policyengine_version=5.2.0", json=_body() + ) + service.created = False + found = client.post("/v2/dynamics?country_id=us", json=_body()) + + assert created.status_code == 201 + assert found.status_code == 200 + assert created.json() == found.json() + item = created.json()["result"]["item"] + assert created.json()["status"] == "ok" + assert item == { + "id": str(DYNAMIC_ID), + "country_id": "us", + "tax_benefit_model_id": str(MODEL_ID), + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:01Z", + "parameter_values": [ + { + "id": str(VALUE_ID), + "parameter_id": str(PARAMETER_ID), + "parameter_name": INCOME_ELASTICITY, + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + "end_date": None, + } + ], + "missing_parameters": [], + } + first_call = service.calls[0] + assert first_call[0] == "create_dynamic" + assert isinstance(first_call[1], NativeDynamicCreationInput) + assert first_call[1].policyengine_version == "5.2.0" + assert flask_calls["count"] == 0 + + +def test_create_refuses_an_empty_dynamic_and_a_country_mismatch_before_service() -> ( + None +): + service = FakeDynamicService() + client, _flask_calls = _client(service) + + empty = client.post("/v2/dynamics?country_id=us", json=_body(parameter_values=[])) + mismatch = client.post("/v2/dynamics?country_id=uk", json=_body()) + unknown_field = client.post("/v2/dynamics?country_id=us", json=_body(name="x")) + oversized = client.post( + "/v2/dynamics?country_id=us", + content=b"{" + b" " * 1_048_577 + b"}", + headers={"content-type": "application/json"}, + ) + + assert empty.status_code == 422 + assert empty.json() == {"status": "error", "message": "Invalid API v2 request"} + assert mismatch.status_code == 400 + assert unknown_field.status_code == 422 + assert oversized.status_code == 413 + assert service.calls == [] + + +def test_create_refuses_policy_parameters_naming_them() -> None: + service = FakeDynamicService() + service.error = DynamicSetsPolicyParametersError(("gov.irs.rate", "gov.irs.x")) + client, _flask_calls = _client(service) + + response = client.post("/v2/dynamics?country_id=us", json=_body()) + + assert response.status_code == 422 + body = response.json() + assert body["status"] == "error" + assert body["code"] == "policy_parameters" + assert body["policy_parameters"] == ["gov.irs.rate", "gov.irs.x"] + assert "/v2/policies" in body["message"] + + +def test_detail_is_country_scoped_and_reports_missing_parameters() -> None: + service = FakeDynamicService() + service.missing_parameters = (INCOME_ELASTICITY,) + client, _flask_calls = _client(service) + dynamic_id = uuid4() + + found = client.get(f"/v2/dynamics/{dynamic_id}?country_id=uk") + service.error = DynamicNotFoundError(f"dynamic {dynamic_id} was not found") + missing = client.get(f"/v2/dynamics/{dynamic_id}?country_id=uk") + + assert found.status_code == 200 + assert found.json()["result"]["item"]["id"] == str(dynamic_id) + assert found.json()["result"]["item"]["missing_parameters"] == [INCOME_ELASTICITY] + assert service.calls[0] == ( + "get_dynamic", + {"country_id": "uk", "dynamic_id": dynamic_id}, + ) + assert missing.status_code == 404 + assert missing.json() == { + "status": "error", + "message": f"dynamic {dynamic_id} was not found", + } + + +@pytest.mark.parametrize( + ("error", "status_code", "message"), + [ + (PolicyCatalogValidationError("wrong model"), 400, "wrong model"), + (MetadataCatalogVersionNotFoundError("no 9.9.9"), 404, "no 9.9.9"), + ( + DynamicContentHashCollisionError("secret detail"), + 409, + "Dynamic content hash conflicts with storage", + ), + ( + DynamicCreationIntegrityError("secret detail"), + 500, + "Stored dynamic integrity failed", + ), + ( + DynamicStorageError("SELECT secret"), + 503, + "V2 dynamic persistence is unavailable", + ), + ( + MetadataCatalogUnavailableError("catalog secret"), + 503, + "V2 dynamic persistence is unavailable", + ), + (RuntimeError("secret detail"), 500, "V2 dynamic operation failed"), + ], +) +def test_dynamic_failures_map_to_secret_safe_typed_errors( + error: Exception, status_code: int, message: str +) -> None: + service = FakeDynamicService() + service.error = error + client, _flask_calls = _client(service) + + response = client.post("/v2/dynamics?country_id=us", json=_body()) + + assert response.status_code == status_code + assert response.json() == {"status": "error", "message": message} + + +@pytest.mark.parametrize( + ("method", "path"), + [ + ("GET", "/v2/dynamics?country_id=us"), + ("PATCH", f"/v2/dynamics/{DYNAMIC_ID}?country_id=us"), + ("DELETE", f"/v2/dynamics/{DYNAMIC_ID}?country_id=us"), + ], +) +def test_dynamics_are_neither_listed_nor_changed(method: str, path: str) -> None: + service = FakeDynamicService() + client, _flask_calls = _client(service) + + response = client.request(method, path) + + assert response.status_code in {404, 405} + assert service.calls == [] + + +def test_openapi_publishes_the_dynamic_routes_and_their_errors() -> None: + client, _flask_calls = _client(FakeDynamicService()) + + schema = client.get("/v2/openapi.json").json() + + create = schema["paths"]["/v2/dynamics"]["post"] + detail = schema["paths"]["/v2/dynamics/{dynamic_id}"]["get"] + assert set(schema["paths"]["/v2/dynamics"]) == {"post"} + assert set(schema["paths"]["/v2/dynamics/{dynamic_id}"]) == {"get"} + assert {"200", "201", "400", "404", "409", "413", "422", "500", "503"} <= set( + create["responses"] + ) + assert {"200", "404", "503"} <= set(detail["responses"]) + query_names = { + parameter["name"] + for parameter in create["parameters"] + if parameter["in"] == "query" + } + assert query_names == {"country_id", "policyengine_version"} + request_schema = schema["components"]["schemas"]["DynamicCreateRequest"] + parameter_values = request_schema["properties"]["parameter_values"] + assert parameter_values["minItems"] == 1 + assert parameter_values["maxItems"] == 1_000 + assert "dynamic_id null" in parameter_values["description"] diff --git a/tests/unit/v2/test_dynamic_table_models.py b/tests/unit/v2/test_dynamic_table_models.py new file mode 100644 index 000000000..ac6fd180c --- /dev/null +++ b/tests/unit/v2/test_dynamic_table_models.py @@ -0,0 +1,282 @@ +"""Dynamics are stored as policies are: once per canonical content hash.""" + +import subprocess +import sys +from datetime import UTC, datetime + +import pytest +import sqlalchemy as sa +from sqlalchemy.exc import IntegrityError +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.models import ( + V2_METADATA, + Dynamic, + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from tests.fixtures.v2_countries import add_listed_countries + +START_DATE = datetime(2026, 1, 1, tzinfo=UTC) + + +def _sqlite_engine_with_foreign_keys() -> sa.Engine: + engine = create_engine("sqlite://") + + @sa.event.listens_for(engine, "connect") + def enable_foreign_keys(dbapi_connection, _connection_record) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + V2_METADATA.create_all(engine) + with Session(engine) as session: + add_listed_countries(session) + return engine + + +def _catalog() -> tuple[TaxBenefitModel, TaxBenefitModelVersion, Parameter]: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, version="2.2.2", current_law_id=1, metadata_time_periods=[2026] + ) + parameter = Parameter( + name="gov.simulation.labor_supply_responses.elasticities.income", + tax_benefit_model_version=version, + ) + return model, version, parameter + + +def _dynamic( + model: TaxBenefitModel, + version: TaxBenefitModelVersion, + *, + content_hash: str = "d" * 64, + country_id: str = "us", +) -> Dynamic: + return Dynamic( + country_id=country_id, + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=version.id, + canonicalization_version=1, + content_hash=content_hash, + ) + + +def test_dynamics_have_policies_columns_and_no_name_or_description() -> None: + dynamics = V2_METADATA.tables["dynamics"] + policies = V2_METADATA.tables["policies"] + + assert set(dynamics.c.keys()) == set(policies.c.keys()) + assert "name" not in dynamics.c + assert "description" not in dynamics.c + for column_name in ( + "country_id", + "tax_benefit_model_id", + "tax_benefit_model_version_id", + "canonicalization_version", + "content_hash", + ): + assert not dynamics.c[column_name].nullable + foreign_keys = { + foreign_key.parent.name: (foreign_key.column.table.name, foreign_key.ondelete) + for foreign_key in dynamics.foreign_keys + } + assert foreign_keys == { + "country_id": ("countries", "RESTRICT"), + "tax_benefit_model_id": ("tax_benefit_models", "RESTRICT"), + "tax_benefit_model_version_id": ("tax_benefit_model_versions", "RESTRICT"), + } + unique_column_sets = { + constraint.name: tuple(column.name for column in constraint.columns) + for constraint in dynamics.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert unique_column_sets == { + "uq_dynamics_id_country": ("id", "country_id"), + "uq_dynamics_canonicalization_content_hash": ( + "canonicalization_version", + "content_hash", + ), + } + check_names = { + constraint.name + for constraint in dynamics.constraints + if isinstance(constraint, sa.CheckConstraint) + } + assert check_names == { + "ck_dynamics_canonicalization_version", + "ck_dynamics_content_hash_length", + } + + +def test_a_dynamic_s_values_are_unique_per_parameter_and_start_date() -> None: + parameter_values = V2_METADATA.tables["parameter_values"] + index = next( + index + for index in parameter_values.indexes + if index.name == "uq_parameter_values_dynamic_parameter_start_date" + ) + + assert index.unique + assert tuple(column.name for column in index.columns) == ( + "dynamic_id", + "parameter_id", + "start_date", + ) + assert str(index.dialect_options["postgresql"]["where"]) == ( + "dynamic_id IS NOT NULL" + ) + assert str(index.dialect_options["sqlite"]["where"]) == "dynamic_id IS NOT NULL" + + +def test_legacy_policy_mappings_name_an_optional_dynamic_of_their_country() -> None: + mappings = V2_METADATA.tables["legacy_policy_mappings"] + + assert mappings.c.dynamic_id.nullable + (dynamic_key,) = ( + constraint + for constraint in mappings.constraints + if isinstance(constraint, sa.ForeignKeyConstraint) + and constraint.referred_table.name == "dynamics" + ) + assert dynamic_key.name == "fk_legacy_policy_mappings_dynamic_country" + assert [column.name for column in dynamic_key.columns] == [ + "dynamic_id", + "country_id", + ] + assert [element.target_fullname for element in dynamic_key.elements] == [ + "dynamics.id", + "dynamics.country_id", + ] + assert dynamic_key.ondelete == "RESTRICT" + assert any( + index.name == "ix_legacy_policy_mappings_dynamic" + and [column.name for column in index.columns] == ["dynamic_id"] + for index in mappings.indexes + ) + + +def test_identical_dynamic_content_is_stored_once() -> None: + engine = _sqlite_engine_with_foreign_keys() + model, version, parameter = _catalog() + + with Session(engine) as session: + session.add_all([model, version, parameter]) + session.commit() + session.add(_dynamic(model, version)) + session.commit() + session.add(_dynamic(model, version)) + with pytest.raises(IntegrityError, match="UNIQUE constraint failed"): + session.commit() + session.rollback() + session.add(_dynamic(model, version, content_hash="e" * 64)) + session.commit() + + engine.dispose() + + +def test_a_dynamic_sets_one_value_per_parameter_and_start_date() -> None: + engine = _sqlite_engine_with_foreign_keys() + model, version, parameter = _catalog() + + with Session(engine) as session: + session.add_all([model, version, parameter]) + session.commit() + dynamic = _dynamic(model, version) + session.add(dynamic) + session.commit() + for value in (0.1, 0.2): + session.add( + ParameterValue( + parameter_id=parameter.id, + dynamic_id=dynamic.id, + value_json=value, + start_date=START_DATE, + ) + ) + with pytest.raises(IntegrityError, match="UNIQUE constraint failed"): + session.commit() + + engine.dispose() + + +def test_a_mapping_cannot_name_another_country_s_dynamic() -> None: + engine = _sqlite_engine_with_foreign_keys() + model, version, _parameter = _catalog() + + with Session(engine) as session: + policy = Policy( + country_id="uk", + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=version.id, + canonicalization_version=1, + content_hash="a" * 64, + ) + session.add_all([model, version, policy]) + session.commit() + us_dynamic = _dynamic(model, version) + session.add(us_dynamic) + session.commit() + session.add( + LegacyPolicyMapping( + country_id="uk", + legacy_policy_id=7, + policy_id=policy.id, + dynamic_id=us_dynamic.id, + source_policy_hash="v1-hash", + ) + ) + with pytest.raises(IntegrityError, match="FOREIGN KEY constraint failed"): + session.commit() + + engine.dispose() + + +def test_the_v2_mappers_configure_without_relationship_warnings() -> None: + # A fresh interpreter, since a process configures its mappers only once + # and an error there would break every later mapper in this one. + configuration = subprocess.run( + [ + sys.executable, + "-c", + "import warnings\n" + "from sqlalchemy.exc import SAWarning\n" + "warnings.simplefilter('error', SAWarning)\n" + "import policyengine_api.data.v2.models\n" + "from sqlalchemy.orm import configure_mappers\n" + "configure_mappers()", + ], + capture_output=True, + text=True, + check=False, + ) + + assert configuration.returncode == 0, configuration.stderr + + +def test_a_dynamic_is_stored_in_one_flush_with_its_new_catalog_version() -> None: + engine = _sqlite_engine_with_foreign_keys() + model, version, parameter = _catalog() + + with Session(engine) as session: + dynamic = Dynamic( + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="d" * 64, + ) + session.add_all([parameter, dynamic]) + session.commit() + stored = session.get(Dynamic, dynamic.id) + + assert stored is not None + assert stored.tax_benefit_model_version_id == version.id + assert stored in version.dynamics + + engine.dispose() diff --git a/tests/unit/v2/test_household_report_submissions.py b/tests/unit/v2/test_household_report_submissions.py index 9d9cd1620..3a8f32d54 100644 --- a/tests/unit/v2/test_household_report_submissions.py +++ b/tests/unit/v2/test_household_report_submissions.py @@ -46,6 +46,7 @@ SERVED_COUNTRIES, STORED_POLICY_ID, StoredCountries, + StoredDynamics, StoredPolicies, ) @@ -134,6 +135,7 @@ def household_submissions( countries=StoredCountries(countries=countries), households=households or StoredHouseholds(), policies=policies or StoredPolicies(), + dynamics=StoredDynamics(), client_factory=client_factory, ) diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index 732fa7d98..cdecc7844 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -288,12 +288,20 @@ def test_policy_content_hash_is_unique_per_canonicalization_version() -> None: def test_policy_parameter_value_owner_period_and_identity_constraints() -> None: engine = _relational_sqlite_engine() - _model, _version, parameter, policy = _policy_graph(content_hash="b" * 64) - dynamic = Dynamic(name="dynamic") + model, version, parameter, policy = _policy_graph(content_hash="b" * 64) + dynamic = Dynamic( + country_id="us", + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=version.id, + canonicalization_version=1, + content_hash="d" * 64, + ) start = datetime(2026, 1, 1, tzinfo=UTC) with Session(engine) as session: - session.add_all([parameter, policy, dynamic]) + session.add_all([parameter, policy]) + session.commit() + session.add(dynamic) session.commit() invalid_owner = ParameterValue( diff --git a/tests/unit/v2/test_policy_legacy_copy.py b/tests/unit/v2/test_policy_legacy_copy.py index e3ea73d7b..7bf5b903a 100644 --- a/tests/unit/v2/test_policy_legacy_copy.py +++ b/tests/unit/v2/test_policy_legacy_copy.py @@ -6,7 +6,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from datetime import UTC, datetime from uuid import UUID @@ -20,28 +20,40 @@ TaxBenefitModel, TaxBenefitModelVersion, ) +from policyengine_api.services.v2.dynamics import services as dynamic_services +from policyengine_api.services.v2.dynamics.types import ( + DynamicCreationResult, + ResolvedDynamicCreationInput, +) from policyengine_api.services.v2.policies import services as policy_services from policyengine_api.services.v2.policies.types import ( LegacyPolicyPersistenceResult, LegacyPolicySnapshot, + LegacyPolicyTranslationReason, PolicyCreationResult, ResolvedPolicyCreationInput, ) from policyengine_api.services.v2.policies.validators import ( LegacyPolicyMappingIntegrityError, + LegacyPolicyTranslationError, ) RUNNING_POLICYENGINE_VERSION = "6.7.5" COPIED_POLICY_ID = UUID("00000000-0000-0000-0000-000000000042") EARLIER_COPY_POLICY_ID = UUID("00000000-0000-0000-0000-000000000041") MAPPING_ID = UUID("00000000-0000-0000-0000-000000000043") +COPIED_DYNAMIC_ID = UUID("00000000-0000-0000-0000-000000000044") +US_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" +UK_INCOME_ELASTICITY = "gov.simulation.labour_supply_responses.income_elasticity" +UK_OLD_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.income_elasticity" @dataclass class RecordedCopyWrites: - """The policies and mappings a copy would have inserted, in order.""" + """The policies, dynamics and mappings a copy would have inserted, in order.""" created_policies: list[ResolvedPolicyCreationInput] = field(default_factory=list) + created_dynamics: list[ResolvedDynamicCreationInput] = field(default_factory=list) created_mappings: list[dict[str, object]] = field(default_factory=list) @@ -53,11 +65,16 @@ def record_policy(_session, policy_input): writes.created_policies.append(policy_input) return PolicyCreationResult(policy_id=COPIED_POLICY_ID, created=True) - def record_mapping(_session, **mapping_fields): - writes.created_mappings.append(mapping_fields) + def record_dynamic(_session, dynamic_input): + writes.created_dynamics.append(dynamic_input) + return DynamicCreationResult(dynamic_id=COPIED_DYNAMIC_ID, created=True) + + def record_mapping(_session, mapping): + writes.created_mappings.append(asdict(mapping)) return MAPPING_ID monkeypatch.setattr(policy_services, "create_resolved_policy", record_policy) + monkeypatch.setattr(dynamic_services, "create_resolved_dynamic", record_dynamic) monkeypatch.setattr(policy_services, "create_legacy_policy_mapping", record_mapping) return writes @@ -185,6 +202,7 @@ def test_policy_saved_under_an_older_country_package_is_copied( "legacy_policy_id": reused_policy.legacy_policy_id, "source_policy_hash": snapshot.source_policy_hash, "policy_id": COPIED_POLICY_ID, + "dynamic_id": None, } ] @@ -298,11 +316,11 @@ def _lose_mapping_race(monkeypatch, *, winning_source_hash: str) -> None: constraint makes it a no-op. """ - def insert_after_concurrent_copy(session, **mapping_fields): + def insert_after_concurrent_copy(session, mapping): session.add( LegacyPolicyMapping( - country_id=mapping_fields["country_id"], - legacy_policy_id=mapping_fields["legacy_policy_id"], + country_id=mapping.country_id, + legacy_policy_id=mapping.legacy_policy_id, policy_id=EARLIER_COPY_POLICY_ID, source_policy_hash=winning_source_hash, ) @@ -350,3 +368,158 @@ def test_concurrent_copy_with_another_source_hash_is_an_integrity_error( with pytest.raises(LegacyPolicyMappingIntegrityError, match="source hash"): _copy(catalog_session, _reused_snapshot()) + + +def test_a_policy_s_model_assumptions_are_copied_into_a_dynamic( + catalog_session: Session, recorded_writes: RecordedCopyWrites +) -> None: + version, parameters = _add_running_catalog( + catalog_session, + country_id="us", + parameter_names=["gov.dol.minimum_wage", US_INCOME_ELASTICITY], + ) + snapshot = _snapshot( + country_id="us", + legacy_policy_id=501, + api_version="1.600.0", + policy_json={ + "gov.dol.minimum_wage": {"2026": 9}, + US_INCOME_ELASTICITY: {"2026": 0.25}, + }, + ) + + result = _copy(catalog_session, snapshot) + + assert result == LegacyPolicyPersistenceResult( + policy_id=COPIED_POLICY_ID, + policy_created=True, + mapping_created=True, + dynamic_id=COPIED_DYNAMIC_ID, + dynamic_created=True, + ) + [copied_policy] = recorded_writes.created_policies + assert [value.parameter_id for value in copied_policy.parameter_values] == [ + parameters["gov.dol.minimum_wage"].id + ] + [copied_dynamic] = recorded_writes.created_dynamics + assert isinstance(copied_dynamic, ResolvedDynamicCreationInput) + assert copied_dynamic.country_id == "us" + assert copied_dynamic.tax_benefit_model_version_id == version.id + assert [ + (value.parameter_id, value.value) for value in copied_dynamic.parameter_values + ] == [(parameters[US_INCOME_ELASTICITY].id, 0.25)] + assert recorded_writes.created_mappings == [ + { + "country_id": "us", + "legacy_policy_id": 501, + "source_policy_hash": snapshot.source_policy_hash, + "policy_id": COPIED_POLICY_ID, + "dynamic_id": COPIED_DYNAMIC_ID, + } + ] + + +def test_a_uk_policy_with_only_old_spelling_assumptions_maps_to_an_empty_policy( + catalog_session: Session, recorded_writes: RecordedCopyWrites +) -> None: + _version, parameters = _add_running_catalog( + catalog_session, country_id="uk", parameter_names=[UK_INCOME_ELASTICITY] + ) + + result = _copy( + catalog_session, + _snapshot( + country_id="uk", + legacy_policy_id=502, + api_version="2.100.0", + policy_json={UK_OLD_INCOME_ELASTICITY: {"2026": 0.3}}, + ), + ) + + assert result.dynamic_id == COPIED_DYNAMIC_ID + [copied_policy] = recorded_writes.created_policies + assert copied_policy.parameter_values == [] + [copied_dynamic] = recorded_writes.created_dynamics + assert [value.parameter_id for value in copied_dynamic.parameter_values] == [ + parameters[UK_INCOME_ELASTICITY].id + ] + + +@pytest.mark.parametrize( + ("policy_json", "reason"), + [ + pytest.param( + {"gov.boe.base_rate": {"2026": 0.05}, US_INCOME_ELASTICITY: {"2026": 1}}, + LegacyPolicyTranslationReason.MISSING_PARAMETERS, + id="assumption-missing-from-the-catalog", + ), + pytest.param( + { + UK_OLD_INCOME_ELASTICITY: {"2026": 0.3}, + UK_INCOME_ELASTICITY: {"2026": 0.4}, + }, + LegacyPolicyTranslationReason.CONFLICTING_VALUES, + id="both-spellings-disagree", + ), + pytest.param( + {UK_INCOME_ELASTICITY: {}}, + LegacyPolicyTranslationReason.EMPTY_PARAMETER_VALUES, + id="assumption-sets-no-values", + ), + ], +) +def test_an_untranslatable_assumption_fails_the_copy_before_anything_is_written( + catalog_session: Session, + recorded_writes: RecordedCopyWrites, + policy_json: dict[str, object], + reason: LegacyPolicyTranslationReason, +) -> None: + _add_running_catalog( + catalog_session, + country_id="uk", + parameter_names=["gov.boe.base_rate", UK_INCOME_ELASTICITY], + ) + + with pytest.raises(LegacyPolicyTranslationError) as failure: + _copy( + catalog_session, + _snapshot( + country_id="uk", + legacy_policy_id=503, + api_version="2.100.0", + policy_json=policy_json, + ), + ) + + assert failure.value.reason == reason + assert recorded_writes == RecordedCopyWrites() + + +def test_a_copy_that_loses_the_mapping_race_reports_the_mapped_dynamic( + catalog_session: Session, + recorded_writes: RecordedCopyWrites, + monkeypatch, +) -> None: + _add_running_catalog( + catalog_session, country_id="us", parameter_names=[US_INCOME_ELASTICITY] + ) + snapshot = _snapshot( + country_id="us", + legacy_policy_id=504, + api_version="1.600.0", + policy_json={US_INCOME_ELASTICITY: {"2026": 0.25}}, + ) + _lose_mapping_race(monkeypatch, winning_source_hash=snapshot.source_policy_hash) + + result = _copy(catalog_session, snapshot) + + # This copy created a dynamic, but the concurrent copy's mapping, made + # before the split existed, names none; the result reports the mapping. + assert len(recorded_writes.created_dynamics) == 1 + assert result == LegacyPolicyPersistenceResult( + policy_id=EARLIER_COPY_POLICY_ID, + policy_created=False, + mapping_created=False, + dynamic_id=None, + dynamic_created=False, + ) diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py index 8dad07c50..752ae1fc2 100644 --- a/tests/unit/v2/test_policy_routes.py +++ b/tests/unit/v2/test_policy_routes.py @@ -33,6 +33,7 @@ PolicyContentHashCollisionError, PolicyCreationIntegrityError, PolicyNotFoundError, + PolicySetsDynamicParametersError, ) POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") @@ -394,6 +395,30 @@ def test_policy_failures_map_to_secret_safe_typed_errors( assert "credential" not in response.text +def test_create_refuses_dynamic_parameters_naming_them() -> None: + service = FakePolicyService() + service.error = PolicySetsDynamicParametersError( + ( + "gov.simulation.labor_supply_responses.elasticities.income", + "simulation.marginal_tax_rate_delta", + ) + ) + client, _flask_calls = _client(service) + + response = client.post("/v2/policies?country_id=us", json=_body()) + + assert response.status_code == 422 + body = response.json() + assert body["status"] == "error" + assert body["code"] == "dynamic_parameters" + assert body["dynamic_parameters"] == [ + "gov.simulation.labor_supply_responses.elasticities.income", + "simulation.marginal_tax_rate_delta", + ] + assert "POST /v2/dynamics" in body["message"] + assert "simulation.marginal_tax_rate_delta" in body["message"] + + @pytest.mark.parametrize("method", ["put", "patch", "delete"]) def test_core_policy_mutations_are_not_exposed(method: str) -> None: service = FakePolicyService() diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 0a20c74a3..6394da3b7 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -480,3 +480,19 @@ def test_the_public_api_records_each_users_report_run_requests() -> None: for role, tables in declared.items(): if role != RUNTIME_ROLE: assert tables["user_report_run_associations"] == NO_ACCESS, role + + +def test_the_public_api_creates_and_reads_dynamics_but_never_changes_them() -> None: + """``POST /v2/dynamics`` and the v1-to-v2 copy create dynamics, as policies. + + A dynamic is immutable once stored, so the Public API reads and inserts + them and never updates or deletes one. The simulation service and its + Modal workers read them for a run; no other role has access. + """ + declared = declared_privileges(TEST_ENVIRONMENT) + + assert declared[RUNTIME_ROLE]["dynamics"] == READ_INSERT + assert declared[RUNTIME_ROLE]["dynamics"] == declared[RUNTIME_ROLE]["policies"] + assert declared[SIMULATION_SERVICE_ROLE]["dynamics"] == READ + assert declared[MODAL_WORKER_ROLE]["dynamics"] == READ + assert declared[CATALOG_WRITER_ROLE]["dynamics"] == NO_ACCESS diff --git a/tests/unit/v2/test_report_rerun_submissions.py b/tests/unit/v2/test_report_rerun_submissions.py index 03eda8a86..6c9552a3e 100644 --- a/tests/unit/v2/test_report_rerun_submissions.py +++ b/tests/unit/v2/test_report_rerun_submissions.py @@ -30,6 +30,7 @@ EconomyCalculationsUnsupportedError, SubmissionStorageError, ) +from tests.unit.v2.test_simulation_submissions import StoredDynamics REPORT_ID = UUID("00000000-0000-0000-0000-00000000e001") STORED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000e002") @@ -138,6 +139,7 @@ def test_a_rerun_is_sent_once_its_country_and_policies_are_checked() -> None: countries=StoredCountries(), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=lambda: client, ) rerun = rerun_input(None, STORED_POLICY_ID) @@ -153,6 +155,7 @@ def test_a_rerun_in_a_household_only_country_is_refused_before_sending() -> None countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -166,6 +169,7 @@ def test_a_rerun_naming_a_policy_no_longer_stored_is_refused() -> None: countries=StoredCountries(), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -178,6 +182,7 @@ def test_a_rerun_whose_policy_sets_parameters_the_catalog_lacks_is_refused() -> countries=StoredCountries(), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -195,6 +200,7 @@ def test_a_rerun_whose_policy_check_cannot_read_storage_is_storage_unavailable() countries=StoredCountries(), households=UnreadHouseholds(), policies=StoredPolicies(error=error), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -212,6 +218,7 @@ def test_a_rerun_where_only_households_are_calculated_is_refused_for_economy_kin countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -228,6 +235,7 @@ def test_a_household_reports_rerun_is_sent_where_only_households_are_calculated( countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=lambda: client, ) rerun = rerun_input(None, STORED_POLICY_ID, kind="household") @@ -245,6 +253,7 @@ def test_a_household_report_of_the_baseline_alone_checks_the_baseline_policy() - countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=lambda: client, ) rerun = rerun_input(STORED_POLICY_ID, None, kind="household") @@ -261,6 +270,7 @@ def test_a_household_reports_rerun_without_household_calculations_is_refused() - countries=StoredCountries(household_enabled=False), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -275,6 +285,7 @@ def test_a_household_reports_rerun_checks_its_policies_as_a_new_one_does() -> No countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -292,6 +303,7 @@ def test_a_household_reports_rerun_whose_countries_cannot_be_read_is_unavailable countries=UnreadableCountries(error), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) diff --git a/tests/unit/v2/test_simulation_submission_database_routes.py b/tests/unit/v2/test_simulation_submission_database_routes.py index 5c3f96a44..b9041099b 100644 --- a/tests/unit/v2/test_simulation_submission_database_routes.py +++ b/tests/unit/v2/test_simulation_submission_database_routes.py @@ -21,6 +21,7 @@ from policyengine_api.data.v2.models import ( V2_METADATA, Country, + Dynamic, Parameter, ParameterValue, Policy, @@ -31,6 +32,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -70,6 +75,9 @@ EARLIER_VERSION = "6.9.0" KEPT_PARAMETER = "gov.example.rate" REMOVED_PARAMETER = "gov.example.removed_rate" +KEPT_ASSUMPTION = "gov.simulation.labor_supply_responses.elasticities.income" +REMOVED_ASSUMPTION = "gov.simulation.labor_supply_responses.removed_elasticity" +UK_ASSUMPTION = "gov.simulation.labour_supply_responses.income_elasticity" STORED_AT = datetime(2026, 11, 10, tzinfo=UTC) REPORTS = "/v2/reports?country_id=us" SIMULATIONS = "/v2/simulations?country_id=us" @@ -181,6 +189,75 @@ def stored_policy_ids(engine: Engine) -> dict[str, UUID]: return {name: policy.id for name, policy in policies.items()} +def store_dynamic( + session: Session, + country_id: str, + parameters: dict[str, Parameter], + content_hash_letter: str, +) -> Dynamic: + """Store a dynamic setting each of ``parameters``, of their catalog version.""" + model_version = next(iter(parameters.values())).tax_benefit_model_version + dynamic = Dynamic( + country_id=country_id, + tax_benefit_model_id=model_version.model_id, + tax_benefit_model_version_id=model_version.id, + canonicalization_version=1, + content_hash=content_hash_letter * 64, + created_at=STORED_AT, + updated_at=STORED_AT, + ) + session.add(dynamic) + session.flush() + session.add_all( + ParameterValue( + parameter_id=parameter.id, + value_json=0.25, + start_date=STORED_AT, + dynamic_id=dynamic.id, + ) + for parameter in parameters.values() + ) + return dynamic + + +@pytest.fixture +def stored_dynamic_ids(engine: Engine) -> dict[str, UUID]: + """Store the US with two catalog versions, the UK with one, and three dynamics. + + ``outdated_dynamic`` sets an assumption the running US catalog version + removed, ``running_dynamic`` was made on the running US version, and + ``uk_dynamic`` is the UK's. + """ + with Session(engine, expire_on_commit=False) as session: + session.add_all( + [ + Country(id="us", household_enabled=True, society_wide_enabled=True), + Country(id="uk", household_enabled=True, society_wide_enabled=True), + ] + ) + us_model = TaxBenefitModel(name="policyengine-us") + earlier = store_catalog_version( + session, us_model, EARLIER_VERSION, [KEPT_ASSUMPTION, REMOVED_ASSUMPTION] + ) + running = store_catalog_version( + session, us_model, RUNNING_VERSION, [KEPT_ASSUMPTION] + ) + uk_running = store_catalog_version( + session, + TaxBenefitModel(name="policyengine-uk"), + RUNNING_VERSION, + [UK_ASSUMPTION], + ) + session.flush() + dynamics = { + "outdated_dynamic": store_dynamic(session, "us", earlier, "d"), + "running_dynamic": store_dynamic(session, "us", running, "e"), + "uk_dynamic": store_dynamic(session, "uk", uk_running, "f"), + } + session.commit() + return {name: dynamic.id for name, dynamic in dynamics.items()} + + @pytest.fixture def simulation_service() -> RecordingSimulationService: return RecordingSimulationService() @@ -198,6 +275,10 @@ def submissions( PolicyDatabaseSession(sessions), running_policyengine_version=RUNNING_VERSION, ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ), client_factory=lambda: simulation_service, ) @@ -333,3 +414,64 @@ def test_a_catalog_that_cannot_be_queried_answers_503( } assert simulation_service.reports == [] assert simulation_service.simulations == [] + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_a_report_naming_another_country_s_dynamic_answers_404( + client: TestClient, + stored_dynamic_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, + side: str, +) -> None: + uk_dynamic_id = stored_dynamic_ids["uk_dynamic"] + + response = client.post( + REPORTS, json=report_body(**{side: {"dynamic_id": str(uk_dynamic_id)}}) + ) + + assert response.status_code == 404, response.text + assert response.json() == { + "status": "error", + "message": f"dynamic {uk_dynamic_id} was not found", + } + assert simulation_service.reports == [] + + +def test_a_simulation_whose_dynamic_sets_parameters_the_catalog_lacks_answers_422( + client: TestClient, + stored_dynamic_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, +) -> None: + outdated_dynamic_id = stored_dynamic_ids["outdated_dynamic"] + body = {**simulation_body(uuid4()), "policy_id": None} + + response = client.post( + SIMULATIONS, json={**body, "dynamic_id": str(outdated_dynamic_id)} + ) + + assert response.status_code == 422, response.text + assert response.json() == { + "status": "error", + "message": ( + f"Dynamic {outdated_dynamic_id} sets parameters the running catalog " + f"version does not have: {REMOVED_ASSUMPTION}" + ), + "code": "missing_parameters", + "missing_parameters": [REMOVED_ASSUMPTION], + } + assert simulation_service.simulations == [] + + +def test_a_dynamic_of_the_country_on_the_running_catalog_is_sent( + submissions: SimulationSubmissionService, + stored_dynamic_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, +) -> None: + running_dynamic = {"dynamic_id": str(stored_dynamic_ids["running_dynamic"])} + report = EconomyReportCreationInput.model_validate( + report_body(baseline=running_dynamic, reform=running_dynamic) + ) + + submissions.submit_report(report) + + assert simulation_service.reports == [report] diff --git a/tests/unit/v2/test_simulation_submissions.py b/tests/unit/v2/test_simulation_submissions.py index 45fb541ab..fff3d940d 100644 --- a/tests/unit/v2/test_simulation_submissions.py +++ b/tests/unit/v2/test_simulation_submissions.py @@ -14,6 +14,12 @@ MetadataCatalogUnavailableError, ) from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.dynamics.types import CheckedDynamicRead +from policyengine_api.services.v2.dynamics.validators import ( + DynamicNotFoundError, + DynamicParametersMissingError, + DynamicStorageError, +) from policyengine_api.services.v2.policies.types import CheckedPolicyRead from policyengine_api.services.v2.policies.validators import ( PolicyNotFoundError, @@ -51,6 +57,18 @@ SECOND_OUTDATED_POLICY_ID: SECOND_REMOVED_PARAMETERS, UNCHECKED_POLICY_ID: None, } +# Stored US dynamics; the second sets parameters the running catalog lacks. +STORED_DYNAMIC_ID = UUID("00000000-0000-0000-0000-00000000c101") +OUTDATED_DYNAMIC_ID = UUID("00000000-0000-0000-0000-00000000c102") +# A stored UK dynamic. +UK_DYNAMIC_ID = UUID("00000000-0000-0000-0000-00000000c103") +REMOVED_ASSUMPTIONS = ("gov.simulation.removed_elasticity",) +# Each stored dynamic's country, and its parameters the running catalog lacks. +STORED_DYNAMICS: dict[UUID, tuple[str, tuple[str, ...] | None]] = { + STORED_DYNAMIC_ID: ("us", ()), + OUTDATED_DYNAMIC_ID: ("us", REMOVED_ASSUMPTIONS), + UK_DYNAMIC_ID: ("uk", ()), +} SERVED_COUNTRIES = [ CountryRead(id="uk", household_enabled=True, society_wide_enabled=True), CountryRead(id="us", household_enabled=True, society_wide_enabled=True), @@ -110,6 +128,37 @@ def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicyRead: return checked_policy(policy_id, country_id, MISSING_PARAMETERS[policy_id]) +@dataclass +class StoredDynamics: + """The dynamics in STORED_DYNAMICS, each stored in its own country. + + With ``error``, every read raises it, as a database or catalog that + cannot be read does. + """ + + error: Exception | None = None + asked: list[tuple[str, UUID]] = field(default_factory=list) + + def get_dynamic(self, *, country_id: str, dynamic_id: UUID) -> CheckedDynamicRead: + self.asked.append((country_id, dynamic_id)) + if self.error is not None: + raise self.error + stored = STORED_DYNAMICS.get(dynamic_id) + if stored is None or stored[0] != country_id: + msg = f"Dynamic {dynamic_id} was not found" + raise DynamicNotFoundError(msg) + return CheckedDynamicRead( + id=dynamic_id, + country_id=country_id, + tax_benefit_model_id=uuid4(), + tax_benefit_model_version_id=uuid4(), + created_at=STORED_AT, + updated_at=STORED_AT, + parameter_values=(), + missing_parameters=stored[1], + ) + + @dataclass class RecordingSimulationService: """The simulation service's client, recording what it is sent.""" @@ -166,12 +215,15 @@ def simulation_input(**changes: Any) -> EconomySimulationCreationInput: def submissions( - client_factory: Any, policies: StoredPolicies | None = None + client_factory: Any, + policies: StoredPolicies | None = None, + dynamics: StoredDynamics | None = None, ) -> SimulationSubmissionService: return SimulationSubmissionService( countries=StoredCountries(), households=UnreadHouseholds(), policies=policies or StoredPolicies(), + dynamics=dynamics or StoredDynamics(), client_factory=client_factory, ) @@ -196,7 +248,7 @@ def test_current_law_needs_no_stored_policy() -> None: policies = StoredPolicies() submissions(lambda: client, policies).submit_report( - report_input(reform={"dynamic_id": str(uuid4())}) + report_input(reform={"dynamic_id": str(STORED_DYNAMIC_ID)}) ) assert policies.asked == [] @@ -208,6 +260,7 @@ def test_a_household_only_country_is_refused_before_sending() -> None: countries=StoredCountries(countries=HOUSEHOLD_ONLY_UK), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -222,6 +275,7 @@ def test_a_country_without_a_countries_row_is_refused() -> None: countries=StoredCountries(countries=SERVED_COUNTRIES[:1]), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -350,3 +404,101 @@ def test_a_policy_check_that_cannot_read_storage_is_storage_unavailable( service.submit_report(report_input()) assert unavailable.value.__cause__ is error + + +def test_a_report_s_dynamics_are_checked_in_its_country_and_sent() -> None: + client = RecordingSimulationService() + dynamics = StoredDynamics() + report = report_input( + baseline={"dynamic_id": str(STORED_DYNAMIC_ID)}, + reform={ + "policy_id": str(STORED_POLICY_ID), + "dynamic_id": str(STORED_DYNAMIC_ID), + }, + ) + + submissions(lambda: client, dynamics=dynamics).submit_report(report) + + assert client.reports == [report] + assert dynamics.asked == [("us", STORED_DYNAMIC_ID)] + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_a_report_naming_another_country_s_dynamic_is_refused(side: str) -> None: + service = submissions(unconfigured_client) + + with pytest.raises(DynamicNotFoundError, match=str(UK_DYNAMIC_ID)): + service.submit_report( + report_input(**{side: {"dynamic_id": str(UK_DYNAMIC_ID)}}) + ) + + +def test_a_simulation_is_sent_once_its_dynamic_is_checked() -> None: + client = RecordingSimulationService() + dynamics = StoredDynamics() + simulation = simulation_input(dynamic_id=str(UK_DYNAMIC_ID)) + + submissions(lambda: client, dynamics=dynamics).submit_simulation(simulation) + + assert client.simulations == [simulation] + assert dynamics.asked == [("uk", UK_DYNAMIC_ID)] + + +def test_a_simulation_naming_another_country_s_or_no_stored_dynamic_is_refused() -> ( + None +): + service = submissions(unconfigured_client) + + for dynamic_id in (STORED_DYNAMIC_ID, uuid4()): + with pytest.raises(DynamicNotFoundError, match=str(dynamic_id)): + service.submit_simulation(simulation_input(dynamic_id=str(dynamic_id))) + + +def test_a_dynamic_setting_parameters_the_running_catalog_lacks_is_refused() -> None: + service = submissions(unconfigured_client) + + with pytest.raises(DynamicParametersMissingError) as refusal: + service.submit_report( + report_input(reform={"dynamic_id": str(OUTDATED_DYNAMIC_ID)}) + ) + + assert isinstance(refusal.value, PolicyParametersMissingError) + assert refusal.value.dynamic_id == OUTDATED_DYNAMIC_ID + assert refusal.value.missing_parameters == REMOVED_ASSUMPTIONS + assert str(refusal.value).startswith(f"Dynamic {OUTDATED_DYNAMIC_ID} sets") + + +def test_policies_are_checked_before_dynamics() -> None: + dynamics = StoredDynamics() + service = submissions(unconfigured_client, dynamics=dynamics) + + with pytest.raises(PolicyNotFoundError): + service.submit_report( + report_input( + reform={ + "policy_id": str(OTHER_POLICY_ID), + "dynamic_id": str(UK_DYNAMIC_ID), + } + ) + ) + + assert dynamics.asked == [] + + +@pytest.mark.parametrize( + "error", + [ + DynamicStorageError("dynamics cannot be read"), + OperationalError("SELECT 1", {}, Exception("connection refused")), + MetadataCatalogUnavailableError("catalog cannot be queried"), + ], +) +def test_a_dynamic_read_that_fails_is_a_storage_error(error: Exception) -> None: + service = submissions(unconfigured_client, dynamics=StoredDynamics(error=error)) + + with pytest.raises(SubmissionStorageError) as failure: + service.submit_simulation( + simulation_input(dynamic_id=str(UK_DYNAMIC_ID), policy_id=None) + ) + + assert failure.value.__cause__ is error