From b872d5fda64819ddd2255104c34ec58963406cb3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:27:34 +0400 Subject: [PATCH 1/6] Test dynamics stored like policies and the gov.simulation split (failing) Dynamics get policies' country, catalog version and content-hash columns, lose name and description, and a dynamic's values are unique per parameter and start date. legacy_policy_mappings names an optional dynamic of the same country. A new pure module says which parameters a dynamic holds and splits a v1 policy's values, rewriting the UK's old labour-supply spelling. Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_dynamic_parameters.py | 147 +++++++++++++ tests/unit/v2/test_dynamic_table_models.py | 229 +++++++++++++++++++++ tests/unit/v2/test_model_persistence.py | 10 +- 3 files changed, 384 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_dynamic_parameters.py create mode 100644 tests/unit/v2/test_dynamic_table_models.py diff --git a/tests/unit/test_dynamic_parameters.py b/tests/unit/test_dynamic_parameters.py new file mode 100644 index 000000000..ee94acc16 --- /dev/null +++ b/tests/unit/test_dynamic_parameters.py @@ -0,0 +1,147 @@ +"""Which parameters a dynamic holds, and how a v1 policy's values are split.""" + +from typing import get_args + +import pytest +from policyengine_api.dynamic_parameters import ( + DYNAMIC_PARAMETER_PREFIXES, + ConflictingParameterSpellingsError, + canonical_parameter_name, + is_dynamic_parameter, + split_legacy_policy_json, +) + +from policyengine_api.supported_countries import SupportedCountryId + +UK_OLD_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.income_elasticity" +UK_INCOME_ELASTICITY = "gov.simulation.labour_supply_responses.income_elasticity" +US_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" + + +def test_every_supported_country_lists_its_prefixes() -> None: + assert set(DYNAMIC_PARAMETER_PREFIXES) == set(get_args(SupportedCountryId)) + assert DYNAMIC_PARAMETER_PREFIXES["us"] == ("gov.simulation.", "simulation.") + assert DYNAMIC_PARAMETER_PREFIXES["uk"] == ("gov.simulation.",) + + +@pytest.mark.parametrize( + ("country_id", "parameter_name"), + [ + ("us", US_INCOME_ELASTICITY), + ("us", "gov.simulation.branch_to_determine_itemization"), + ("us", "simulation.marginal_tax_rate_delta"), + ("uk", UK_INCOME_ELASTICITY), + ("uk", "gov.simulation.marginal_tax_rate_delta"), + ], +) +def test_parameters_under_the_country_s_prefixes_are_dynamic( + country_id: SupportedCountryId, parameter_name: str +) -> None: + assert is_dynamic_parameter(country_id, parameter_name) + + +@pytest.mark.parametrize( + ("country_id", "parameter_name"), + [ + ("us", "gov.irs.credits.ctc.amount.base[0].amount"), + # A name that only starts with the same letters is not under the prefix. + ("us", "gov.simulations_count"), + ("us", "simulationX.rate"), + # The UK has no top-level simulation tree. + ("uk", "simulation.marginal_tax_rate_delta"), + ("uk", "gov.hmrc.income_tax.allowances.personal_allowance.amount"), + ], +) +def test_other_parameters_are_not_dynamic( + country_id: SupportedCountryId, parameter_name: str +) -> None: + assert not is_dynamic_parameter(country_id, parameter_name) + + +def test_only_the_uk_old_labour_supply_spelling_is_rewritten() -> None: + assert canonical_parameter_name("uk", UK_OLD_INCOME_ELASTICITY) == ( + UK_INCOME_ELASTICITY + ) + assert canonical_parameter_name("uk", UK_INCOME_ELASTICITY) == ( + UK_INCOME_ELASTICITY + ) + # In the US, labor_supply_responses is the parameter's own name. + assert canonical_parameter_name("us", US_INCOME_ELASTICITY) == ( + US_INCOME_ELASTICITY + ) + assert canonical_parameter_name("uk", "gov.hmrc.rate") == "gov.hmrc.rate" + + +def test_a_us_policy_is_split_into_its_policy_and_dynamic_values() -> None: + policy_json = { + "gov.irs.rate": {"2026-01-01.2100-12-31": 0.2}, + US_INCOME_ELASTICITY: {"2026": 0.1}, + "simulation.marginal_tax_rate_delta": {"2026": 500}, + } + + split = split_legacy_policy_json("us", policy_json) + + assert split.policy_values == {"gov.irs.rate": {"2026-01-01.2100-12-31": 0.2}} + assert split.dynamic_values == { + US_INCOME_ELASTICITY: {"2026": 0.1}, + "simulation.marginal_tax_rate_delta": {"2026": 500}, + } + + +def test_a_uk_policy_s_old_spelling_is_rewritten_into_its_dynamic() -> None: + split = split_legacy_policy_json( + "uk", + { + "gov.hmrc.rate": {"2026": 0.2}, + UK_OLD_INCOME_ELASTICITY: {"2026": 0.3}, + }, + ) + + assert split.policy_values == {"gov.hmrc.rate": {"2026": 0.2}} + assert split.dynamic_values == {UK_INCOME_ELASTICITY: {"2026": 0.3}} + + +def test_both_uk_spellings_of_one_parameter_merge_their_periods() -> None: + split = split_legacy_policy_json( + "uk", + { + UK_OLD_INCOME_ELASTICITY: {"2026": 0.3, "2027": 0.4}, + UK_INCOME_ELASTICITY: {"2027": 0.4, "2028": 0.5}, + }, + ) + + assert split.policy_values == {} + assert split.dynamic_values == { + UK_INCOME_ELASTICITY: {"2026": 0.3, "2027": 0.4, "2028": 0.5} + } + + +@pytest.mark.parametrize( + ("old_spelling_values", "new_spelling_values"), + [ + ({"2026": 0.3}, {"2026": 0.4}), + ({"2026": 0.3}, 0.4), + ("0.3", {"2027": 0.4}), + ], +) +def test_both_uk_spellings_that_disagree_are_refused( + old_spelling_values: object, new_spelling_values: object +) -> None: + with pytest.raises(ConflictingParameterSpellingsError): + split_legacy_policy_json( + "uk", + { + UK_OLD_INCOME_ELASTICITY: old_spelling_values, + UK_INCOME_ELASTICITY: new_spelling_values, + }, + ) + + +def test_a_policy_with_no_dynamic_values_is_left_whole() -> None: + policy_json = {"gov.irs.rate": {"2026": 0.2}} + + split = split_legacy_policy_json("us", policy_json) + + assert split.policy_values == policy_json + assert split.dynamic_values == {} + assert split_legacy_policy_json("us", {}) == split_legacy_policy_json("uk", {}) diff --git a/tests/unit/v2/test_dynamic_table_models.py b/tests/unit/v2/test_dynamic_table_models.py new file mode 100644 index 000000000..012a3efdf --- /dev/null +++ b/tests/unit/v2/test_dynamic_table_models.py @@ -0,0 +1,229 @@ +"""Dynamics are stored as policies are: once per canonical content hash.""" + +from datetime import UTC, datetime + +import pytest +import sqlalchemy as sa +from sqlalchemy.exc import IntegrityError +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.models import ( + V2_METADATA, + Dynamic, + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from tests.fixtures.v2_countries import add_listed_countries + +START_DATE = datetime(2026, 1, 1, tzinfo=UTC) + + +def _sqlite_engine_with_foreign_keys() -> sa.Engine: + engine = create_engine("sqlite://") + + @sa.event.listens_for(engine, "connect") + def enable_foreign_keys(dbapi_connection, _connection_record) -> None: + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + V2_METADATA.create_all(engine) + with Session(engine) as session: + add_listed_countries(session) + return engine + + +def _catalog() -> tuple[TaxBenefitModel, TaxBenefitModelVersion, Parameter]: + model = TaxBenefitModel(name="policyengine-us") + version = TaxBenefitModelVersion( + model=model, version="2.2.2", current_law_id=1, metadata_time_periods=[2026] + ) + parameter = Parameter( + name="gov.simulation.labor_supply_responses.elasticities.income", + tax_benefit_model_version=version, + ) + return model, version, parameter + + +def _dynamic( + model: TaxBenefitModel, + version: TaxBenefitModelVersion, + *, + content_hash: str = "d" * 64, + country_id: str = "us", +) -> Dynamic: + return Dynamic( + country_id=country_id, + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=version.id, + canonicalization_version=1, + content_hash=content_hash, + ) + + +def test_dynamics_have_policies_columns_and_no_name_or_description() -> None: + dynamics = V2_METADATA.tables["dynamics"] + policies = V2_METADATA.tables["policies"] + + assert set(dynamics.c.keys()) == set(policies.c.keys()) + assert "name" not in dynamics.c + assert "description" not in dynamics.c + for column_name in ( + "country_id", + "tax_benefit_model_id", + "tax_benefit_model_version_id", + "canonicalization_version", + "content_hash", + ): + assert not dynamics.c[column_name].nullable + foreign_keys = { + foreign_key.parent.name: (foreign_key.column.table.name, foreign_key.ondelete) + for foreign_key in dynamics.foreign_keys + } + assert foreign_keys == { + "country_id": ("countries", "RESTRICT"), + "tax_benefit_model_id": ("tax_benefit_models", "RESTRICT"), + "tax_benefit_model_version_id": ("tax_benefit_model_versions", "RESTRICT"), + } + unique_column_sets = { + constraint.name: tuple(column.name for column in constraint.columns) + for constraint in dynamics.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert unique_column_sets == { + "uq_dynamics_id_country": ("id", "country_id"), + "uq_dynamics_canonicalization_content_hash": ( + "canonicalization_version", + "content_hash", + ), + } + check_names = { + constraint.name + for constraint in dynamics.constraints + if isinstance(constraint, sa.CheckConstraint) + } + assert check_names == { + "ck_dynamics_canonicalization_version", + "ck_dynamics_content_hash_length", + } + + +def test_a_dynamic_s_values_are_unique_per_parameter_and_start_date() -> None: + parameter_values = V2_METADATA.tables["parameter_values"] + index = next( + index + for index in parameter_values.indexes + if index.name == "uq_parameter_values_dynamic_parameter_start_date" + ) + + assert index.unique + assert tuple(column.name for column in index.columns) == ( + "dynamic_id", + "parameter_id", + "start_date", + ) + assert str(index.dialect_options["postgresql"]["where"]) == ( + "dynamic_id IS NOT NULL" + ) + assert str(index.dialect_options["sqlite"]["where"]) == "dynamic_id IS NOT NULL" + + +def test_legacy_policy_mappings_name_an_optional_dynamic_of_their_country() -> None: + mappings = V2_METADATA.tables["legacy_policy_mappings"] + + assert mappings.c.dynamic_id.nullable + (dynamic_key,) = ( + constraint + for constraint in mappings.constraints + if isinstance(constraint, sa.ForeignKeyConstraint) + and constraint.referred_table.name == "dynamics" + ) + assert dynamic_key.name == "fk_legacy_policy_mappings_dynamic_country" + assert [column.name for column in dynamic_key.columns] == [ + "dynamic_id", + "country_id", + ] + assert [element.target_fullname for element in dynamic_key.elements] == [ + "dynamics.id", + "dynamics.country_id", + ] + assert dynamic_key.ondelete == "RESTRICT" + assert any( + index.name == "ix_legacy_policy_mappings_dynamic" + and [column.name for column in index.columns] == ["dynamic_id"] + for index in mappings.indexes + ) + + +def test_identical_dynamic_content_is_stored_once() -> None: + engine = _sqlite_engine_with_foreign_keys() + model, version, parameter = _catalog() + + with Session(engine) as session: + session.add_all([model, version, parameter, _dynamic(model, version)]) + session.commit() + session.add(_dynamic(model, version)) + with pytest.raises(IntegrityError, match="UNIQUE constraint failed"): + session.commit() + session.rollback() + session.add(_dynamic(model, version, content_hash="e" * 64)) + session.commit() + + engine.dispose() + + +def test_a_dynamic_sets_one_value_per_parameter_and_start_date() -> None: + engine = _sqlite_engine_with_foreign_keys() + model, version, parameter = _catalog() + + with Session(engine) as session: + dynamic = _dynamic(model, version) + session.add_all([model, version, parameter, dynamic]) + session.commit() + for value in (0.1, 0.2): + session.add( + ParameterValue( + parameter_id=parameter.id, + dynamic_id=dynamic.id, + value_json=value, + start_date=START_DATE, + ) + ) + with pytest.raises(IntegrityError, match="UNIQUE constraint failed"): + session.commit() + + engine.dispose() + + +def test_a_mapping_cannot_name_another_country_s_dynamic() -> None: + engine = _sqlite_engine_with_foreign_keys() + model, version, _parameter = _catalog() + + with Session(engine) as session: + policy = Policy( + country_id="uk", + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=version.id, + canonicalization_version=1, + content_hash="a" * 64, + ) + us_dynamic = _dynamic(model, version) + session.add_all([model, version, policy, us_dynamic]) + session.commit() + session.add( + LegacyPolicyMapping( + country_id="uk", + legacy_policy_id=7, + policy_id=policy.id, + dynamic_id=us_dynamic.id, + source_policy_hash="v1-hash", + ) + ) + with pytest.raises(IntegrityError, match="FOREIGN KEY constraint failed"): + session.commit() + + engine.dispose() diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index 732fa7d98..4341e2800 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -288,8 +288,14 @@ def test_policy_content_hash_is_unique_per_canonicalization_version() -> None: def test_policy_parameter_value_owner_period_and_identity_constraints() -> None: engine = _relational_sqlite_engine() - _model, _version, parameter, policy = _policy_graph(content_hash="b" * 64) - dynamic = Dynamic(name="dynamic") + model, version, parameter, policy = _policy_graph(content_hash="b" * 64) + dynamic = Dynamic( + country_id="us", + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=version.id, + canonicalization_version=1, + content_hash="d" * 64, + ) start = datetime(2026, 1, 1, tzinfo=UTC) with Session(engine) as session: From 525cd9f75e946c569b81f8696365223b73e34d1c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 15:34:49 +0400 Subject: [PATCH 2/6] Store dynamics like policies, and map v1 policies to their dynamic Revision 9d1d12a7ac58 (on f519a77ba122, autogenerated with no corrections apart from ruff format) gives dynamics policies' country, tax-benefit model, catalog version, canonicalization version and content hash, unique per version and hash, and drops name and description, which nothing wrote or read. parameter_values gains a partial unique index on a dynamic's values per parameter and start date, and legacy_policy_mappings a nullable dynamic_id, a RESTRICT key to a dynamic of the mapping's country. The runtime role may read and insert dynamics. policyengine_api.dynamic_parameters says which parameters a dynamic holds (every gov.simulation. parameter, plus the US simulation. tree), rewrites the UK's old labor_supply_responses spelling, and splits a v1 policy's values into the policy's and the dynamic's. Co-Authored-By: Claude Opus 5.5 --- ...tore_dynamics_like_policies_and_map_v1_.py | 207 +++++++++++ policyengine_api/data/v2/migration_target.py | 2 +- policyengine_api/data/v2/models/metadata.py | 17 +- policyengine_api/data/v2/models/policies.py | 53 ++- .../data/v2/models/policy_mappings.py | 24 +- policyengine_api/data/v2/privileges.py | 9 +- policyengine_api/dynamic_parameters.py | 136 +++++++ tests/integration/test_alembic_v2_dynamics.py | 339 ++++++++++++++++++ tests/unit/v2/test_alembic_v2_revisions.py | 79 +++- tests/unit/v2/test_dynamic_table_models.py | 12 +- tests/unit/v2/test_model_persistence.py | 4 +- tests/unit/v2/test_privilege_manifest.py | 16 + 12 files changed, 879 insertions(+), 19 deletions(-) create mode 100644 migrations/v2/versions/9d1d12a7ac58_store_dynamics_like_policies_and_map_v1_.py create mode 100644 policyengine_api/dynamic_parameters.py create mode 100644 tests/integration/test_alembic_v2_dynamics.py diff --git a/migrations/v2/versions/9d1d12a7ac58_store_dynamics_like_policies_and_map_v1_.py b/migrations/v2/versions/9d1d12a7ac58_store_dynamics_like_policies_and_map_v1_.py new file mode 100644 index 000000000..9ce710982 --- /dev/null +++ b/migrations/v2/versions/9d1d12a7ac58_store_dynamics_like_policies_and_map_v1_.py @@ -0,0 +1,207 @@ +"""store dynamics like policies and map v1 policies to them + +Revision ID: 9d1d12a7ac58 +Revises: f519a77ba122 +Create Date: 2026-11-17 15:29:53.770002 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate + +Stores dynamics as policies are stored. dynamics drops name and description, +which nothing wrote, and gains policies' columns: the country, the +tax-benefit model and catalog version (each RESTRICT), and the +canonicalization version and content hash, unique together. Checks require a +positive canonicalization version and a 64-character hash, and (id, +country_id) is unique so other tables can name a dynamic of one country. +parameter_values gains a partial unique index on (dynamic_id, parameter_id, +start_date) for rows with a dynamic: a dynamic's values are unique per +parameter and start date, as a policy's are, and the index holds only +dynamics' rows. legacy_policy_mappings gains a nullable dynamic_id, the +dynamic the v1-to-v2 copy split from the v1 policy, a foreign key (with the +country) to dynamics, RESTRICT, and indexed. The operations are exactly as +generated, apart from ruff format. + +Nothing wrote dynamics before this revision, and its new columns are +required with no default, so the upgrade fails, changing nothing, while +dynamics has a row. The previous Public API revision maps the columns this +revision drops, but never reads or writes dynamics, and its v1-to-v2 copy +writes mappings without dynamic_id, which stays null. The simulation service +selects only dynamics.id. The downgrade drops the new columns, their +constraints and the mappings' dynamic_id with its values, and adds name back +as required, so it fails while dynamics has a row. +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "9d1d12a7ac58" +down_revision: Union[str, None] = "f519a77ba122" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "dynamics", + sa.Column( + "country_id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False + ), + ) + op.add_column( + "dynamics", sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False) + ) + op.add_column( + "dynamics", sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False) + ) + op.add_column( + "dynamics", sa.Column("canonicalization_version", sa.Integer(), nullable=False) + ) + op.add_column( + "dynamics", + sa.Column( + "content_hash", sqlmodel.sql.sqltypes.AutoString(length=64), nullable=False + ), + ) + op.create_index( + op.f("ix_dynamics_tax_benefit_model_id"), + "dynamics", + ["tax_benefit_model_id"], + unique=False, + ) + op.create_index( + op.f("ix_dynamics_tax_benefit_model_version_id"), + "dynamics", + ["tax_benefit_model_version_id"], + unique=False, + ) + op.create_unique_constraint( + "uq_dynamics_canonicalization_content_hash", + "dynamics", + ["canonicalization_version", "content_hash"], + ) + op.create_unique_constraint( + "uq_dynamics_id_country", "dynamics", ["id", "country_id"] + ) + op.create_foreign_key( + op.f("fk_dynamics_tax_benefit_model_version_id_tax_benefit_model_versions"), + "dynamics", + "tax_benefit_model_versions", + ["tax_benefit_model_version_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + op.f("fk_dynamics_tax_benefit_model_id_tax_benefit_models"), + "dynamics", + "tax_benefit_models", + ["tax_benefit_model_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + op.f("fk_dynamics_country_id_countries"), + "dynamics", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_check_constraint( + op.f("ck_dynamics_canonicalization_version"), + "dynamics", + "canonicalization_version > 0", + ) + op.create_check_constraint( + op.f("ck_dynamics_content_hash_length"), "dynamics", "length(content_hash) = 64" + ) + op.drop_column("dynamics", "description") + op.drop_column("dynamics", "name") + op.add_column( + "legacy_policy_mappings", sa.Column("dynamic_id", sa.Uuid(), nullable=True) + ) + op.create_index( + "ix_legacy_policy_mappings_dynamic", + "legacy_policy_mappings", + ["dynamic_id"], + unique=False, + ) + op.create_foreign_key( + "fk_legacy_policy_mappings_dynamic_country", + "legacy_policy_mappings", + "dynamics", + ["dynamic_id", "country_id"], + ["id", "country_id"], + ondelete="RESTRICT", + ) + op.create_index( + "uq_parameter_values_dynamic_parameter_start_date", + "parameter_values", + ["dynamic_id", "parameter_id", "start_date"], + unique=True, + postgresql_where=sa.text("dynamic_id IS NOT NULL"), + sqlite_where=sa.text("dynamic_id IS NOT NULL"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "uq_parameter_values_dynamic_parameter_start_date", + table_name="parameter_values", + postgresql_where=sa.text("dynamic_id IS NOT NULL"), + sqlite_where=sa.text("dynamic_id IS NOT NULL"), + ) + op.drop_constraint( + "fk_legacy_policy_mappings_dynamic_country", + "legacy_policy_mappings", + type_="foreignkey", + ) + op.drop_index( + "ix_legacy_policy_mappings_dynamic", table_name="legacy_policy_mappings" + ) + op.drop_column("legacy_policy_mappings", "dynamic_id") + op.add_column( + "dynamics", + sa.Column("name", sa.VARCHAR(length=255), autoincrement=False, nullable=False), + ) + op.add_column( + "dynamics", + sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), + ) + op.drop_constraint( + op.f("ck_dynamics_content_hash_length"), "dynamics", type_="check" + ) + op.drop_constraint( + op.f("ck_dynamics_canonicalization_version"), "dynamics", type_="check" + ) + op.drop_constraint( + op.f("fk_dynamics_country_id_countries"), "dynamics", type_="foreignkey" + ) + op.drop_constraint( + op.f("fk_dynamics_tax_benefit_model_id_tax_benefit_models"), + "dynamics", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_dynamics_tax_benefit_model_version_id_tax_benefit_model_versions"), + "dynamics", + type_="foreignkey", + ) + op.drop_constraint("uq_dynamics_id_country", "dynamics", type_="unique") + op.drop_constraint( + "uq_dynamics_canonicalization_content_hash", "dynamics", type_="unique" + ) + op.drop_index( + op.f("ix_dynamics_tax_benefit_model_version_id"), table_name="dynamics" + ) + op.drop_index(op.f("ix_dynamics_tax_benefit_model_id"), table_name="dynamics") + op.drop_column("dynamics", "content_hash") + op.drop_column("dynamics", "canonicalization_version") + op.drop_column("dynamics", "tax_benefit_model_version_id") + op.drop_column("dynamics", "tax_benefit_model_id") + op.drop_column("dynamics", "country_id") + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index b51fd461c..9c4fcbf26 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "f519a77ba122" +V2_ALEMBIC_HEAD_REVISION = "9d1d12a7ac58" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 30115f15a..8777d00d8 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -418,8 +418,9 @@ class Parameter(IdentifiedModel, table=True): class ParameterValue(IdentifiedModel, table=True): """A parameter's value from ``start_date`` until ``end_date``, or indefinitely. - A value owned by a policy or dynamic is deleted with its owner. A value with - neither is current law, and a parameter has at most one per start date. + A value owned by a policy or dynamic is deleted with its owner, and each + owner sets at most one per parameter and start date. A value with neither + is current law, and a parameter has at most one per start date. """ __tablename__ = "parameter_values" @@ -438,6 +439,18 @@ class ParameterValue(IdentifiedModel, table=True): "start_date", name="uq_parameter_values_policy_parameter_start_date", ), + # A dynamic's values are unique as a policy's are. The index covers + # only dynamics' rows, so it doesn't index every current-law value; + # rows without a dynamic never conflict in a unique key anyway. + sa.Index( + "uq_parameter_values_dynamic_parameter_start_date", + "dynamic_id", + "parameter_id", + "start_date", + unique=True, + postgresql_where=sa.text("dynamic_id IS NOT NULL"), + sqlite_where=sa.text("dynamic_id IS NOT NULL"), + ), sa.Index( "ix_parameter_values_parameter_period", "parameter_id", diff --git a/policyengine_api/data/v2/models/policies.py b/policyengine_api/data/v2/models/policies.py index f0da6e8cc..e7f4e5b86 100644 --- a/policyengine_api/data/v2/models/policies.py +++ b/policyengine_api/data/v2/models/policies.py @@ -96,15 +96,55 @@ class Policy(TimestampedModel, table=True): class Dynamic(TimestampedModel, table=True): - """Named parameter values, separate from any policy, that a simulation can apply. - - Its parameter values are deleted with it. + """Immutable model assumptions, stored once per canonicalization version and hash. + + A dynamic holds parameter values a simulation applies alongside its + policy: every parameter under ``gov.simulation.``, plus the US's + top-level ``simulation.`` (``policyengine_api.dynamic_parameters``). It + is stored and hashed as a policy is, bound to one catalog version, with + no name or description: a user's saved dynamics name it. Its parameter + values are deleted with it. """ __tablename__ = "dynamics" + __table_args__ = ( + sa.UniqueConstraint( + "id", + "country_id", + name="uq_dynamics_id_country", + ), + sa.UniqueConstraint( + "canonicalization_version", + "content_hash", + name="uq_dynamics_canonicalization_content_hash", + ), + sa.CheckConstraint( + "canonicalization_version > 0", + name="ck_dynamics_canonicalization_version", + ), + sa.CheckConstraint( + "length(content_hash) = 64", + name="ck_dynamics_content_hash_length", + ), + ) - name: str = Field(max_length=255) - description: str | None = None + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) + tax_benefit_model_id: UUID = Field( + foreign_key="tax_benefit_models.id", + ondelete="RESTRICT", + index=True, + ) + tax_benefit_model_version_id: UUID = Field( + foreign_key="tax_benefit_model_versions.id", + ondelete="RESTRICT", + index=True, + ) + canonicalization_version: int + content_hash: str = Field(max_length=64) parameter_values: list["ParameterValue"] = Relationship( back_populates="dynamic", @@ -112,3 +152,6 @@ class Dynamic(TimestampedModel, table=True): ) simulations: list["Simulation"] = Relationship(back_populates="dynamic") household_jobs: list["HouseholdJob"] = Relationship(back_populates="dynamic") + legacy_mappings: list["LegacyPolicyMapping"] = Relationship( + back_populates="dynamic" + ) diff --git a/policyengine_api/data/v2/models/policy_mappings.py b/policyengine_api/data/v2/models/policy_mappings.py index 7cdbc91b9..8e5475561 100644 --- a/policyengine_api/data/v2/models/policy_mappings.py +++ b/policyengine_api/data/v2/models/policy_mappings.py @@ -1,7 +1,7 @@ """Durable source-identity mappings for immediate v1 policy mirroring.""" from datetime import datetime -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional from uuid import UUID import sqlalchemy as sa @@ -11,7 +11,7 @@ if TYPE_CHECKING: from policyengine_api.data.v2.models.associations import UserPolicy - from policyengine_api.data.v2.models.policies import Policy + from policyengine_api.data.v2.models.policies import Dynamic, Policy class LegacyUserMapping(SQLModel, table=True): @@ -39,7 +39,13 @@ class LegacyUserMapping(SQLModel, table=True): class LegacyPolicyMapping(TimestampedModel, table=True): - """Map one country-scoped v1 policy ID to deduplicated v2 content.""" + """Map one country-scoped v1 policy ID to deduplicated v2 content. + + The copy splits a v1 policy's model assumptions into a dynamic of the same + country (``dynamic_id``), and keeps the rest in the policy. A v1 policy + with no such values, and every v1 policy copied before the split existed, + maps to no dynamic. + """ __tablename__ = "legacy_policy_mappings" __table_args__ = ( @@ -58,18 +64,30 @@ class LegacyPolicyMapping(TimestampedModel, table=True): name="fk_legacy_policy_mappings_policy_country", ondelete="RESTRICT", ), + sa.ForeignKeyConstraint( + ["dynamic_id", "country_id"], + ["dynamics.id", "dynamics.country_id"], + name="fk_legacy_policy_mappings_dynamic_country", + ondelete="RESTRICT", + ), sa.Index( "ix_legacy_policy_mappings_policy", "policy_id", ), + sa.Index( + "ix_legacy_policy_mappings_dynamic", + "dynamic_id", + ), ) country_id: str = Field(max_length=2) legacy_policy_id: int = Field(sa_type=sa.BigInteger) policy_id: UUID + dynamic_id: UUID | None = None source_policy_hash: str = Field(max_length=255) policy: "Policy" = Relationship(back_populates="legacy_mappings") + dynamic: Optional["Dynamic"] = Relationship(back_populates="legacy_mappings") class LegacyUserPolicyMapping(TimestampedModel, table=True): diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index ba116de22..5bc260267 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -245,8 +245,10 @@ def _run_table( # Custom totals, unused until they return. "aggregates": _access(), "change_aggregates": _access(), - # A simulation's or household job's dynamic, which a run reads. - "dynamics": _access(**_RUN_INPUT_READS), + # A simulation's or household job's dynamic, which a run reads. The Public + # API creates them, as it does policies, through /v2/dynamics and the + # v1-to-v2 policy copy; a dynamic never changes once stored. + "dynamics": _access(runtime=READ_INSERT, **_RUN_INPUT_READS), # Simulations, reports, their runs and household jobs. The catalog writer # checks that publication leaves the rows of three of them untouched. "budget_window_runs": _access( @@ -299,7 +301,8 @@ def _run_table( "parameter_tree_children": _access( runtime=READ, catalog_writer=READ_INSERT_DELETE | _ANALYZE ), - # The Public API also inserts a v2 policy's own parameter values. Catalog + # The Public API also inserts a v2 policy's or dynamic's own parameter + # values. Catalog # pruning deletes an older version's current-law values (those with no # policy or dynamic), never a policy's or a dynamic's. "parameter_values": _access( diff --git a/policyengine_api/dynamic_parameters.py b/policyengine_api/dynamic_parameters.py new file mode 100644 index 000000000..753e53a84 --- /dev/null +++ b/policyengine_api/dynamic_parameters.py @@ -0,0 +1,136 @@ +"""Which parameters a dynamic holds, and how a v1 policy's values are split. + +A dynamic holds the model's assumptions: every parameter under +``gov.simulation.``, in every country, and the US's top-level +``simulation.`` tree (C1 ยท 35, Q1). These are behavioural responses, model +settings and data assumptions, which a report applies alongside a policy. +Parameters a country package adds later under these prefixes belong to +dynamics with no code change. + +The rule is used in three places: + +- ``POST /v2/policies`` refuses a policy that sets any such parameter, and + ``POST /v2/dynamics`` refuses a dynamic that sets any other. +- The copy of each v1 policy into v2 splits the v1 policy's values with + ``split_legacy_policy_json``: these values become a dynamic and the rest + the policy. +- The one-time migration splits v1 policies, and v2 copies made before the + split existed, with the same functions. + +The UK renamed ``gov.simulation.labor_supply_responses`` to +``gov.simulation.labour_supply_responses`` on 2026-05-01. Its catalog has +only the new names, so a UK v1 policy saved with the old spelling is +rewritten before its parameters are looked up. In the US, +``labor_supply_responses`` is the parameter's own name and is never +rewritten. + +The module imports nothing from the package, so validators, transformations +and command-line code can all import it. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Final + +if TYPE_CHECKING: + from collections.abc import Mapping + + from policyengine_api.supported_countries import SupportedCountryId + +GOV_SIMULATION_PREFIX: Final = "gov.simulation." +DYNAMIC_PARAMETER_PREFIXES: Final[Mapping[SupportedCountryId, tuple[str, ...]]] = { + "uk": (GOV_SIMULATION_PREFIX,), + "us": (GOV_SIMULATION_PREFIX, "simulation."), +} +UK_OLD_LABOUR_SUPPLY_PREFIX: Final = "gov.simulation.labor_supply_responses." +UK_LABOUR_SUPPLY_PREFIX: Final = "gov.simulation.labour_supply_responses." + + +class ConflictingParameterSpellingsError(ValueError): + """A v1 policy sets one parameter under both UK spellings, and they disagree. + + Both spellings' values are period objects, merged into one; the policy is + refused when either is not a period object or both give one period + different values. + """ + + +@dataclass(frozen=True) +class LegacyPolicySplit: + """A v1 policy's ``{path: {period: value}}`` object, split in two. + + ``dynamic_values`` holds the parameters a dynamic holds, and + ``policy_values`` the rest, each keyed by the parameter's current name. + Either may be empty. + """ + + policy_values: dict[str, object] + dynamic_values: dict[str, object] + + +def is_dynamic_parameter(country_id: SupportedCountryId, parameter_name: str) -> bool: + """Return whether the country's parameter belongs in a dynamic, not a policy.""" + return parameter_name.startswith(DYNAMIC_PARAMETER_PREFIXES[country_id]) + + +def canonical_parameter_name( + country_id: SupportedCountryId, parameter_name: str +) -> str: + """Return the parameter's current name: the UK's old spelling rewritten.""" + if country_id == "uk" and parameter_name.startswith(UK_OLD_LABOUR_SUPPLY_PREFIX): + return UK_LABOUR_SUPPLY_PREFIX + parameter_name.removeprefix( + UK_OLD_LABOUR_SUPPLY_PREFIX + ) + return parameter_name + + +def _merged_period_values( + parameter_name: str, earlier: object, later: object +) -> dict[object, object]: + """Merge two spellings' period objects; raise when they disagree.""" + if type(earlier) is not dict or type(later) is not dict: + msg = f"parameter {parameter_name!r} is set under both spellings" + raise ConflictingParameterSpellingsError(msg) + for period, value in later.items(): + if period in earlier and earlier[period] != value: + msg = ( + f"parameter {parameter_name!r} is set under both spellings " + f"with different values for {period!r}" + ) + raise ConflictingParameterSpellingsError(msg) + return {**earlier, **later} + + +def split_legacy_policy_json( + country_id: SupportedCountryId, policy_json: Mapping[str, object] +) -> LegacyPolicySplit: + """Split a v1 policy's values into the policy's and the dynamic's. + + Each parameter path is first given its current name + (``canonical_parameter_name``). A UK policy that sets one parameter under + both spellings has their period objects merged; raise + ConflictingParameterSpellingsError when they can't be. The values + themselves are left as they are, for the translation to check. + """ + renamed: dict[str, object] = {} + for parameter_name, period_values in policy_json.items(): + current_name = canonical_parameter_name(country_id, parameter_name) + if current_name in renamed: + renamed[current_name] = _merged_period_values( + current_name, renamed[current_name], period_values + ) + else: + renamed[current_name] = period_values + return LegacyPolicySplit( + policy_values={ + name: values + for name, values in renamed.items() + if not is_dynamic_parameter(country_id, name) + }, + dynamic_values={ + name: values + for name, values in renamed.items() + if is_dynamic_parameter(country_id, name) + }, + ) diff --git a/tests/integration/test_alembic_v2_dynamics.py b/tests/integration/test_alembic_v2_dynamics.py new file mode 100644 index 000000000..3da55d7cf --- /dev/null +++ b/tests/integration/test_alembic_v2_dynamics.py @@ -0,0 +1,339 @@ +"""Check the revision that stores dynamics like policies against disposable Postgres. + +The revision gives ``dynamics`` policies' columns and drops ``name`` and +``description``, adds a partial unique index on a dynamic's values, and adds +``legacy_policy_mappings.dynamic_id``, a key to a dynamic of the mapping's +country. Nothing else changes. Its new required columns have no default, so +the upgrade fails while ``dynamics`` has a row, and the downgrade, which adds +``name`` back as required, fails the same way. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import pytest +from alembic import command +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.exc import IntegrityError + +from tests.fixtures.integration.alembic_v2 import alembic_v2_config, disposable_v2_url +from tests.integration.simulation_rows import ( + PublicApiRows, + delete_public_api_rows, + insert_public_api_rows, + insert_row, + refused, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Connection, Engine + +PREVIOUS_REVISION = "f519a77ba122" +DYNAMICS_REVISION = "9d1d12a7ac58" +CHANGED_TABLES = frozenset({"dynamics", "legacy_policy_mappings", "parameter_values"}) + + +@pytest.fixture +def engine() -> Iterator[Engine]: + """Yield an engine on a database at head, and leave it at head.""" + command.upgrade(alembic_v2_config(), "head") + database_engine = create_engine(disposable_v2_url()) + try: + yield database_engine + finally: + command.upgrade(alembic_v2_config(), "head") + database_engine.dispose() + + +@pytest.fixture +def connection(engine: Engine) -> Iterator[Connection]: + """Yield a connection at head whose writes are rolled back after the test.""" + with engine.connect() as database_connection: + transaction = database_connection.begin() + try: + yield database_connection + finally: + transaction.rollback() + + +def _schema(engine: Engine) -> dict[str, dict[str, Any]]: + """Return each table's columns, keys, checks and indexes, comparably.""" + inspector = inspect(engine) + return { + table_name: { + "columns": sorted( + (column["name"], str(column["type"]), column["nullable"]) + for column in inspector.get_columns(table_name) + ), + "foreign_keys": sorted( + ( + str(key["name"]), + tuple(key["constrained_columns"]), + key["referred_table"], + tuple(key["referred_columns"]), + key["options"].get("ondelete"), + ) + for key in inspector.get_foreign_keys(table_name) + ), + "unique": sorted( + (str(constraint["name"]), tuple(constraint["column_names"])) + for constraint in inspector.get_unique_constraints(table_name) + ), + "checks": sorted( + (str(check["name"]), check["sqltext"]) + for check in inspector.get_check_constraints(table_name) + ), + "indexes": sorted( + ( + str(index["name"]), + tuple(str(column) for column in index["column_names"]), + bool(index["unique"]), + str(index.get("dialect_options", {}).get("postgresql_where")), + ) + for index in inspector.get_indexes(table_name) + ), + } + for table_name in inspector.get_table_names() + } + + +def _current_revision(engine: Engine) -> str: + with engine.connect() as connection: + return str( + connection.execute( + text("SELECT version_num FROM alembic_version") + ).scalar_one() + ) + + +def _insert_dynamic( + connection: Connection, + rows: PublicApiRows, + *, + content_hash: str = "d" * 64, + country_id: str = "us", +) -> object: + dynamic_id = uuid4() + insert_row( + connection, + "dynamics", + id=dynamic_id, + country_id=country_id, + tax_benefit_model_id=rows.model_id, + tax_benefit_model_version_id=rows.version_id, + canonicalization_version=1, + content_hash=content_hash, + ) + return dynamic_id + + +def _insert_parameter(connection: Connection, rows: PublicApiRows) -> object: + parameter_id = uuid4() + insert_row( + connection, + "parameters", + id=parameter_id, + name="gov.simulation.labor_supply_responses.elasticities.income", + tax_benefit_model_version_id=rows.version_id, + ) + return parameter_id + + +def _insert_policy( + connection: Connection, rows: PublicApiRows, *, country_id: str +) -> object: + policy_id = uuid4() + insert_row( + connection, + "policies", + id=policy_id, + country_id=country_id, + tax_benefit_model_id=rows.model_id, + tax_benefit_model_version_id=rows.version_id, + canonicalization_version=1, + content_hash=uuid4().hex * 2, + ) + return policy_id + + +def test_the_revision_changes_only_dynamics_their_values_and_mappings( + engine: Engine, +) -> None: + config = alembic_v2_config() + command.downgrade(config, PREVIOUS_REVISION) + before = _schema(engine) + command.upgrade(config, DYNAMICS_REVISION) + after = _schema(engine) + + assert {name: before[name] for name in before if name not in CHANGED_TABLES} == { + name: after[name] for name in after if name not in CHANGED_TABLES + } + assert {column[0] for column in after["dynamics"]["columns"]} == { + column[0] for column in after["policies"]["columns"] + } + assert {column[0] for column in before["dynamics"]["columns"]} == { + "id", + "created_at", + "updated_at", + "name", + "description", + } + assert set(after["legacy_policy_mappings"]["columns"]) - set( + before["legacy_policy_mappings"]["columns"] + ) == {("dynamic_id", "UUID", True)} + assert set(after["legacy_policy_mappings"]["foreign_keys"]) - set( + before["legacy_policy_mappings"]["foreign_keys"] + ) == { + ( + "fk_legacy_policy_mappings_dynamic_country", + ("dynamic_id", "country_id"), + "dynamics", + ("id", "country_id"), + "RESTRICT", + ) + } + assert set(after["parameter_values"]["indexes"]) - set( + before["parameter_values"]["indexes"] + ) == { + ( + "uq_parameter_values_dynamic_parameter_start_date", + ("dynamic_id", "parameter_id", "start_date"), + True, + "(dynamic_id IS NOT NULL)", + ) + } + assert { + key: value + for key, value in after["parameter_values"].items() + if key != "indexes" + } == { + key: value + for key, value in before["parameter_values"].items() + if key != "indexes" + } + + command.downgrade(config, PREVIOUS_REVISION) + assert _schema(engine) == before + command.upgrade(config, "head") + command.check(config) + + +def test_a_dynamic_is_stored_once_and_sets_one_value_per_parameter_and_date( + connection: Connection, +) -> None: + rows = insert_public_api_rows(connection) + parameter_id = _insert_parameter(connection, rows) + dynamic_id = _insert_dynamic(connection, rows) + + with refused(connection, match="uq_dynamics_canonicalization_content_hash"): + _insert_dynamic(connection, rows) + with refused(connection, match="ck_dynamics_content_hash_length"): + _insert_dynamic(connection, rows, content_hash="short") + + value = { + "parameter_id": parameter_id, + "dynamic_id": dynamic_id, + "value_json": "0.1", + "start_date": "2026-01-01T00:00:00Z", + } + insert_row(connection, "parameter_values", id=uuid4(), **value) + with refused(connection, match="uq_parameter_values_dynamic_parameter_start_date"): + insert_row(connection, "parameter_values", id=uuid4(), **value) + # Another dynamic may set the same parameter from the same date. + other_dynamic_id = _insert_dynamic(connection, rows, content_hash="e" * 64) + insert_row( + connection, + "parameter_values", + id=uuid4(), + **{**value, "dynamic_id": other_dynamic_id}, + ) + + +def test_a_mapping_names_a_dynamic_of_its_country_which_stays_while_named( + connection: Connection, +) -> None: + rows = insert_public_api_rows(connection) + us_dynamic_id = _insert_dynamic(connection, rows) + us_policy_id = _insert_policy(connection, rows, country_id="us") + uk_policy_id = _insert_policy(connection, rows, country_id="uk") + + with refused(connection, match="fk_legacy_policy_mappings_dynamic_country"): + insert_row( + connection, + "legacy_policy_mappings", + id=uuid4(), + country_id="uk", + legacy_policy_id=1, + policy_id=uk_policy_id, + dynamic_id=us_dynamic_id, + source_policy_hash="v1-hash", + ) + insert_row( + connection, + "legacy_policy_mappings", + id=uuid4(), + country_id="us", + legacy_policy_id=1, + policy_id=us_policy_id, + dynamic_id=us_dynamic_id, + source_policy_hash="v1-hash", + ) + # A mapping without a dynamic is unaffected. + insert_row( + connection, + "legacy_policy_mappings", + id=uuid4(), + country_id="us", + legacy_policy_id=2, + policy_id=us_policy_id, + source_policy_hash="v1-hash-2", + ) + with refused(connection, match="fk_legacy_policy_mappings_dynamic_country"): + connection.execute( + text("DELETE FROM dynamics WHERE id = :id"), {"id": us_dynamic_id} + ) + + +def test_the_upgrade_fails_changing_nothing_while_a_dynamic_is_stored( + engine: Engine, +) -> None: + config = alembic_v2_config() + command.downgrade(config, PREVIOUS_REVISION) + dynamic_id = uuid4() + with engine.begin() as connection: + insert_row(connection, "dynamics", id=dynamic_id, name="stored before") + try: + with pytest.raises(IntegrityError, match="country_id"): + command.upgrade(config, DYNAMICS_REVISION) + assert _current_revision(engine) == PREVIOUS_REVISION + dynamics_columns = inspect(engine).get_columns("dynamics") + assert "name" in {column["name"] for column in dynamics_columns} + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM dynamics WHERE id = :id"), {"id": dynamic_id} + ) + + +def test_the_downgrade_fails_changing_nothing_while_a_dynamic_is_stored( + engine: Engine, +) -> None: + config = alembic_v2_config() + command.upgrade(config, DYNAMICS_REVISION) + with engine.begin() as connection: + rows = insert_public_api_rows(connection) + dynamic_id = _insert_dynamic(connection, rows) + try: + with pytest.raises(IntegrityError, match="name"): + command.downgrade(config, PREVIOUS_REVISION) + assert _current_revision(engine) == DYNAMICS_REVISION + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM dynamics WHERE id = :id"), {"id": dynamic_id} + ) + delete_public_api_rows(connection, rows) diff --git a/tests/unit/v2/test_alembic_v2_revisions.py b/tests/unit/v2/test_alembic_v2_revisions.py index 2c484b86d..da9be4836 100644 --- a/tests/unit/v2/test_alembic_v2_revisions.py +++ b/tests/unit/v2/test_alembic_v2_revisions.py @@ -66,8 +66,11 @@ # The shared reports revision, on the withheld area codes revision, and parent # of the user invitations revision. SHARED_REPORTS_REVISION = "94dc43079186" -# The user invitations revision, on the shared reports revision. +# The user invitations revision, on the shared reports revision, and parent +# of the dynamics revision. USER_INVITATIONS_REVISION = "f519a77ba122" +# The dynamics revision, on the user invitations revision. +DYNAMICS_REVISION = "9d1d12a7ac58" def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> None: @@ -76,6 +79,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + USER_INVITATIONS_REVISION, SHARED_REPORTS_REVISION, WITHHELD_AREA_CODES_REVISION, SPM_AVAILABILITY_REVISION, @@ -1132,3 +1136,76 @@ def test_user_invitations_revision_adds_its_table_and_the_dismissal_column() -> ) assert "ck_user_type_changes_source_allowed" in upgrade assert "source IN ('admin_route', 'script', 'invitation')" in upgrade + + +def test_dynamics_revision_stores_dynamics_like_policies_and_maps_them() -> None: + """The revision after user invitations stores dynamics as policies are stored. + + It is generated with no corrections: ``dynamics`` gains policies' five + columns, their two indexes, two unique constraints, three foreign keys + and two checks, and loses ``name`` and ``description``; + ``legacy_policy_mappings`` gains ``dynamic_id`` with its index and key; + ``parameter_values`` gains the partial unique index on dynamics' values. + The downgrade undoes each, and neither writes data. + """ + (path,) = (REPO / "migrations/v2/versions").glob(f"{DYNAMICS_REVISION}_*.py") + revision = path.read_text(encoding="utf-8") + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert V2_ALEMBIC_HEAD_REVISION == DYNAMICS_REVISION + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert ( + f'down_revision: Union[str, None] = "{USER_INVITATIONS_REVISION}"' in revision + ) + assert "correction" not in revision.lower() + assert sorted(re.findall(r"op\.\w+", upgrade)) == sorted( + [ + *["op.add_column"] * 6, + *["op.create_index"] * 4, + *["op.create_unique_constraint"] * 2, + *["op.create_foreign_key"] * 4, + *["op.create_check_constraint"] * 2, + *["op.drop_column"] * 2, + # Two generated index names, three key names and two check names. + *["op.f"] * 7, + ] + ) + assert sorted(re.findall(r"op\.\w+", downgrade)) == sorted( + [ + *["op.drop_index"] * 4, + *["op.drop_constraint"] * 8, + *["op.drop_column"] * 6, + *["op.add_column"] * 2, + *["op.f"] * 7, + ] + ) + for column_name in ( + "country_id", + "tax_benefit_model_id", + "tax_benefit_model_version_id", + "canonicalization_version", + "content_hash", + ): + assert re.search( + rf'op\.add_column\(\s*"dynamics",\s*sa\.Column\(\s*"{column_name}",' + r"[^)]*\)?[^)]*nullable=False", + upgrade, + ) + assert re.search(r'op\.drop_column\("dynamics", "name"\)', upgrade) + assert re.search(r'op\.drop_column\("dynamics", "description"\)', upgrade) + assert re.search( + r'op\.add_column\(\s*"legacy_policy_mappings",\s*' + r'sa\.Column\("dynamic_id", sa\.Uuid\(\), nullable=True\)', + upgrade, + ) + assert "fk_legacy_policy_mappings_dynamic_country" in upgrade + assert "uq_parameter_values_dynamic_parameter_start_date" in upgrade + assert 'postgresql_where=sa.text("dynamic_id IS NOT NULL")' in upgrade + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision diff --git a/tests/unit/v2/test_dynamic_table_models.py b/tests/unit/v2/test_dynamic_table_models.py index 012a3efdf..1253aa9f4 100644 --- a/tests/unit/v2/test_dynamic_table_models.py +++ b/tests/unit/v2/test_dynamic_table_models.py @@ -164,7 +164,9 @@ def test_identical_dynamic_content_is_stored_once() -> None: model, version, parameter = _catalog() with Session(engine) as session: - session.add_all([model, version, parameter, _dynamic(model, version)]) + session.add_all([model, version, parameter]) + session.commit() + session.add(_dynamic(model, version)) session.commit() session.add(_dynamic(model, version)) with pytest.raises(IntegrityError, match="UNIQUE constraint failed"): @@ -181,8 +183,10 @@ def test_a_dynamic_sets_one_value_per_parameter_and_start_date() -> None: model, version, parameter = _catalog() with Session(engine) as session: + session.add_all([model, version, parameter]) + session.commit() dynamic = _dynamic(model, version) - session.add_all([model, version, parameter, dynamic]) + session.add(dynamic) session.commit() for value in (0.1, 0.2): session.add( @@ -211,8 +215,10 @@ def test_a_mapping_cannot_name_another_country_s_dynamic() -> None: canonicalization_version=1, content_hash="a" * 64, ) + session.add_all([model, version, policy]) + session.commit() us_dynamic = _dynamic(model, version) - session.add_all([model, version, policy, us_dynamic]) + session.add(us_dynamic) session.commit() session.add( LegacyPolicyMapping( diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index 4341e2800..cdecc7844 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -299,7 +299,9 @@ def test_policy_parameter_value_owner_period_and_identity_constraints() -> None: start = datetime(2026, 1, 1, tzinfo=UTC) with Session(engine) as session: - session.add_all([parameter, policy, dynamic]) + session.add_all([parameter, policy]) + session.commit() + session.add(dynamic) session.commit() invalid_owner = ParameterValue( diff --git a/tests/unit/v2/test_privilege_manifest.py b/tests/unit/v2/test_privilege_manifest.py index 0a20c74a3..6394da3b7 100644 --- a/tests/unit/v2/test_privilege_manifest.py +++ b/tests/unit/v2/test_privilege_manifest.py @@ -480,3 +480,19 @@ def test_the_public_api_records_each_users_report_run_requests() -> None: for role, tables in declared.items(): if role != RUNTIME_ROLE: assert tables["user_report_run_associations"] == NO_ACCESS, role + + +def test_the_public_api_creates_and_reads_dynamics_but_never_changes_them() -> None: + """``POST /v2/dynamics`` and the v1-to-v2 copy create dynamics, as policies. + + A dynamic is immutable once stored, so the Public API reads and inserts + them and never updates or deletes one. The simulation service and its + Modal workers read them for a run; no other role has access. + """ + declared = declared_privileges(TEST_ENVIRONMENT) + + assert declared[RUNTIME_ROLE]["dynamics"] == READ_INSERT + assert declared[RUNTIME_ROLE]["dynamics"] == declared[RUNTIME_ROLE]["policies"] + assert declared[SIMULATION_SERVICE_ROLE]["dynamics"] == READ + assert declared[MODAL_WORKER_ROLE]["dynamics"] == READ + assert declared[CATALOG_WRITER_ROLE]["dynamics"] == NO_ACCESS From 48affaec053f3ec8a2f605e0ee96ab044f5f8de3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 16:57:08 +0400 Subject: [PATCH 3/6] Test /v2/dynamics, the /v2/policies refusal, dynamic checks and the split copy (failing) POST /v2/dynamics finds or creates a dynamic of gov.simulation parameters (201 or 200), refuses an empty one and one with other parameters (422), and GET /v2/dynamics/{id} reads one in its country. POST /v2/policies refuses gov.simulation and US simulation parameters with 422, naming them. /v2/simulations and /v2/reports refuse a dynamic not stored in the country (404) or setting parameters the running catalog lacks (422). The v1-to-v2 copy splits a v1 policy's model assumptions into a dynamic, rewrites the UK's old spelling, maps the v1 policy to both, and logs the dynamic. Co-Authored-By: Claude Opus 5.5 --- .../test_app_v2_workflow_contracts.py | 2 + .../test_v2_dynamic_persistence.py | 396 ++++++++++++++++++ .../unit/services/test_mirror_log_payloads.py | 7 + .../unit/v2/test_budget_window_submissions.py | 27 +- tests/unit/v2/test_dynamic_catalog.py | 218 ++++++++++ tests/unit/v2/test_dynamic_routes.py | 321 ++++++++++++++ .../v2/test_household_report_submissions.py | 2 + tests/unit/v2/test_policy_legacy_copy.py | 144 ++++++- tests/unit/v2/test_policy_routes.py | 25 ++ .../unit/v2/test_report_rerun_submissions.py | 12 + ...t_simulation_submission_database_routes.py | 142 +++++++ tests/unit/v2/test_simulation_submissions.py | 148 ++++++- 12 files changed, 1440 insertions(+), 4 deletions(-) create mode 100644 tests/integration/test_v2_dynamic_persistence.py create mode 100644 tests/unit/v2/test_dynamic_catalog.py create mode 100644 tests/unit/v2/test_dynamic_routes.py diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py index 8642f0542..cfe8c896d 100644 --- a/tests/contract/test_app_v2_workflow_contracts.py +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -10,6 +10,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): assert {workflow.name for workflow in APP_V2_WORKFLOW_CONTRACTS} == { "policy_save_search", "policy_resources_v2", + "dynamic_resources_v2", "user_policy_associations_v2", "household_save_read", "household_resources_v2", @@ -37,6 +38,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "detailed_save_reports_v2", "metadata_resources_v2_preview", "policy_resources_v2", + "dynamic_resources_v2", "user_policy_associations_v2", "household_resources_v2", "household_calculations_v2", diff --git a/tests/integration/test_v2_dynamic_persistence.py b/tests/integration/test_v2_dynamic_persistence.py new file mode 100644 index 000000000..ddd2fc89b --- /dev/null +++ b/tests/integration/test_v2_dynamic_persistence.py @@ -0,0 +1,396 @@ +"""PostgreSQL tests for dynamics, created by ``/v2/dynamics`` and the v1-to-v2 copy. + +The services run as the Public API's runtime role, which proves its declared +privileges cover them (rule ``api.v2.runtime-privileges-tested-as-runtime-role``); +the catalog is written and everything removed as the superuser. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +import pytest +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService +from policyengine_api.services.v2.dynamics.types import NativeDynamicCreationInput +from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError +from sqlalchemy import create_engine, delete, or_, select +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session + +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import ( + Dynamic, + LegacyPolicyMapping, + Parameter, + ParameterValue, + Policy, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.services.v2.policies.database_session import ( + PolicyDatabaseSession, +) +from policyengine_api.services.v2.policies.services import V2PolicyService +from policyengine_api.services.v2.policies.types import LegacyPolicySnapshot + +if TYPE_CHECKING: + from collections.abc import Iterator + +RUNNING_VERSION = "5.2.0" +US_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" +US_MTR_DELTA = "simulation.marginal_tax_rate_delta" +US_TAX_RATE = "gov.irs.income.rate" +UK_INCOME_ELASTICITY = "gov.simulation.labour_supply_responses.income_elasticity" +UK_OLD_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.income_elasticity" +UK_TAX_RATE = "gov.hmrc.income_tax.rate" + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get(V2_ALEMBIC_DISPOSABLE_TEST, ""), + } + ) + if not settings.disposable_test: + pytest.fail("dynamic persistence tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +@dataclass(frozen=True) +class StoredCatalog: + """A country's running catalog version and its parameters' IDs by name.""" + + model_id: UUID + version_id: UUID + parameter_ids: dict[str, UUID] + + +def _store_catalog( + session: Session, country_id: str, names: list[str] +) -> StoredCatalog: + model = TaxBenefitModel(name=f"policyengine-{country_id}") + version = TaxBenefitModelVersion( + model=model, + version=RUNNING_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + parameters = [ + Parameter(name=name, tax_benefit_model_version=version) for name in names + ] + session.add_all(parameters) + session.flush() + return StoredCatalog( + model_id=model.id, + version_id=version.id, + parameter_ids={parameter.name: parameter.id for parameter in parameters}, + ) + + +def _remove_catalog(engine, catalog: StoredCatalog) -> None: + with engine.begin() as connection: + dynamic_ids = select(Dynamic.id).where( + Dynamic.tax_benefit_model_id == catalog.model_id + ) + policy_ids = select(Policy.id).where( + Policy.tax_benefit_model_id == catalog.model_id + ) + connection.execute( + delete(LegacyPolicyMapping).where( + or_( + LegacyPolicyMapping.policy_id.in_(policy_ids), + LegacyPolicyMapping.dynamic_id.in_(dynamic_ids), + ) + ) + ) + connection.execute( + delete(ParameterValue).where( + or_( + ParameterValue.policy_id.in_(policy_ids), + ParameterValue.dynamic_id.in_(dynamic_ids), + ) + ) + ) + connection.execute( + delete(Dynamic).where(Dynamic.tax_benefit_model_id == catalog.model_id) + ) + connection.execute( + delete(Policy).where(Policy.tax_benefit_model_id == catalog.model_id) + ) + connection.execute( + delete(Parameter).where( + Parameter.tax_benefit_model_version_id == catalog.version_id + ) + ) + connection.execute( + delete(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.id == catalog.version_id + ) + ) + connection.execute( + delete(TaxBenefitModel).where(TaxBenefitModel.id == catalog.model_id) + ) + + +@pytest.fixture +def superuser_engine() -> Iterator[object]: + engine = create_engine(_disposable_url()) + try: + yield engine + finally: + engine.dispose() + + +@pytest.fixture +def runtime_sessions( + managed_v2_roles, v2_engine_as_role +) -> Iterator[sessionmaker[Session]]: + """Open sessions as the Public API's runtime role.""" + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime) + try: + yield sessionmaker(runtime_engine, class_=Session, expire_on_commit=False) + finally: + runtime_engine.dispose() + + +@pytest.fixture +def us_catalog(superuser_engine) -> Iterator[StoredCatalog]: + with Session(superuser_engine) as session, session.begin(): + catalog = _store_catalog( + session, "us", [US_INCOME_ELASTICITY, US_MTR_DELTA, US_TAX_RATE] + ) + try: + yield catalog + finally: + _remove_catalog(superuser_engine, catalog) + + +@pytest.fixture +def uk_catalog(superuser_engine) -> Iterator[StoredCatalog]: + with Session(superuser_engine) as session, session.begin(): + catalog = _store_catalog(session, "uk", [UK_INCOME_ELASTICITY, UK_TAX_RATE]) + try: + yield catalog + finally: + _remove_catalog(superuser_engine, catalog) + + +def _dynamic_service(sessions: sessionmaker[Session]) -> V2DynamicService: + return V2DynamicService( + DynamicDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ) + + +def _policy_service(sessions: sessionmaker[Session]) -> V2PolicyService: + return V2PolicyService( + PolicyDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ) + + +def _dynamic_input(catalog: StoredCatalog, value: float) -> NativeDynamicCreationInput: + return NativeDynamicCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model_id, + parameter_values=[ + { + "parameter_id": catalog.parameter_ids[US_INCOME_ELASTICITY], + "value": value, + "start_date": "2026-01-01T00:00:00Z", + }, + { + "parameter_id": catalog.parameter_ids[US_MTR_DELTA], + "value": 500, + "start_date": "2026-01-01T00:00:00Z", + }, + ], + ) + + +def _snapshot( + country_id: str, legacy_policy_id: int, policy_json: dict[str, object] +) -> LegacyPolicySnapshot: + return LegacyPolicySnapshot( + country_id=country_id, + legacy_policy_id=legacy_policy_id, + label=None, + api_version="1.0.0", + policy_json=policy_json, + source_policy_hash=f"v1-hash-{legacy_policy_id}", + ) + + +def _stored_names(session: Session, *, policy_id=None, dynamic_id=None) -> set[str]: + owner = ( + ParameterValue.policy_id == policy_id + if policy_id is not None + else ParameterValue.dynamic_id == dynamic_id + ) + return set( + session.scalars( + select(Parameter.name) + .join(ParameterValue, ParameterValue.parameter_id == Parameter.id) + .where(owner) + ) + ) + + +def test_identical_dynamic_content_is_stored_once_and_read_back( + runtime_sessions, us_catalog, superuser_engine +) -> None: + service = _dynamic_service(runtime_sessions) + + created = service.create_dynamic(_dynamic_input(us_catalog, 0.25)) + found = service.create_dynamic(_dynamic_input(us_catalog, 0.25)) + other = service.create_dynamic(_dynamic_input(us_catalog, 0.5)) + read = service.get_dynamic(country_id="us", dynamic_id=created.item.id) + + assert created.created is True + assert found.created is False + assert found.item.id == created.item.id + assert other.created is True + assert other.item.id != created.item.id + assert read.tax_benefit_model_version_id == us_catalog.version_id + assert read.missing_parameters == () + assert {(value.parameter_name, value.value) for value in read.parameter_values} == { + (US_INCOME_ELASTICITY, 0.25), + (US_MTR_DELTA, 500), + } + with Session(superuser_engine) as session: + stored = session.scalars( + select(Dynamic).where(Dynamic.tax_benefit_model_id == us_catalog.model_id) + ).all() + assert len(stored) == 2 + assert _stored_names(session, dynamic_id=created.item.id) == { + US_INCOME_ELASTICITY, + US_MTR_DELTA, + } + + +def test_a_dynamic_is_read_only_in_its_own_country( + runtime_sessions, us_catalog +) -> None: + service = _dynamic_service(runtime_sessions) + created = service.create_dynamic(_dynamic_input(us_catalog, 0.25)) + + with pytest.raises(DynamicNotFoundError): + service.get_dynamic(country_id="uk", dynamic_id=created.item.id) + with pytest.raises(DynamicNotFoundError): + service.get_dynamic(country_id="us", dynamic_id=uuid4()) + + +def test_the_copy_splits_a_us_policy_into_a_policy_and_a_dynamic( + runtime_sessions, us_catalog, superuser_engine +) -> None: + service = _policy_service(runtime_sessions) + snapshot = _snapshot( + "us", + 9101, + { + US_TAX_RATE: {"2026": 0.2}, + US_INCOME_ELASTICITY: {"2026": 0.25}, + US_MTR_DELTA: {"2026": 500}, + }, + ) + + first = service.mirror_legacy_policy(snapshot) + retry = service.mirror_legacy_policy(snapshot) + + assert first.policy_created is True + assert first.dynamic_created is True + assert first.dynamic_id is not None + assert retry.policy_id == first.policy_id + assert retry.dynamic_id == first.dynamic_id + assert retry.mapping_created is False + with Session(superuser_engine) as session: + assert _stored_names(session, policy_id=first.policy_id) == {US_TAX_RATE} + assert _stored_names(session, dynamic_id=first.dynamic_id) == { + US_INCOME_ELASTICITY, + US_MTR_DELTA, + } + mapping = session.scalars( + select(LegacyPolicyMapping).where( + LegacyPolicyMapping.country_id == "us", + LegacyPolicyMapping.legacy_policy_id == 9101, + ) + ).one() + assert (mapping.policy_id, mapping.dynamic_id) == ( + first.policy_id, + first.dynamic_id, + ) + + +def test_the_copy_rewrites_the_uk_old_spelling_into_the_dynamic( + runtime_sessions, uk_catalog, superuser_engine +) -> None: + service = _policy_service(runtime_sessions) + dynamic_values_only = service.mirror_legacy_policy( + _snapshot("uk", 9201, {UK_OLD_INCOME_ELASTICITY: {"2026": 0.3}}) + ) + same_values_new_spelling = service.mirror_legacy_policy( + _snapshot("uk", 9202, {UK_INCOME_ELASTICITY: {"2026": 0.3}}) + ) + no_dynamic_values = service.mirror_legacy_policy( + _snapshot("uk", 9203, {UK_TAX_RATE: {"2026": 0.4}}) + ) + + assert dynamic_values_only.dynamic_id is not None + assert same_values_new_spelling.dynamic_id == dynamic_values_only.dynamic_id + assert same_values_new_spelling.dynamic_created is False + assert no_dynamic_values.dynamic_id is None + assert no_dynamic_values.dynamic_created is False + with Session(superuser_engine) as session: + # A v1 policy holding only dynamic values maps to an empty policy. + assert _stored_names(session, policy_id=dynamic_values_only.policy_id) == set() + assert _stored_names(session, dynamic_id=dynamic_values_only.dynamic_id) == { + UK_INCOME_ELASTICITY + } + dynamic = session.get(Dynamic, dynamic_values_only.dynamic_id) + assert dynamic is not None + assert dynamic.country_id == "uk" + assert dynamic.tax_benefit_model_version_id == uk_catalog.version_id + + +def test_a_copy_made_before_the_split_keeps_its_mapping_without_a_dynamic( + runtime_sessions, us_catalog, superuser_engine +) -> None: + snapshot = _snapshot("us", 9301, {US_INCOME_ELASTICITY: {"2026": 0.25}}) + with Session(superuser_engine) as session, session.begin(): + earlier_copy = Policy( + country_id="us", + tax_benefit_model_id=us_catalog.model_id, + tax_benefit_model_version_id=us_catalog.version_id, + canonicalization_version=1, + content_hash=uuid4().hex * 2, + ) + session.add(earlier_copy) + session.flush() + session.add( + LegacyPolicyMapping( + country_id="us", + legacy_policy_id=9301, + policy_id=earlier_copy.id, + source_policy_hash=snapshot.source_policy_hash, + ) + ) + + result = _policy_service(runtime_sessions).mirror_legacy_policy(snapshot) + + assert result.policy_id == earlier_copy.id + assert result.dynamic_id is None + assert result.mapping_created is False diff --git a/tests/unit/services/test_mirror_log_payloads.py b/tests/unit/services/test_mirror_log_payloads.py index 20cf5e693..27b95233c 100644 --- a/tests/unit/services/test_mirror_log_payloads.py +++ b/tests/unit/services/test_mirror_log_payloads.py @@ -45,6 +45,7 @@ ) DESTINATION_ID = UUID("00000000-0000-0000-0000-000000000010") +DESTINATION_DYNAMIC_ID = UUID("00000000-0000-0000-0000-000000000011") ASSOCIATION_ID = UUID("00000000-0000-0000-0000-000000000060") DATABASE_ERROR = OperationalError("statement", {}, Exception("database")) @@ -324,6 +325,8 @@ def test_policy_mirror_success_logs_one_complete_record() -> None: policy_id=DESTINATION_ID, policy_created=False, mapping_created=True, + dynamic_id=DESTINATION_DYNAMIC_ID, + dynamic_created=True, ) with patch("policyengine_api.services.policy_mirroring.logger") as logger: @@ -340,10 +343,12 @@ def test_policy_mirror_success_logs_one_complete_record() -> None: "country_id": "uk", "legacy_policy_id": 5, "destination_policy_id": str(DESTINATION_ID), + "destination_dynamic_id": str(DESTINATION_DYNAMIC_ID), "outcome": "ok", "failure_category": None, "failure_reason": None, "policy_created": False, + "dynamic_created": True, "mapping_created": True, } assert list(payload) == list(expected) @@ -372,10 +377,12 @@ def test_policy_mirror_failure_logs_one_complete_record() -> None: "country_id": "uk", "legacy_policy_id": 5, "destination_policy_id": None, + "destination_dynamic_id": None, "outcome": "error", "failure_category": "database", "failure_reason": "database_error", "policy_created": None, + "dynamic_created": None, "mapping_created": None, } assert list(payload) == list(expected) diff --git a/tests/unit/v2/test_budget_window_submissions.py b/tests/unit/v2/test_budget_window_submissions.py index b85336f75..846d9c649 100644 --- a/tests/unit/v2/test_budget_window_submissions.py +++ b/tests/unit/v2/test_budget_window_submissions.py @@ -13,6 +13,7 @@ from uuid import UUID, uuid4 import pytest +from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError from pydantic import ValidationError from policyengine_api.services.v2.countries.types import CountryRead @@ -31,7 +32,12 @@ from policyengine_api.services.v2.simulation_service.validators import ( EconomyCalculationsUnsupportedError, ) -from tests.unit.v2.test_simulation_submissions import UnreadHouseholds +from tests.unit.v2.test_simulation_submissions import ( + STORED_DYNAMIC_ID, + UK_DYNAMIC_ID, + StoredDynamics, + UnreadHouseholds, +) STORED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c101") OUTDATED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000c102") @@ -111,6 +117,7 @@ def submissions( countries=StoredCountries(), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=lambda: client, ) @@ -188,3 +195,21 @@ def test_a_window_may_end_in_2200() -> None: window = window_input(year=2191, budget_window_size=10) assert (window.year, window.budget_window_size) == (2191, 10) + + +def test_a_budget_window_s_dynamics_are_checked_in_its_country() -> None: + client = RecordingSimulationService() + stored_dynamic = { + "policy_id": str(STORED_POLICY_ID), + "dynamic_id": str(STORED_DYNAMIC_ID), + } + + submissions(client, StoredPolicies()).submit_budget_window( + window_input(baseline=stored_dynamic, reform=stored_dynamic) + ) + with pytest.raises(DynamicNotFoundError, match=str(UK_DYNAMIC_ID)): + submissions(client, StoredPolicies()).submit_budget_window( + window_input(reform={"dynamic_id": str(UK_DYNAMIC_ID)}) + ) + + assert len(client.windows) == 1 diff --git a/tests/unit/v2/test_dynamic_catalog.py b/tests/unit/v2/test_dynamic_catalog.py new file mode 100644 index 000000000..69584284a --- /dev/null +++ b/tests/unit/v2/test_dynamic_catalog.py @@ -0,0 +1,218 @@ +"""Binding dynamics and policies to a catalog, and keeping their parameters apart. + +A dynamic holds only parameters under ``gov.simulation.`` (plus the US's +``simulation.``), and a policy holds none of them; each is refused with the +parameter names it may not set. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from uuid import UUID + +import pytest +from policyengine_api.services.v2.dynamics.services import ( + resolve_dynamic_creation_input, +) +from policyengine_api.services.v2.dynamics.types import ( + DynamicCreationInput, + ResolvedDynamicCreationInput, +) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicSetsPolicyParametersError, +) +from pydantic import ValidationError +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.models import ( + V2_METADATA, + Parameter, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.policies.services import ( + resolve_policy_creation_input, +) +from policyengine_api.services.v2.policies.types import PolicyCreationInput +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, + PolicySetsDynamicParametersError, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + +INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" +MARGINAL_TAX_RATE_DELTA = "simulation.marginal_tax_rate_delta" +TAX_RATE = "gov.irs.income.rate" +RUNNING_VERSION = "5.2.0" + + +class Catalog: + """A US catalog version with two dynamic parameters and one policy parameter.""" + + def __init__(self, session: Session) -> None: + self.model = TaxBenefitModel(name="policyengine-us") + self.version = TaxBenefitModelVersion( + model=self.model, + version=RUNNING_VERSION, + current_law_id=1, + metadata_time_periods=[2026], + ) + self.parameter_ids = {} + for name in (INCOME_ELASTICITY, MARGINAL_TAX_RATE_DELTA, TAX_RATE): + parameter = Parameter(name=name, tax_benefit_model_version=self.version) + session.add(parameter) + self.parameter_ids[name] = parameter.id + session.commit() + + +@pytest.fixture +def session() -> Iterator[Session]: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + database_session = Session(engine) + try: + yield database_session + finally: + database_session.close() + engine.dispose() + + +@pytest.fixture +def catalog(session: Session) -> Catalog: + return Catalog(session) + + +def _values(catalog: Catalog, *names: str) -> list[dict[str, object]]: + return [ + { + "parameter_id": catalog.parameter_ids[name], + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + } + for name in names + ] + + +def _dynamic_input(catalog: Catalog, *names: str) -> DynamicCreationInput: + return DynamicCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model.id, + parameter_values=_values(catalog, *names), + ) + + +def test_a_dynamic_is_bound_to_the_running_catalog_version( + session: Session, catalog: Catalog +) -> None: + resolved = resolve_dynamic_creation_input( + session, + _dynamic_input(catalog, INCOME_ELASTICITY, MARGINAL_TAX_RATE_DELTA), + running_policyengine_version=RUNNING_VERSION, + ) + + assert isinstance(resolved, ResolvedDynamicCreationInput) + assert resolved.tax_benefit_model_version_id == catalog.version.id + assert resolved.policyengine_version == RUNNING_VERSION + assert {value.parameter_id for value in resolved.parameter_values} == { + catalog.parameter_ids[INCOME_ELASTICITY], + catalog.parameter_ids[MARGINAL_TAX_RATE_DELTA], + } + + +def test_a_dynamic_setting_policy_parameters_is_refused_naming_them( + session: Session, catalog: Catalog +) -> None: + with pytest.raises(DynamicSetsPolicyParametersError) as refusal: + resolve_dynamic_creation_input( + session, + _dynamic_input(catalog, INCOME_ELASTICITY, TAX_RATE), + running_policyengine_version=RUNNING_VERSION, + ) + + assert refusal.value.parameter_names == (TAX_RATE,) + assert TAX_RATE in str(refusal.value) + assert "/v2/policies" in str(refusal.value) + + +def test_a_dynamic_with_a_parameter_of_another_catalog_is_refused( + session: Session, catalog: Catalog +) -> None: + dynamic_input = DynamicCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model.id, + parameter_values=[ + { + "parameter_id": UUID(int=7), + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + } + ], + ) + + with pytest.raises(PolicyCatalogValidationError, match="every parameter_id"): + resolve_dynamic_creation_input( + session, dynamic_input, running_policyengine_version=RUNNING_VERSION + ) + + +@pytest.mark.parametrize( + "dynamic_type", [DynamicCreationInput, ResolvedDynamicCreationInput] +) +def test_a_dynamic_with_no_values_is_refused(dynamic_type: type) -> None: + fields: dict[str, object] = { + "country_id": "us", + "tax_benefit_model_id": UUID(int=1), + "parameter_values": [], + } + if dynamic_type is ResolvedDynamicCreationInput: + fields |= { + "tax_benefit_model_version_id": UUID(int=2), + "policyengine_version": RUNNING_VERSION, + } + + with pytest.raises(ValidationError, match="at least 1 item"): + dynamic_type(**fields) + + +def test_a_policy_setting_dynamic_parameters_is_refused_naming_them( + session: Session, catalog: Catalog +) -> None: + policy_input = PolicyCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model.id, + parameter_values=_values( + catalog, TAX_RATE, MARGINAL_TAX_RATE_DELTA, INCOME_ELASTICITY + ), + ) + + with pytest.raises(PolicySetsDynamicParametersError) as refusal: + resolve_policy_creation_input( + session, policy_input, running_policyengine_version=RUNNING_VERSION + ) + + assert refusal.value.parameter_names == ( + INCOME_ELASTICITY, + MARGINAL_TAX_RATE_DELTA, + ) + assert "/v2/dynamics" in str(refusal.value) + + +def test_a_policy_without_dynamic_parameters_is_still_bound( + session: Session, catalog: Catalog +) -> None: + resolved = resolve_policy_creation_input( + session, + PolicyCreationInput( + country_id="us", + tax_benefit_model_id=catalog.model.id, + parameter_values=_values(catalog, TAX_RATE), + ), + running_policyengine_version=RUNNING_VERSION, + ) + + assert resolved.tax_benefit_model_version_id == catalog.version.id + assert [value.parameter_id for value in resolved.parameter_values] == [ + catalog.parameter_ids[TAX_RATE] + ] diff --git a/tests/unit/v2/test_dynamic_routes.py b/tests/unit/v2/test_dynamic_routes.py new file mode 100644 index 000000000..1e0edbc74 --- /dev/null +++ b/tests/unit/v2/test_dynamic_routes.py @@ -0,0 +1,321 @@ +"""Native FastAPI contract tests for v2 dynamics, stored as policies are.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from uuid import UUID, uuid4 + +import pytest +from fastapi.testclient import TestClient +from flask import Flask, jsonify +from policyengine_api.services.v2.dynamics.types import ( + CheckedDynamicRead, + NativeDynamicCreation, + NativeDynamicCreationInput, +) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, + DynamicNotFoundError, + DynamicSetsPolicyParametersError, +) +from sqlalchemy.exc import OperationalError + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.policies.types import PolicyParameterValueRead +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, +) + +DYNAMIC_ID = UUID("00000000-0000-0000-0000-000000000011") +MODEL_ID = UUID("00000000-0000-0000-0000-000000000020") +MODEL_VERSION_ID = UUID("00000000-0000-0000-0000-000000000030") +PARAMETER_ID = UUID("00000000-0000-0000-0000-000000000040") +VALUE_ID = UUID("00000000-0000-0000-0000-000000000050") +CREATED_AT = datetime(2026, 1, 1, tzinfo=UTC) +INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" + + +def _dynamic_read( + *, dynamic_id: UUID = DYNAMIC_ID, missing_parameters: tuple[str, ...] | None = () +) -> CheckedDynamicRead: + return CheckedDynamicRead( + id=dynamic_id, + country_id="us", + tax_benefit_model_id=MODEL_ID, + tax_benefit_model_version_id=MODEL_VERSION_ID, + created_at=CREATED_AT, + updated_at=CREATED_AT + timedelta(seconds=1), + parameter_values=( + PolicyParameterValueRead( + id=VALUE_ID, + parameter_id=PARAMETER_ID, + parameter_name=INCOME_ELASTICITY, + value=0.25, + start_date=CREATED_AT, + end_date=None, + ), + ), + missing_parameters=missing_parameters, + ) + + +class FakeDynamicService: + """Records each call; raises ``error`` when set.""" + + def __init__(self) -> None: + self.created = True + self.error: Exception | None = None + self.missing_parameters: tuple[str, ...] | None = () + self.calls: list[tuple[str, object]] = [] + + def create_dynamic( + self, dynamic_input: NativeDynamicCreationInput + ) -> NativeDynamicCreation: + self.calls.append(("create_dynamic", dynamic_input)) + if self.error is not None: + raise self.error + return NativeDynamicCreation(item=_dynamic_read(), created=self.created) + + def get_dynamic(self, **filters: object) -> CheckedDynamicRead: + self.calls.append(("get_dynamic", filters)) + if self.error is not None: + raise self.error + dynamic_id = filters["dynamic_id"] + assert isinstance(dynamic_id, UUID) + return _dynamic_read( + dynamic_id=dynamic_id, missing_parameters=self.missing_parameters + ) + + +def _client(service: FakeDynamicService) -> tuple[TestClient, dict[str, int]]: + flask_calls = {"count": 0} + flask_app = Flask(__name__) + + @flask_app.route("/", methods=["GET", "POST", "PATCH", "DELETE"]) + def fallback(resource: str): + flask_calls["count"] += 1 + return jsonify({"source": "flask", "resource": resource}) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + v2_dynamic_service_factory=lambda: service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app(flask_app, dependencies=dependencies, route_settings=settings), + raise_server_exceptions=False, + ) + return client, flask_calls + + +def _body(**changes: object) -> dict[str, object]: + body: dict[str, object] = { + "country_id": "us", + "tax_benefit_model_id": str(MODEL_ID), + "parameter_values": [ + { + "parameter_id": str(PARAMETER_ID), + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + } + ], + } + body.update(changes) + return body + + +def test_create_answers_201_for_a_new_dynamic_and_200_for_stored_content() -> None: + service = FakeDynamicService() + client, flask_calls = _client(service) + + created = client.post( + "/v2/dynamics?country_id=us&policyengine_version=5.2.0", json=_body() + ) + service.created = False + found = client.post("/v2/dynamics?country_id=us", json=_body()) + + assert created.status_code == 201 + assert found.status_code == 200 + assert created.json() == found.json() + item = created.json()["result"]["item"] + assert created.json()["status"] == "ok" + assert item == { + "id": str(DYNAMIC_ID), + "country_id": "us", + "tax_benefit_model_id": str(MODEL_ID), + "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:01Z", + "parameter_values": [ + { + "id": str(VALUE_ID), + "parameter_id": str(PARAMETER_ID), + "parameter_name": INCOME_ELASTICITY, + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + "end_date": None, + } + ], + "missing_parameters": [], + } + first_call = service.calls[0] + assert first_call[0] == "create_dynamic" + assert isinstance(first_call[1], NativeDynamicCreationInput) + assert first_call[1].policyengine_version == "5.2.0" + assert flask_calls["count"] == 0 + + +def test_create_refuses_an_empty_dynamic_and_a_country_mismatch_before_service() -> ( + None +): + service = FakeDynamicService() + client, _flask_calls = _client(service) + + empty = client.post("/v2/dynamics?country_id=us", json=_body(parameter_values=[])) + mismatch = client.post("/v2/dynamics?country_id=uk", json=_body()) + unknown_field = client.post("/v2/dynamics?country_id=us", json=_body(name="x")) + oversized = client.post( + "/v2/dynamics?country_id=us", + content=b"{" + b" " * 1_048_577 + b"}", + headers={"content-type": "application/json"}, + ) + + assert empty.status_code == 422 + assert empty.json() == {"status": "error", "message": "Invalid API v2 request"} + assert mismatch.status_code == 400 + assert unknown_field.status_code == 422 + assert oversized.status_code == 413 + assert service.calls == [] + + +def test_create_refuses_policy_parameters_naming_them() -> None: + service = FakeDynamicService() + service.error = DynamicSetsPolicyParametersError(("gov.irs.rate", "gov.irs.x")) + client, _flask_calls = _client(service) + + response = client.post("/v2/dynamics?country_id=us", json=_body()) + + assert response.status_code == 422 + body = response.json() + assert body["status"] == "error" + assert body["code"] == "policy_parameters" + assert body["policy_parameters"] == ["gov.irs.rate", "gov.irs.x"] + assert "/v2/policies" in body["message"] + + +def test_detail_is_country_scoped_and_reports_missing_parameters() -> None: + service = FakeDynamicService() + service.missing_parameters = (INCOME_ELASTICITY,) + client, _flask_calls = _client(service) + dynamic_id = uuid4() + + found = client.get(f"/v2/dynamics/{dynamic_id}?country_id=uk") + service.error = DynamicNotFoundError(f"dynamic {dynamic_id} was not found") + missing = client.get(f"/v2/dynamics/{dynamic_id}?country_id=uk") + + assert found.status_code == 200 + assert found.json()["result"]["item"]["id"] == str(dynamic_id) + assert found.json()["result"]["item"]["missing_parameters"] == [INCOME_ELASTICITY] + assert service.calls[0] == ( + "get_dynamic", + {"country_id": "uk", "dynamic_id": dynamic_id}, + ) + assert missing.status_code == 404 + assert missing.json() == { + "status": "error", + "message": f"dynamic {dynamic_id} was not found", + } + + +@pytest.mark.parametrize( + ("error", "status_code", "message"), + [ + (PolicyCatalogValidationError("wrong model"), 400, "wrong model"), + (MetadataCatalogVersionNotFoundError("no 9.9.9"), 404, "no 9.9.9"), + ( + DynamicContentHashCollisionError("secret detail"), + 409, + "Dynamic content hash conflicts with storage", + ), + ( + DynamicCreationIntegrityError("secret detail"), + 500, + "Stored dynamic integrity failed", + ), + ( + OperationalError("SELECT secret", {}, Exception("password")), + 503, + "V2 dynamic persistence is unavailable", + ), + (RuntimeError("secret detail"), 500, "V2 dynamic operation failed"), + ], +) +def test_dynamic_failures_map_to_secret_safe_typed_errors( + error: Exception, status_code: int, message: str +) -> None: + service = FakeDynamicService() + service.error = error + client, _flask_calls = _client(service) + + response = client.post("/v2/dynamics?country_id=us", json=_body()) + + assert response.status_code == status_code + assert response.json() == {"status": "error", "message": message} + + +@pytest.mark.parametrize( + ("method", "path"), + [ + ("GET", "/v2/dynamics?country_id=us"), + ("PATCH", f"/v2/dynamics/{DYNAMIC_ID}?country_id=us"), + ("DELETE", f"/v2/dynamics/{DYNAMIC_ID}?country_id=us"), + ], +) +def test_dynamics_are_neither_listed_nor_changed(method: str, path: str) -> None: + service = FakeDynamicService() + client, _flask_calls = _client(service) + + response = client.request(method, path) + + assert response.status_code in {404, 405} + assert service.calls == [] + + +def test_openapi_publishes_the_dynamic_routes_and_their_errors() -> None: + client, _flask_calls = _client(FakeDynamicService()) + + schema = client.get("/openapi.json").json() + + create = schema["paths"]["/v2/dynamics"]["post"] + detail = schema["paths"]["/v2/dynamics/{dynamic_id}"]["get"] + assert set(schema["paths"]["/v2/dynamics"]) == {"post"} + assert set(schema["paths"]["/v2/dynamics/{dynamic_id}"]) == {"get"} + assert {"200", "201", "400", "404", "409", "413", "422", "500", "503"} <= set( + create["responses"] + ) + assert {"200", "404", "503"} <= set(detail["responses"]) + query_names = { + parameter["name"] + for parameter in create["parameters"] + if parameter["in"] == "query" + } + assert query_names == {"country_id", "policyengine_version"} + request_schema = schema["components"]["schemas"]["DynamicCreateRequest"] + parameter_values = request_schema["properties"]["parameter_values"] + assert parameter_values["minItems"] == 1 + assert parameter_values["maxItems"] == 1_000 + assert "dynamic_id null" in parameter_values["description"] diff --git a/tests/unit/v2/test_household_report_submissions.py b/tests/unit/v2/test_household_report_submissions.py index 9d9cd1620..3a8f32d54 100644 --- a/tests/unit/v2/test_household_report_submissions.py +++ b/tests/unit/v2/test_household_report_submissions.py @@ -46,6 +46,7 @@ SERVED_COUNTRIES, STORED_POLICY_ID, StoredCountries, + StoredDynamics, StoredPolicies, ) @@ -134,6 +135,7 @@ def household_submissions( countries=StoredCountries(countries=countries), households=households or StoredHouseholds(), policies=policies or StoredPolicies(), + dynamics=StoredDynamics(), client_factory=client_factory, ) diff --git a/tests/unit/v2/test_policy_legacy_copy.py b/tests/unit/v2/test_policy_legacy_copy.py index e3ea73d7b..33f43a2aa 100644 --- a/tests/unit/v2/test_policy_legacy_copy.py +++ b/tests/unit/v2/test_policy_legacy_copy.py @@ -11,6 +11,10 @@ from uuid import UUID import pytest +from policyengine_api.services.v2.dynamics.types import ( + DynamicCreationResult, + ResolvedDynamicCreationInput, +) from sqlmodel import Session, create_engine from policyengine_api.data.v2.models import ( @@ -24,24 +28,31 @@ from policyengine_api.services.v2.policies.types import ( LegacyPolicyPersistenceResult, LegacyPolicySnapshot, + LegacyPolicyTranslationReason, PolicyCreationResult, ResolvedPolicyCreationInput, ) from policyengine_api.services.v2.policies.validators import ( LegacyPolicyMappingIntegrityError, + LegacyPolicyTranslationError, ) RUNNING_POLICYENGINE_VERSION = "6.7.5" COPIED_POLICY_ID = UUID("00000000-0000-0000-0000-000000000042") EARLIER_COPY_POLICY_ID = UUID("00000000-0000-0000-0000-000000000041") MAPPING_ID = UUID("00000000-0000-0000-0000-000000000043") +COPIED_DYNAMIC_ID = UUID("00000000-0000-0000-0000-000000000044") +US_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.elasticities.income" +UK_INCOME_ELASTICITY = "gov.simulation.labour_supply_responses.income_elasticity" +UK_OLD_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.income_elasticity" @dataclass class RecordedCopyWrites: - """The policies and mappings a copy would have inserted, in order.""" + """The policies, dynamics and mappings a copy would have inserted, in order.""" created_policies: list[ResolvedPolicyCreationInput] = field(default_factory=list) + created_dynamics: list[ResolvedDynamicCreationInput] = field(default_factory=list) created_mappings: list[dict[str, object]] = field(default_factory=list) @@ -53,11 +64,16 @@ def record_policy(_session, policy_input): writes.created_policies.append(policy_input) return PolicyCreationResult(policy_id=COPIED_POLICY_ID, created=True) + def record_dynamic(_session, dynamic_input): + writes.created_dynamics.append(dynamic_input) + return DynamicCreationResult(dynamic_id=COPIED_DYNAMIC_ID, created=True) + def record_mapping(_session, **mapping_fields): writes.created_mappings.append(mapping_fields) return MAPPING_ID monkeypatch.setattr(policy_services, "create_resolved_policy", record_policy) + monkeypatch.setattr(policy_services, "create_resolved_dynamic", record_dynamic) monkeypatch.setattr(policy_services, "create_legacy_policy_mapping", record_mapping) return writes @@ -185,6 +201,7 @@ def test_policy_saved_under_an_older_country_package_is_copied( "legacy_policy_id": reused_policy.legacy_policy_id, "source_policy_hash": snapshot.source_policy_hash, "policy_id": COPIED_POLICY_ID, + "dynamic_id": None, } ] @@ -350,3 +367,128 @@ def test_concurrent_copy_with_another_source_hash_is_an_integrity_error( with pytest.raises(LegacyPolicyMappingIntegrityError, match="source hash"): _copy(catalog_session, _reused_snapshot()) + + +def test_a_policy_s_model_assumptions_are_copied_into_a_dynamic( + catalog_session: Session, recorded_writes: RecordedCopyWrites +) -> None: + version, parameters = _add_running_catalog( + catalog_session, + country_id="us", + parameter_names=["gov.dol.minimum_wage", US_INCOME_ELASTICITY], + ) + snapshot = _snapshot( + country_id="us", + legacy_policy_id=501, + api_version="1.600.0", + policy_json={ + "gov.dol.minimum_wage": {"2026": 9}, + US_INCOME_ELASTICITY: {"2026": 0.25}, + }, + ) + + result = _copy(catalog_session, snapshot) + + assert result == LegacyPolicyPersistenceResult( + policy_id=COPIED_POLICY_ID, + policy_created=True, + mapping_created=True, + dynamic_id=COPIED_DYNAMIC_ID, + dynamic_created=True, + ) + [copied_policy] = recorded_writes.created_policies + assert [value.parameter_id for value in copied_policy.parameter_values] == [ + parameters["gov.dol.minimum_wage"].id + ] + [copied_dynamic] = recorded_writes.created_dynamics + assert isinstance(copied_dynamic, ResolvedDynamicCreationInput) + assert copied_dynamic.country_id == "us" + assert copied_dynamic.tax_benefit_model_version_id == version.id + assert [ + (value.parameter_id, value.value) for value in copied_dynamic.parameter_values + ] == [(parameters[US_INCOME_ELASTICITY].id, 0.25)] + assert recorded_writes.created_mappings == [ + { + "country_id": "us", + "legacy_policy_id": 501, + "source_policy_hash": snapshot.source_policy_hash, + "policy_id": COPIED_POLICY_ID, + "dynamic_id": COPIED_DYNAMIC_ID, + } + ] + + +def test_a_uk_policy_with_only_old_spelling_assumptions_maps_to_an_empty_policy( + catalog_session: Session, recorded_writes: RecordedCopyWrites +) -> None: + _version, parameters = _add_running_catalog( + catalog_session, country_id="uk", parameter_names=[UK_INCOME_ELASTICITY] + ) + + result = _copy( + catalog_session, + _snapshot( + country_id="uk", + legacy_policy_id=502, + api_version="2.100.0", + policy_json={UK_OLD_INCOME_ELASTICITY: {"2026": 0.3}}, + ), + ) + + assert result.dynamic_id == COPIED_DYNAMIC_ID + [copied_policy] = recorded_writes.created_policies + assert copied_policy.parameter_values == [] + [copied_dynamic] = recorded_writes.created_dynamics + assert [value.parameter_id for value in copied_dynamic.parameter_values] == [ + parameters[UK_INCOME_ELASTICITY].id + ] + + +@pytest.mark.parametrize( + ("policy_json", "reason"), + [ + pytest.param( + {"gov.boe.base_rate": {"2026": 0.05}, US_INCOME_ELASTICITY: {"2026": 1}}, + LegacyPolicyTranslationReason.MISSING_PARAMETERS, + id="assumption-missing-from-the-catalog", + ), + pytest.param( + { + UK_OLD_INCOME_ELASTICITY: {"2026": 0.3}, + UK_INCOME_ELASTICITY: {"2026": 0.4}, + }, + LegacyPolicyTranslationReason.CONFLICTING_VALUES, + id="both-spellings-disagree", + ), + pytest.param( + {UK_INCOME_ELASTICITY: {}}, + LegacyPolicyTranslationReason.EMPTY_PARAMETER_VALUES, + id="assumption-sets-no-values", + ), + ], +) +def test_an_untranslatable_assumption_fails_the_copy_before_anything_is_written( + catalog_session: Session, + recorded_writes: RecordedCopyWrites, + policy_json: dict[str, object], + reason: LegacyPolicyTranslationReason, +) -> None: + _add_running_catalog( + catalog_session, + country_id="uk", + parameter_names=["gov.boe.base_rate", UK_INCOME_ELASTICITY], + ) + + with pytest.raises(LegacyPolicyTranslationError) as failure: + _copy( + catalog_session, + _snapshot( + country_id="uk", + legacy_policy_id=503, + api_version="2.100.0", + policy_json=policy_json, + ), + ) + + assert failure.value.reason == reason + assert recorded_writes == RecordedCopyWrites() diff --git a/tests/unit/v2/test_policy_routes.py b/tests/unit/v2/test_policy_routes.py index 8dad07c50..752ae1fc2 100644 --- a/tests/unit/v2/test_policy_routes.py +++ b/tests/unit/v2/test_policy_routes.py @@ -33,6 +33,7 @@ PolicyContentHashCollisionError, PolicyCreationIntegrityError, PolicyNotFoundError, + PolicySetsDynamicParametersError, ) POLICY_ID = UUID("00000000-0000-0000-0000-000000000010") @@ -394,6 +395,30 @@ def test_policy_failures_map_to_secret_safe_typed_errors( assert "credential" not in response.text +def test_create_refuses_dynamic_parameters_naming_them() -> None: + service = FakePolicyService() + service.error = PolicySetsDynamicParametersError( + ( + "gov.simulation.labor_supply_responses.elasticities.income", + "simulation.marginal_tax_rate_delta", + ) + ) + client, _flask_calls = _client(service) + + response = client.post("/v2/policies?country_id=us", json=_body()) + + assert response.status_code == 422 + body = response.json() + assert body["status"] == "error" + assert body["code"] == "dynamic_parameters" + assert body["dynamic_parameters"] == [ + "gov.simulation.labor_supply_responses.elasticities.income", + "simulation.marginal_tax_rate_delta", + ] + assert "POST /v2/dynamics" in body["message"] + assert "simulation.marginal_tax_rate_delta" in body["message"] + + @pytest.mark.parametrize("method", ["put", "patch", "delete"]) def test_core_policy_mutations_are_not_exposed(method: str) -> None: service = FakePolicyService() diff --git a/tests/unit/v2/test_report_rerun_submissions.py b/tests/unit/v2/test_report_rerun_submissions.py index 03eda8a86..6c9552a3e 100644 --- a/tests/unit/v2/test_report_rerun_submissions.py +++ b/tests/unit/v2/test_report_rerun_submissions.py @@ -30,6 +30,7 @@ EconomyCalculationsUnsupportedError, SubmissionStorageError, ) +from tests.unit.v2.test_simulation_submissions import StoredDynamics REPORT_ID = UUID("00000000-0000-0000-0000-00000000e001") STORED_POLICY_ID = UUID("00000000-0000-0000-0000-00000000e002") @@ -138,6 +139,7 @@ def test_a_rerun_is_sent_once_its_country_and_policies_are_checked() -> None: countries=StoredCountries(), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=lambda: client, ) rerun = rerun_input(None, STORED_POLICY_ID) @@ -153,6 +155,7 @@ def test_a_rerun_in_a_household_only_country_is_refused_before_sending() -> None countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -166,6 +169,7 @@ def test_a_rerun_naming_a_policy_no_longer_stored_is_refused() -> None: countries=StoredCountries(), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -178,6 +182,7 @@ def test_a_rerun_whose_policy_sets_parameters_the_catalog_lacks_is_refused() -> countries=StoredCountries(), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -195,6 +200,7 @@ def test_a_rerun_whose_policy_check_cannot_read_storage_is_storage_unavailable() countries=StoredCountries(), households=UnreadHouseholds(), policies=StoredPolicies(error=error), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -212,6 +218,7 @@ def test_a_rerun_where_only_households_are_calculated_is_refused_for_economy_kin countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -228,6 +235,7 @@ def test_a_household_reports_rerun_is_sent_where_only_households_are_calculated( countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=lambda: client, ) rerun = rerun_input(None, STORED_POLICY_ID, kind="household") @@ -245,6 +253,7 @@ def test_a_household_report_of_the_baseline_alone_checks_the_baseline_policy() - countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=lambda: client, ) rerun = rerun_input(STORED_POLICY_ID, None, kind="household") @@ -261,6 +270,7 @@ def test_a_household_reports_rerun_without_household_calculations_is_refused() - countries=StoredCountries(household_enabled=False), households=UnreadHouseholds(), policies=policies, + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -275,6 +285,7 @@ def test_a_household_reports_rerun_checks_its_policies_as_a_new_one_does() -> No countries=StoredCountries(society_wide_enabled=False), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -292,6 +303,7 @@ def test_a_household_reports_rerun_whose_countries_cannot_be_read_is_unavailable countries=UnreadableCountries(error), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) diff --git a/tests/unit/v2/test_simulation_submission_database_routes.py b/tests/unit/v2/test_simulation_submission_database_routes.py index 5c3f96a44..890d2ae2f 100644 --- a/tests/unit/v2/test_simulation_submission_database_routes.py +++ b/tests/unit/v2/test_simulation_submission_database_routes.py @@ -13,6 +13,10 @@ from uuid import UUID, uuid4 import pytest +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from sqlalchemy import text from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool @@ -21,6 +25,7 @@ from policyengine_api.data.v2.models import ( V2_METADATA, Country, + Dynamic, Parameter, ParameterValue, Policy, @@ -70,6 +75,9 @@ EARLIER_VERSION = "6.9.0" KEPT_PARAMETER = "gov.example.rate" REMOVED_PARAMETER = "gov.example.removed_rate" +KEPT_ASSUMPTION = "gov.simulation.labor_supply_responses.elasticities.income" +REMOVED_ASSUMPTION = "gov.simulation.labor_supply_responses.removed_elasticity" +UK_ASSUMPTION = "gov.simulation.labour_supply_responses.income_elasticity" STORED_AT = datetime(2026, 11, 10, tzinfo=UTC) REPORTS = "/v2/reports?country_id=us" SIMULATIONS = "/v2/simulations?country_id=us" @@ -181,6 +189,75 @@ def stored_policy_ids(engine: Engine) -> dict[str, UUID]: return {name: policy.id for name, policy in policies.items()} +def store_dynamic( + session: Session, + country_id: str, + parameters: dict[str, Parameter], + content_hash_letter: str, +) -> Dynamic: + """Store a dynamic setting each of ``parameters``, of their catalog version.""" + model_version = next(iter(parameters.values())).tax_benefit_model_version + dynamic = Dynamic( + country_id=country_id, + tax_benefit_model_id=model_version.model_id, + tax_benefit_model_version_id=model_version.id, + canonicalization_version=1, + content_hash=content_hash_letter * 64, + created_at=STORED_AT, + updated_at=STORED_AT, + ) + session.add(dynamic) + session.flush() + session.add_all( + ParameterValue( + parameter_id=parameter.id, + value_json=0.25, + start_date=STORED_AT, + dynamic_id=dynamic.id, + ) + for parameter in parameters.values() + ) + return dynamic + + +@pytest.fixture +def stored_dynamic_ids(engine: Engine) -> dict[str, UUID]: + """Store the US with two catalog versions, the UK with one, and three dynamics. + + ``outdated_dynamic`` sets an assumption the running US catalog version + removed, ``running_dynamic`` was made on the running US version, and + ``uk_dynamic`` is the UK's. + """ + with Session(engine, expire_on_commit=False) as session: + session.add_all( + [ + Country(id="us", household_enabled=True, society_wide_enabled=True), + Country(id="uk", household_enabled=True, society_wide_enabled=True), + ] + ) + us_model = TaxBenefitModel(name="policyengine-us") + earlier = store_catalog_version( + session, us_model, EARLIER_VERSION, [KEPT_ASSUMPTION, REMOVED_ASSUMPTION] + ) + running = store_catalog_version( + session, us_model, RUNNING_VERSION, [KEPT_ASSUMPTION] + ) + uk_running = store_catalog_version( + session, + TaxBenefitModel(name="policyengine-uk"), + RUNNING_VERSION, + [UK_ASSUMPTION], + ) + session.flush() + dynamics = { + "outdated_dynamic": store_dynamic(session, "us", earlier, "d"), + "running_dynamic": store_dynamic(session, "us", running, "e"), + "uk_dynamic": store_dynamic(session, "uk", uk_running, "f"), + } + session.commit() + return {name: dynamic.id for name, dynamic in dynamics.items()} + + @pytest.fixture def simulation_service() -> RecordingSimulationService: return RecordingSimulationService() @@ -198,6 +275,10 @@ def submissions( PolicyDatabaseSession(sessions), running_policyengine_version=RUNNING_VERSION, ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), + running_policyengine_version=RUNNING_VERSION, + ), client_factory=lambda: simulation_service, ) @@ -333,3 +414,64 @@ def test_a_catalog_that_cannot_be_queried_answers_503( } assert simulation_service.reports == [] assert simulation_service.simulations == [] + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_a_report_naming_another_country_s_dynamic_answers_404( + client: TestClient, + stored_dynamic_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, + side: str, +) -> None: + uk_dynamic_id = stored_dynamic_ids["uk_dynamic"] + + response = client.post( + REPORTS, json=report_body(**{side: {"dynamic_id": str(uk_dynamic_id)}}) + ) + + assert response.status_code == 404, response.text + assert response.json() == { + "status": "error", + "message": f"dynamic {uk_dynamic_id} was not found", + } + assert simulation_service.reports == [] + + +def test_a_simulation_whose_dynamic_sets_parameters_the_catalog_lacks_answers_422( + client: TestClient, + stored_dynamic_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, +) -> None: + outdated_dynamic_id = stored_dynamic_ids["outdated_dynamic"] + body = {**simulation_body(uuid4()), "policy_id": None} + + response = client.post( + SIMULATIONS, json={**body, "dynamic_id": str(outdated_dynamic_id)} + ) + + assert response.status_code == 422, response.text + assert response.json() == { + "status": "error", + "message": ( + f"Dynamic {outdated_dynamic_id} sets parameters the running catalog " + f"version does not have: {REMOVED_ASSUMPTION}" + ), + "code": "missing_parameters", + "missing_parameters": [REMOVED_ASSUMPTION], + } + assert simulation_service.simulations == [] + + +def test_a_dynamic_of_the_country_on_the_running_catalog_is_sent( + submissions: SimulationSubmissionService, + stored_dynamic_ids: dict[str, UUID], + simulation_service: RecordingSimulationService, +) -> None: + running_dynamic = {"dynamic_id": str(stored_dynamic_ids["running_dynamic"])} + report = EconomyReportCreationInput.model_validate( + report_body(baseline=running_dynamic, reform=running_dynamic) + ) + + submissions.submit_report(report) + + assert simulation_service.reports == [report] diff --git a/tests/unit/v2/test_simulation_submissions.py b/tests/unit/v2/test_simulation_submissions.py index 45fb541ab..67713277a 100644 --- a/tests/unit/v2/test_simulation_submissions.py +++ b/tests/unit/v2/test_simulation_submissions.py @@ -8,6 +8,11 @@ from uuid import UUID, uuid4 import pytest +from policyengine_api.services.v2.dynamics.types import CheckedDynamicRead +from policyengine_api.services.v2.dynamics.validators import ( + DynamicNotFoundError, + DynamicParametersMissingError, +) from sqlalchemy.exc import OperationalError from policyengine_api.data.v2.catalog.catalog_selection_validation import ( @@ -51,6 +56,18 @@ SECOND_OUTDATED_POLICY_ID: SECOND_REMOVED_PARAMETERS, UNCHECKED_POLICY_ID: None, } +# Stored US dynamics; the second sets parameters the running catalog lacks. +STORED_DYNAMIC_ID = UUID("00000000-0000-0000-0000-00000000c101") +OUTDATED_DYNAMIC_ID = UUID("00000000-0000-0000-0000-00000000c102") +# A stored UK dynamic. +UK_DYNAMIC_ID = UUID("00000000-0000-0000-0000-00000000c103") +REMOVED_ASSUMPTIONS = ("gov.simulation.removed_elasticity",) +# Each stored dynamic's country, and its parameters the running catalog lacks. +STORED_DYNAMICS: dict[UUID, tuple[str, tuple[str, ...] | None]] = { + STORED_DYNAMIC_ID: ("us", ()), + OUTDATED_DYNAMIC_ID: ("us", REMOVED_ASSUMPTIONS), + UK_DYNAMIC_ID: ("uk", ()), +} SERVED_COUNTRIES = [ CountryRead(id="uk", household_enabled=True, society_wide_enabled=True), CountryRead(id="us", household_enabled=True, society_wide_enabled=True), @@ -110,6 +127,37 @@ def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicyRead: return checked_policy(policy_id, country_id, MISSING_PARAMETERS[policy_id]) +@dataclass +class StoredDynamics: + """The dynamics in STORED_DYNAMICS, each stored in its own country. + + With ``error``, every read raises it, as a database or catalog that + cannot be read does. + """ + + error: Exception | None = None + asked: list[tuple[str, UUID]] = field(default_factory=list) + + def get_dynamic(self, *, country_id: str, dynamic_id: UUID) -> CheckedDynamicRead: + self.asked.append((country_id, dynamic_id)) + if self.error is not None: + raise self.error + stored = STORED_DYNAMICS.get(dynamic_id) + if stored is None or stored[0] != country_id: + msg = f"Dynamic {dynamic_id} was not found" + raise DynamicNotFoundError(msg) + return CheckedDynamicRead( + id=dynamic_id, + country_id=country_id, + tax_benefit_model_id=uuid4(), + tax_benefit_model_version_id=uuid4(), + created_at=STORED_AT, + updated_at=STORED_AT, + parameter_values=(), + missing_parameters=stored[1], + ) + + @dataclass class RecordingSimulationService: """The simulation service's client, recording what it is sent.""" @@ -166,12 +214,15 @@ def simulation_input(**changes: Any) -> EconomySimulationCreationInput: def submissions( - client_factory: Any, policies: StoredPolicies | None = None + client_factory: Any, + policies: StoredPolicies | None = None, + dynamics: StoredDynamics | None = None, ) -> SimulationSubmissionService: return SimulationSubmissionService( countries=StoredCountries(), households=UnreadHouseholds(), policies=policies or StoredPolicies(), + dynamics=dynamics or StoredDynamics(), client_factory=client_factory, ) @@ -196,7 +247,7 @@ def test_current_law_needs_no_stored_policy() -> None: policies = StoredPolicies() submissions(lambda: client, policies).submit_report( - report_input(reform={"dynamic_id": str(uuid4())}) + report_input(reform={"dynamic_id": str(STORED_DYNAMIC_ID)}) ) assert policies.asked == [] @@ -208,6 +259,7 @@ def test_a_household_only_country_is_refused_before_sending() -> None: countries=StoredCountries(countries=HOUSEHOLD_ONLY_UK), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -222,6 +274,7 @@ def test_a_country_without_a_countries_row_is_refused() -> None: countries=StoredCountries(countries=SERVED_COUNTRIES[:1]), households=UnreadHouseholds(), policies=StoredPolicies(), + dynamics=StoredDynamics(), client_factory=unconfigured_client, ) @@ -350,3 +403,94 @@ def test_a_policy_check_that_cannot_read_storage_is_storage_unavailable( service.submit_report(report_input()) assert unavailable.value.__cause__ is error + + +def test_a_report_s_dynamics_are_checked_in_its_country_and_sent() -> None: + client = RecordingSimulationService() + dynamics = StoredDynamics() + report = report_input( + baseline={"dynamic_id": str(STORED_DYNAMIC_ID)}, + reform={ + "policy_id": str(STORED_POLICY_ID), + "dynamic_id": str(STORED_DYNAMIC_ID), + }, + ) + + submissions(lambda: client, dynamics=dynamics).submit_report(report) + + assert client.reports == [report] + assert dynamics.asked == [("us", STORED_DYNAMIC_ID)] + + +@pytest.mark.parametrize("side", ["baseline", "reform"]) +def test_a_report_naming_another_country_s_dynamic_is_refused(side: str) -> None: + service = submissions(unconfigured_client) + + with pytest.raises(DynamicNotFoundError, match=str(UK_DYNAMIC_ID)): + service.submit_report( + report_input(**{side: {"dynamic_id": str(UK_DYNAMIC_ID)}}) + ) + + +def test_a_simulation_is_sent_once_its_dynamic_is_checked() -> None: + client = RecordingSimulationService() + dynamics = StoredDynamics() + simulation = simulation_input(dynamic_id=str(UK_DYNAMIC_ID)) + + submissions(lambda: client, dynamics=dynamics).submit_simulation(simulation) + + assert client.simulations == [simulation] + assert dynamics.asked == [("uk", UK_DYNAMIC_ID)] + + +def test_a_simulation_naming_another_country_s_or_no_stored_dynamic_is_refused() -> ( + None +): + service = submissions(unconfigured_client) + + for dynamic_id in (STORED_DYNAMIC_ID, uuid4()): + with pytest.raises(DynamicNotFoundError, match=str(dynamic_id)): + service.submit_simulation(simulation_input(dynamic_id=str(dynamic_id))) + + +def test_a_dynamic_setting_parameters_the_running_catalog_lacks_is_refused() -> None: + service = submissions(unconfigured_client) + + with pytest.raises(DynamicParametersMissingError) as refusal: + service.submit_report( + report_input(reform={"dynamic_id": str(OUTDATED_DYNAMIC_ID)}) + ) + + assert isinstance(refusal.value, PolicyParametersMissingError) + assert refusal.value.dynamic_id == OUTDATED_DYNAMIC_ID + assert refusal.value.missing_parameters == REMOVED_ASSUMPTIONS + assert str(refusal.value).startswith(f"Dynamic {OUTDATED_DYNAMIC_ID} sets") + + +def test_policies_are_checked_before_dynamics() -> None: + dynamics = StoredDynamics() + service = submissions(unconfigured_client, dynamics=dynamics) + + with pytest.raises(PolicyNotFoundError): + service.submit_report( + report_input( + reform={ + "policy_id": str(OTHER_POLICY_ID), + "dynamic_id": str(UK_DYNAMIC_ID), + } + ) + ) + + assert dynamics.asked == [] + + +def test_a_dynamic_read_that_fails_is_a_storage_error() -> None: + error = OperationalError("SELECT 1", {}, Exception("connection refused")) + service = submissions(unconfigured_client, dynamics=StoredDynamics(error=error)) + + with pytest.raises(SubmissionStorageError) as failure: + service.submit_simulation( + simulation_input(dynamic_id=str(UK_DYNAMIC_ID), policy_id=None) + ) + + assert failure.value.__cause__ is error From 5616ae3accabd97c22e3ac8b4493178eac33f732 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:17:10 +0400 Subject: [PATCH 4/6] Create dynamics through /v2/dynamics, refuse them in policies, and split the copy POST /v2/dynamics binds model assumptions (parameters under gov.simulation, and in the US simulation) to a catalog version and stores them once per content hash with the policies' canonical document, answering 201 or 200; GET /v2/dynamics/{dynamic_id} reads one in its country with its missing_parameters. A dynamic setting any other parameter answers 422 with code policy_parameters, and an empty one fails request validation. The dynamic service reports database failures as DynamicStorageError (503). POST /v2/policies answers 422 with code dynamic_parameters for a policy that sets model assumptions. Economy simulations, economy reports and budget windows refuse a dynamic the country does not store (404) or one setting parameters the running catalog lacks (422 missing_parameters). The v1-to-v2 policy copy splits a v1 policy's model assumptions into a dynamic, rewriting the UK's old labor_supply_responses spelling first, and records the dynamic in legacy_policy_mappings.dynamic_id; its log record names the dynamic, and classifies the dynamic's integrity failures. Co-Authored-By: Claude Opus 5.5 --- .../v2-dynamics-like-policies.added.md | 1 + docs/engineering/migration-contracts.md | 16 +- docs/engineering/skills/alembic-migrations.md | 62 +++- docs/generated/migration_contracts.json | 48 ++- docs/migration/stage-10-v2-policies.md | 27 +- .../fastapi_routes/dependencies.py | 19 ++ .../resource_service_protocols.py | 27 ++ .../fastapi_routes/v2/dynamics/__init__.py | 1 + .../v2/dynamics/request_models.py | 14 + .../v2/dynamics/response_models.py | 111 +++++++ .../fastapi_routes/v2/dynamics/routes.py | 208 +++++++++++++ .../fastapi_routes/v2/errors_support.py | 10 +- .../v2/policies/response_models.py | 20 +- .../fastapi_routes/v2/policies/routes.py | 30 +- policyengine_api/fastapi_routes/v2/routes.py | 4 + .../v2/simulation_resources_support.py | 11 +- policyengine_api/migration_registry.py | 8 +- policyengine_api/query_parameters.py | 8 + policyengine_api/services/policy_mirroring.py | 34 +- .../services/v2/dynamics/__init__.py | 1 + .../dynamics/database_connectors/__init__.py | 1 + .../dynamics/database_connectors/creates.py | 69 +++++ .../v2/dynamics/database_connectors/reads.py | 66 ++++ .../services/v2/dynamics/database_session.py | 32 ++ .../services/v2/dynamics/services.py | 291 ++++++++++++++++++ .../services/v2/dynamics/transformations.py | 86 ++++++ .../services/v2/dynamics/types.py | 95 ++++++ .../services/v2/dynamics/validators.py | 106 +++++++ .../policies/database_connectors/creates.py | 23 +- .../v2/policies/database_connectors/reads.py | 12 +- .../services/v2/policies/services.py | 75 +++-- .../services/v2/policies/transformations.py | 120 ++++++-- .../services/v2/policies/types.py | 50 ++- .../services/v2/policies/validators.py | 40 +++ .../v2/simulation_service/services.py | 89 ++++-- .../services/v2/simulation_service/types.py | 12 +- policyengine_api/supported_countries.py | 4 + pyproject.toml | 2 + tests/contract/registry.py | 41 +++ .../test_v2_budget_window_report_creation.py | 7 + .../test_v2_catalog_publication_pruning.py | 14 +- .../test_v2_dynamic_persistence.py | 17 +- .../integration/test_v2_household_reports.py | 7 + tests/integration/test_v2_report_creation.py | 7 + tests/integration/test_v2_report_runs.py | 7 + tests/integration/user_report_run_support.py | 7 + tests/unit/services/test_policy_mirroring.py | 16 + tests/unit/test_dynamic_parameters.py | 2 +- .../unit/v2/test_budget_window_submissions.py | 2 +- tests/unit/v2/test_dynamic_catalog.py | 60 +++- tests/unit/v2/test_dynamic_routes.py | 32 +- tests/unit/v2/test_policy_legacy_copy.py | 23 +- ...t_simulation_submission_database_routes.py | 8 +- tests/unit/v2/test_simulation_submissions.py | 22 +- 54 files changed, 1927 insertions(+), 178 deletions(-) create mode 100644 changelog.d/v2-dynamics-like-policies.added.md create mode 100644 policyengine_api/fastapi_routes/v2/dynamics/__init__.py create mode 100644 policyengine_api/fastapi_routes/v2/dynamics/request_models.py create mode 100644 policyengine_api/fastapi_routes/v2/dynamics/response_models.py create mode 100644 policyengine_api/fastapi_routes/v2/dynamics/routes.py create mode 100644 policyengine_api/services/v2/dynamics/__init__.py create mode 100644 policyengine_api/services/v2/dynamics/database_connectors/__init__.py create mode 100644 policyengine_api/services/v2/dynamics/database_connectors/creates.py create mode 100644 policyengine_api/services/v2/dynamics/database_connectors/reads.py create mode 100644 policyengine_api/services/v2/dynamics/database_session.py create mode 100644 policyengine_api/services/v2/dynamics/services.py create mode 100644 policyengine_api/services/v2/dynamics/transformations.py create mode 100644 policyengine_api/services/v2/dynamics/types.py create mode 100644 policyengine_api/services/v2/dynamics/validators.py diff --git a/changelog.d/v2-dynamics-like-policies.added.md b/changelog.d/v2-dynamics-like-policies.added.md new file mode 100644 index 000000000..ce2f7b9f4 --- /dev/null +++ b/changelog.d/v2-dynamics-like-policies.added.md @@ -0,0 +1 @@ +Store dynamics, a simulation's model assumptions, as policies are stored: once per country, catalog version and content hash, with no name or description. `POST /v2/dynamics` finds or creates one (201 or 200) from parameters under `gov.simulation`, and in the US also `simulation`, refusing other parameters and an empty dynamic with 422, and `GET /v2/dynamics/{dynamic_id}` reads one in its country. `POST /v2/policies` refuses those parameters with 422 (code `dynamic_parameters`, naming them). `/v2/simulations` and `/v2/reports` refuse a dynamic the country does not store (404) or one that sets parameters the running catalog lacks (422 `missing_parameters`). The v1-to-v2 policy copy splits a v1 policy's `gov.simulation` and US `simulation` values into a dynamic, rewriting the UK's old `labor_supply_responses` spelling, and maps each v1 policy to its policy and dynamic (`legacy_policy_mappings.dynamic_id`). diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index cd6308cbc..3533d11c3 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -7,8 +7,8 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | Metric | Count | | --- | ---: | | route group count | 9 | -| workflow count | 18 | -| request count | 101 | +| workflow count | 19 | +| request count | 103 | | db entity count | 6 | | sim flow count | 3 | @@ -19,7 +19,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check`, `deployment` | `none` | `none` | | `specification` | `specification` | `none` | `none` | | `metadata` | `metadata`, `countries`, `datasets`, `economy-options`, `model-catalog`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | -| `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | +| `policy` | `policy`, `policies`, `dynamics`, `user-policy`, `user-policies` | `policy` | `none` | | `household` | `household`, `households`, `household-calculations`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | | `simulation` | `simulation`, `simulations`, `user-simulations` | `simulation` | `economy` | @@ -49,6 +49,16 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/v2/policies/{policy_id}?country_id=us` | 200 | `policy` | none | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at`, `result.item.missing_parameters` | | `GET` | `/v2/policies?country_id=us` | 200 | `policy` | none | `status`, `message`, `result.items`, `result.offset`, `result.limit`, `result.has_more` | +### `dynamic_resources_v2` + +- Current contract: `typed_v2_resources` +- Future owner: Dynamics stored like policies + +| Method | Path | Status | Route group | Required headers | Stable response fields | +| --- | --- | ---: | --- | --- | --- | +| `POST` | `/v2/dynamics?country_id=us` | 201 | `policy` | none | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at`, `result.item.missing_parameters` | +| `GET` | `/v2/dynamics/{dynamic_id}?country_id=us` | 200 | `policy` | none | `status`, `message`, `result.item.id`, `result.item.country_id`, `result.item.tax_benefit_model_id`, `result.item.parameter_values`, `result.item.created_at`, `result.item.updated_at`, `result.item.missing_parameters` | + ### `user_policy_associations_v2` - Current contract: `typed_v2_resources` diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 157f8aecf..9646144c0 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -310,8 +310,9 @@ production, so that its absence fails the deploy again. The declarations include write privileges for code paths production doesn't turn on: the dual-write copies into `legacy_policy_mappings` and -`legacy_household_mappings`, and the writes behind the native `/v2/policies` -and `/v2/households` routes. Narrow them when those tables are removed. +`legacy_household_mappings`, and the writes behind the native `/v2/policies`, +`/v2/dynamics` and `/v2/households` routes. Narrow them when those tables are +removed. When a pull request adds or changes a v2 table, or a connector's use of one: @@ -456,6 +457,8 @@ it publishes that version, with nothing to restore it. Downgrading past the revision is therefore safe only while no version is pruned (no `tax_benefit_model_versions` row has `catalog_values_pruned_at` set). From the head, such a downgrade first runs the later revisions' downgrades: +`9d1d12a7ac58`'s needs `dynamics` empty (see "API v2 dynamics stored like +policies"), `f519a77ba122`'s drops `user_invitations` and needs no recorded change with source `invitation` (see "API v2 user invitations"), `94dc43079186`'s drops every user's report-run record, the simulation and @@ -1012,3 +1015,58 @@ change's source is `invitation`. `test_user_invitations_revision_adds_invitations_and_dismissals_only` and `test_user_invitations_revision_records_invitation_changes_until_downgraded` in `tests/integration/test_alembic_v2_user_types.py` check both. + +### API v2 dynamics stored like policies + +Revision `9d1d12a7ac58` (parent `f519a77ba122`) was autogenerated, with no +post-generation corrections apart from `ruff format`, after these model +changes, so that dynamics are stored as policies are: + +- `dynamics` drops `name` and `description`, which nothing wrote or read: a + user's saved dynamic will hold its name, as `user_policies` holds a + policy's. It gains `policies`' columns: `country_id` (a key to + `countries`), `tax_benefit_model_id` and `tax_benefit_model_version_id` + (indexed), each `RESTRICT`, and `canonicalization_version` and + `content_hash`, unique together + (`uq_dynamics_canonicalization_content_hash`), with the checks policies + have (a positive version, a 64-character hash). `(id, country_id)` is + unique (`uq_dynamics_id_country`), so other tables can name a dynamic of + one country. +- `parameter_values` gains `uq_parameter_values_dynamic_parameter_start_date`, + a partial unique index on `(dynamic_id, parameter_id, start_date)` for rows + with a dynamic: a dynamic sets one value per parameter and start date, as a + policy does. Rows without a dynamic never conflict in a unique key, so the + partial index refuses exactly what a full unique constraint would, but it + holds only dynamics' rows instead of every current-law value. It is also + declared for SQLite, so the unit tests' databases keep the same rule. +- `legacy_policy_mappings` gains the nullable `dynamic_id`, the dynamic the + v1-to-v2 policy copy split from a v1 policy: a key with the mapping's + country to `dynamics` (`fk_legacy_policy_mappings_dynamic_country`, + `RESTRICT`), indexed. A mapping without a dynamic is unaffected. + +A dynamic is hashed by the policies' function (`canonical_policy_document`) +with the same canonicalization version, so identical content in a policy +and a dynamic has the same hash; they are in different tables. The runtime +role may now read and insert `dynamics`; the simulation service and its +Modal workers keep reading them. Which parameters a dynamic holds, and how +the copy splits a v1 policy, is in `policyengine_api/dynamic_parameters.py`. + +Nothing wrote `dynamics` before this revision, and its new columns are +required with no server default, so the upgrade fails, changing nothing, +while `dynamics` has a row. Before deploying it to an environment, count +them read-only: + +```sql +SELECT count(*) FROM dynamics; +``` + +The previous Public API revision maps `name` and `description` but never +reads or writes `dynamics`, and its copy writes mappings without +`dynamic_id`, which stays null, so it keeps working at this revision. The +simulation service selects only `dynamics.id`. The downgrade drops the new +columns, constraints and indexes and the mappings' `dynamic_id` with its +values, and adds `name` back as required, so it fails while `dynamics` has a +row. `tests/integration/test_alembic_v2_dynamics.py` checks that nothing else +changes, the downgrade and upgrade, both refusals, and the keys and unique +rules. + diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 4c5d23ceb..52e8312de 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,10 +1,10 @@ { "metadata": { "db_entity_count": 6, - "request_count": 101, + "request_count": 103, "route_group_count": 9, "sim_flow_count": 3, - "workflow_count": 18 + "workflow_count": 19 }, "route_groups": [ { @@ -51,6 +51,7 @@ "path_segments": [ "policy", "policies", + "dynamics", "user-policy", "user-policies" ], @@ -201,6 +202,49 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "Dynamics stored like policies", + "name": "dynamic_resources_v2", + "requests": [ + { + "expected_status": 201, + "headers": [], + "method": "POST", + "path": "/v2/dynamics?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + "result.item.missing_parameters" + ] + }, + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/dynamics/{dynamic_id}?country_id=us", + "route_group": "policy", + "stable_response_fields": [ + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + "result.item.missing_parameters" + ] + } + ] + }, { "current_contract": "typed_v2_resources", "future_owner_pr": "PR 12A: User Login and v1-Anchored User Associations", diff --git a/docs/migration/stage-10-v2-policies.md b/docs/migration/stage-10-v2-policies.md index d83aa2187..ca5f2834e 100644 --- a/docs/migration/stage-10-v2-policies.md +++ b/docs/migration/stage-10-v2-policies.md @@ -134,7 +134,26 @@ the running catalog whatever country package version the v1 row was saved under (its stored `api_version`), the same way the Public API simulates an old policy with the running country package. A policy is copied when every parameter path it sets exists in the running catalog and its periods and values -are valid. A saved-policy mutation commits +are valid. + +The copy splits off the v1 policy's model assumptions: every value under +`gov.simulation.`, and in the US also `simulation.` +(`policyengine_api/dynamic_parameters.py`). It stores them as a v2 dynamic, +hashed and deduplicated as a policy is, and keeps the rest in the v2 policy, +which is empty when the v1 policy set nothing else. Before looking up the +parameters it rewrites the UK's old spelling +`gov.simulation.labor_supply_responses.` to +`gov.simulation.labour_supply_responses.`, the only one the UK catalog has; a +v1 policy that sets one parameter under both spellings with different values +fails as `conflicting_values`. The mapping in `legacy_policy_mappings` names the +policy and the dynamic (`dynamic_id`), or no dynamic when the v1 policy sets no +assumptions. A v1 policy copied before the split existed keeps its copy: its +assumptions stay policy values and its mapping names no dynamic, until the +one-time migration splits it again. Native `POST /v2/policies` refuses such +values with 422 instead (code `dynamic_parameters`), and they are created +through `POST /v2/dynamics`. + +A saved-policy mutation commits its source row, incremented revision, and complete event in one Cloud SQL transaction. The same request then processes that source's pending events in revision order and records `processed_at` only after the corresponding @@ -206,14 +225,16 @@ Monitor structured events with metric names `v1_policy_mirror_operations` and | `failure_category` | `failure_reason` | | --- | --- | | `catalog_or_translation` | `missing_parameters` (the policy sets a parameter the running catalog lacks), `invalid_parameter_values`, `empty_parameter_values`, `invalid_period`, `conflicting_values`, `too_many_values` (more than 1,000 dated values), `catalog_absent` (no catalog for the running PolicyEngine.py version), `catalog_query_failed`, `catalog_version_not_found`, `catalog_mismatch` | -| `integrity` | `mapping_conflict`, `content_hash_collision`, `stored_policy_integrity`, `saved_policy_integrity` (any integrity check of a saved-policy copy) | +| `integrity` | `mapping_conflict`, `content_hash_collision` (of the policy or the dynamic), `stored_policy_integrity`, `stored_dynamic_integrity`, `saved_policy_integrity` (any integrity check of a saved-policy copy) | | `database` | `database_error` | | `configuration` | `configuration` | | `unexpected` | `missing_reform_policy` (a saved policy's v1 reform could not be loaded), `unexpected` | | `missing_mirror_snapshot`, `missing_mirror_revision` | the same value as the category | Events include the legacy integer ID, -destination UUID when committed, attempted and completed database sources, and +destination UUID when committed (a policy copy's record also names its +dynamic's, `destination_dynamic_id`, and whether it created it, +`dynamic_created`), attempted and completed database sources, and duration. They do not include policy JSON, presentation data, database URLs, or credentials. diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index aeb4411e2..e886b9250 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -18,6 +18,7 @@ from policyengine_api.fastapi_routes.resource_service_protocols import ( V2AccountResourceService, V2CountryResourceService, + V2DynamicResourceService, V2HouseholdCalculationResourceService, V2HouseholdResourceService, V2PolicyResourceService, @@ -71,6 +72,7 @@ "SimulationGatewayProbe", "V2AccountResourceService", "V2CountryResourceService", + "V2DynamicResourceService", "V2HouseholdCalculationResourceService", "V2HouseholdResourceService", "V2MetadataResourceReader", @@ -174,6 +176,21 @@ def _default_v2_policy_service_factory() -> V2PolicyResourceService: ) +def _default_v2_dynamic_service_factory() -> V2DynamicResourceService: + from policyengine_api.data.v2.database import ( + get_v2_session_factory, + ) + from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, + ) + from policyengine_api.services.v2.dynamics.services import V2DynamicService + + return V2DynamicService( + DynamicDatabaseSession(get_v2_session_factory()), + running_policyengine_version=_running_policyengine_version(), + ) + + def _default_v2_household_service_factory() -> V2HouseholdResourceService: from policyengine_api.data.v2.database import ( get_v2_session_factory, @@ -234,6 +251,7 @@ def _default_simulation_submissions() -> SimulationServiceClient: countries=_default_v2_country_service_factory(), households=_default_v2_household_service_factory(), policies=_default_v2_policy_service_factory(), + dynamics=_default_v2_dynamic_service_factory(), client_factory=process_simulation_service_client, ) @@ -545,6 +563,7 @@ class NativeRouteDependencies: v2_metadata_reader_factory: Callable[[], V2MetadataResourceReader] | None = None v2_country_service_factory: Callable[[], V2CountryResourceService] | None = None v2_policy_service_factory: Callable[[], V2PolicyResourceService] | None = None + v2_dynamic_service_factory: Callable[[], V2DynamicResourceService] | None = None v2_household_service_factory: Callable[[], V2HouseholdResourceService] | None = None v2_household_calculation_service_factory: ( Callable[[], V2HouseholdCalculationResourceService] | None diff --git a/policyengine_api/fastapi_routes/resource_service_protocols.py b/policyengine_api/fastapi_routes/resource_service_protocols.py index 5f3e25951..3a45cf89c 100644 --- a/policyengine_api/fastapi_routes/resource_service_protocols.py +++ b/policyengine_api/fastapi_routes/resource_service_protocols.py @@ -10,6 +10,11 @@ from policyengine_api.data.v2.models import UserType from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.dynamics.types import ( + CheckedDynamicRead, + NativeDynamicCreation, + NativeDynamicCreationInput, + ) from policyengine_api.services.v2.household_calculations.types import ( HouseholdCalculationInput, HouseholdCalculationRead, @@ -130,6 +135,28 @@ def list_policies( ... +class V2DynamicResourceService(Protocol): + """Route-independent native dynamic operations for one request.""" + + def create_dynamic( + self, + dynamic_input: NativeDynamicCreationInput, + ) -> NativeDynamicCreation: + """Store the dynamic, or return the identical one already stored. + + Dynamics are immutable and deduplicated by canonical content hash, as + policies are; ``created`` says whether this call inserted the dynamic. + """ + ... + + def get_dynamic(self, *, country_id: str, dynamic_id: UUID) -> CheckedDynamicRead: + """Return one dynamic of the country with its parameter values. + + Raise ``DynamicNotFoundError`` when no such dynamic exists in that country. + """ + ... + + class V2HouseholdResourceService(Protocol): """Route-independent native household operations for one request.""" diff --git a/policyengine_api/fastapi_routes/v2/dynamics/__init__.py b/policyengine_api/fastapi_routes/v2/dynamics/__init__.py new file mode 100644 index 000000000..657ea9884 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/dynamics/__init__.py @@ -0,0 +1 @@ +"""FastAPI adapters for immutable v2 dynamics.""" diff --git a/policyengine_api/fastapi_routes/v2/dynamics/request_models.py b/policyengine_api/fastapi_routes/v2/dynamics/request_models.py new file mode 100644 index 000000000..310d7e529 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/dynamics/request_models.py @@ -0,0 +1,14 @@ +"""Strict HTTP request models for the native v2 dynamic API.""" + +from policyengine_api.services.v2.dynamics.types import DynamicCreationInput + +MAXIMUM_DYNAMIC_REQUEST_BYTES = 1_048_576 + + +class DynamicCreateRequest(DynamicCreationInput): + """Native body containing immutable dynamic content only. + + A dynamic holds the model's assumptions: parameters under + ``gov.simulation``, and in the US also ``simulation``. Any other + parameter belongs in a policy (``POST /v2/policies``). + """ diff --git a/policyengine_api/fastapi_routes/v2/dynamics/response_models.py b/policyengine_api/fastapi_routes/v2/dynamics/response_models.py new file mode 100644 index 000000000..bb1f29f57 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/dynamics/response_models.py @@ -0,0 +1,111 @@ +"""Strict HTTP response models for the native v2 dynamic API.""" + +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING, Annotated, Any, Literal +from uuid import UUID + +from pydantic import Field + +from policyengine_api.fastapi_routes.v2.errors_support import V2ErrorResponse +from policyengine_api.fastapi_routes.v2.policies.response_models import ( + PolicyParameterValueItem, + PolicySuccessResponse, + StrictPolicyAPIModel, +) + +if TYPE_CHECKING: + from policyengine_api.services.v2.dynamics.types import CheckedDynamicRead + + +class DynamicItem(StrictPolicyAPIModel): + """A stored dynamic: the model assumptions a simulation applies with its policy. + + Dynamics are stored as policies are: immutable, bound to one catalog + version and deduplicated, so creating one with identical content returns + the existing dynamic and its ``id``. A dynamic has no name; a user's saved + dynamic names it. + """ + + id: UUID + country_id: str + tax_benefit_model_id: UUID + created_at: datetime + updated_at: datetime + parameter_values: list[PolicyParameterValueItem] + missing_parameters: Annotated[ + list[str] | None, + Field( + description=( + "The parameter paths this dynamic sets that the current " + "tax-benefit model (the catalog of the running PolicyEngine.py " + "version) no longer has, sorted; empty when it has them all. " + "Null when that catalog is not published, so nothing could be " + "checked. A dynamic with missing parameters cannot be run on " + "the current model." + ) + ), + ] + + @classmethod + def from_read(cls, item: CheckedDynamicRead) -> DynamicItem: + """Build the response item by validating the service read record's fields.""" + return cls.model_validate(item) + + +class DynamicDetailResult(StrictPolicyAPIModel): + """Result holding one dynamic in ``item``.""" + + item: DynamicItem + + +class DynamicDetailResponse(PolicySuccessResponse[DynamicDetailResult]): + """Success response holding one dynamic.""" + + +class V2PolicyParametersErrorResponse(V2ErrorResponse): + """A refused dynamic that sets parameters belonging in a policy. + + ``policy_parameters`` names each parameter the dynamic sets that is not + one of the model's assumptions, sorted. + """ + + code: Literal["policy_parameters"] = "policy_parameters" + policy_parameters: list[str] + + +DYNAMIC_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 400: { + "model": V2ErrorResponse, + "description": "The dynamic content or country selection is invalid.", + }, + 404: { + "model": V2ErrorResponse, + "description": "The selected dynamic or catalog does not exist.", + }, + 409: { + "model": V2ErrorResponse, + "description": "Immutable dynamic identity conflicts with stored state.", + }, + 413: { + "model": V2ErrorResponse, + "description": "The dynamic request body exceeds 1 MiB.", + }, + 422: { + "model": V2PolicyParametersErrorResponse | V2ErrorResponse, + "description": ( + "The request does not match the dynamic schema (a dynamic sets at " + "least one value), or the dynamic sets parameters that belong in a " + "policy (code policy_parameters, naming them)." + ), + }, + 500: { + "model": V2ErrorResponse, + "description": "Stored dynamic integrity validation failed.", + }, + 503: { + "model": V2ErrorResponse, + "description": "Supabase dynamic persistence is unavailable.", + }, +} diff --git a/policyengine_api/fastapi_routes/v2/dynamics/routes.py b/policyengine_api/fastapi_routes/v2/dynamics/routes.py new file mode 100644 index 000000000..72d5ff401 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/dynamics/routes.py @@ -0,0 +1,208 @@ +"""Native FastAPI routes for immutable v2 dynamics. + +A dynamic holds the model's assumptions a simulation applies alongside its +policy. It is created and read as a policy is, needs no user credential (a +user's saved dynamics name it), and is never listed, changed or deleted. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated, TypeVar +from uuid import UUID + +from fastapi import APIRouter, Depends +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_dynamic_service_factory, +) +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.fastapi_routes.v2.dynamics.request_models import ( + MAXIMUM_DYNAMIC_REQUEST_BYTES, + DynamicCreateRequest, +) +from policyengine_api.fastapi_routes.v2.dynamics.response_models import ( + DYNAMIC_ERROR_RESPONSES, + DynamicDetailResponse, + DynamicDetailResult, + DynamicItem, + V2PolicyParametersErrorResponse, +) +from policyengine_api.fastapi_routes.v2.errors_support import ( + V2ErrorMapping, + run_v2_operation, + v2_error_response, +) +from policyengine_api.fastapi_routes.v2.user_resources_support import ( + request_size_limit, +) +from policyengine_api.query_parameters import DynamicCreateQuery, DynamicDetailQuery +from policyengine_api.services.v2.dynamics.types import NativeDynamicCreationInput +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, + DynamicNotFoundError, + DynamicSetsPolicyParametersError, + DynamicStorageError, +) +from policyengine_api.services.v2.policies.validators import ( + PolicyCatalogValidationError, +) + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2DynamicResourceService, + ) + + +DynamicCreateQueryDependency = Annotated[ + DynamicCreateQuery, Depends(query_dependency(DynamicCreateQuery)) +] +DynamicDetailQueryDependency = Annotated[ + DynamicDetailQuery, Depends(query_dependency(DynamicDetailQuery)) +] +enforce_dynamic_request_size = request_size_limit( + MAXIMUM_DYNAMIC_REQUEST_BYTES, label="1 MiB" +) + +OperationT = TypeVar("OperationT") + +# Checked in order, so the collision is matched before its integrity base class. +DYNAMIC_ERROR_MAPPINGS = ( + V2ErrorMapping((PolicyCatalogValidationError,), 400), + V2ErrorMapping((MetadataCatalogVersionNotFoundError, DynamicNotFoundError), 404), + V2ErrorMapping( + (DynamicContentHashCollisionError,), + 409, + "Dynamic content hash conflicts with storage", + ), + V2ErrorMapping( + (DynamicCreationIntegrityError,), 500, "Stored dynamic integrity failed" + ), + V2ErrorMapping( + (V2ConfigurationError, MetadataCatalogUnavailableError, DynamicStorageError), + 503, + "V2 dynamic persistence is unavailable", + ), +) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2DynamicResourceService]: + if dependencies.v2_dynamic_service_factory is not None: + return dependencies.v2_dynamic_service_factory + return _default_v2_dynamic_service_factory + + +def _policy_parameters_response( + error: DynamicSetsPolicyParametersError, +) -> JSONResponse: + """Serialize the 422 refusing a dynamic that sets a policy's parameters.""" + refusal = V2PolicyParametersErrorResponse( + message=str(error), policy_parameters=list(error.parameter_names) + ) + return JSONResponse(status_code=422, content=refusal.model_dump(mode="json")) + + +def _dynamic_operation( + operation: Callable[[], OperationT], +) -> OperationT | JSONResponse: + def refused() -> OperationT | JSONResponse: + try: + return operation() + except DynamicSetsPolicyParametersError as error: + return _policy_parameters_response(error) + + return run_v2_operation( + refused, + DYNAMIC_ERROR_MAPPINGS, + fallback_message="V2 dynamic operation failed", + ) + + +def build_v2_dynamic_router(dependencies: NativeRouteDependencies) -> APIRouter: + """Build native dynamic routes without opening a database connection.""" + router = APIRouter(prefix="/v2", responses=DYNAMIC_ERROR_RESPONSES) + service_factory = _service_factory(dependencies) + + @router.post( + "/dynamics", + response_model=DynamicDetailResponse, + status_code=201, + responses={ + 200: { + "model": DynamicDetailResponse, + "description": "Equivalent immutable content already exists.", + } + }, + summary="Create or find an immutable dynamic", + description=( + "Binds the model assumptions to the catalog of policyengine_version, " + "or of the running PolicyEngine.py version when it names none, and " + "stores them once per content hash, as policies are stored. Every " + "parameter must be under gov.simulation, or in the US also " + "simulation; any other answers 422 with code policy_parameters. " + "Answers 201 when it creates the dynamic and 200 when identical " + "content is already stored. There is no list route." + ), + ) + def create_dynamic( + body: DynamicCreateRequest, + query: DynamicCreateQueryDependency, + _size: Annotated[None, Depends(enforce_dynamic_request_size)], + ) -> DynamicDetailResponse | JSONResponse: + if body.country_id != query.country_id: + return v2_error_response(400, "Body country_id must match query country_id") + + def create() -> DynamicDetailResponse | JSONResponse: + result = service_factory().create_dynamic( + NativeDynamicCreationInput( + **body.model_dump(), + policyengine_version=query.policyengine_version, + ) + ) + response = DynamicDetailResponse( + result=DynamicDetailResult(item=DynamicItem.from_read(result.item)) + ) + if result.created: + return response + return JSONResponse( + status_code=200, content=response.model_dump(mode="json") + ) + + return _dynamic_operation(create) + + @router.get( + "/dynamics/{dynamic_id}", + response_model=DynamicDetailResponse, + summary="Read one immutable dynamic", + description=( + "Returns the country's dynamic with its values, and the parameters " + "it sets that the running catalog no longer has. Another country's " + "dynamic answers 404." + ), + ) + def get_dynamic( + dynamic_id: UUID, + query: DynamicDetailQueryDependency, + ) -> DynamicDetailResponse | JSONResponse: + def read() -> DynamicDetailResponse: + item = service_factory().get_dynamic( + country_id=query.country_id, dynamic_id=dynamic_id + ) + return DynamicDetailResponse( + result=DynamicDetailResult(item=DynamicItem.from_read(item)) + ) + + return _dynamic_operation(read) + + return router diff --git a/policyengine_api/fastapi_routes/v2/errors_support.py b/policyengine_api/fastapi_routes/v2/errors_support.py index 632442309..d809815a0 100644 --- a/policyengine_api/fastapi_routes/v2/errors_support.py +++ b/policyengine_api/fastapi_routes/v2/errors_support.py @@ -57,11 +57,11 @@ class V2CodedErrorResponse(V2ErrorResponse): class V2MissingParametersErrorResponse(V2ErrorResponse): - """A refused calculation whose policy sets parameters the catalog lacks. + """A refused calculation whose policy or dynamic sets parameters the catalog lacks. - ``missing_parameters`` names each parameter the policy sets that the - running catalog version does not have, sorted; the calculation would - otherwise run without those values. + ``missing_parameters`` names each parameter the policy or dynamic sets + that the running catalog version does not have, sorted; the calculation + would otherwise run without those values. """ code: Literal["missing_parameters"] = "missing_parameters" @@ -102,7 +102,7 @@ def v2_coded_error_response( def v2_missing_parameters_response( message: str, missing_parameters: Sequence[str] ) -> JSONResponse: - """Serialize a 422 refusing a policy whose parameters the catalog lacks.""" + """Serialize a 422 refusing a policy or dynamic the catalog lacks parameters of.""" error = V2MissingParametersErrorResponse( message=message, missing_parameters=list(missing_parameters) ) diff --git a/policyengine_api/fastapi_routes/v2/policies/response_models.py b/policyengine_api/fastapi_routes/v2/policies/response_models.py index a73c24ce8..45b88a983 100644 --- a/policyengine_api/fastapi_routes/v2/policies/response_models.py +++ b/policyengine_api/fastapi_routes/v2/policies/response_models.py @@ -123,6 +123,18 @@ class PolicyPageResponse(PolicySuccessResponse[PolicyPageResult]): """Success response holding one page of policies.""" +class V2DynamicParametersErrorResponse(V2ErrorResponse): + """A refused policy that sets parameters only a dynamic may hold. + + ``dynamic_parameters`` names each of the model's assumptions the policy + sets, sorted: parameters under ``gov.simulation``, and in the US also + ``simulation``. Send them as a dynamic through ``POST /v2/dynamics``. + """ + + code: Literal["dynamic_parameters"] = "dynamic_parameters" + dynamic_parameters: list[str] + + POLICY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { 400: { "model": V2ErrorResponse, @@ -141,8 +153,12 @@ class PolicyPageResponse(PolicySuccessResponse[PolicyPageResult]): "description": "The policy request body exceeds 1 MiB.", }, 422: { - "model": V2ErrorResponse, - "description": "The request does not match the policy schema.", + "model": V2DynamicParametersErrorResponse | V2ErrorResponse, + "description": ( + "The request does not match the policy schema, or the policy sets " + "parameters only a dynamic may hold (code dynamic_parameters, " + "naming them)." + ), }, 500: { "model": V2ErrorResponse, diff --git a/policyengine_api/fastapi_routes/v2/policies/routes.py b/policyengine_api/fastapi_routes/v2/policies/routes.py index 8b07feeaa..14d57f665 100644 --- a/policyengine_api/fastapi_routes/v2/policies/routes.py +++ b/policyengine_api/fastapi_routes/v2/policies/routes.py @@ -35,6 +35,7 @@ PolicyItem, PolicyPageResponse, PolicyPageResult, + V2DynamicParametersErrorResponse, ) from policyengine_api.query_parameters import ( PolicyCollectionQuery, @@ -47,6 +48,7 @@ PolicyContentHashCollisionError, PolicyCreationIntegrityError, PolicyNotFoundError, + PolicySetsDynamicParametersError, ) if TYPE_CHECKING: @@ -116,11 +118,27 @@ def _service_factory( ) +def _dynamic_parameters_response( + error: PolicySetsDynamicParametersError, +) -> JSONResponse: + """Serialize the 422 refusing a policy that sets a dynamic's parameters.""" + refusal = V2DynamicParametersErrorResponse( + message=str(error), dynamic_parameters=list(error.parameter_names) + ) + return JSONResponse(status_code=422, content=refusal.model_dump(mode="json")) + + def _policy_operation( operation: Callable[[], OperationT], ) -> OperationT | JSONResponse: + def refused() -> OperationT | JSONResponse: + try: + return operation() + except PolicySetsDynamicParametersError as error: + return _dynamic_parameters_response(error) + return run_v2_operation( - operation, + refused, POLICY_ERROR_MAPPINGS, fallback_message="V2 policy operation failed", ) @@ -144,6 +162,16 @@ def build_v2_policy_router( } }, summary="Create or find an immutable policy", + description=( + "Binds the policy to the catalog of policyengine_version, or of the " + "running PolicyEngine.py version when it names none, and stores it " + "once per content hash. A policy may not set the model's " + "assumptions, the parameters under gov.simulation (and in the US " + "simulation): those answer 422 with code dynamic_parameters, and " + "go in a dynamic through POST /v2/dynamics. Answers 201 when it " + "creates the policy and 200 when identical content is already " + "stored." + ), ) def create_policy( body: PolicyCreateRequest, diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 51e96ee25..589bcc73d 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -16,6 +16,9 @@ from policyengine_api.fastapi_routes.v2.countries.routes import ( build_v2_country_router, ) +from policyengine_api.fastapi_routes.v2.dynamics.routes import ( + build_v2_dynamic_router, +) from policyengine_api.fastapi_routes.v2.errors_support import V2ErrorResponse from policyengine_api.fastapi_routes.v2.household_calculations.routes import ( build_v2_household_calculation_router, @@ -79,6 +82,7 @@ def build_v2_router( router.include_router(build_v2_household_router(dependencies)) router.include_router(build_v2_household_calculation_router(dependencies)) router.include_router(build_v2_policy_router(dependencies)) + router.include_router(build_v2_dynamic_router(dependencies)) router.include_router(build_v2_report_router(dependencies)) router.include_router(build_v2_simulation_router(dependencies)) router.include_router(build_v2_country_router(dependencies)) diff --git a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py index fbb15814e..611cc8594 100644 --- a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py +++ b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py @@ -22,6 +22,7 @@ run_v2_operation, v2_missing_parameters_response, ) +from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError from policyengine_api.services.v2.household_calculations.validators import ( HouseholdCalculationCountryError, HouseholdCalculationYearError, @@ -52,6 +53,7 @@ # not logged. SUBMISSION_REFUSALS: tuple[type[Exception], ...] = ( PolicyNotFoundError, + DynamicNotFoundError, HouseholdNotFoundError, EconomyCalculationsUnsupportedError, HouseholdCalculationCountryError, @@ -69,7 +71,9 @@ # the record it created and failed is still a 503, and each create route # answers it with the record's ID (``unavailable_after_creation_answer``). SUBMISSION_ERROR_MAPPINGS = ( - V2ErrorMapping((PolicyNotFoundError, HouseholdNotFoundError), 404), + V2ErrorMapping( + (PolicyNotFoundError, DynamicNotFoundError, HouseholdNotFoundError), 404 + ), V2ErrorMapping( ( EconomyCalculationsUnsupportedError, @@ -154,8 +158,9 @@ def simulation_resource_operation( ) -> OperationT | JSONResponse: """Run one service call and answer its failure with a typed v2 error. - A policy that sets parameters the running catalog version lacks answers - 422 with code ``missing_parameters``, naming them, whatever the route. A + A policy or dynamic that sets parameters the running catalog version + lacks answers 422 with code ``missing_parameters``, naming them, whatever + the route. A failure of a type in ``refusals`` is the caller's to act on and is not logged, nor is that 422; any other is logged under ``event``. """ diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index 93c5874b5..6c7e808cb 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -56,7 +56,13 @@ class RouteGroupConfig: ), RouteGroupConfig( name="policy", - path_segments=("policy", "policies", "user-policy", "user-policies"), + path_segments=( + "policy", + "policies", + "dynamics", + "user-policy", + "user-policies", + ), db_entity="policy", ), RouteGroupConfig( diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index 2e3ee8d27..dc76f9f29 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -345,6 +345,14 @@ class PolicyDetailQuery(CountryQuery): """Query contract for country-scoped policy detail reads.""" +class DynamicCreateQuery(CatalogQuery): + """Query contract for native immutable dynamic creation.""" + + +class DynamicDetailQuery(CountryQuery): + """Query contract for country-scoped dynamic detail reads.""" + + class PolicyCollectionQuery(CountryQuery, PaginationQuery): """Query contract for an exact-filtered policy collection.""" diff --git a/policyengine_api/services/policy_mirroring.py b/policyengine_api/services/policy_mirroring.py index b46299a6f..c26844aad 100644 --- a/policyengine_api/services/policy_mirroring.py +++ b/policyengine_api/services/policy_mirroring.py @@ -17,6 +17,10 @@ ) from policyengine_api.data.v2.settings import V2ConfigurationError from policyengine_api.gcp_logging import logger +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, +) from policyengine_api.services.v2.policies.validators import ( LegacyPolicyMappingIntegrityError, LegacyPolicyTranslationError, @@ -91,6 +95,7 @@ class PolicyCopyFailureReason(StrEnum): MAPPING_CONFLICT = "mapping_conflict" CONTENT_HASH_COLLISION = "content_hash_collision" STORED_POLICY_INTEGRITY = "stored_policy_integrity" + STORED_DYNAMIC_INTEGRITY = "stored_dynamic_integrity" SAVED_POLICY_INTEGRITY = "saved_policy_integrity" MISSING_REFORM_POLICY = "missing_reform_policy" MISSING_MIRROR_SNAPSHOT = "missing_mirror_snapshot" @@ -121,7 +126,7 @@ class PolicyCopyFailure: ) -# Checked in order, so the hash collision is matched before its integrity base +# Checked in order, so each hash collision is matched before its integrity base # class. _FAILURES_BY_ERROR_TYPE: tuple[tuple[type[Exception], PolicyCopyFailure], ...] = ( ( @@ -166,6 +171,20 @@ class PolicyCopyFailure: PolicyCopyFailureReason.STORED_POLICY_INTEGRITY, ), ), + ( + DynamicContentHashCollisionError, + PolicyCopyFailure( + PolicyCopyFailureCategory.INTEGRITY, + PolicyCopyFailureReason.CONTENT_HASH_COLLISION, + ), + ), + ( + DynamicCreationIntegrityError, + PolicyCopyFailure( + PolicyCopyFailureCategory.INTEGRITY, + PolicyCopyFailureReason.STORED_DYNAMIC_INTEGRITY, + ), + ), ( SQLAlchemyError, PolicyCopyFailure( @@ -207,8 +226,10 @@ class _MirrorOutcome: outcome: str actual_write_sources: list[str] destination_policy_id: object | None = None + destination_dynamic_id: object | None = None failure: PolicyCopyFailure | None = None policy_created: bool | None = None + dynamic_created: bool | None = None mapping_created: bool | None = None @@ -219,6 +240,7 @@ def _log_mirror_event( mirror_outcome: _MirrorOutcome, ) -> None: destination_policy_id = mirror_outcome.destination_policy_id + destination_dynamic_id = mirror_outcome.destination_dynamic_id failure = mirror_outcome.failure # Metric logging must not change the mirror outcome. with contextlib.suppress(Exception): @@ -237,10 +259,16 @@ def _log_mirror_event( if destination_policy_id is not None else None ), + "destination_dynamic_id": ( + str(destination_dynamic_id) + if destination_dynamic_id is not None + else None + ), "outcome": mirror_outcome.outcome, "failure_category": failure.category.value if failure else None, "failure_reason": failure.reason.value if failure else None, "policy_created": mirror_outcome.policy_created, + "dynamic_created": mirror_outcome.dynamic_created, "mapping_created": mirror_outcome.mapping_created, "duration_ms": round((time.perf_counter() - started_at) * 1000, 3), }, @@ -253,7 +281,7 @@ def mirror_policy_after_commit( *, mirror_factory: Callable[[], LegacyPolicyMirror] | None = None, ) -> LegacyPolicyPersistenceResult: - """Copy a committed v1 policy and its mapping into v2 in one Supabase transaction. + """Copy a committed v1 policy, its dynamic and its mapping into v2 at once. Log the outcome. On failure, raise PolicyMirrorUnavailableError after logging it; the v1 route still returns its normal response. @@ -283,7 +311,9 @@ def mirror_policy_after_commit( outcome="ok", actual_write_sources=["cloud_sql", "supabase"], destination_policy_id=result.policy_id, + destination_dynamic_id=result.dynamic_id, policy_created=result.policy_created, + dynamic_created=result.dynamic_created, mapping_created=result.mapping_created, ), ) diff --git a/policyengine_api/services/v2/dynamics/__init__.py b/policyengine_api/services/v2/dynamics/__init__.py new file mode 100644 index 000000000..b7311d76a --- /dev/null +++ b/policyengine_api/services/v2/dynamics/__init__.py @@ -0,0 +1 @@ +"""Application services for immutable v2 dynamics, stored as policies are.""" diff --git a/policyengine_api/services/v2/dynamics/database_connectors/__init__.py b/policyengine_api/services/v2/dynamics/database_connectors/__init__.py new file mode 100644 index 000000000..e9b9241cb --- /dev/null +++ b/policyengine_api/services/v2/dynamics/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for v2 dynamic services.""" diff --git a/policyengine_api/services/v2/dynamics/database_connectors/creates.py b/policyengine_api/services/v2/dynamics/database_connectors/creates.py new file mode 100644 index 000000000..142e79d06 --- /dev/null +++ b/policyengine_api/services/v2/dynamics/database_connectors/creates.py @@ -0,0 +1,69 @@ +"""Database inserts used by immutable v2 dynamic operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +from sqlalchemy.dialects.postgresql import insert +from sqlmodel import Session, col + +from policyengine_api.data.v2.models import Dynamic, ParameterValue + +if TYPE_CHECKING: + from policyengine_api.services.v2.dynamics.types import ( + ResolvedDynamicCreationInput, + ) + + +def create_dynamic( + session: Session, + dynamic_input: ResolvedDynamicCreationInput, + *, + canonicalization_version: int, + content_hash: str, +) -> UUID | None: + """Insert a dynamic unless one with the same canonical version and hash exists. + + Return the new id, or None when the unique content-hash constraint made the + insert a no-op; the caller then reads the existing row. Parameter values are + inserted separately by create_dynamic_parameter_values. + """ + dynamic_id = uuid4() + statement = ( + insert(Dynamic) + .values( + id=dynamic_id, + country_id=dynamic_input.country_id, + tax_benefit_model_id=dynamic_input.tax_benefit_model_id, + tax_benefit_model_version_id=dynamic_input.tax_benefit_model_version_id, + canonicalization_version=canonicalization_version, + content_hash=content_hash, + ) + .on_conflict_do_nothing(constraint="uq_dynamics_canonicalization_content_hash") + .returning(col(Dynamic.id)) + ) + return session.execute(statement).scalar_one_or_none() + + +def create_dynamic_parameter_values( + session: Session, + *, + dynamic_id: UUID, + dynamic_input: ResolvedDynamicCreationInput, +) -> None: + """Insert a new dynamic's parameter values and flush them for later reads.""" + session.add_all( + [ + ParameterValue( + policy_id=None, + dynamic_id=dynamic_id, + parameter_id=value.parameter_id, + value_json=value.value, + start_date=value.start_date, + end_date=value.end_date, + ) + for value in dynamic_input.parameter_values + ] + ) + session.flush() diff --git a/policyengine_api/services/v2/dynamics/database_connectors/reads.py b/policyengine_api/services/v2/dynamics/database_connectors/reads.py new file mode 100644 index 000000000..003ae1a8a --- /dev/null +++ b/policyengine_api/services/v2/dynamics/database_connectors/reads.py @@ -0,0 +1,66 @@ +"""Database selections used by immutable v2 dynamic operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Dynamic, Parameter, ParameterValue + +if TYPE_CHECKING: + from uuid import UUID + + +def read_dynamic_by_content_identity( + session: Session, *, canonicalization_version: int, content_hash: str +) -> Dynamic | None: + """Return the dynamic with this canonical version and content hash, or None.""" + return session.exec( + select(Dynamic).where( + Dynamic.canonicalization_version == canonicalization_version, + Dynamic.content_hash == content_hash, + ) + ).one_or_none() + + +def read_dynamic_parameter_values( + session: Session, dynamic_id: UUID +) -> list[ParameterValue]: + """Return every parameter value stored for one dynamic, in no particular order.""" + return list( + session.exec( + select(ParameterValue).where(ParameterValue.dynamic_id == dynamic_id) + ).all() + ) + + +def read_dynamic_row( + session: Session, *, country_id: str, dynamic_id: UUID +) -> Dynamic | None: + """Return the dynamic with this id in this country, or None.""" + return session.exec( + select(Dynamic).where( + Dynamic.id == dynamic_id, Dynamic.country_id == country_id + ) + ).one_or_none() + + +def read_dynamic_parameter_values_with_names( + session: Session, dynamic_ids: list[UUID] +) -> list[tuple[ParameterValue, str]]: + """Return the dynamics' values with parameter names, by name then start date.""" + if not dynamic_ids: + return [] + return list( + session.exec( + select(ParameterValue, Parameter.name) + .join(Parameter, col(Parameter.id) == col(ParameterValue.parameter_id)) + .where(col(ParameterValue.dynamic_id).in_(dynamic_ids)) + .order_by( + col(Parameter.name), + col(ParameterValue.start_date), + col(ParameterValue.id), + ) + ).all() + ) diff --git a/policyengine_api/services/v2/dynamics/database_session.py b/policyengine_api/services/v2/dynamics/database_session.py new file mode 100644 index 000000000..192433c2d --- /dev/null +++ b/policyengine_api/services/v2/dynamics/database_session.py @@ -0,0 +1,32 @@ +"""Database session lifetime management for v2 dynamic services.""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.orm import sessionmaker + from sqlmodel import Session + + +class DynamicDatabaseSession: + """Open read sessions and transaction-scoped sessions for dynamic services.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + """Use ``session_factory`` to open every session.""" + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + """Yield a session for reads; nothing is committed.""" + with self._session_factory() as session: + yield session + + @contextmanager + def transaction(self) -> Iterator[Session]: + """Yield a session whose transaction commits on exit and rolls back on error.""" + with self._session_factory.begin() as session: + yield session diff --git a/policyengine_api/services/v2/dynamics/services.py b/policyengine_api/services/v2/dynamics/services.py new file mode 100644 index 000000000..28d8c7d81 --- /dev/null +++ b/policyengine_api/services/v2/dynamics/services.py @@ -0,0 +1,291 @@ +"""Router-facing services for v2 dynamics, stored as policies are. + +A dynamic is bound to a catalog version and deduplicated by the policies' +canonical content hash, with the policies' catalog reads. This module never +imports ``policies.services``: the v1-to-v2 policy copy there creates +dynamics through ``create_resolved_dynamic``. +""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogNotPublishedError, +) +from policyengine_api.services.v2.dynamics.database_connectors.creates import ( + create_dynamic, + create_dynamic_parameter_values, +) +from policyengine_api.services.v2.dynamics.database_connectors.reads import ( + read_dynamic_by_content_identity, + read_dynamic_parameter_values, + read_dynamic_parameter_values_with_names, + read_dynamic_row, +) +from policyengine_api.services.v2.dynamics.transformations import ( + checked_dynamic_read, + dynamic_creation_input, + dynamic_parameter_values_by_dynamic, + dynamic_read, +) +from policyengine_api.services.v2.dynamics.types import ( + CheckedDynamicRead, + DynamicCreationInput, + DynamicCreationResult, + DynamicRead, + NativeDynamicCreation, + NativeDynamicCreationInput, + ResolvedDynamicCreationInput, +) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, + DynamicNotFoundError, + DynamicStorageError, + require_dynamic_parameters, +) +from policyengine_api.services.v2.policies.database_connectors.reads import ( + read_model_version, + read_parameters_by_name, + read_policy_catalog, + read_version_parameter_names, +) +from policyengine_api.services.v2.policies.transformations import ( + canonical_policy_document, + canonicalize_policy, + policy_missing_parameters, + stored_policy_creation_input, +) +from policyengine_api.services.v2.policies.validators import validate_policy_catalog + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + from uuid import UUID + + from sqlmodel import Session + + from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, + ) + from policyengine_api.services.v2.policies.types import ( + CanonicalPolicyContent, + ResolvedPolicyCreationInput, + ) + + +@contextmanager +def _storage_failures() -> Iterator[None]: + """Raise DynamicStorageError for a database read or write that failed.""" + try: + yield + except SQLAlchemyError as error: + msg = "the v2 database could not be read or written for a dynamic" + raise DynamicStorageError(msg) from error + + +def resolve_dynamic_creation_input( + session: Session, + dynamic_input: DynamicCreationInput, + *, + policyengine_version: str | None = None, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> ResolvedDynamicCreationInput: + """Bind dynamic content to the selected catalog's model version. + + Raise PolicyCatalogValidationError if the model or any parameter id is not + part of the selected catalog, and DynamicSetsPolicyParametersError if any + parameter is not one of the model's assumptions. + """ + selected = read_policy_catalog( + session, + dynamic_input.country_id, + policyengine_version=policyengine_version, + running_policyengine_version=running_policyengine_version, + ) + parameter_names = read_version_parameter_names( + session, + model_version_id=selected.model_version.id, + requested_ids={value.parameter_id for value in dynamic_input.parameter_values}, + ) + resolved = validate_policy_catalog( + dynamic_input, + selected=selected, + resolved_parameter_ids=set(parameter_names), + ) + require_dynamic_parameters(dynamic_input.country_id, parameter_names.values()) + return dynamic_creation_input(resolved) + + +def create_resolved_dynamic( + session: Session, + dynamic_input: ResolvedDynamicCreationInput, + *, + canonicalizer: Callable[ + [ResolvedPolicyCreationInput], CanonicalPolicyContent + ] = canonicalize_policy, +) -> DynamicCreationResult: + """Store a resolved dynamic, or return the existing one with identical content. + + Raise DynamicCreationIntegrityError if the hash conflict finds no usable + row, and DynamicContentHashCollisionError if the stored row's canonical + bytes differ. + """ + content = canonicalizer(dynamic_input) + created_dynamic_id = create_dynamic( + session, + dynamic_input, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if created_dynamic_id is not None: + create_dynamic_parameter_values( + session, dynamic_id=created_dynamic_id, dynamic_input=dynamic_input + ) + return DynamicCreationResult(dynamic_id=created_dynamic_id, created=True) + + existing = read_dynamic_by_content_identity( + session, + canonicalization_version=content.version, + content_hash=content.content_hash, + ) + if existing is None: + msg = "dynamic hash conflict did not resolve to a stored dynamic" + raise DynamicCreationIntegrityError(msg) + model_version = read_model_version(session, existing.tax_benefit_model_version_id) + if model_version is None: + msg = "stored dynamic references an absent model version" + raise DynamicCreationIntegrityError(msg) + stored_input = stored_policy_creation_input( + existing, + model_version, + read_dynamic_parameter_values(session, existing.id), + ) + if canonical_policy_document(stored_input) != content.document: + msg = "stored dynamic content differs for the same canonical version and hash" + raise DynamicContentHashCollisionError(msg) + return DynamicCreationResult(dynamic_id=existing.id, created=False) + + +def read_complete_dynamic( + session: Session, *, country_id: str, dynamic_id: UUID +) -> DynamicRead: + """Return one dynamic of this country, or raise DynamicNotFoundError.""" + dynamic = read_dynamic_row(session, country_id=country_id, dynamic_id=dynamic_id) + if dynamic is None: + msg = f"dynamic {dynamic_id} was not found" + raise DynamicNotFoundError(msg) + values = dynamic_parameter_values_by_dynamic( + [dynamic.id], read_dynamic_parameter_values_with_names(session, [dynamic.id]) + ) + return dynamic_read(dynamic, values) + + +def with_missing_dynamic_parameters( + session: Session, + dynamic: DynamicRead, + *, + country_id: str, + running_policyengine_version: str = POLICYENGINE_VERSION, +) -> CheckedDynamicRead: + """Return the dynamic with the parameters it sets that the running catalog lacks. + + As for a policy, ``missing_parameters`` is None when the running + PolicyEngine.py version has no published catalog for the country. A + catalog that cannot be queried raises, as any other read would. + """ + try: + selected = read_policy_catalog( + session, + country_id, + running_policyengine_version=running_policyengine_version, + ) + except MetadataCatalogNotPublishedError: + return checked_dynamic_read(dynamic, None) + running_model_version_id = selected.model_version.id + names_to_check = ( + set() + if dynamic.tax_benefit_model_version_id == running_model_version_id + else {value.parameter_name for value in dynamic.parameter_values} + ) + running_parameters = read_parameters_by_name( + session, model_version_id=running_model_version_id, names=names_to_check + ) + return checked_dynamic_read( + dynamic, + policy_missing_parameters( + dynamic, + running_model_version_id=running_model_version_id, + running_parameter_names=running_parameters, + ), + ) + + +class V2DynamicService: + """Sequence dynamic operations while delegating database lifetime management.""" + + def __init__( + self, + database_session: DynamicDatabaseSession, + *, + running_policyengine_version: str = POLICYENGINE_VERSION, + ) -> None: + """Open sessions through ``database_session``. + + ``running_policyengine_version`` selects the catalog when a request names none. + """ + self._database_session = database_session + self._running_policyengine_version = running_policyengine_version + + def create_dynamic( + self, dynamic_input: NativeDynamicCreationInput + ) -> NativeDynamicCreation: + """Store native dynamic content, deduplicated by content hash. + + Return the stored dynamic and whether this call created it. Raise + DynamicStorageError when the database cannot be read or written. + """ + content = DynamicCreationInput.model_validate( + dynamic_input.model_dump(exclude={"policyengine_version"}) + ) + with _storage_failures(), self._database_session.transaction() as session: + resolved = resolve_dynamic_creation_input( + session, + content, + policyengine_version=dynamic_input.policyengine_version, + running_policyengine_version=self._running_policyengine_version, + ) + persisted = create_resolved_dynamic(session, resolved) + item = read_complete_dynamic( + session, + country_id=dynamic_input.country_id, + dynamic_id=persisted.dynamic_id, + ) + checked = with_missing_dynamic_parameters( + session, + item, + country_id=dynamic_input.country_id, + running_policyengine_version=self._running_policyengine_version, + ) + return NativeDynamicCreation(item=checked, created=persisted.created) + + def get_dynamic(self, *, country_id: str, dynamic_id: UUID) -> CheckedDynamicRead: + """Return one dynamic of this country, or raise DynamicNotFoundError. + + Its ``missing_parameters`` are checked against the running catalog. + Raise DynamicStorageError when the database cannot be read. + """ + with _storage_failures(), self._database_session.read() as session: + item = read_complete_dynamic( + session, country_id=country_id, dynamic_id=dynamic_id + ) + return with_missing_dynamic_parameters( + session, + item, + country_id=country_id, + running_policyengine_version=self._running_policyengine_version, + ) diff --git a/policyengine_api/services/v2/dynamics/transformations.py b/policyengine_api/services/v2/dynamics/transformations.py new file mode 100644 index 000000000..11aa3f42a --- /dev/null +++ b/policyengine_api/services/v2/dynamics/transformations.py @@ -0,0 +1,86 @@ +"""Pure representation transformations for v2 dynamic operations. + +A dynamic's canonical bytes and content hash are a policy's +(``policies.transformations.canonicalize_policy``), so a dynamic needs no +canonicalization of its own. +""" + +from __future__ import annotations + +from dataclasses import fields as dataclass_fields +from typing import TYPE_CHECKING + +from policyengine_api.services.v2.dynamics.types import ( + CheckedDynamicRead, + DynamicRead, + ResolvedDynamicCreationInput, +) +from policyengine_api.services.v2.policies.types import PolicyParameterValueRead + +if TYPE_CHECKING: + from uuid import UUID + + from policyengine_api.data.v2.models import Dynamic, ParameterValue + from policyengine_api.services.v2.policies.types import ( + ResolvedPolicyCreationInput, + ) + + +def dynamic_creation_input( + content: ResolvedPolicyCreationInput, +) -> ResolvedDynamicCreationInput: + """Return resolved content as a dynamic's; it must set at least one value.""" + return ResolvedDynamicCreationInput.model_validate(dict(content)) + + +def dynamic_parameter_values_by_dynamic( + dynamic_ids: list[UUID], + rows: list[tuple[ParameterValue, str]], +) -> dict[UUID, tuple[PolicyParameterValueRead, ...]]: + """Group named value rows by dynamic id; every requested dynamic gets an entry.""" + grouped: dict[UUID, list[PolicyParameterValueRead]] = { + dynamic_id: [] for dynamic_id in dynamic_ids + } + for value, parameter_name in rows: + if value.dynamic_id is None: + continue + grouped[value.dynamic_id].append( + PolicyParameterValueRead( + id=value.id, + parameter_id=value.parameter_id, + parameter_name=parameter_name, + value=value.value_json, + start_date=value.start_date, + end_date=value.end_date, + ) + ) + return {dynamic_id: tuple(values) for dynamic_id, values in grouped.items()} + + +def dynamic_read( + dynamic: Dynamic, + values: dict[UUID, tuple[PolicyParameterValueRead, ...]], +) -> DynamicRead: + """Convert a stored dynamic and its grouped values to the service's read model.""" + return DynamicRead( + id=dynamic.id, + country_id=dynamic.country_id, + tax_benefit_model_id=dynamic.tax_benefit_model_id, + tax_benefit_model_version_id=dynamic.tax_benefit_model_version_id, + created_at=dynamic.created_at, + updated_at=dynamic.updated_at, + parameter_values=values.get(dynamic.id, ()), + ) + + +def checked_dynamic_read( + dynamic: DynamicRead, missing_parameters: tuple[str, ...] | None +) -> CheckedDynamicRead: + """Return the dynamic read with the result of its missing-parameter check.""" + return CheckedDynamicRead( + **{ + field.name: getattr(dynamic, field.name) + for field in dataclass_fields(DynamicRead) + }, + missing_parameters=missing_parameters, + ) diff --git a/policyengine_api/services/v2/dynamics/types.py b/policyengine_api/services/v2/dynamics/types.py new file mode 100644 index 000000000..aa173ef7c --- /dev/null +++ b/policyengine_api/services/v2/dynamics/types.py @@ -0,0 +1,95 @@ +"""Framework-independent data exchanged by v2 dynamic layers. + +A dynamic's content has a policy's fields, is bound to one catalog version +as a policy is, and is hashed by the same canonical document +(``policies.transformations.canonical_policy_document``). It differs in two +ways: it sets at least one value, since "no assumptions beyond the model's +defaults" is ``dynamic_id`` null, and it holds only the model's assumptions +(``policyengine_api.dynamic_parameters``), which ``dynamics.validators`` +checks once the catalog has named the parameters. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Annotated + +from pydantic import Field + +from policyengine_api.query_parameters import PolicyEngineVersion +from policyengine_api.services.v2.policies.types import ( + ParameterValueInputs, + PolicyCreationInput, + PolicyRead, + ResolvedPolicyCreationInput, +) + +if TYPE_CHECKING: + from uuid import UUID + +# A dynamic's values: at least one, and at most as many as a policy's. +DynamicParameterValueInputs = Annotated[ + ParameterValueInputs, + Field( + min_length=1, + description=( + "The model assumptions the dynamic sets, by catalog parameter ID: " + "parameters under gov.simulation, and in the US also simulation. " + "At least one: for none, send dynamic_id null." + ), + ), +] + + +class DynamicCreationInput(PolicyCreationInput): + """Complete immutable dynamic content accepted from any source.""" + + parameter_values: DynamicParameterValueInputs + + +class NativeDynamicCreationInput(DynamicCreationInput): + """Native dynamic content plus an optional catalog-version selection.""" + + policyengine_version: PolicyEngineVersion | None = None + + +class ResolvedDynamicCreationInput(ResolvedPolicyCreationInput): + """Dynamic content bound to one exact initialized catalog version.""" + + parameter_values: DynamicParameterValueInputs + + +@dataclass(frozen=True) +class DynamicCreationResult: + """New or deduplicated immutable dynamic identity.""" + + dynamic_id: UUID + created: bool + + +@dataclass(frozen=True) +class DynamicRead(PolicyRead): + """A stored v2 dynamic with its parameter values, as returned to routes. + + It has a policy's fields, read from ``dynamics`` instead of ``policies``. + """ + + +@dataclass(frozen=True) +class CheckedDynamicRead(DynamicRead): + """A stored dynamic checked against the running catalog, as routes return it. + + ``missing_parameters`` lists the parameter names the dynamic sets that the + running catalog lacks, sorted; it is None only when the running + PolicyEngine.py version has no published catalog for the country. + """ + + missing_parameters: tuple[str, ...] | None + + +@dataclass(frozen=True) +class NativeDynamicCreation: + """Complete dynamic read plus whether this request inserted it.""" + + item: CheckedDynamicRead + created: bool diff --git a/policyengine_api/services/v2/dynamics/validators.py b/policyengine_api/services/v2/dynamics/validators.py new file mode 100644 index 000000000..b6557f298 --- /dev/null +++ b/policyengine_api/services/v2/dynamics/validators.py @@ -0,0 +1,106 @@ +"""Database-independent validation for v2 dynamic operations.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.dynamic_parameters import is_dynamic_parameter +from policyengine_api.services.v2.policies.validators import ( + PolicyParametersMissingError, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + from uuid import UUID + + from policyengine_api.supported_countries import SupportedCountryId + + +class DynamicNotFoundError(LookupError): + """Raised when a dynamic UUID is absent from the selected country.""" + + +class DynamicStorageError(RuntimeError): + """Raised when the v2 database could not be read or written for a dynamic.""" + + +class DynamicCreationIntegrityError(RuntimeError): + """Raised when stored dynamic content cannot support safe deduplication.""" + + +class DynamicContentHashCollisionError(DynamicCreationIntegrityError): + """Raised when equal version/hash keys identify different canonical bytes.""" + + +class DynamicSetsPolicyParametersError(ValueError): + """Raised when a dynamic sets parameters that are not model assumptions. + + A dynamic holds only parameters under ``gov.simulation.``, plus the US's + ``simulation.`` (``policyengine_api.dynamic_parameters``); the others + belong in a policy. ``parameter_names`` lists them, sorted. + """ + + def __init__(self, parameter_names: tuple[str, ...]) -> None: + """Name each parameter the dynamic may not set.""" + self.parameter_names = parameter_names + super().__init__(parameter_names) + + def __str__(self) -> str: + """Say which parameters to send as a policy, and where.""" + return ( + "A dynamic holds only the model's assumptions, the parameters under " + "gov.simulation (and, in the US, simulation): send the others as a " + "policy through POST /v2/policies. This dynamic sets: " + + ", ".join(self.parameter_names) + ) + + +class DynamicParametersMissingError(PolicyParametersMissingError): + """Raised when a dynamic sets parameters the running catalog version lacks. + + A calculation refuses it as it refuses such a policy, with the same 422 + and ``missing_parameters``; ``dynamic_id`` names the dynamic (also as + ``policy_id``, the base error's field). + """ + + @property + def dynamic_id(self) -> UUID: + """The dynamic that sets the missing parameters.""" + return self.policy_id + + def __str__(self) -> str: + """Name the dynamic and each missing parameter.""" + return ( + f"Dynamic {self.policy_id} sets parameters the running catalog " + f"version does not have: {', '.join(self.missing_parameters)}" + ) + + +def require_dynamic_parameters( + country_id: SupportedCountryId, parameter_names: Iterable[str] +) -> None: + """Raise DynamicSetsPolicyParametersError unless each parameter is an assumption.""" + policy_parameter_names = tuple( + sorted( + { + name + for name in parameter_names + if not is_dynamic_parameter(country_id, name) + } + ) + ) + if policy_parameter_names: + raise DynamicSetsPolicyParametersError(policy_parameter_names) + + +def require_dynamic_parameters_present( + dynamic_id: UUID, missing_parameters: tuple[str, ...] | None +) -> None: + """Raise DynamicParametersMissingError when the dynamic sets missing parameters. + + ``missing_parameters`` is a checked dynamic's (``CheckedDynamicRead``): + None when the running PolicyEngine.py version has no published catalog + for the country, so nothing could be checked, which passes. + """ + if missing_parameters: + raise DynamicParametersMissingError(dynamic_id, missing_parameters) diff --git a/policyengine_api/services/v2/policies/database_connectors/creates.py b/policyengine_api/services/v2/policies/database_connectors/creates.py index 18e81b853..8627aaf55 100644 --- a/policyengine_api/services/v2/policies/database_connectors/creates.py +++ b/policyengine_api/services/v2/policies/database_connectors/creates.py @@ -11,7 +11,10 @@ from policyengine_api.data.v2.models import LegacyPolicyMapping, ParameterValue, Policy if TYPE_CHECKING: - from policyengine_api.services.v2.policies.types import ResolvedPolicyCreationInput + from policyengine_api.services.v2.policies.types import ( + LegacyPolicyMappingInput, + ResolvedPolicyCreationInput, + ) def create_policy( @@ -68,14 +71,9 @@ def create_parameter_values( def create_legacy_policy_mapping( - session: Session, - *, - country_id: str, - legacy_policy_id: int, - source_policy_hash: str, - policy_id: UUID, + session: Session, mapping: LegacyPolicyMappingInput ) -> UUID | None: - """Insert the mapping from a v1 policy id to its v2 policy. + """Insert the mapping from a v1 policy id to its v2 policy and dynamic. Return the new mapping id, or None when a mapping for this country and v1 id already exists (for example, a concurrent copy won the race). @@ -83,10 +81,11 @@ def create_legacy_policy_mapping( return session.execute( insert(LegacyPolicyMapping) .values( - country_id=country_id, - legacy_policy_id=legacy_policy_id, - policy_id=policy_id, - source_policy_hash=source_policy_hash, + country_id=mapping.country_id, + legacy_policy_id=mapping.legacy_policy_id, + policy_id=mapping.policy_id, + dynamic_id=mapping.dynamic_id, + source_policy_hash=mapping.source_policy_hash, ) .on_conflict_do_nothing(constraint="uq_legacy_policy_mappings_country_legacy") .returning(col(LegacyPolicyMapping.id)) diff --git a/policyengine_api/services/v2/policies/database_connectors/reads.py b/policyengine_api/services/v2/policies/database_connectors/reads.py index 1eba1685b..d26e33dfc 100644 --- a/policyengine_api/services/v2/policies/database_connectors/reads.py +++ b/policyengine_api/services/v2/policies/database_connectors/reads.py @@ -46,15 +46,15 @@ def read_policy_catalog( ) -def read_version_parameter_ids( +def read_version_parameter_names( session: Session, *, model_version_id: UUID, requested_ids: set[UUID] -) -> set[UUID]: - """Return the requested parameter ids that exist in this model version.""" +) -> dict[UUID, str]: + """Return the names of the requested parameters this model version has, by id.""" if not requested_ids: - return set() - return set( + return {} + return dict( session.exec( - select(Parameter.id).where( + select(Parameter.id, Parameter.name).where( Parameter.tax_benefit_model_version_id == model_version_id, col(Parameter.id).in_(requested_ids), ) diff --git a/policyengine_api/services/v2/policies/services.py b/policyengine_api/services/v2/policies/services.py index 6829d826b..9327d7b85 100644 --- a/policyengine_api/services/v2/policies/services.py +++ b/policyengine_api/services/v2/policies/services.py @@ -8,6 +8,9 @@ from policyengine_api.data.v2.catalog.catalog_selection_validation import ( MetadataCatalogNotPublishedError, ) +from policyengine_api.services.v2.dynamics.transformations import ( + dynamic_creation_input, +) from policyengine_api.services.v2.policies.database_connectors.creates import ( create_legacy_policy_mapping, create_parameter_values, @@ -23,7 +26,7 @@ read_policy_catalog, read_policy_row, read_policy_rows, - read_version_parameter_ids, + read_version_parameter_names, ) from policyengine_api.services.v2.policies.transformations import ( canonical_policy_document, @@ -33,6 +36,7 @@ policy_page, policy_parameter_values_by_policy, policy_read, + split_legacy_policy, stored_policy_creation_input, translate_legacy_policy, ) @@ -40,6 +44,7 @@ CanonicalPolicyContent, CheckedPolicyPage, CheckedPolicyRead, + LegacyPolicyMappingInput, LegacyPolicyPersistenceResult, LegacyPolicySnapshot, NativePolicyCreation, @@ -55,6 +60,7 @@ PolicyContentHashCollisionError, PolicyCreationIntegrityError, PolicyNotFoundError, + refuse_dynamic_parameters, validate_policy_catalog, verify_legacy_policy_mapping, ) @@ -80,7 +86,8 @@ def resolve_policy_creation_input( """Bind policy content to the selected catalog's model version. Raise PolicyCatalogValidationError if the model or any parameter id is not - part of the selected catalog. + part of the selected catalog, and PolicySetsDynamicParametersError if any + parameter is one only a dynamic may hold. """ selected = read_policy_catalog( session, @@ -91,16 +98,18 @@ def resolve_policy_creation_input( requested_parameter_ids = { value.parameter_id for value in policy_input.parameter_values } - resolved_parameter_ids = read_version_parameter_ids( + parameter_names = read_version_parameter_names( session, model_version_id=selected.model_version.id, requested_ids=requested_parameter_ids, ) - return validate_policy_catalog( + resolved = validate_policy_catalog( policy_input, selected=selected, - resolved_parameter_ids=resolved_parameter_ids, + resolved_parameter_ids=set(parameter_names), ) + refuse_dynamic_parameters(policy_input.country_id, parameter_names.values()) + return resolved def create_resolved_policy( @@ -245,12 +254,22 @@ def mirror_legacy_policy_in_session( """Copy one committed v1 policy into v2 and map its v1 id to the result. The policy is translated against the running catalog, whichever country - package version it was saved under. A v1 policy that already has a v2 copy - returns that copy without being translated again, so writing it again after - a PolicyEngine.py or country package upgrade still succeeds. Raise + package version it was saved under. Its model assumptions (every + ``gov.simulation.`` value, and the US's ``simulation.``) are copied into + a dynamic, with the UK's old labour-supply spelling rewritten, and the + rest into the policy, which is empty when nothing else remains; the + mapping names both. A v1 policy that sets no assumptions maps to no + dynamic. A v1 policy that already has a v2 copy returns that copy without + being translated again, so writing it again after a PolicyEngine.py or + country package upgrade still succeeds; a copy made before the split + existed keeps its assumptions in its policy and maps to no dynamic. Raise LegacyPolicyMappingIntegrityError if the v1 id is already mapped with a different source hash. """ + from policyengine_api.services.v2.dynamics.services import ( + create_resolved_dynamic, + ) + existing = read_legacy_policy_mapping( session, country_id=snapshot.country_id, @@ -267,6 +286,7 @@ def mirror_legacy_policy_in_session( policy_id=existing.policy_id, policy_created=False, mapping_created=False, + dynamic_id=existing.dynamic_id, ) selected = read_policy_catalog( @@ -274,33 +294,43 @@ def mirror_legacy_policy_in_session( snapshot.country_id, running_policyengine_version=running_policyengine_version, ) - policy_json = snapshot.policy_json - if not isinstance(policy_json, dict): - msg = "legacy policy_json must be an object" - raise TypeError(msg) + split = split_legacy_policy(snapshot) parameters = read_parameters_by_name( session, model_version_id=selected.model_version.id, - names=set(policy_json), + names={*split.policy_values, *split.dynamic_values}, ) - policy_input = translate_legacy_policy( + translation = translate_legacy_policy( snapshot, selected=selected, parameters=parameters, ) - policy_result = create_resolved_policy(session, policy_input) + policy_result = create_resolved_policy(session, translation.policy) + dynamic_result = ( + None + if translation.dynamic is None + else create_resolved_dynamic( + session, dynamic_creation_input(translation.dynamic) + ) + ) + dynamic_id = None if dynamic_result is None else dynamic_result.dynamic_id mapping_id = create_legacy_policy_mapping( session, - country_id=snapshot.country_id, - legacy_policy_id=snapshot.legacy_policy_id, - source_policy_hash=snapshot.source_policy_hash, - policy_id=policy_result.policy_id, + LegacyPolicyMappingInput( + country_id=snapshot.country_id, + legacy_policy_id=snapshot.legacy_policy_id, + source_policy_hash=snapshot.source_policy_hash, + policy_id=policy_result.policy_id, + dynamic_id=dynamic_id, + ), ) if mapping_id is not None: return LegacyPolicyPersistenceResult( policy_id=policy_result.policy_id, policy_created=policy_result.created, mapping_created=True, + dynamic_id=dynamic_id, + dynamic_created=dynamic_result is not None and dynamic_result.created, ) concurrent = read_legacy_policy_mapping( session, @@ -314,9 +344,9 @@ def mirror_legacy_policy_in_session( # A concurrent copy mapped the policy first. As with a mapping found at the # start, the same source hash makes its v2 policy the copy, even when it was # translated with another catalog version than this one. If this copy's - # translation differed, the v2 policy it inserted commits without a mapping, - # and the result's policy_created is false because it reports the mapped - # policy, not that one. + # translation differed, the v2 policy and dynamic it inserted commit + # without a mapping, and the result's policy_created and dynamic_created + # are false because it reports the mapped ones, not those. verify_legacy_policy_mapping( concurrent, source_policy_hash=snapshot.source_policy_hash ) @@ -324,6 +354,7 @@ def mirror_legacy_policy_in_session( policy_id=concurrent.policy_id, policy_created=False, mapping_created=False, + dynamic_id=concurrent.dynamic_id, ) diff --git a/policyengine_api/services/v2/policies/transformations.py b/policyengine_api/services/v2/policies/transformations.py index 799fac982..2259f334b 100644 --- a/policyengine_api/services/v2/policies/transformations.py +++ b/policyengine_api/services/v2/policies/transformations.py @@ -13,10 +13,16 @@ period as parse_policyengine_period, ) +from policyengine_api.dynamic_parameters import ( + ConflictingParameterSpellingsError, + LegacyPolicySplit, + split_legacy_policy_json, +) from policyengine_api.services.v2.policies.types import ( CanonicalPolicyContent, CheckedPolicyRead, LegacyPolicySnapshot, + LegacyPolicyTranslation, LegacyPolicyTranslationReason, PolicyCreationInput, PolicyPage, @@ -40,11 +46,13 @@ SelectedCatalog, ) from policyengine_api.data.v2.models import ( + Dynamic, Parameter, ParameterValue, Policy, TaxBenefitModelVersion, ) + from policyengine_api.supported_countries import SupportedCountryId POLICY_CANONICALIZATION_VERSION = 1 # A v1 date-range period key is "start.end": two dates joined by one dot. @@ -139,11 +147,11 @@ def canonicalize_policy( def stored_policy_creation_input( - policy: Policy, + policy: Policy | Dynamic, model_version: TaxBenefitModelVersion, values: list[ParameterValue], ) -> ResolvedPolicyCreationInput: - """Rebuild a stored policy's input so its canonical bytes can be compared.""" + """Rebuild a stored policy's or dynamic's input to compare its canonical bytes.""" return ResolvedPolicyCreationInput.model_validate( { "country_id": policy.country_id, @@ -296,36 +304,43 @@ def parse_legacy_period(value: str) -> tuple[datetime, datetime]: return start_date, end_date -def translate_legacy_policy( - snapshot: LegacyPolicySnapshot, - *, - selected: SelectedCatalog, - parameters: Mapping[str, Parameter], -) -> ResolvedPolicyCreationInput: - """Translate a v1 policy into policy content bound to the running catalog. - - The country package version the v1 policy was saved under is not compared: - the Public API simulates an old policy with the running package, so the copy - binds it to the running catalog the same way. +def split_legacy_policy(snapshot: LegacyPolicySnapshot) -> LegacyPolicySplit: + """Split a v1 policy's values into the policy's and its model assumptions. - ``parameters`` holds the running catalog's parameters with the policy's - parameter paths, keyed by path. Raise LegacyPolicyTranslationError, with a - fixed reason, if the v1 policy sets a parameter the catalog lacks, sets no - value for a parameter, or has invalid or conflicting periods. + The split is ``split_legacy_policy_json``'s, which also rewrites the UK's + old labour-supply spelling. Raise LegacyPolicyTranslationError, with + reason ``conflicting_values``, when a UK policy sets one parameter under + both spellings and they disagree. """ policy_json = snapshot.policy_json if not isinstance(policy_json, dict): msg = "legacy policy_json must be an object" raise TypeError(msg) - if parameters_missing_from_model(policy_json, parameters): - msg = "every legacy parameter path must exist in the running catalog" + try: + return split_legacy_policy_json(snapshot.country_id, policy_json) + except ConflictingParameterSpellingsError as error: + msg = "legacy parameter spellings conflict" raise LegacyPolicyTranslationError( - msg, reason=LegacyPolicyTranslationReason.MISSING_PARAMETERS - ) + msg, reason=LegacyPolicyTranslationReason.CONFLICTING_VALUES + ) from error + + +def _translate_legacy_values( + country_id: SupportedCountryId, + values_by_name: Mapping[str, object], + *, + selected: SelectedCatalog, + parameters: Mapping[str, Parameter], +) -> ResolvedPolicyCreationInput: + """Translate one part of a v1 policy's values into content bound to the catalog. + Every name in ``values_by_name`` is in ``parameters``. Raise + LegacyPolicyTranslationError for values that are not period objects, + invalid or conflicting periods, or more values than v2 content may hold. + """ parameter_values: list[PolicyParameterValueInput] = [] - for parameter_name in sorted(policy_json): - period_values = policy_json[parameter_name] + for parameter_name in sorted(values_by_name): + period_values = values_by_name[parameter_name] if type(period_values) is not dict: msg = ( f"legacy parameter {parameter_name!r} must contain period/value entries" @@ -354,13 +369,13 @@ def translate_legacy_policy( ) ) if len(parameter_values) > MAXIMUM_POLICY_PARAMETER_VALUES: - msg = "legacy policy has more values than a v2 policy may hold" + msg = "legacy policy has more values than a v2 policy or dynamic may hold" raise LegacyPolicyTranslationError( msg, reason=LegacyPolicyTranslationReason.TOO_MANY_VALUES ) try: - policy_input = PolicyCreationInput( - country_id=snapshot.country_id, + content = PolicyCreationInput( + country_id=country_id, tax_benefit_model_id=selected.model.id, parameter_values=parameter_values, ) @@ -372,7 +387,56 @@ def translate_legacy_policy( msg, reason=LegacyPolicyTranslationReason.CONFLICTING_VALUES ) from error return validate_policy_catalog( - policy_input, + content, + selected=selected, + resolved_parameter_ids={parameters[name].id for name in values_by_name}, + ) + + +def translate_legacy_policy( + snapshot: LegacyPolicySnapshot, + *, + selected: SelectedCatalog, + parameters: Mapping[str, Parameter], +) -> LegacyPolicyTranslation: + """Translate a v1 policy into a policy and a dynamic bound to the running catalog. + + The v1 policy's model assumptions (``policyengine_api.dynamic_parameters``) + become the dynamic's content, None when it sets none, and the rest the + policy's, which may be empty. The UK's old labour-supply spelling is + rewritten first (``split_legacy_policy``). + + The country package version the v1 policy was saved under is not compared: + the Public API simulates an old policy with the running package, so the copy + binds it to the running catalog the same way. + + ``parameters`` holds the running catalog's parameters with the split's + parameter names, keyed by name. Raise LegacyPolicyTranslationError, with a + fixed reason, if the v1 policy sets a parameter the catalog lacks, sets no + value for a parameter, or has invalid or conflicting periods or spellings. + """ + split = split_legacy_policy(snapshot) + if parameters_missing_from_model( + [*split.policy_values, *split.dynamic_values], parameters + ): + msg = "every legacy parameter path must exist in the running catalog" + raise LegacyPolicyTranslationError( + msg, reason=LegacyPolicyTranslationReason.MISSING_PARAMETERS + ) + policy = _translate_legacy_values( + snapshot.country_id, + split.policy_values, selected=selected, - resolved_parameter_ids={parameter.id for parameter in parameters.values()}, + parameters=parameters, + ) + dynamic = ( + _translate_legacy_values( + snapshot.country_id, + split.dynamic_values, + selected=selected, + parameters=parameters, + ) + if split.dynamic_values + else None ) + return LegacyPolicyTranslation(policy=policy, dynamic=dynamic) diff --git a/policyengine_api/services/v2/policies/types.py b/policyengine_api/services/v2/policies/types.py index d5ef827d1..ba007a913 100644 --- a/policyengine_api/services/v2/policies/types.py +++ b/policyengine_api/services/v2/policies/types.py @@ -66,15 +66,22 @@ def validate_period(self) -> PolicyParameterValueInput: return self +# A policy's or a dynamic's values, at most MAXIMUM_POLICY_PARAMETER_VALUES. +ParameterValueInputs = Annotated[ + list[PolicyParameterValueInput], + Field(max_length=MAXIMUM_POLICY_PARAMETER_VALUES), +] + + class PolicyCreationInput(StrictPolicyInput): - """Complete immutable policy content accepted from any source.""" + """Complete immutable policy content accepted from any source. + + A dynamic's content has the same fields (``dynamics.types``). + """ country_id: CountryId tax_benefit_model_id: UUID - parameter_values: Annotated[ - list[PolicyParameterValueInput], - Field(max_length=MAXIMUM_POLICY_PARAMETER_VALUES), - ] + parameter_values: ParameterValueInputs @field_validator("parameter_values") @classmethod @@ -143,13 +150,44 @@ class PolicyCreationResult: created: bool +@dataclass(frozen=True) +class LegacyPolicyTranslation: + """A v1 policy translated against the running catalog, split in two. + + ``policy`` holds the values that stay in the policy, possibly none. + ``dynamic`` holds the model's assumptions, the content of the dynamic + the copy stores, or is None when the v1 policy sets none. + """ + + policy: ResolvedPolicyCreationInput + dynamic: ResolvedPolicyCreationInput | None + + +@dataclass(frozen=True) +class LegacyPolicyMappingInput: + """The mapping a copy records from a v1 policy to its v2 policy and dynamic.""" + + country_id: str + legacy_policy_id: int + source_policy_hash: str + policy_id: UUID + dynamic_id: UUID | None + + @dataclass(frozen=True) class LegacyPolicyPersistenceResult: - """Destination identity and insertion outcomes for one mirror attempt.""" + """Destination identity and insertion outcomes for one mirror attempt. + + ``dynamic_id`` is the dynamic the v1 policy's model assumptions were + copied into, None when it sets none or was copied before the split + existed; ``dynamic_created`` says whether this attempt inserted it. + """ policy_id: UUID policy_created: bool mapping_created: bool + dynamic_id: UUID | None = None + dynamic_created: bool = False @dataclass(frozen=True) diff --git a/policyengine_api/services/v2/policies/validators.py b/policyengine_api/services/v2/policies/validators.py index 05786fdb7..cb1145832 100644 --- a/policyengine_api/services/v2/policies/validators.py +++ b/policyengine_api/services/v2/policies/validators.py @@ -6,6 +6,8 @@ from datetime import UTC, datetime from typing import TYPE_CHECKING, Any +from policyengine_api.dynamic_parameters import is_dynamic_parameter + if TYPE_CHECKING: from collections.abc import Container, Iterable from uuid import UUID @@ -19,6 +21,7 @@ PolicyCreationInput, ResolvedPolicyCreationInput, ) + from policyengine_api.supported_countries import SupportedCountryId MAXIMUM_POLICY_PARAMETER_VALUES = 1_000 @@ -58,6 +61,43 @@ def __init__(self, message: str, *, reason: LegacyPolicyTranslationReason) -> No self.reason = reason +class PolicySetsDynamicParametersError(ValueError): + """Raised when a policy sets parameters only a dynamic may hold. + + These are the model's assumptions, every parameter under + ``gov.simulation.`` plus the US's ``simulation.`` + (``policyengine_api.dynamic_parameters``), sent as a dynamic through + ``POST /v2/dynamics``. ``parameter_names`` lists them, sorted. + """ + + def __init__(self, parameter_names: tuple[str, ...]) -> None: + """Name each parameter the policy may not set.""" + self.parameter_names = parameter_names + super().__init__(parameter_names) + + def __str__(self) -> str: + """Say which parameters to send as a dynamic, and where.""" + return ( + "A policy cannot set the model's assumptions, the parameters under " + "gov.simulation (and, in the US, simulation): send them as a " + "dynamic through POST /v2/dynamics. This policy sets: " + + ", ".join(self.parameter_names) + ) + + +def refuse_dynamic_parameters( + country_id: SupportedCountryId, parameter_names: Iterable[str] +) -> None: + """Raise PolicySetsDynamicParametersError if any parameter belongs in a dynamic.""" + dynamic_parameter_names = tuple( + sorted( + {name for name in parameter_names if is_dynamic_parameter(country_id, name)} + ) + ) + if dynamic_parameter_names: + raise PolicySetsDynamicParametersError(dynamic_parameter_names) + + class PolicyParametersMissingError(ValueError): """Raised when a policy sets parameters the running catalog version lacks. diff --git a/policyengine_api/services/v2/simulation_service/services.py b/policyengine_api/services/v2/simulation_service/services.py index 908d80134..488fbc9a6 100644 --- a/policyengine_api/services/v2/simulation_service/services.py +++ b/policyengine_api/services/v2/simulation_service/services.py @@ -8,10 +8,14 @@ country does not), that each policy the submission names is stored in the country, through the same service as ``/v2/policies`` (current law names none), and that none of those policies sets a parameter the running catalog -version lacks (``require_runnable_policies``, which re-runs call too). -Only then is the simulation service asked. The dynamics a submission names, -its region and its dataset are checked by the simulation service, which reads -them in the catalog version its default worker runs. +version lacks (``require_runnable_policies``, which re-runs call too). An +economy simulation, economy report or budget window then has each dynamic it +names checked the same way, through the same service as ``/v2/dynamics`` +(none means no assumptions beyond the model's defaults): +``require_runnable_dynamics``. +Only then is the simulation service asked. The submission's region and its +dataset are checked by the simulation service, which reads them in the +catalog version its default worker runs. A household report is checked the same way, except that its country's row must allow household calculations instead (a household-only country passes), @@ -22,7 +26,9 @@ Running a stored report again (``POST /reports/{report_id}/runs``) checks its country as a new report of its kind is checked, then the policies its -simulations name. +simulations name. Its dynamics are not checked again: each was checked to be +the country's when the report was created, and the simulation service refuses +a re-run whose dynamic sets parameters its catalog version lacks. """ from __future__ import annotations @@ -35,6 +41,10 @@ from policyengine_api.data.v2.catalog.catalog_selection_validation import ( MetadataCatalogUnavailableError, ) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicStorageError, + require_dynamic_parameters_present, +) from policyengine_api.services.v2.household_calculations.validators import ( require_default_year, require_household_country, @@ -58,6 +68,7 @@ from policyengine_api.services.v2.simulation_service.types import ( BudgetWindowReportCreationInput, CountrySource, + DynamicSource, EconomyReportCreationInput, EconomySimulationCreationInput, HouseholdReportCreationInput, @@ -79,12 +90,17 @@ def _storage_failures() -> Iterator[None]: """Raise SubmissionStorageError for a read that failed. - That includes the policy check's catalog query, which the policy service - reports as MetadataCatalogUnavailableError. + That includes the policy and dynamic checks' catalog query, which their + services report as MetadataCatalogUnavailableError, and the dynamic + service's DynamicStorageError. """ try: yield - except (SQLAlchemyError, MetadataCatalogUnavailableError) as error: + except ( + SQLAlchemyError, + MetadataCatalogUnavailableError, + DynamicStorageError, + ) as error: msg = "V2 storage is unavailable to check the submission" raise SubmissionStorageError(msg) from error @@ -113,6 +129,28 @@ def require_runnable_policies( require_policy_parameters_present(policy.id, policy.missing_parameters) +def require_runnable_dynamics( + dynamics: DynamicSource, *, country_id: str, dynamic_ids: Iterable[UUID | None] +) -> None: + """Raise unless each named dynamic is the country's and its parameters all exist. + + As for policies, every dynamic is read before any is checked, so + DynamicNotFoundError for a dynamic the country does not store comes + before DynamicParametersMissingError for one that sets parameters the + running catalog version lacks. None names no dynamic, and a dynamic the + running catalog could not check passes. Raise SubmissionStorageError when + the database or its catalog cannot be read. + """ + with _storage_failures(): + stored_dynamics = [ + dynamics.get_dynamic(country_id=country_id, dynamic_id=dynamic_id) + for dynamic_id in dict.fromkeys(dynamic_ids) + if dynamic_id is not None + ] + for dynamic in stored_dynamics: + require_dynamic_parameters_present(dynamic.id, dynamic.missing_parameters) + + class SimulationSubmissionService: """Check submissions, then send them to the simulation service.""" @@ -122,33 +160,37 @@ def __init__( countries: CountrySource, households: HouseholdSource, policies: PolicySource, + dynamics: DynamicSource, client_factory: Callable[[], SimulationServiceClient], ) -> None: """Check through these sources; send through ``client_factory``'s client. - ``households`` is read only for a household report. The client is - built only once a submission passes the checks, so a refused - submission gets its own answer even when the simulation service is - not configured. + ``households`` is read only for a household report, and ``dynamics`` + only for a submission that names a dynamic. The client is built only + once a submission passes the checks, so a refused submission gets its + own answer even when the simulation service is not configured. """ self._countries = countries self._households = households self._policies = policies + self._dynamics = dynamics self._client_factory = client_factory def _check( self, country_id: str, policy_ids: Iterable[UUID | None], + dynamic_ids: Iterable[UUID | None] = (), *, require_country: CountryCheck = require_economy_country, ) -> None: - """Raise unless the country passes ``require_country`` and the policies can run. + """Raise unless the country passes ``require_country`` and all can run. By default the country must have economy calculations. Raise the country check's error (EconomyCalculationsUnsupportedError by default), then PolicyNotFoundError, then PolicyParametersMissingError, - and SubmissionStorageError when the database or its catalog cannot be + then DynamicNotFoundError, then DynamicParametersMissingError, and + SubmissionStorageError when the database or its catalog cannot be read. """ with _storage_failures(): @@ -156,6 +198,9 @@ def _check( require_runnable_policies( self._policies, country_id=country_id, policy_ids=policy_ids ) + require_runnable_dynamics( + self._dynamics, country_id=country_id, dynamic_ids=dynamic_ids + ) def submit_simulation( self, simulation: EconomySimulationCreationInput @@ -164,18 +209,22 @@ def submit_simulation( Raise the checks' errors, then the simulation service's. """ - self._check(simulation.country_id, (simulation.policy_id,)) + self._check( + simulation.country_id, (simulation.policy_id,), (simulation.dynamic_id,) + ) return self._client_factory().submit_simulation(simulation) def submit_report(self, report: EconomyReportCreationInput) -> SubmittedReport: """Check the report, then have the simulation service create and start it. Both the baseline's and the reform's policies are checked, the - baseline's first. Raise the checks' errors, then the simulation - service's. + baseline's first, then their dynamics in the same order. Raise the + checks' errors, then the simulation service's. """ self._check( - report.country_id, (report.baseline.policy_id, report.reform.policy_id) + report.country_id, + (report.baseline.policy_id, report.reform.policy_id), + (report.baseline.dynamic_id, report.reform.dynamic_id), ) return self._client_factory().submit_report(report) @@ -218,7 +267,9 @@ def submit_budget_window( simulation service's. """ self._check( - report.country_id, (report.baseline.policy_id, report.reform.policy_id) + report.country_id, + (report.baseline.policy_id, report.reform.policy_id), + (report.baseline.dynamic_id, report.reform.dynamic_id), ) return self._client_factory().submit_budget_window(report) diff --git a/policyengine_api/services/v2/simulation_service/types.py b/policyengine_api/services/v2/simulation_service/types.py index 80bad5ebf..85030aeaf 100644 --- a/policyengine_api/services/v2/simulation_service/types.py +++ b/policyengine_api/services/v2/simulation_service/types.py @@ -42,6 +42,7 @@ from uuid import UUID from policyengine_api.services.v2.countries.types import CountryRead + from policyengine_api.services.v2.dynamics.types import CheckedDynamicRead from policyengine_api.services.v2.policies.types import CheckedPolicyRead # The bounds the simulation service and the tables accept. @@ -442,7 +443,8 @@ class SimulationServiceClient(Protocol): it through a client, also raises EconomyCalculationsUnsupportedError, HouseholdCalculationCountryError, HouseholdNotFoundError, HouseholdCalculationYearError, PolicyNotFoundError, - PolicyParametersMissingError and + PolicyParametersMissingError, DynamicNotFoundError, + DynamicParametersMissingError and SubmissionStorageError from its checks. """ @@ -499,3 +501,11 @@ class PolicySource(Protocol): def get_policy(self, *, country_id: str, policy_id: UUID) -> CheckedPolicyRead: """Return the country's policy, or raise ``PolicyNotFoundError``.""" ... + + +class DynamicSource(Protocol): + """The stored dynamics a submission names.""" + + def get_dynamic(self, *, country_id: str, dynamic_id: UUID) -> CheckedDynamicRead: + """Return the country's dynamic, or raise ``DynamicNotFoundError``.""" + ... diff --git a/policyengine_api/supported_countries.py b/policyengine_api/supported_countries.py index 70dcce58a..2efe6d226 100644 --- a/policyengine_api/supported_countries.py +++ b/policyengine_api/supported_countries.py @@ -24,6 +24,10 @@ v1 ``household_mirror_events`` table, which the v1 household insert writes in its own transaction. +A new country also needs its dynamic parameter prefixes, the parameters +only a dynamic may hold (``DYNAMIC_PARAMETER_PREFIXES`` in +``policyengine_api.dynamic_parameters``); a unit test fails until it has them. + Catalog publication also needs the country's checked-in region reference file (``policyengine_api/data/v2/catalog/region_reference_files``) and its region scoping rules (``REGION_SCOPING_RULES`` in diff --git a/pyproject.toml b/pyproject.toml index 68a37a0cb..f486d817d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -402,6 +402,8 @@ banned-module-level-imports = [ "policyengine_api.libs.simulation_reports", "policyengine_api.services.v2.countries.database_session", "policyengine_api.services.v2.countries.services", + "policyengine_api.services.v2.dynamics.database_session", + "policyengine_api.services.v2.dynamics.services", "policyengine_api.services.v2.household_calculations.database_session", "policyengine_api.services.v2.household_calculations.services", "policyengine_api.services.v2.households.database_session", diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 75b3220fa..0bf798576 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -211,6 +211,47 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="dynamic_resources_v2", + current_contract="typed_v2_resources", + future_owner_pr="Dynamics stored like policies", + requests=( + ContractRequest( + method="POST", + path="/v2/dynamics?country_id=us", + expected_status=201, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + "result.item.missing_parameters", + ), + route_group="policy", + ), + ContractRequest( + method="GET", + path="/v2/dynamics/{dynamic_id}?country_id=us", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.item.id", + "result.item.country_id", + "result.item.tax_benefit_model_id", + "result.item.parameter_values", + "result.item.created_at", + "result.item.updated_at", + "result.item.missing_parameters", + ), + route_group="policy", + ), + ), + ), WorkflowContract( name="user_policy_associations_v2", current_contract="typed_v2_resources", diff --git a/tests/integration/test_v2_budget_window_report_creation.py b/tests/integration/test_v2_budget_window_report_creation.py index f1913b40f..25f8d1909 100644 --- a/tests/integration/test_v2_budget_window_report_creation.py +++ b/tests/integration/test_v2_budget_window_report_creation.py @@ -35,6 +35,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -157,6 +161,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) dependencies = NativeRouteDependencies( diff --git a/tests/integration/test_v2_catalog_publication_pruning.py b/tests/integration/test_v2_catalog_publication_pruning.py index 21f192827..075ebe715 100644 --- a/tests/integration/test_v2_catalog_publication_pruning.py +++ b/tests/integration/test_v2_catalog_publication_pruning.py @@ -75,7 +75,6 @@ US_MODEL = "policyengine-us" UK_MODEL = "policyengine-uk" MODEL_NAMES = (US_MODEL, UK_MODEL) -PROBE_DYNAMIC_NAME = "catalog-pruning-probe" # Each fixture catalog version holds, per country, two current-law values, # three parameter tree rows and one search file. FIXTURE_PRUNED_ROWS = {"parameter_values": 2, "tree_rows": 3, "search_files": 1} @@ -99,12 +98,9 @@ def _disposable_url() -> str: def _clear_catalog(engine: Engine) -> None: + # Policies and dynamics name a model, so the truncation removes them too. with engine.begin() as connection: connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) - connection.execute( - text("DELETE FROM dynamics WHERE name = :name"), - {"name": PROBE_DYNAMIC_NAME}, - ) @pytest.fixture @@ -267,7 +263,13 @@ def _add_saved_reform_and_dynamic(engine: Engine, version: str) -> None: canonicalization_version=1, content_hash=uuid4().hex * 2, ) - dynamic = Dynamic(name=PROBE_DYNAMIC_NAME) + dynamic = Dynamic( + country_id="us", + tax_benefit_model_id=model.id, + tax_benefit_model_version_id=model_version.id, + canonicalization_version=1, + content_hash=uuid4().hex * 2, + ) session.add_all([policy, dynamic]) session.flush() session.add_all( diff --git a/tests/integration/test_v2_dynamic_persistence.py b/tests/integration/test_v2_dynamic_persistence.py index ddd2fc89b..34d4ce312 100644 --- a/tests/integration/test_v2_dynamic_persistence.py +++ b/tests/integration/test_v2_dynamic_persistence.py @@ -13,12 +13,6 @@ from uuid import UUID, uuid4 import pytest -from policyengine_api.services.v2.dynamics.database_session import ( - DynamicDatabaseSession, -) -from policyengine_api.services.v2.dynamics.services import V2DynamicService -from policyengine_api.services.v2.dynamics.types import NativeDynamicCreationInput -from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError from sqlalchemy import create_engine, delete, or_, select from sqlalchemy.orm import sessionmaker from sqlmodel import Session @@ -37,6 +31,12 @@ TaxBenefitModelVersion, ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService +from policyengine_api.services.v2.dynamics.types import NativeDynamicCreationInput +from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError from policyengine_api.services.v2.policies.database_session import ( PolicyDatabaseSession, ) @@ -380,17 +380,18 @@ def test_a_copy_made_before_the_split_keeps_its_mapping_without_a_dynamic( ) session.add(earlier_copy) session.flush() + earlier_copy_id = earlier_copy.id session.add( LegacyPolicyMapping( country_id="us", legacy_policy_id=9301, - policy_id=earlier_copy.id, + policy_id=earlier_copy_id, source_policy_hash=snapshot.source_policy_hash, ) ) result = _policy_service(runtime_sessions).mirror_legacy_policy(snapshot) - assert result.policy_id == earlier_copy.id + assert result.policy_id == earlier_copy_id assert result.dynamic_id is None assert result.mapping_created is False diff --git a/tests/integration/test_v2_household_reports.py b/tests/integration/test_v2_household_reports.py index 4d72c15c6..a64d971e8 100644 --- a/tests/integration/test_v2_household_reports.py +++ b/tests/integration/test_v2_household_reports.py @@ -39,6 +39,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -343,6 +347,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) dependencies = NativeRouteDependencies( diff --git a/tests/integration/test_v2_report_creation.py b/tests/integration/test_v2_report_creation.py index 7ce1d32b7..98ef13935 100644 --- a/tests/integration/test_v2_report_creation.py +++ b/tests/integration/test_v2_report_creation.py @@ -33,6 +33,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -182,6 +186,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) dependencies = NativeRouteDependencies( diff --git a/tests/integration/test_v2_report_runs.py b/tests/integration/test_v2_report_runs.py index ca4612001..96e17b9b0 100644 --- a/tests/integration/test_v2_report_runs.py +++ b/tests/integration/test_v2_report_runs.py @@ -35,6 +35,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -287,6 +291,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) signed_in = {} if requester is None else {"optional_user": lambda: requester} diff --git a/tests/integration/user_report_run_support.py b/tests/integration/user_report_run_support.py index 9aac21c8a..4ffeb2c9b 100644 --- a/tests/integration/user_report_run_support.py +++ b/tests/integration/user_report_run_support.py @@ -29,6 +29,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) @@ -239,6 +243,9 @@ def client_for( policies=V2PolicyService( PolicyDatabaseSession(sessions), running_policyengine_version="4.2.0" ), + dynamics=V2DynamicService( + DynamicDatabaseSession(sessions), running_policyengine_version="4.2.0" + ), client_factory=lambda: simulation_service, ) signed_in = {} if requester is None else {"optional_user": lambda: requester} diff --git a/tests/unit/services/test_policy_mirroring.py b/tests/unit/services/test_policy_mirroring.py index de695a56d..3e34a0d14 100644 --- a/tests/unit/services/test_policy_mirroring.py +++ b/tests/unit/services/test_policy_mirroring.py @@ -22,6 +22,10 @@ classify_policy_copy_failure, mirror_policy_after_commit, ) +from policyengine_api.services.v2.dynamics.validators import ( + DynamicContentHashCollisionError, + DynamicCreationIntegrityError, +) from policyengine_api.services.v2.policies.types import ( LegacyPolicyPersistenceResult, LegacyPolicySnapshot, @@ -214,6 +218,18 @@ def _translation_failure( "stored_policy_integrity", id="stored-policy-integrity", ), + pytest.param( + DynamicContentHashCollisionError("secret"), + "integrity", + "content_hash_collision", + id="dynamic-content-hash-collision", + ), + pytest.param( + DynamicCreationIntegrityError("secret"), + "integrity", + "stored_dynamic_integrity", + id="stored-dynamic-integrity", + ), pytest.param( PoolTimeoutError("pool timeout secret"), "database", diff --git a/tests/unit/test_dynamic_parameters.py b/tests/unit/test_dynamic_parameters.py index ee94acc16..5e3935cdf 100644 --- a/tests/unit/test_dynamic_parameters.py +++ b/tests/unit/test_dynamic_parameters.py @@ -3,6 +3,7 @@ from typing import get_args import pytest + from policyengine_api.dynamic_parameters import ( DYNAMIC_PARAMETER_PREFIXES, ConflictingParameterSpellingsError, @@ -10,7 +11,6 @@ is_dynamic_parameter, split_legacy_policy_json, ) - from policyengine_api.supported_countries import SupportedCountryId UK_OLD_INCOME_ELASTICITY = "gov.simulation.labor_supply_responses.income_elasticity" diff --git a/tests/unit/v2/test_budget_window_submissions.py b/tests/unit/v2/test_budget_window_submissions.py index 846d9c649..91be8837a 100644 --- a/tests/unit/v2/test_budget_window_submissions.py +++ b/tests/unit/v2/test_budget_window_submissions.py @@ -13,10 +13,10 @@ from uuid import UUID, uuid4 import pytest -from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError from pydantic import ValidationError from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.dynamics.validators import DynamicNotFoundError from policyengine_api.services.v2.policies.types import CheckedPolicyRead from policyengine_api.services.v2.policies.validators import ( PolicyNotFoundError, diff --git a/tests/unit/v2/test_dynamic_catalog.py b/tests/unit/v2/test_dynamic_catalog.py index 69584284a..518079cd9 100644 --- a/tests/unit/v2/test_dynamic_catalog.py +++ b/tests/unit/v2/test_dynamic_catalog.py @@ -11,24 +11,35 @@ from uuid import UUID import pytest +from pydantic import ValidationError +from sqlalchemy.exc import OperationalError +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, +) +from policyengine_api.data.v2.models import ( + V2_METADATA, + Parameter, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) from policyengine_api.services.v2.dynamics.services import ( + V2DynamicService, resolve_dynamic_creation_input, ) from policyengine_api.services.v2.dynamics.types import ( DynamicCreationInput, + NativeDynamicCreationInput, ResolvedDynamicCreationInput, ) from policyengine_api.services.v2.dynamics.validators import ( DynamicSetsPolicyParametersError, -) -from pydantic import ValidationError -from sqlmodel import Session, create_engine - -from policyengine_api.data.v2.models import ( - V2_METADATA, - Parameter, - TaxBenefitModel, - TaxBenefitModelVersion, + DynamicStorageError, ) from policyengine_api.services.v2.policies.services import ( resolve_policy_creation_input, @@ -216,3 +227,34 @@ def test_a_policy_without_dynamic_parameters_is_still_bound( assert [value.parameter_id for value in resolved.parameter_values] == [ catalog.parameter_ids[TAX_RATE] ] + + +def test_a_dynamic_storage_that_cannot_be_read_is_a_storage_error() -> None: + # A database without the v2 tables fails every read with OperationalError. + # A create's first read is the catalog query, which reports its own error. + engine = create_engine("sqlite://") + service = V2DynamicService( + DynamicDatabaseSession(sessionmaker(engine, class_=Session)), + running_policyengine_version=RUNNING_VERSION, + ) + try: + with pytest.raises(DynamicStorageError) as failure: + service.get_dynamic(country_id="us", dynamic_id=UUID(int=3)) + with pytest.raises(MetadataCatalogUnavailableError): + service.create_dynamic( + NativeDynamicCreationInput( + country_id="us", + tax_benefit_model_id=UUID(int=1), + parameter_values=[ + { + "parameter_id": UUID(int=2), + "value": 0.25, + "start_date": "2026-01-01T00:00:00Z", + } + ], + ) + ) + finally: + engine.dispose() + + assert isinstance(failure.value.__cause__, OperationalError) diff --git a/tests/unit/v2/test_dynamic_routes.py b/tests/unit/v2/test_dynamic_routes.py index 1e0edbc74..048305261 100644 --- a/tests/unit/v2/test_dynamic_routes.py +++ b/tests/unit/v2/test_dynamic_routes.py @@ -8,6 +8,17 @@ import pytest from fastapi.testclient import TestClient from flask import Flask, jsonify + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.catalog.catalog_selection_validation import ( + MetadataCatalogUnavailableError, + MetadataCatalogVersionNotFoundError, +) +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) from policyengine_api.services.v2.dynamics.types import ( CheckedDynamicRead, NativeDynamicCreation, @@ -18,17 +29,7 @@ DynamicCreationIntegrityError, DynamicNotFoundError, DynamicSetsPolicyParametersError, -) -from sqlalchemy.exc import OperationalError - -from policyengine_api.asgi_factory import create_asgi_app -from policyengine_api.data.v2.catalog.catalog_selection_validation import ( - MetadataCatalogVersionNotFoundError, -) -from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies -from policyengine_api.migration_flags import ( - RouteImplementation, - RouteImplementationSettings, + DynamicStorageError, ) from policyengine_api.services.v2.policies.types import PolicyParameterValueRead from policyengine_api.services.v2.policies.validators import ( @@ -257,7 +258,12 @@ def test_detail_is_country_scoped_and_reports_missing_parameters() -> None: "Stored dynamic integrity failed", ), ( - OperationalError("SELECT secret", {}, Exception("password")), + DynamicStorageError("SELECT secret"), + 503, + "V2 dynamic persistence is unavailable", + ), + ( + MetadataCatalogUnavailableError("catalog secret"), 503, "V2 dynamic persistence is unavailable", ), @@ -298,7 +304,7 @@ def test_dynamics_are_neither_listed_nor_changed(method: str, path: str) -> None def test_openapi_publishes_the_dynamic_routes_and_their_errors() -> None: client, _flask_calls = _client(FakeDynamicService()) - schema = client.get("/openapi.json").json() + schema = client.get("/v2/openapi.json").json() create = schema["paths"]["/v2/dynamics"]["post"] detail = schema["paths"]["/v2/dynamics/{dynamic_id}"]["get"] diff --git a/tests/unit/v2/test_policy_legacy_copy.py b/tests/unit/v2/test_policy_legacy_copy.py index 33f43a2aa..6198f4f65 100644 --- a/tests/unit/v2/test_policy_legacy_copy.py +++ b/tests/unit/v2/test_policy_legacy_copy.py @@ -6,15 +6,11 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import asdict, dataclass, field from datetime import UTC, datetime from uuid import UUID import pytest -from policyengine_api.services.v2.dynamics.types import ( - DynamicCreationResult, - ResolvedDynamicCreationInput, -) from sqlmodel import Session, create_engine from policyengine_api.data.v2.models import ( @@ -24,6 +20,11 @@ TaxBenefitModel, TaxBenefitModelVersion, ) +from policyengine_api.services.v2.dynamics import services as dynamic_services +from policyengine_api.services.v2.dynamics.types import ( + DynamicCreationResult, + ResolvedDynamicCreationInput, +) from policyengine_api.services.v2.policies import services as policy_services from policyengine_api.services.v2.policies.types import ( LegacyPolicyPersistenceResult, @@ -68,12 +69,12 @@ def record_dynamic(_session, dynamic_input): writes.created_dynamics.append(dynamic_input) return DynamicCreationResult(dynamic_id=COPIED_DYNAMIC_ID, created=True) - def record_mapping(_session, **mapping_fields): - writes.created_mappings.append(mapping_fields) + def record_mapping(_session, mapping): + writes.created_mappings.append(asdict(mapping)) return MAPPING_ID monkeypatch.setattr(policy_services, "create_resolved_policy", record_policy) - monkeypatch.setattr(policy_services, "create_resolved_dynamic", record_dynamic) + monkeypatch.setattr(dynamic_services, "create_resolved_dynamic", record_dynamic) monkeypatch.setattr(policy_services, "create_legacy_policy_mapping", record_mapping) return writes @@ -315,11 +316,11 @@ def _lose_mapping_race(monkeypatch, *, winning_source_hash: str) -> None: constraint makes it a no-op. """ - def insert_after_concurrent_copy(session, **mapping_fields): + def insert_after_concurrent_copy(session, mapping): session.add( LegacyPolicyMapping( - country_id=mapping_fields["country_id"], - legacy_policy_id=mapping_fields["legacy_policy_id"], + country_id=mapping.country_id, + legacy_policy_id=mapping.legacy_policy_id, policy_id=EARLIER_COPY_POLICY_ID, source_policy_hash=winning_source_hash, ) diff --git a/tests/unit/v2/test_simulation_submission_database_routes.py b/tests/unit/v2/test_simulation_submission_database_routes.py index 890d2ae2f..b9041099b 100644 --- a/tests/unit/v2/test_simulation_submission_database_routes.py +++ b/tests/unit/v2/test_simulation_submission_database_routes.py @@ -13,10 +13,6 @@ from uuid import UUID, uuid4 import pytest -from policyengine_api.services.v2.dynamics.database_session import ( - DynamicDatabaseSession, -) -from policyengine_api.services.v2.dynamics.services import V2DynamicService from sqlalchemy import text from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import StaticPool @@ -36,6 +32,10 @@ CountryDatabaseSession, ) from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.dynamics.database_session import ( + DynamicDatabaseSession, +) +from policyengine_api.services.v2.dynamics.services import V2DynamicService from policyengine_api.services.v2.households.database_session import ( HouseholdDatabaseSession, ) diff --git a/tests/unit/v2/test_simulation_submissions.py b/tests/unit/v2/test_simulation_submissions.py index 67713277a..fff3d940d 100644 --- a/tests/unit/v2/test_simulation_submissions.py +++ b/tests/unit/v2/test_simulation_submissions.py @@ -8,17 +8,18 @@ from uuid import UUID, uuid4 import pytest -from policyengine_api.services.v2.dynamics.types import CheckedDynamicRead -from policyengine_api.services.v2.dynamics.validators import ( - DynamicNotFoundError, - DynamicParametersMissingError, -) from sqlalchemy.exc import OperationalError from policyengine_api.data.v2.catalog.catalog_selection_validation import ( MetadataCatalogUnavailableError, ) from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.dynamics.types import CheckedDynamicRead +from policyengine_api.services.v2.dynamics.validators import ( + DynamicNotFoundError, + DynamicParametersMissingError, + DynamicStorageError, +) from policyengine_api.services.v2.policies.types import CheckedPolicyRead from policyengine_api.services.v2.policies.validators import ( PolicyNotFoundError, @@ -484,8 +485,15 @@ def test_policies_are_checked_before_dynamics() -> None: assert dynamics.asked == [] -def test_a_dynamic_read_that_fails_is_a_storage_error() -> None: - error = OperationalError("SELECT 1", {}, Exception("connection refused")) +@pytest.mark.parametrize( + "error", + [ + DynamicStorageError("dynamics cannot be read"), + OperationalError("SELECT 1", {}, Exception("connection refused")), + MetadataCatalogUnavailableError("catalog cannot be queried"), + ], +) +def test_a_dynamic_read_that_fails_is_a_storage_error(error: Exception) -> None: service = submissions(unconfigured_client, dynamics=StoredDynamics(error=error)) with pytest.raises(SubmissionStorageError) as failure: From 960e10eacffa834fc9a30d170175422845bb406d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:22:14 +0400 Subject: [PATCH 5/6] Say that /v2/simulations and /v2/reports check the dynamics they name Co-Authored-By: Claude Opus 5.5 --- .../fastapi_routes/v2/reports/response_models.py | 13 +++++++------ .../fastapi_routes/v2/reports/routes.py | 4 ++-- .../v2/simulations/response_models.py | 7 ++++--- .../fastapi_routes/v2/simulations/routes.py | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/policyengine_api/fastapi_routes/v2/reports/response_models.py b/policyengine_api/fastapi_routes/v2/reports/response_models.py index 3b9e3c760..468b30d51 100644 --- a/policyengine_api/fastapi_routes/v2/reports/response_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/response_models.py @@ -485,8 +485,8 @@ class ReportRunNotRecordedResponse(V2ErrorResponse): 401: error_response(UNAUTHENTICATED_REQUESTER), 403: error_response(MACHINE_REQUESTER), 404: error_response( - "A policy the report names, or a household report's household, is not " - "stored in the country." + "A policy or dynamic the report names, or a household report's " + "household, is not stored in the country." ), 413: error_response("The request body exceeds 16 KiB."), 422: { @@ -497,10 +497,11 @@ class ReportRunNotRecordedResponse(V2ErrorResponse): "country has no economy reports (such as a household-only country), " "for an economy report or budget window, or no household " "calculations, for a household report; a household " - "report's household has no default year; a policy the report names " - "sets parameters the running catalog version does not have " - "(code missing_parameters, which names them; the baseline's policy is " - "checked first); or the simulation service cannot run the report: its " + "report's household has no default year; a policy or dynamic the " + "report names sets parameters the running catalog version does not " + "have (code missing_parameters, which names them; the policies are " + "checked before the dynamics, the baseline's first); or the " + "simulation service cannot run the report: its " "422, or its 400 for a region, dataset or dynamic it does not have. The " "message says which; for the simulation service's refusal, it gives " "the service's code, when it gave one, and its reason, but not its " diff --git a/policyengine_api/fastapi_routes/v2/reports/routes.py b/policyengine_api/fastapi_routes/v2/reports/routes.py index 2628f3053..b78448dd4 100644 --- a/policyengine_api/fastapi_routes/v2/reports/routes.py +++ b/policyengine_api/fastapi_routes/v2/reports/routes.py @@ -465,8 +465,8 @@ def build_v2_report_router(dependencies: NativeRouteDependencies) -> APIRouter: summary="Create an economy report, a household report or a budget window", description=( "Checks that the country has economy reports and stores each named " - "policy, and that the running catalog version has every parameter " - "those policies set, then has the simulation service create the " + "policy and dynamic, and that the running catalog version has every " + "parameter they set, then has the simulation service create the " "report, its two " "simulations and their runs on its default worker's PolicyEngine.py " "version, and start the run. With data_source local_area_dataset, " diff --git a/policyengine_api/fastapi_routes/v2/simulations/response_models.py b/policyengine_api/fastapi_routes/v2/simulations/response_models.py index 7b95a579e..56b5fd39b 100644 --- a/policyengine_api/fastapi_routes/v2/simulations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/simulations/response_models.py @@ -126,15 +126,16 @@ class UnavailableAfterSimulationCreatedResponse(V2ErrorResponse): CREATE_RESPONSES: dict[int | str, dict[str, Any]] = { 400: error_response("The body and query name different countries."), 404: error_response( - "The policy the simulation names is not stored in the country." + "The policy or dynamic the simulation names is not stored in the country." ), 413: error_response("The request body exceeds 16 KiB."), 422: { "model": V2MissingParametersErrorResponse | V2ErrorResponse, "description": ( "The request does not match the schema; the country has no economy " - "calculations; the policy sets parameters the running catalog version " - "does not have (code missing_parameters, which names them); or the " + "calculations; the policy or dynamic sets parameters the running " + "catalog version does not have (code missing_parameters, which names " + "them; the policy is checked first); or the " "simulation service cannot run the simulation: its 422, or its 400 " "for a region, dataset or dynamic it does not have. The message says " "which; for the simulation service's refusal, it gives the service's " diff --git a/policyengine_api/fastapi_routes/v2/simulations/routes.py b/policyengine_api/fastapi_routes/v2/simulations/routes.py index 043f61402..0486307cf 100644 --- a/policyengine_api/fastapi_routes/v2/simulations/routes.py +++ b/policyengine_api/fastapi_routes/v2/simulations/routes.py @@ -170,8 +170,8 @@ def build_v2_simulation_router(dependencies: NativeRouteDependencies) -> APIRout summary="Create an economy simulation", description=( "Checks that the country has economy calculations and stores the " - "named policy, and that the running catalog version has every " - "parameter it sets, then has the simulation service create the " + "named policy and dynamic, and that the running catalog version has " + "every parameter they set, then has the simulation service create the " "simulation and its first run on its default worker's " "PolicyEngine.py version, and start the run. Answers the simulation " "as stored, with its pending run. A created simulation that cannot " From 7f44328ec526d4452a9bb4acb02b8d597f9ce895 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 17 Nov 2026 17:47:42 +0400 Subject: [PATCH 6/6] Fix review round 1's code-quality findings Dynamic gains Policy's relationships to its tax-benefit model and catalog version, so the ORM stores a new catalog version before a dynamic in one flush, and the unused relationships between dynamics and legacy policy mappings are gone, so mapper configuration emits no SAWarning again. Tests cover both, the copy losing the mapping race after creating a dynamic, and a dynamic that cannot be written. The mapping input's country is a CountryId. Docstrings and descriptions name the dynamic errors, the re-run description says that the simulation service refuses a dynamic's missing parameters, and the migration skill says that building the new index reads all of parameter_values and holds back inserts until it finishes. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/skills/alembic-migrations.md | 7 ++- policyengine_api/data/v2/models/metadata.py | 2 + policyengine_api/data/v2/models/policies.py | 7 +-- .../data/v2/models/policy_mappings.py | 5 +- policyengine_api/data/v2/privileges.py | 5 +- .../resource_service_protocols.py | 10 ++-- .../v2/reports/response_models.py | 2 +- .../fastapi_routes/v2/reports/run_models.py | 3 +- .../v2/simulation_resources_support.py | 5 +- .../v2/simulations/response_models.py | 2 +- .../services/v2/policies/types.py | 2 +- tests/unit/v2/test_dynamic_catalog.py | 24 +++++++++- tests/unit/v2/test_dynamic_table_models.py | 47 +++++++++++++++++++ tests/unit/v2/test_policy_legacy_copy.py | 30 ++++++++++++ 14 files changed, 127 insertions(+), 24 deletions(-) diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 9646144c0..487a17850 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -1037,8 +1037,11 @@ changes, so that dynamics are stored as policies are: with a dynamic: a dynamic sets one value per parameter and start date, as a policy does. Rows without a dynamic never conflict in a unique key, so the partial index refuses exactly what a full unique constraint would, but it - holds only dynamics' rows instead of every current-law value. It is also - declared for SQLite, so the unit tests' databases keep the same rule. + holds only dynamics' rows instead of every current-law value. Building it + still reads every row of `parameter_values` (millions of current-law + values), and, as Alembic creates it without `CONCURRENTLY`, inserts into + that table wait until the build finishes. It is also declared for SQLite, + so the unit tests' databases keep the same rule. - `legacy_policy_mappings` gains the nullable `dynamic_id`, the dynamic the v1-to-v2 policy copy split from a v1 policy: a key with the mapping's country to `dynamics` (`fk_legacy_policy_mappings_dynamic_country`, diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index 8777d00d8..018bb6edb 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -80,6 +80,7 @@ class TaxBenefitModel(TimestampedModel, table=True): back_populates="tax_benefit_model" ) policies: list["Policy"] = Relationship(back_populates="tax_benefit_model") + dynamics: list["Dynamic"] = Relationship(back_populates="tax_benefit_model") reports: list["Report"] = Relationship(back_populates="tax_benefit_model") @@ -156,6 +157,7 @@ class TaxBenefitModelVersion(IdentifiedModel, table=True): datasets: list["Dataset"] = Relationship(back_populates="tax_benefit_model_version") regions: list["Region"] = Relationship(back_populates="tax_benefit_model_version") policies: list["Policy"] = Relationship(back_populates="tax_benefit_model_version") + dynamics: list["Dynamic"] = Relationship(back_populates="tax_benefit_model_version") class Region(TimestampedModel, table=True): diff --git a/policyengine_api/data/v2/models/policies.py b/policyengine_api/data/v2/models/policies.py index e7f4e5b86..ba134ed0f 100644 --- a/policyengine_api/data/v2/models/policies.py +++ b/policyengine_api/data/v2/models/policies.py @@ -146,12 +146,13 @@ class Dynamic(TimestampedModel, table=True): canonicalization_version: int content_hash: str = Field(max_length=64) + tax_benefit_model: TaxBenefitModel = Relationship(back_populates="dynamics") + tax_benefit_model_version: "TaxBenefitModelVersion" = Relationship( + back_populates="dynamics" + ) parameter_values: list["ParameterValue"] = Relationship( back_populates="dynamic", cascade_delete=True, ) simulations: list["Simulation"] = Relationship(back_populates="dynamic") household_jobs: list["HouseholdJob"] = Relationship(back_populates="dynamic") - legacy_mappings: list["LegacyPolicyMapping"] = Relationship( - back_populates="dynamic" - ) diff --git a/policyengine_api/data/v2/models/policy_mappings.py b/policyengine_api/data/v2/models/policy_mappings.py index 8e5475561..64dd79d93 100644 --- a/policyengine_api/data/v2/models/policy_mappings.py +++ b/policyengine_api/data/v2/models/policy_mappings.py @@ -1,7 +1,7 @@ """Durable source-identity mappings for immediate v1 policy mirroring.""" from datetime import datetime -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from uuid import UUID import sqlalchemy as sa @@ -11,7 +11,7 @@ if TYPE_CHECKING: from policyengine_api.data.v2.models.associations import UserPolicy - from policyengine_api.data.v2.models.policies import Dynamic, Policy + from policyengine_api.data.v2.models.policies import Policy class LegacyUserMapping(SQLModel, table=True): @@ -87,7 +87,6 @@ class LegacyPolicyMapping(TimestampedModel, table=True): source_policy_hash: str = Field(max_length=255) policy: "Policy" = Relationship(back_populates="legacy_mappings") - dynamic: Optional["Dynamic"] = Relationship(back_populates="legacy_mappings") class LegacyUserPolicyMapping(TimestampedModel, table=True): diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 5bc260267..45fef0ec1 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -302,9 +302,8 @@ def _run_table( runtime=READ, catalog_writer=READ_INSERT_DELETE | _ANALYZE ), # The Public API also inserts a v2 policy's or dynamic's own parameter - # values. Catalog - # pruning deletes an older version's current-law values (those with no - # policy or dynamic), never a policy's or a dynamic's. + # values. Catalog pruning deletes an older version's current-law values + # (those with no policy or dynamic), never a policy's or a dynamic's. "parameter_values": _access( runtime=READ_INSERT, catalog_writer=READ_WRITE | _ANALYZE, **_RUN_INPUT_READS ), diff --git a/policyengine_api/fastapi_routes/resource_service_protocols.py b/policyengine_api/fastapi_routes/resource_service_protocols.py index 325250fb2..cb710ff6b 100644 --- a/policyengine_api/fastapi_routes/resource_service_protocols.py +++ b/policyengine_api/fastapi_routes/resource_service_protocols.py @@ -287,10 +287,10 @@ def create_report( country without household calculations, ``HouseholdNotFoundError`` for a household the country does not store, ``HouseholdCalculationYearError`` for one with no default year, - ``PolicyNotFoundError`` - for a policy the country does not store, - ``PolicyParametersMissingError`` for a policy that sets parameters the - running catalog version lacks, ``SubmissionStorageError`` when the + ``PolicyNotFoundError`` or ``DynamicNotFoundError`` for a policy or + dynamic the country does not store, ``PolicyParametersMissingError`` + (``DynamicParametersMissingError`` for a dynamic) for one setting + parameters the running catalog lacks, ``SubmissionStorageError`` when the database or its catalog cannot be read for those checks, the simulation service's errors, ``ReportRunNotRecordedError`` when the request could not be recorded, and ``CreatedReportUnreadableError`` @@ -357,7 +357,7 @@ def create_simulation( Raise the checks' and the simulation service's errors as ``V2StoredReportResourceService.create_report`` does, for the - simulation's one policy, and ``CreatedSimulationUnreadableError`` when + simulation's policy and dynamic, and ``CreatedSimulationUnreadableError`` when the simulation it created cannot be read back. """ ... diff --git a/policyengine_api/fastapi_routes/v2/reports/response_models.py b/policyengine_api/fastapi_routes/v2/reports/response_models.py index 468b30d51..c21f0188c 100644 --- a/policyengine_api/fastapi_routes/v2/reports/response_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/response_models.py @@ -502,7 +502,7 @@ class ReportRunNotRecordedResponse(V2ErrorResponse): "have (code missing_parameters, which names them; the policies are " "checked before the dynamics, the baseline's first); or the " "simulation service cannot run the report: its " - "422, or its 400 for a region, dataset or dynamic it does not have. The " + "422, or its 400 for a region or dataset it does not have. The " "message says which; for the simulation service's refusal, it gives " "the service's code, when it gave one, and its reason, but not its " "status." diff --git a/policyengine_api/fastapi_routes/v2/reports/run_models.py b/policyengine_api/fastapi_routes/v2/reports/run_models.py index 4680fa289..8b00b970c 100644 --- a/policyengine_api/fastapi_routes/v2/reports/run_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/run_models.py @@ -182,7 +182,8 @@ class RerunReportUnreadableResponse(V2ErrorResponse): "reports, or no household calculations for a household report; a " "policy of the report's simulations sets " "parameters the running catalog version does not have (code " - "missing_parameters, which names them); or the simulation service " + "missing_parameters, which names them), where a dynamic that does is " + "the simulation service's 422 instead; or the simulation service " "cannot run the report as stored: its 422, or its 400 for a region, " "dataset or dynamic the report names that the current catalog version " "does not have, with its code and reason." diff --git a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py index 611cc8594..36e517939 100644 --- a/policyengine_api/fastapi_routes/v2/simulation_resources_support.py +++ b/policyengine_api/fastapi_routes/v2/simulation_resources_support.py @@ -160,9 +160,8 @@ def simulation_resource_operation( A policy or dynamic that sets parameters the running catalog version lacks answers 422 with code ``missing_parameters``, naming them, whatever - the route. A - failure of a type in ``refusals`` is the caller's to act on and is not - logged, nor is that 422; any other is logged under ``event``. + the route. A failure of a type in ``refusals`` is the caller's to act on + and is not logged, nor is that 422; any other is logged under ``event``. """ def logged() -> OperationT | JSONResponse: diff --git a/policyengine_api/fastapi_routes/v2/simulations/response_models.py b/policyengine_api/fastapi_routes/v2/simulations/response_models.py index 56b5fd39b..f3279ab03 100644 --- a/policyengine_api/fastapi_routes/v2/simulations/response_models.py +++ b/policyengine_api/fastapi_routes/v2/simulations/response_models.py @@ -137,7 +137,7 @@ class UnavailableAfterSimulationCreatedResponse(V2ErrorResponse): "catalog version does not have (code missing_parameters, which names " "them; the policy is checked first); or the " "simulation service cannot run the simulation: its 422, or its 400 " - "for a region, dataset or dynamic it does not have. The message says " + "for a region or dataset it does not have. The message says " "which; for the simulation service's refusal, it gives the service's " "code, when it gave one, and its reason, but not its status." ), diff --git a/policyengine_api/services/v2/policies/types.py b/policyengine_api/services/v2/policies/types.py index ba007a913..80d9aa8c6 100644 --- a/policyengine_api/services/v2/policies/types.py +++ b/policyengine_api/services/v2/policies/types.py @@ -167,7 +167,7 @@ class LegacyPolicyTranslation: class LegacyPolicyMappingInput: """The mapping a copy records from a v1 policy to its v2 policy and dynamic.""" - country_id: str + country_id: CountryId legacy_policy_id: int source_policy_hash: str policy_id: UUID diff --git a/tests/unit/v2/test_dynamic_catalog.py b/tests/unit/v2/test_dynamic_catalog.py index 518079cd9..b53267f43 100644 --- a/tests/unit/v2/test_dynamic_catalog.py +++ b/tests/unit/v2/test_dynamic_catalog.py @@ -11,8 +11,9 @@ from uuid import UUID import pytest +import sqlalchemy as sa from pydantic import ValidationError -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import OperationalError, SQLAlchemyError from sqlalchemy.orm import sessionmaker from sqlmodel import Session, create_engine @@ -258,3 +259,24 @@ def test_a_dynamic_storage_that_cannot_be_read_is_a_storage_error() -> None: engine.dispose() assert isinstance(failure.value.__cause__, OperationalError) + + +def test_a_dynamic_that_cannot_be_written_is_a_storage_error( + session: Session, catalog: Catalog +) -> None: + engine = session.get_bind() + with engine.begin() as connection: + connection.execute(sa.text("DROP TABLE dynamics")) + service = V2DynamicService( + DynamicDatabaseSession(sessionmaker(engine, class_=Session)), + running_policyengine_version=RUNNING_VERSION, + ) + + with pytest.raises(DynamicStorageError) as failure: + service.create_dynamic( + NativeDynamicCreationInput( + **_dynamic_input(catalog, INCOME_ELASTICITY).model_dump() + ) + ) + + assert isinstance(failure.value.__cause__, SQLAlchemyError) diff --git a/tests/unit/v2/test_dynamic_table_models.py b/tests/unit/v2/test_dynamic_table_models.py index 1253aa9f4..ac6fd180c 100644 --- a/tests/unit/v2/test_dynamic_table_models.py +++ b/tests/unit/v2/test_dynamic_table_models.py @@ -1,5 +1,7 @@ """Dynamics are stored as policies are: once per canonical content hash.""" +import subprocess +import sys from datetime import UTC, datetime import pytest @@ -233,3 +235,48 @@ def test_a_mapping_cannot_name_another_country_s_dynamic() -> None: session.commit() engine.dispose() + + +def test_the_v2_mappers_configure_without_relationship_warnings() -> None: + # A fresh interpreter, since a process configures its mappers only once + # and an error there would break every later mapper in this one. + configuration = subprocess.run( + [ + sys.executable, + "-c", + "import warnings\n" + "from sqlalchemy.exc import SAWarning\n" + "warnings.simplefilter('error', SAWarning)\n" + "import policyengine_api.data.v2.models\n" + "from sqlalchemy.orm import configure_mappers\n" + "configure_mappers()", + ], + capture_output=True, + text=True, + check=False, + ) + + assert configuration.returncode == 0, configuration.stderr + + +def test_a_dynamic_is_stored_in_one_flush_with_its_new_catalog_version() -> None: + engine = _sqlite_engine_with_foreign_keys() + model, version, parameter = _catalog() + + with Session(engine) as session: + dynamic = Dynamic( + country_id="us", + tax_benefit_model=model, + tax_benefit_model_version=version, + canonicalization_version=1, + content_hash="d" * 64, + ) + session.add_all([parameter, dynamic]) + session.commit() + stored = session.get(Dynamic, dynamic.id) + + assert stored is not None + assert stored.tax_benefit_model_version_id == version.id + assert stored in version.dynamics + + engine.dispose() diff --git a/tests/unit/v2/test_policy_legacy_copy.py b/tests/unit/v2/test_policy_legacy_copy.py index 6198f4f65..7bf5b903a 100644 --- a/tests/unit/v2/test_policy_legacy_copy.py +++ b/tests/unit/v2/test_policy_legacy_copy.py @@ -493,3 +493,33 @@ def test_an_untranslatable_assumption_fails_the_copy_before_anything_is_written( assert failure.value.reason == reason assert recorded_writes == RecordedCopyWrites() + + +def test_a_copy_that_loses_the_mapping_race_reports_the_mapped_dynamic( + catalog_session: Session, + recorded_writes: RecordedCopyWrites, + monkeypatch, +) -> None: + _add_running_catalog( + catalog_session, country_id="us", parameter_names=[US_INCOME_ELASTICITY] + ) + snapshot = _snapshot( + country_id="us", + legacy_policy_id=504, + api_version="1.600.0", + policy_json={US_INCOME_ELASTICITY: {"2026": 0.25}}, + ) + _lose_mapping_race(monkeypatch, winning_source_hash=snapshot.source_policy_hash) + + result = _copy(catalog_session, snapshot) + + # This copy created a dynamic, but the concurrent copy's mapping, made + # before the split existed, names none; the result reports the mapping. + assert len(recorded_writes.created_dynamics) == 1 + assert result == LegacyPolicyPersistenceResult( + policy_id=EARLIER_COPY_POLICY_ID, + policy_created=False, + mapping_created=False, + dynamic_id=None, + dynamic_created=False, + )