diff --git a/changelog.d/c1-30-data-source-and-keys.added.md b/changelog.d/c1-30-data-source-and-keys.added.md new file mode 100644 index 000000000..f2991edbb --- /dev/null +++ b/changelog.d/c1-30-data-source-and-keys.added.md @@ -0,0 +1 @@ +`POST /v2/reports` takes `data_source: "local_area_dataset"` on an economy report and passes it to the simulation service, which calculates a UK-wide or nation report on the dataset its catalog gives local areas. Every report read answers its `dataset_independent_key`, `GET /v2/reports?dataset_independent_key=` finds the country's reports that share one, and a local-area report's read says `data_source` and lists the constituency and local-authority codes disclosure control withheld. diff --git a/policyengine_api/fastapi_routes/resource_service_protocols.py b/policyengine_api/fastapi_routes/resource_service_protocols.py index 5f3e25951..aa2818879 100644 --- a/policyengine_api/fastapi_routes/resource_service_protocols.py +++ b/policyengine_api/fastapi_routes/resource_service_protocols.py @@ -235,6 +235,20 @@ def list_reports( """ ... + def search_reports( + self, + *, + country_id: str, + dataset_independent_key: str, + requester: AccountPrincipal | None = None, + ) -> tuple[StoredReportSummaryRead, ...]: + """Return the country's reports with the dataset-independent key, newest first. + + At most ``MAXIMUM_LISTED_REPORTS``, the ones created last, each read + as ``list_reports`` reads it. + """ + ... + def create_report( self, report_input: ReportCreationInput, diff --git a/policyengine_api/fastapi_routes/v2/reports/response_models.py b/policyengine_api/fastapi_routes/v2/reports/response_models.py index 3bdb6d4d2..3b9e3c760 100644 --- a/policyengine_api/fastapi_routes/v2/reports/response_models.py +++ b/policyengine_api/fastapi_routes/v2/reports/response_models.py @@ -18,6 +18,7 @@ from policyengine_api.fastapi_routes.v2.simulation_resources_support import ( error_response, ) +from policyengine_api.services.v2.simulation_service.types import ReportDataSource from policyengine_api.services.v2.stored_reports.budget_window_results.types import ( BudgetWindowResult, ) @@ -90,13 +91,18 @@ class ReportDefinitionItem(StrictReportAPIModel): a report the simulation service shares between requests records no one caller's. ``current_versions`` are the versions the Public API runs now, so a run on other versions is out of date. ``options`` are the options - the report was created with. + the report was created with. ``dataset_independent_key`` is the + simulation service's key for the report's definition without its + dataset, which a national report shares with its local-area report: find + reports by it with ``GET /v2/reports?dataset_independent_key=``. It is + null for a report stored before the keys. """ id: UUID country_id: str label: str | None description: str | None + dataset_independent_key: str | None year: int | None baseline_simulation: ReportSimulationItem | None reform_simulation: ReportSimulationItem | None @@ -108,16 +114,23 @@ class ReportDefinitionItem(StrictReportAPIModel): class ReportSummaryItem(ReportDefinitionItem): """An economy report comparing a baseline and a reform simulation. - Both simulations use the region and dataset named. ``run`` is the run - the read answers: the one it names, else the requesting user's newest - recorded run, else the newest succeeded run that wasn't forced, else the - newest run that wasn't forced, whatever its versions. ``status`` is that - run's, or ``not_run`` without one. + Both simulations use the region and dataset named. ``data_source`` is + ``local_area_dataset`` for a local-area report: a UK-wide or nation + report on the dataset the catalog version gives local areas, whose + result also has each constituency's and local authority's impact and the + codes of those withheld. A request with that data_source creates one, + but the request's field isn't stored: the read derives it from the + report's region and dataset. It is null for every other report. + ``run`` is the run the read answers: the one it names, else the + requesting user's newest recorded run, else the newest succeeded run + that wasn't forced, else the newest run that wasn't forced, whatever its + versions. ``status`` is that run's, or ``not_run`` without one. """ kind: Literal["economy"] region_code: str | None dataset_name: str | None + data_source: ReportDataSource | None status: Literal["not_run", "pending", "running", "succeeded", "failed"] run: ReportRunItem | None @@ -272,7 +285,8 @@ class BudgetWindowReportSummaryItem(StrictReportAPIModel): ``current_versions`` are the versions the Public API runs now, so a run on other versions is out of date. ``options`` are the options the report was created with. ``label`` and ``description`` are the requesting user's - saved report's, null otherwise. + saved report's, null otherwise. ``dataset_independent_key`` is as + ``ReportDefinitionItem`` describes it. """ id: UUID @@ -280,6 +294,7 @@ class BudgetWindowReportSummaryItem(StrictReportAPIModel): kind: Literal["budget_window"] label: str | None description: str | None + dataset_independent_key: str | None year: int | None budget_window_size: int region_code: str | None @@ -348,7 +363,11 @@ class ReportDetailResult(StrictReportAPIModel): class ReportListResult(StrictReportAPIModel): - """The listed reports in ``items``, in the order named, without results.""" + """The listed reports in ``items``, without results. + + Reports chosen by ``report_id`` are in the order named; reports found by + ``dataset_independent_key`` are newest created first. + """ items: list[ Annotated[ diff --git a/policyengine_api/fastapi_routes/v2/reports/routes.py b/policyengine_api/fastapi_routes/v2/reports/routes.py index c4b888fe9..2628f3053 100644 --- a/policyengine_api/fastapi_routes/v2/reports/routes.py +++ b/policyengine_api/fastapi_routes/v2/reports/routes.py @@ -212,6 +212,38 @@ def _requester_refused( ) +def _list_or_search( + service: V2StoredReportResourceService, + query: ReportCollectionQuery, + requester: AccountPrincipal | None, +) -> ReportListResponse | JSONResponse: + """List the reports the query names, or find those with its key. + + Each is read as the requester sees it; a browser credential naming a + signed-in user is refused with a 401. + """ + try: + if query.dataset_independent_key is None: + reports = service.list_reports( + country_id=query.country_id, + report_ids=query.report_id, + requester=requester, + ) + else: + reports = service.search_reports( + country_id=query.country_id, + dataset_independent_key=query.dataset_independent_key, + requester=requester, + ) + except StandardUserRequiresSignInError as error: + return _requester_refused(error) + return ReportListResponse( + result=ReportListResult( + items=[report_summary_item(report) for report in reports] + ) + ) + + # The two routes that record a request take its user's credential, but # optionally: FastAPI lists the schemes a credential may use, and an empty # requirement says a request may send none. @@ -437,7 +469,14 @@ def build_v2_report_router(dependencies: NativeRouteDependencies) -> APIRouter: "those policies set, then has the simulation service create the " "report, its two " "simulations and their runs on its default worker's PolicyEngine.py " - "version, and start the run. A household report (kind household) " + "version, and start the run. With data_source local_area_dataset, " + "an economy report calculates on the dataset the catalog version " + "gives local areas, and its result also has each constituency's and " + "local authority's impact and the codes of those withheld; the " + "simulation service takes it for uk and its four nations only, on a " + "release that gives local areas a dataset of their own, and the " + "Public API answers its refusal with 422. A household report (kind " + "household) " "is checked the same way, except that the country must have " "household calculations instead (a household-only country does), " "and the household it names must be stored in the country, with a " @@ -479,7 +518,7 @@ def create() -> ReportDetailResponse | JSONResponse: "/reports", response_model=ReportListResponse, responses={401: READ_RESPONSES[401], 403: READ_RESPONSES[403]}, - summary="List chosen reports", + summary="List chosen reports, or find reports by their dataset-independent key", description=( "Lists the country's reports among the report_id values (at most " "25), in the order named and once each, each with the run a read of " @@ -488,7 +527,13 @@ def create() -> ReportDetailResponse | JSONResponse: "description are that user's saved report's, and a report the user " "asked to run shows the run of their newest request; without one, " "the label is null. Unknown reports and other countries' are left " - "out." + "out. With dataset_independent_key in place of report_id, lists the " + "same way the country's reports whose dataset_independent_key it " + "is, newest created first, at most the 25 created last: reports " + "that differ only in their dataset, such as a national report and " + "its local-area report (data_source local_area_dataset), and the " + "same reports on another data release. Give one of the two, never " + "both." ), openapi_extra=OPTIONAL_USER_CREDENTIAL, ) @@ -497,19 +542,7 @@ def list_reports( requester: Annotated[AccountPrincipal | None, Depends(optional_user)], ) -> ReportListResponse | JSONResponse: def read() -> ReportListResponse | JSONResponse: - try: - reports = service_factory().list_reports( - country_id=query.country_id, - report_ids=query.report_id, - requester=requester, - ) - except StandardUserRequiresSignInError as error: - return _requester_refused(error) - return ReportListResponse( - result=ReportListResult( - items=[report_summary_item(report) for report in reports] - ) - ) + return _list_or_search(service_factory(), query, requester) return _report_operation(read) diff --git a/policyengine_api/libs/simulation_reports.py b/policyengine_api/libs/simulation_reports.py index fb5b4f1a8..5549c4b8b 100644 --- a/policyengine_api/libs/simulation_reports.py +++ b/policyengine_api/libs/simulation_reports.py @@ -14,18 +14,20 @@ current law). A report request names the country, ``report_type`` (``economy``), the year and region its two simulations share, each simulation's policy and dynamic, ``include_cliffs``, and the SPM settings -only when the caller gave them. No request names a dataset: the simulation -service calculates on the dataset its catalog version gives the region. A -budget window goes to the same ``POST /reports`` as ``report_type`` -``budget_window``, naming its first year and ``budget_window_size`` in place -of the year, and no ``include_cliffs``. A household report goes there too, -as ``report_type`` ``household``, naming the stored household, each -simulation's policy (the reform null for a report of the baseline alone), -but no year: the simulation service calculates the household's default -year, adding every variable the household lacks. None names a -PolicyEngine.py version: the simulation service runs its default worker's. -None names a label or a description, which belong to users' saved reports. -The simulation service answers: +and ``data_source`` only when the caller gave them. No request names a +dataset: the simulation service calculates on the dataset its catalog +version gives the region, or, for ``data_source`` ``local_area_dataset``, +the one it gives local areas. A budget window goes to the same +``POST /reports`` as ``report_type`` ``budget_window``, naming its first +year and ``budget_window_size`` in place of the year, and no +``include_cliffs``. A household report goes there too, as ``report_type`` +``household``, naming the stored household, each simulation's policy (the +reform null for a report of the baseline alone), but no year: the +simulation service calculates the household's default year, adding every +variable the household lacks. None names a PolicyEngine.py version: the +simulation service runs its default worker's. None names a label or a +description, which belong to users' saved reports. The simulation service +answers: - 201 or 202 with the created record's ID and its first run's ID: for a budget window, its budget-window run's ID (``budget_window_run_id``). One @@ -253,9 +255,12 @@ def _report_definition( def report_request(report: EconomyReportCreationInput) -> dict[str, Any]: """Return the simulation service's request body for the report. - ``spm`` is sent only when the caller gave it. + ``spm`` and ``data_source`` are each sent only when the caller gave them. """ - return {**_report_definition(report), "include_cliffs": report.include_cliffs} + body = {**_report_definition(report), "include_cliffs": report.include_cliffs} + if report.data_source is not None: + body["data_source"] = report.data_source + return body def budget_window_request(report: BudgetWindowReportCreationInput) -> dict[str, Any]: diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index 2e3ee8d27..c39281438 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -86,6 +86,17 @@ def validate_legacy_user_id(value: str) -> str: ), ] ResourceId = Annotated[UUID, Field(description="Exact resource UUID")] +# A report's dataset-independent key, as the simulation service stores it: the +# SHA-256 of its definition without the dataset, in lowercase hexadecimal. +DatasetIndependentKey = Annotated[ + str, + Field( + pattern=r"^[0-9a-f]{64}$", + description=( + "A report's dataset_independent_key: 64 lowercase hexadecimal digits" + ), + ), +] DefaultYear = Annotated[ int, Field( @@ -459,16 +470,37 @@ class ReportRunsQuery(CountryQuery): class ReportCollectionQuery(CountryQuery): - """Query contract for listing chosen reports of a country. + """Query contract for listing chosen reports of a country, or finding them. - ``report_id`` is repeated once per report, at most 25 times. + Give either ``report_id``, repeated once per report and at most 25 + times, or one ``dataset_independent_key``, which finds the country's + reports that share it; never both. """ report_id: list[ResourceId] = Field( - min_length=1, + default_factory=list, max_length=MAXIMUM_LISTED_REPORTS, - description="A report to list; repeat the key for each report", + description=( + "A report to list; repeat the key for each report. Not with " + "dataset_independent_key" + ), ) + dataset_independent_key: DatasetIndependentKey | None = Field( + default=None, + description=( + "Find the country's reports with this dataset_independent_key, which " + "a national report shares with its local-area report. Not with " + "report_id" + ), + ) + + @model_validator(mode="after") + def require_one_filter(self) -> ReportCollectionQuery: + """Refuse a query naming both report_id and a key, or neither.""" + if bool(self.report_id) == (self.dataset_independent_key is not None): + msg = "Give either report_id or dataset_independent_key" + raise ValueError(msg) + return self class SimulationCreateQuery(CountryQuery): diff --git a/policyengine_api/services/v2/simulation_service/types.py b/policyengine_api/services/v2/simulation_service/types.py index 016d97525..4691ba88e 100644 --- a/policyengine_api/services/v2/simulation_service/types.py +++ b/policyengine_api/services/v2/simulation_service/types.py @@ -202,8 +202,9 @@ class EconomySimulationCreationInput(EconomySimulationFields): country_id: CountryId -def _omit_spm_default(schema: dict[str, Any]) -> None: - # Omitting spm sends no SPM settings; an explicit null is not a selection. +def _omit_null_default(schema: dict[str, Any]) -> None: + # Omitting the field (spm, data_source) sends nothing; an explicit null is + # refused, so the schema documents no null default. schema.pop("default", None) @@ -233,7 +234,7 @@ class ReportDefinitionFields(StrictSubmissionModel): baseline: PolicyChoice reform: PolicyChoice spm: SPMSelection | SkipJsonSchema[None] = Field( - default=None, json_schema_extra=_omit_spm_default + default=None, json_schema_extra=_omit_null_default ) @field_validator("spm", mode="before") @@ -246,16 +247,44 @@ def require_spm_object(cls, value: object) -> object: return value +# The dataset an economy report may ask for in place of its region's: the one +# the catalog version gives local areas. A report read names it for a UK-wide +# or nation report on that dataset (``stored_reports.types.ReportSummaryRead``), +# never for a local area's own report, which is on it too. +ReportDataSource = Literal["local_area_dataset"] +LOCAL_AREA_DATASET: ReportDataSource = "local_area_dataset" + + class EconomyReportFields(ReportDefinitionFields): """An economy report: a baseline and a reform simulation of one region's data. ``include_cliffs`` also calculates the baseline's and reform's cliff - impact; the other fields are as ``ReportDefinitionFields`` describes + impact. ``data_source`` ``local_area_dataset`` calculates both + simulations on the dataset the catalog version gives local areas + instead of the region's, so the result also has each constituency's and + local authority's impact and the codes of those disclosure control + withheld; omit it, rather than sending null, for the region's dataset. + The simulation service takes it only for ``uk`` and the four + ``country/*`` nations, on a release that gives local areas a dataset of + their own, and refuses it with 422 otherwise: the Public API checks only + its value. The other fields are as ``ReportDefinitionFields`` describes them. """ kind: Literal["economy"] include_cliffs: StrictBool = False + data_source: ReportDataSource | SkipJsonSchema[None] = Field( + default=None, json_schema_extra=_omit_null_default + ) + + @field_validator("data_source", mode="before") + @classmethod + def require_data_source_value(cls, value: object) -> object: + """Refuse an explicit null ``data_source``; omit the field instead.""" + if value is None: + msg = "data_source must be local_area_dataset when supplied" + raise ValueError(msg) + return value class EconomyReportCreationInput(EconomyReportFields): diff --git a/policyengine_api/services/v2/stored_reports/database_connectors/reads.py b/policyengine_api/services/v2/stored_reports/database_connectors/reads.py index 51929f32c..73291b49e 100644 --- a/policyengine_api/services/v2/stored_reports/database_connectors/reads.py +++ b/policyengine_api/services/v2/stored_reports/database_connectors/reads.py @@ -5,6 +5,8 @@ year. A report's default run, the one a read answers without naming one, is its newest succeeded run that wasn't forced (a window's: not recalculated), else its newest such run of any status; a forced run is never the default. +A report read also says whether it is a local-area report: a UK-wide or +nation report on the dataset its catalog version gives local areas. """ from __future__ import annotations @@ -19,6 +21,7 @@ BudgetWindowRun, Dataset, Region, + RegionType, Report, ReportRun, ReportRunStatus, @@ -30,9 +33,14 @@ from collections.abc import Collection, Sequence from uuid import UUID - from sqlalchemy import Label + from sqlalchemy import ColumnElement, Label from sqlmodel import Session +# The regions a report on the local-area dataset may cover: UK-wide and the +# nations. A constituency's or local authority's own report is on that +# dataset too, but calculates only its one area. +LOCAL_AREA_REPORT_REGION_TYPES = (RegionType.NATIONAL, RegionType.COUNTRY) + def _default_first(run_model: type[ReportRun | BudgetWindowRun]) -> Label[int]: """Return a label numbering each report's runs from 1, its default run first. @@ -57,19 +65,50 @@ def _default_first(run_model: type[ReportRun | BudgetWindowRun]) -> Label[int]: ) +def _is_default_dataset_of_a(region_type: RegionType) -> ColumnElement[bool]: + """Return whether some region of the type has the report's dataset as default.""" + area_region = aliased(Region) + return ( + select(col(area_region.id)) + .where( + col(area_region.region_type) == region_type, + col(area_region.default_dataset_id) == Report.dataset_id, + ) + .exists() + ) + + +def _on_local_area_dataset() -> Label[bool]: + """Label whether the report is a local-area report. + + It is when its region is UK-wide or a nation and its dataset is the one + its catalog version gives both the constituency and the local-authority + regions, as the simulation service decides which dataset local areas use. + A release whose two levels name different datasets has no local-area + dataset. Datasets belong to one catalog version, as the regions naming + them do. A report whose region is gone reads NULL, which is not one. + """ + return and_( + col(Region.region_type).in_(LOCAL_AREA_REPORT_REGION_TYPES), + _is_default_dataset_of_a(RegionType.CONSTITUENCY), + _is_default_dataset_of_a(RegionType.LOCAL_AUTHORITY), + ).label("on_local_area_dataset") + + def read_report_rows( session: Session, *, country_id: str, report_ids: Collection[UUID] ) -> list[StoredReportRows]: """Return the country's reports among ``report_ids``, in two queries. - Each comes with its region's code, its dataset's name, its two simulations - and its default run: the newest succeeded run that wasn't forced, else - the newest run that wasn't forced, whatever its status and versions - (newest is created last, ties broken by the larger ID). Runs of a budget - window's years are not the report's own runs, so they are left out. Each - also comes with its default budget-window run, chosen the same way among - runs that weren't recalculated, and that run's years, which a second - query reads for every report at once. Reports of other + Each comes with its region's code, its dataset's name, whether it is a + local-area report, its two simulations and its default run: the newest + succeeded run that wasn't forced, else the newest run that wasn't + forced, whatever its status and versions (newest is created last, ties + broken by the larger ID). Runs of a budget window's years are not the + report's own runs, so they are left out. Each also comes with its + default budget-window run, chosen the same way among runs that weren't + recalculated, and that run's years, which a second query reads for every + report at once. Reports of other countries, and IDs no report has, are absent from the list, which has no set order. """ @@ -99,6 +138,7 @@ def read_report_rows( Report, col(Region.code), col(Dataset.name), + _on_local_area_dataset(), baseline_simulation, reform_simulation, default_run, @@ -139,6 +179,7 @@ def read_report_rows( report=report, region_code=region_code, dataset_name=dataset_name, + on_local_area_dataset=bool(on_local_area_dataset), baseline_simulation=baseline, reform_simulation=reform, run=run, @@ -151,6 +192,7 @@ def read_report_rows( report, region_code, dataset_name, + on_local_area_dataset, baseline, reform, run, @@ -159,6 +201,26 @@ def read_report_rows( ] +def search_report_ids( + session: Session, *, country_id: str, dataset_independent_key: str, limit: int +) -> list[UUID]: + """Return the IDs of the country's reports with the dataset-independent key. + + They are the ``limit`` reports created last, newest first, ties going to + the larger ID, found through ``ix_reports_dataset_independent_key``. + """ + statement = ( + select(col(Report.id)) + .where( + col(Report.country) == country_id, + col(Report.dataset_independent_key) == dataset_independent_key, + ) + .order_by(col(Report.created_at).desc(), col(Report.id).desc()) + .limit(limit) + ) + return list(session.execute(statement).scalars()) + + def read_budget_window_years( session: Session, budget_window_run_ids: Sequence[UUID] ) -> dict[UUID, tuple[ReportRun, ...]]: diff --git a/policyengine_api/services/v2/stored_reports/database_connectors/reads_results.py b/policyengine_api/services/v2/stored_reports/database_connectors/reads_results.py index adae01e21..c5285fb41 100644 --- a/policyengine_api/services/v2/stored_reports/database_connectors/reads_results.py +++ b/policyengine_api/services/v2/stored_reports/database_connectors/reads_results.py @@ -18,6 +18,7 @@ LocalAuthorityImpact, Poverty, ProgramStatistics, + WithheldAreaCode, ) from policyengine_api.services.v2.stored_reports.economy_results.types import ( EconomyResultRows, @@ -38,11 +39,14 @@ def read_economy_result_rows( country_id: str, report_run: ReportRun, reform_simulation_run_id: UUID, + on_local_area_dataset: bool, ) -> EconomyResultRows: """Return every result row of ``report_run``, one query per result table. The comparison tables' rows are the report run's; the poverty and inequality rows are those of its baseline and reform simulation runs. + The withheld area codes are read only for a local-area report's run + (``on_local_area_dataset``), whose writer records them. """ report_run_id = report_run.id simulation_run_ids = ( @@ -137,4 +141,16 @@ def read_economy_result_rows( ) ) ), + on_local_area_dataset=on_local_area_dataset, + withheld_area_codes=( + tuple( + session.exec( + select(WithheldAreaCode).where( + WithheldAreaCode.report_run_id == report_run_id + ) + ) + ) + if on_local_area_dataset + else () + ), ) diff --git a/policyengine_api/services/v2/stored_reports/economy_results/transformations.py b/policyengine_api/services/v2/stored_reports/economy_results/transformations.py index 73c84588d..4909fad68 100644 --- a/policyengine_api/services/v2/stored_reports/economy_results/transformations.py +++ b/policyengine_api/services/v2/stored_reports/economy_results/transformations.py @@ -22,13 +22,17 @@ - ``congressional_district_impact``, ``constituency_impact`` and ``local_authority_impact`` from their tables, in district or area code order. +- ``constituency_withheld_codes`` and ``local_authority_withheld_codes``, + for a local-area report only, from ``withheld_area_codes``, sorted. - ``cliff_impact``, ``spm_provenance``, ``data_version`` and ``model_version`` from the report run's own columns; the SPM receipts are read as ``SPMComparisonProvenance``. An optional part with no rows is null. The writer stores no rows for an empty list either, so a UK result whose areas were all withheld reads back with -null areas, and a UK wealth breakdown without deciles as no breakdown. +null areas, and a UK wealth breakdown without deciles as no breakdown. A +local-area report's run always calculates both area levels, so its levels +read as lists, empty when every area was withheld. """ from __future__ import annotations @@ -247,6 +251,8 @@ def economy_result(rows: EconomyResultRows) -> EconomyResult: constituency_impact=_constituencies(rows), local_authority_impact=_local_authorities(rows), congressional_district_impact=_districts(rows.congressional_district_impacts), + constituency_withheld_codes=_withheld_codes(rows, "constituency"), + local_authority_withheld_codes=_withheld_codes(rows, "local_authority"), cliff_impact=_cliff_impact(report_run), ) @@ -515,9 +521,28 @@ def _area_values(row: object) -> dict[str, Any]: } +def _no_area_rows(rows: EconomyResultRows) -> list[Any] | None: + """Return a level with no rows: empty for a local-area report, else null.""" + return [] if rows.on_local_area_dataset else None + + +def _withheld_codes( + rows: EconomyResultRows, area_type: Literal["constituency", "local_authority"] +) -> list[str] | None: + """Return a local-area report's sorted withheld codes of a level, else None. + + A row's ``area_type`` is a ``StrEnum``, equal to its value. + """ + if not rows.on_local_area_dataset: + return None + return sorted( + row.code for row in rows.withheld_area_codes if row.area_type == area_type + ) + + def _constituencies(rows: EconomyResultRows) -> list[ConstituencyImpact] | None: if not rows.constituency_impacts: - return None + return _no_area_rows(rows) return [ ConstituencyImpact( constituency_code=row.constituency_code, @@ -532,7 +557,7 @@ def _constituencies(rows: EconomyResultRows) -> list[ConstituencyImpact] | None: def _local_authorities(rows: EconomyResultRows) -> list[LocalAuthorityImpact] | None: if not rows.local_authority_impacts: - return None + return _no_area_rows(rows) return [ LocalAuthorityImpact( local_authority_code=row.local_authority_code, diff --git a/policyengine_api/services/v2/stored_reports/economy_results/types.py b/policyengine_api/services/v2/stored_reports/economy_results/types.py index 4160a85ad..fb25dd75c 100644 --- a/policyengine_api/services/v2/stored_reports/economy_results/types.py +++ b/policyengine_api/services/v2/stored_reports/economy_results/types.py @@ -8,9 +8,12 @@ ``EconomyResultRows`` holds the rows of one report run, and ``economy_results.transformations`` builds the result from them again. -The UK local-area breakdown (``LocalAreaBreakdownOutput``), the result of a -separate run that computes only constituency and local-authority impacts, is -not part of it. +A local-area report's result (a UK-wide or nation report on the dataset the +catalog version gives local areas) also lists the codes of the areas +disclosure control withheld at each level, from ``withheld_area_codes``, as +``SingleYearMacroOutput``'s ``constituency_withheld_codes`` and +``local_authority_withheld_codes`` do; every other result leaves the two +lists out. """ from __future__ import annotations @@ -18,7 +21,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Literal -from pydantic import BaseModel, ConfigDict, JsonValue +from pydantic import BaseModel, ConfigDict, Field, JsonValue from policyengine_api.spm import SPMComparisonProvenance @@ -51,6 +54,9 @@ from policyengine_api.data.v2.models import ( ProgramStatistics as ProgramStatisticsRow, ) + from policyengine_api.data.v2.models import ( + WithheldAreaCode as WithheldAreaCodeRow, + ) class EconomyResultUnreadableError(ValueError): @@ -299,6 +305,10 @@ class CongressionalDistrictImpacts(EconomyResultModel): districts: list[CongressionalDistrictImpact] +def _is_none(value: object) -> bool: + return value is None + + class EconomyResult(EconomyResultModel): """A report run's single-year economy result, in the format the app reads. @@ -310,8 +320,13 @@ class EconomyResult(EconomyResultModel): keys), or null. ``wealth_decile`` and ``intra_wealth_decile`` are UK-only; ``poverty_by_race`` and ``congressional_district_impact`` US-only; the constituency and local-authority impacts are UK-only and null when the - run did not compute them. ``labor_supply_response`` and ``cliff_impact`` - are null when the run did not compute them. + run did not compute them; a local-area report's run computes both, so a + level whose every area was withheld is empty. ``constituency_withheld_codes`` + and ``local_authority_withheld_codes`` are a local-area report's sorted + codes of the areas each level withheld, empty when it withheld none; for + every other report they are null and left out of the JSON, as the + simulation service leaves them out. ``labor_supply_response`` and + ``cliff_impact`` are null when the run did not compute them. """ spm_provenance: SPMComparisonProvenance | None @@ -331,6 +346,12 @@ class EconomyResult(EconomyResultModel): constituency_impact: list[ConstituencyImpact] | None local_authority_impact: list[LocalAuthorityImpact] | None congressional_district_impact: CongressionalDistrictImpacts | None + constituency_withheld_codes: list[str] | None = Field( + default=None, exclude_if=_is_none + ) + local_authority_withheld_codes: list[str] | None = Field( + default=None, exclude_if=_is_none + ) cliff_impact: CliffImpact | None @@ -357,6 +378,9 @@ class EconomyResultRows: The comparison tables' rows are the report run's; the poverty and inequality rows are those of its baseline and reform simulation runs. + ``on_local_area_dataset`` says whether the run is a local-area report's, + whose ``withheld_area_codes`` rows are read and list every area its + writer withheld; for any other run they are not read. """ country_id: str @@ -374,3 +398,5 @@ class EconomyResultRows: congressional_district_impacts: tuple[CongressionalDistrictImpactRow, ...] constituency_impacts: tuple[ConstituencyImpactRow, ...] local_authority_impacts: tuple[LocalAuthorityImpactRow, ...] + on_local_area_dataset: bool = False + withheld_area_codes: tuple[WithheldAreaCodeRow, ...] = () diff --git a/policyengine_api/services/v2/stored_reports/services.py b/policyengine_api/services/v2/stored_reports/services.py index 588af9d12..9107b5192 100644 --- a/policyengine_api/services/v2/stored_reports/services.py +++ b/policyengine_api/services/v2/stored_reports/services.py @@ -15,7 +15,9 @@ run, as a report migrated from v1 has none, answers its definition with no result. -The result is rebuilt from the run's result rows once it has succeeded. A +The result is rebuilt from the run's result rows once it has succeeded; a +local-area report's (a UK-wide or nation report on the dataset its catalog +version gives local areas) also lists its withheld area codes. A budget-window report's run is a budget-window run, read with its years; once that run has succeeded, its result is the budget-window format, rebuilt from each year's budget rows. Reading a @@ -26,7 +28,10 @@ succeeded, and each side's failure once it has failed. Lists name at most 25 reports and read them as a read without a run does, with any window's years, without results: two queries, and with a user three or four more for -their labels and their newest recorded runs. +their labels and their newest recorded runs. A search finds the country's +reports that share a ``dataset_independent_key``, which a national report +shares with its local-area report: the 25 created last, newest first, read +as a list reads them. Creating a report or budget window checks the country and the policies it names (and a household report's household), asks the simulation service to @@ -61,6 +66,7 @@ from policyengine_api.data.v2.models import ReportRunStatus, ReportRunTrigger from policyengine_api.data.v2.models.base import utc_now +from policyengine_api.query_parameters import MAXIMUM_LISTED_REPORTS from policyengine_api.services.v2.identity.services import ( find_principal_user_in_session, ) @@ -87,6 +93,7 @@ from policyengine_api.services.v2.stored_reports.database_connectors.reads import ( read_budget_window_years, read_report_rows, + search_report_ids, ) from policyengine_api.services.v2.stored_reports.database_connectors.reads_runs import ( read_budget_window_run, @@ -461,6 +468,7 @@ def _read_report( country_id=country_id, report_run=run, reform_simulation_run_id=require_reform_run(run), + on_local_area_dataset=stored.on_local_area_dataset, ) ) return report_read( @@ -549,6 +557,41 @@ def list_reports( for stored in listed_reports(rows, report_ids=report_ids) ) + def search_reports( + self, + *, + country_id: str, + dataset_independent_key: str, + requester: AccountPrincipal | None = None, + ) -> tuple[StoredReportSummaryRead, ...]: + """Return the country's reports that share a dataset-independent key. + + They are read as ``list_reports`` reads them, without results: the + ``MAXIMUM_LISTED_REPORTS`` reports created last, newest first (ties + going to the larger ID), of every kind. Raise + StandardUserRequiresSignInError for an unauthenticated credential + that names a signed-in user, and ReportStorageError when the database + cannot be read. + """ + with _storage_failures(), self._database_session.read() as session: + report_ids = search_report_ids( + session, + country_id=country_id, + dataset_independent_key=dataset_independent_key, + limit=MAXIMUM_LISTED_REPORTS, + ) + rows = _requesters_view( + session, + read_report_rows(session, country_id=country_id, report_ids=report_ids), + requester=requester, + with_requested_runs=True, + ) + current_versions = self._current_versions(country_id) + return tuple( + _summary_read(stored, current_versions=current_versions) + for stored in listed_reports(rows, report_ids=report_ids) + ) + def create_report( self, report_input: ReportCreationInput, diff --git a/policyengine_api/services/v2/stored_reports/transformations.py b/policyengine_api/services/v2/stored_reports/transformations.py index 2dbcdafc4..dae16644f 100644 --- a/policyengine_api/services/v2/stored_reports/transformations.py +++ b/policyengine_api/services/v2/stored_reports/transformations.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING +from policyengine_api.services.v2.simulation_service.types import LOCAL_AREA_DATASET from policyengine_api.services.v2.stored_reports.types import ( BudgetWindowReportRead, BudgetWindowReportSummaryRead, @@ -97,6 +98,7 @@ def _report_definition( country_id=report.country, label=rows.saved_report_name, description=rows.saved_report_description, + dataset_independent_key=report.dataset_independent_key, year=report.year, baseline_simulation=_simulation(rows.baseline_simulation), reform_simulation=_simulation(rows.reform_simulation), @@ -126,6 +128,7 @@ def report_summary_read( kind="economy", region_code=rows.region_code, dataset_name=rows.dataset_name, + data_source=LOCAL_AREA_DATASET if rows.on_local_area_dataset else None, status=status, run=run, ) diff --git a/policyengine_api/services/v2/stored_reports/types.py b/policyengine_api/services/v2/stored_reports/types.py index 175072fc0..b6d8e2e90 100644 --- a/policyengine_api/services/v2/stored_reports/types.py +++ b/policyengine_api/services/v2/stored_reports/types.py @@ -30,6 +30,7 @@ ) from policyengine_api.services.v2.simulation_service.types import ( PackageVersions, + ReportDataSource, ) from policyengine_api.services.v2.stored_reports.budget_window_results import ( types as window_result_types, @@ -93,12 +94,17 @@ class ReportDefinitionRead: ``current_versions`` are the country package and PolicyEngine.py versions the Public API serves now; a run on other versions is out of date. ``options`` are the options the report was created with, as stored. + ``dataset_independent_key`` is the key the simulation service stored for + the report's definition without its dataset, which a national report + shares with its local-area report, or None for a report stored before + the keys. """ id: UUID country_id: str label: str | None description: str | None + dataset_independent_key: str | None year: int | None baseline_simulation: ReportSimulationRead | None reform_simulation: ReportSimulationRead | None @@ -112,12 +118,17 @@ class ReportSummaryRead(ReportDefinitionRead): """An economy report's definition, the run read and the current versions. ``region_code`` and ``dataset_name`` are the region and dataset both - simulations use, null when the report names none. + simulations use, null when the report names none. ``data_source`` is + ``local_area_dataset`` for a local-area report, a UK-wide or nation + report on the dataset its catalog version gives local areas, and None + for every other. It is read from the report's region and dataset rows, + since nothing stores the creation request's ``data_source``. """ kind: Literal["economy"] region_code: str | None dataset_name: str | None + data_source: ReportDataSource | None status: ReportStatusName run: ReportRunRead | None @@ -264,7 +275,9 @@ class StoredReportRows: """A report's row, the run a read answers and the rows it names. ``region_code`` and ``dataset_name`` are those of the report's region and - dataset, and null when it names none; a simulation is None once deleted. + dataset, and null when it names none; ``on_local_area_dataset`` says + whether it is a local-area report (see ``ReportSummaryRead``); a + simulation is None once deleted. ``run`` is a report run that is not a year of a budget window, and ``budget_window_run`` a budget-window run, with ``budget_window_years`` its years' report runs in year order, or None. Read without naming a @@ -279,6 +292,7 @@ class StoredReportRows: report: Report region_code: str | None dataset_name: str | None + on_local_area_dataset: bool baseline_simulation: Simulation | None reform_simulation: Simulation | None run: ReportRun | None diff --git a/tests/fixtures/economy_results.py b/tests/fixtures/economy_results.py index a47b68547..d5ee9f6a4 100644 --- a/tests/fixtures/economy_results.py +++ b/tests/fixtures/economy_results.py @@ -4,7 +4,10 @@ ``libs/policyengine-stage12-persistence/src/policyengine_stage12_persistence/ report_run_results.py`` in PolicyEngine/policyengine-sim-api (last changed in ``c02145fd``), field for field, so the Public API's rebuild of a result from -its rows is tested against the same mapping. Each row is a dict of the +its rows is tested against the same mapping. Its ``withheld_area_codes`` rows +follow the writer as PolicyEngine/policyengine-sim-api#1068 changes it +(``46c38f07``): one row per code a result's ``constituency_withheld_codes`` +and ``local_authority_withheld_codes`` list. Each row is a dict of the table's columns; ``id`` and ``created_at`` are left for the caller. """ @@ -339,6 +342,19 @@ def uk_economy_result() -> dict[str, Any]: } +def uk_local_area_economy_result() -> dict[str, Any]: + """Return a UK-wide report's result on the local-area dataset. + + It is a UK result with both area levels, and the sorted codes of the + areas disclosure control withheld at each level. + """ + return { + **uk_economy_result(), + "constituency_withheld_codes": ["S14000045"], + "local_authority_withheld_codes": ["E06000064", "W06000015"], + } + + @dataclass(frozen=True) class StoredEconomyResult: """What an economy result is stored as: report run columns and table rows.""" @@ -389,6 +405,7 @@ def economy_result_rows( "local_authority_impacts": _area_rows( output.get("local_authority_impact"), report_keys, "local_authority" ), + "withheld_area_codes": _withheld_rows(output, report_keys), }, ) @@ -654,3 +671,18 @@ def _area_rows( } for area in areas ] + + +def _withheld_rows( + output: dict[str, Any], keys: dict[str, Any] +) -> list[dict[str, Any]]: + """Return a row per area code each level's withheld list names.""" + levels = ( + ("constituency", output.get("constituency_withheld_codes")), + ("local_authority", output.get("local_authority_withheld_codes")), + ) + return [ + {**keys, "area_type": area_type, "code": code} + for area_type, codes in levels + for code in codes or () + ] diff --git a/tests/integration/stored_report_rows.py b/tests/integration/stored_report_rows.py index c3fddbe6c..9c5c16f9a 100644 --- a/tests/integration/stored_report_rows.py +++ b/tests/integration/stored_report_rows.py @@ -29,6 +29,8 @@ REGION_CODES = {"us": "state/ca", "uk": "country/wales"} DATASET_NAMES = {"us": "enhanced_cps_2024", "uk": "enhanced_frs_2023_24"} +# The dataset .py 6.17.1's UK bundle gives constituencies and local authorities. +LOCAL_AREA_DATASET_NAME = "microcosm_uk_2024_25_dense" STARTED_AT = datetime(2026, 11, 1, 9, tzinfo=UTC) # The year of a report store_report stores, unless it is given another. STORED_REPORT_YEAR = 2026 @@ -39,7 +41,9 @@ class CatalogRows: """A country's model, a catalog version of it, a dataset, a region and a policy. ``created_model`` says whether the model row was stored here, so cleanup - leaves a model another test stored. + leaves a model another test stored. ``area_region_ids`` and + ``area_dataset_ids`` are the UK-wide and area regions and the local-area + dataset ``store_local_area_catalog`` adds. """ country: str @@ -52,6 +56,8 @@ class CatalogRows: report_ids: list[UUID] = field(default_factory=list) simulation_ids: list[UUID] = field(default_factory=list) household_ids: list[UUID] = field(default_factory=list) + area_region_ids: list[UUID] = field(default_factory=list) + area_dataset_ids: list[UUID] = field(default_factory=list) def store_catalog(connection: Connection, country: str) -> CatalogRows: @@ -116,6 +122,53 @@ def store_catalog(connection: Connection, country: str) -> CatalogRows: return rows +def store_local_area_catalog( + connection: Connection, catalog: CatalogRows +) -> CatalogRows: + """Give a UK catalog version its UK-wide region and local areas, as .py 6.17.1. + + UK-wide uses the catalog's national dataset; a constituency and a local + authority use the local-area dataset, which this stores. Return the + catalog with the UK-wide region and the local-area dataset in place of + its own, for storing a local-area report; it records what it stores in + ``catalog``'s lists, so ``delete_catalog(catalog)`` deletes it. + """ + local_area_dataset_id = uuid4() + insert_row( + connection, + "datasets", + id=local_area_dataset_id, + name=LOCAL_AREA_DATASET_NAME, + year=2026, + is_output_dataset=False, + tax_benefit_model_version_id=catalog.version_id, + ) + catalog.area_dataset_ids.append(local_area_dataset_id) + regions = { + "uk": ("national", catalog.dataset_id), + "constituency/W07000081": ("constituency", local_area_dataset_id), + "local_authority/W06000015": ("local_authority", local_area_dataset_id), + } + region_ids = {} + for code, (region_type, dataset_id) in regions.items(): + region_ids[code] = uuid4() + insert_row( + connection, + "regions", + id=region_ids[code], + code=code, + label=code, + region_type=region_type, + requires_filter=False, + tax_benefit_model_version_id=catalog.version_id, + default_dataset_id=dataset_id, + ) + catalog.area_region_ids.append(region_ids[code]) + return replace( + catalog, dataset_id=local_area_dataset_id, region_id=region_ids["uk"] + ) + + def store_simulation( connection: Connection, catalog: CatalogRows, @@ -216,18 +269,23 @@ class StoredReport: year: int = STORED_REPORT_YEAR -def store_report( +def store_report( # noqa: PLR0913 # the connection and catalog, then each column a test varies connection: Connection, catalog: CatalogRows, *, household_id: UUID | None = None, budget_window_size: int | None = None, year: int = STORED_REPORT_YEAR, + dataset_independent_key: str | None = None, + created_at: datetime | None = None, ) -> StoredReport: """Store a report of ``year`` comparing current law with the catalog's policy. A budget window's ``year`` is its first year, whose simulations the - report names. + report names. With ``dataset_independent_key``, the report has keys, as + the simulation service stores them from C1 · 27 on: that key, and a + definition key of its own; without it, it has none. Without + ``created_at``, the report is created now. """ report = StoredReport( report_id=uuid4(), @@ -244,10 +302,13 @@ def store_report( "INSERT INTO reports (id, label, description, country, type, " "tax_benefit_model_id, policy_id, baseline_simulation_id, " "reform_simulation_id, household_id, dataset_id, region_id, year, " - "budget_window_size, inputs) VALUES (:id, :label, :description, " - ":country, :type, :model_id, :policy_id, :baseline, :reform, " - ":household_id, :dataset_id, :region_id, :year, :budget_window_size, " - "CAST(:inputs AS JSON))" + "budget_window_size, inputs, key_version, definition_key, " + "dataset_independent_key, created_at) VALUES (:id, :label, " + ":description, :country, :type, :model_id, :policy_id, :baseline, " + ":reform, :household_id, :dataset_id, :region_id, :year, " + ":budget_window_size, CAST(:inputs AS JSON), :key_version, " + ":definition_key, :dataset_independent_key, " + "COALESCE(CAST(:created_at AS timestamptz), now()))" ), { "id": report.report_id, @@ -265,6 +326,12 @@ def store_report( "budget_window_size": budget_window_size, "year": year, "inputs": '{"include_cliffs": true}', + "key_version": None if dataset_independent_key is None else 1, + "definition_key": ( + None if dataset_independent_key is None else uuid4().hex * 2 + ), + "dataset_independent_key": dataset_independent_key, + "created_at": created_at, }, ) catalog.report_ids.append(report.report_id) @@ -408,6 +475,14 @@ def delete_catalog(connection: Connection, catalog: CatalogRows) -> None: connection.execute( text("DELETE FROM households WHERE id = :id"), {"id": household_id} ) + for region_id in catalog.area_region_ids: + connection.execute( + text("DELETE FROM regions WHERE id = :id"), {"id": region_id} + ) + for dataset_id in catalog.area_dataset_ids: + connection.execute( + text("DELETE FROM datasets WHERE id = :id"), {"id": dataset_id} + ) for table, row_id in ( ("policies", catalog.policy_id), ("regions", catalog.region_id), diff --git a/tests/integration/test_v2_report_reads.py b/tests/integration/test_v2_report_reads.py index 2b4e11817..dcf12be22 100644 --- a/tests/integration/test_v2_report_reads.py +++ b/tests/integration/test_v2_report_reads.py @@ -10,6 +10,7 @@ from __future__ import annotations import json +from dataclasses import replace from datetime import timedelta from typing import TYPE_CHECKING, Any from uuid import UUID, uuid4 @@ -42,13 +43,19 @@ from policyengine_api.services.v2.stored_simulations.services import ( V2StoredSimulationService, ) -from tests.fixtures.economy_results import uk_economy_result, us_economy_result +from tests.fixtures.economy_results import ( + uk_economy_result, + uk_local_area_economy_result, + us_economy_result, +) from tests.integration.stored_report_rows import ( + LOCAL_AREA_DATASET_NAME, REGION_CODES, STARTED_AT, RunSettings, delete_catalog, store_catalog, + store_local_area_catalog, store_report, store_report_run, store_simulation, @@ -476,3 +483,135 @@ def test_of_two_simulation_runs_created_together_the_larger_id_is_the_newest( item = response.json()["result"]["item"] assert item["run"]["id"] == str(LARGER_RUN_ID) assert item["status"] == "pending" + + +DATASET_INDEPENDENT_KEY = "bcabc98b4e0d3eff" * 4 + + +def test_a_search_finds_the_countrys_reports_with_the_key_newest_first( + owner_engine: Engine, catalogs: dict[str, CatalogRows], client: TestClient +) -> None: + with owner_engine.begin() as connection: + older, newer = ( + store_report( + connection, + catalogs["uk"], + dataset_independent_key=DATASET_INDEPENDENT_KEY, + created_at=STARTED_AT + timedelta(hours=hours), + ) + for hours in (1, 2) + ) + store_report(connection, catalogs["uk"], dataset_independent_key="0" * 64) + store_report(connection, catalogs["uk"]) + store_report( + connection, catalogs["us"], dataset_independent_key=DATASET_INDEPENDENT_KEY + ) + + response = client.get( + f"/v2/reports?country_id=uk&dataset_independent_key={DATASET_INDEPENDENT_KEY}" + ) + + assert response.status_code == 200, response.text + items = response.json()["result"]["items"] + assert [item["id"] for item in items] == [ + str(newer.report_id), + str(older.report_id), + ] + assert {item["dataset_independent_key"] for item in items} == { + DATASET_INDEPENDENT_KEY + } + detail = client.get(f"/v2/reports/{newer.report_id}?country_id=uk").json() + assert detail["result"]["item"]["dataset_independent_key"] == ( + DATASET_INDEPENDENT_KEY + ) + + +def test_a_local_area_report_reads_its_areas_and_withheld_codes( + owner_engine: Engine, catalogs: dict[str, CatalogRows], client: TestClient +) -> None: + output = uk_local_area_economy_result() + with owner_engine.begin() as connection: + local_areas = store_local_area_catalog(connection, catalogs["uk"]) + report = store_report(connection, local_areas) + store_report_run( + connection, + local_areas, + report, + RunSettings(versions=CURRENT_VERSIONS), + result=output, + ) + + response = client.get(f"/v2/reports/{report.report_id}?country_id=uk") + + assert response.status_code == 200, response.text + item = response.json()["result"]["item"] + expected = _as_json(output) + expected["model_version"] = CURRENT_VERSIONS["country_package_version"] + assert item["data_source"] == "local_area_dataset" + assert item["region_code"] == "uk" + assert item["dataset_name"] == LOCAL_AREA_DATASET_NAME + assert item["result"] == expected + + +@pytest.mark.parametrize("region_code", ["uk", "country/wales"], ids=["uk", "nation"]) +def test_a_uk_wide_or_nation_report_on_its_own_dataset_reads_no_withheld_codes( + owner_engine: Engine, + catalogs: dict[str, CatalogRows], + client: TestClient, + region_code: str, +) -> None: + with owner_engine.begin() as connection: + local_areas = store_local_area_catalog(connection, catalogs["uk"]) + # UK-wide on the national dataset, or the catalog's Wales region. + on_national_dataset = ( + replace(local_areas, dataset_id=catalogs["uk"].dataset_id) + if region_code == "uk" + else catalogs["uk"] + ) + report = store_report(connection, on_national_dataset) + store_report_run( + connection, + on_national_dataset, + report, + RunSettings(versions=CURRENT_VERSIONS), + result=uk_economy_result(), + ) + + response = client.get(f"/v2/reports/{report.report_id}?country_id=uk") + + assert response.status_code == 200, response.text + item = response.json()["result"]["item"] + assert item["data_source"] is None + assert item["region_code"] == region_code + assert "constituency_withheld_codes" not in item["result"] + assert "local_authority_withheld_codes" not in item["result"] + + +def test_a_local_area_report_with_every_area_of_a_level_withheld_reads_it_empty( + owner_engine: Engine, catalogs: dict[str, CatalogRows], client: TestClient +) -> None: + output = { + **uk_local_area_economy_result(), + "constituency_impact": [], + "constituency_withheld_codes": ["E14001063", "W07000081"], + "local_authority_withheld_codes": [], + } + with owner_engine.begin() as connection: + local_areas = store_local_area_catalog(connection, catalogs["uk"]) + report = store_report(connection, local_areas) + store_report_run( + connection, + local_areas, + report, + RunSettings(versions=CURRENT_VERSIONS), + result=output, + ) + + response = client.get(f"/v2/reports/{report.report_id}?country_id=uk") + + assert response.status_code == 200, response.text + result = response.json()["result"]["item"]["result"] + assert result["constituency_impact"] == [] + assert result["constituency_withheld_codes"] == ["E14001063", "W07000081"] + assert result["local_authority_withheld_codes"] == [] + assert len(result["local_authority_impact"]) == 2 diff --git a/tests/integration/test_v2_user_report_views.py b/tests/integration/test_v2_user_report_views.py index 1a146f5f3..9d47fc198 100644 --- a/tests/integration/test_v2_user_report_views.py +++ b/tests/integration/test_v2_user_report_views.py @@ -320,6 +320,39 @@ def test_a_reports_label_and_description_are_the_users_saved_reports( assert (anonymous_item["label"], anonymous_item["description"]) == (None, None) +def test_a_search_by_key_reads_each_report_as_the_user_sees_it( + owner_engine: Engine, + catalog: CatalogRows, + runtime_sessions: Any, + browsers: list[UUID], +) -> None: + saver, _ = browsers + dataset_independent_key = "b" * 64 + with owner_engine.begin() as connection: + report = store_report( + connection, catalog, dataset_independent_key=dataset_independent_key + ) + store_saved_report(owner_engine, user_id=saver, report_id=report.report_id) + client = client_for( + runtime_sessions, JoiningSimulationService(owner_engine, catalog) + ) + search = ( + f"/v2/reports?country_id=us&dataset_independent_key={dataset_independent_key}" + ) + + savers_found = client.get(search, headers=as_browser(saver)).json()["result"][ + "items" + ] + anonymous_found = client.get(search).json()["result"]["items"] + + assert [(item["id"], item["label"]) for item in savers_found] == [ + (str(report.report_id), "My report") + ] + assert [(item["id"], item["label"]) for item in anonymous_found] == [ + (str(report.report_id), None) + ] + + def test_a_malformed_browser_credential_on_a_read_is_refused( owner_engine: Engine, catalog: CatalogRows, runtime_sessions: Any ) -> None: diff --git a/tests/unit/libs/test_simulation_reports.py b/tests/unit/libs/test_simulation_reports.py index 7308b0911..bf3a0befc 100644 --- a/tests/unit/libs/test_simulation_reports.py +++ b/tests/unit/libs/test_simulation_reports.py @@ -779,3 +779,114 @@ def test_partly_configured_gateway_credentials_are_unavailable(monkeypatch) -> N with pytest.raises(SimulationServiceUnavailableError, match="not configured"): SimulationServiceReportClient.from_environment() + + +def test_a_report_sends_its_data_source_only_when_given() -> None: + on_local_areas = RecordingService() + on_the_regions_dataset = RecordingService() + + on_local_areas.client().submit_report( + report_input( + country_id="uk", region_code="uk", data_source="local_area_dataset" + ) + ) + on_the_regions_dataset.client().submit_report(report_input()) + + local_area_body = on_local_areas.sent_body() + assert local_area_body["data_source"] == "local_area_dataset" + assert "dataset_name" not in local_area_body + assert "data_source" not in on_the_regions_dataset.sent_body() + + +@pytest.mark.parametrize( + ("region_code", "body", "message"), + [ + # The simulation service's answers to these UK requests, as + # PolicyEngine/policyengine-sim-api's C1 · 29a builds them: a code with + # one reason located at data_source for a UK-wide report on a release + # that gives local areas no dataset of their own, and one reason with + # no code for a region other than UK-wide or a nation. + pytest.param( + "uk", + { + "code": "no_local_area_dataset", + "detail": [ + { + "loc": ["body", "data_source"], + "msg": ( + "This release gives local areas no dataset of their own." + ), + "type": "no_local_area_dataset", + } + ], + }, + "The simulation service cannot run this request (no_local_area_dataset): " + "data_source: This release gives local areas no dataset of their own.", + id="release-without-a-local-area-dataset", + ), + pytest.param( + "constituency/W07000081", + { + "detail": [ + { + "loc": ["body", "data_source"], + "msg": ( + "Value error, the local-area dataset needs a UK-wide or " + "nation region (country/england, country/northern_ireland, " + "country/scotland, country/wales, uk), not " + "'constituency/W07000081'" + ), + "type": "value_error", + } + ] + }, + "The simulation service cannot run this request: data_source: Value " + "error, the local-area dataset needs a UK-wide or nation region " + "(country/england, country/northern_ireland, country/scotland, " + "country/wales, uk), not 'constituency/W07000081'", + id="another-region", + ), + ], +) +def test_a_data_source_the_service_refuses_is_refused_with_its_reasons( + region_code: str, body: dict[str, object], message: str +) -> None: + # The Public API checks only the field's value; the simulation service + # checks the country, the region and the release it runs. + service = RecordingService(status_code=422, body=body) + + with pytest.raises(SimulationServiceInvalidRequestError) as refused: + service.client().submit_report( + report_input( + country_id="uk", + region_code=region_code, + data_source="local_area_dataset", + ) + ) + + assert _report_route_status(refused.value) == 422 + assert str(refused.value) == message + + +@pytest.mark.parametrize( + "code", ["local_areas_unsupported", "local_area_dataset_mismatch"] +) +def test_the_services_own_local_area_dataset_state_is_unavailable(code: str) -> None: + # Its worker declares no local-area dataset, or a dataset the catalog + # version doesn't give the local areas: nothing the caller can change. + service = RecordingService( + status_code=400, body={"code": code, "detail": "The worker's dataset differs."} + ) + + with pytest.raises(SimulationServiceUnavailableError) as unavailable: + service.client().submit_report( + report_input( + country_id="uk", region_code="uk", data_source="local_area_dataset" + ) + ) + + assert _report_route_status(unavailable.value) == 503 + assert str(unavailable.value) == ( + f"The simulation service refused the report ({code}): " + "The worker's dataset differs." + ) diff --git a/tests/unit/v2/budget_window_report_support.py b/tests/unit/v2/budget_window_report_support.py index f9c28ba0b..b364168c4 100644 --- a/tests/unit/v2/budget_window_report_support.py +++ b/tests/unit/v2/budget_window_report_support.py @@ -99,6 +99,7 @@ def budget_window_summary_fields(**changes: Any) -> dict[str, Any]: "kind": "budget_window", "label": "Expanded child tax credit, 2026 to 2027", "description": None, + "dataset_independent_key": None, "year": 2026, "budget_window_size": 2, "region_code": "us", diff --git a/tests/unit/v2/sqlite_report_rows.py b/tests/unit/v2/sqlite_report_rows.py new file mode 100644 index 000000000..6a3efa5e8 --- /dev/null +++ b/tests/unit/v2/sqlite_report_rows.py @@ -0,0 +1,129 @@ +"""SQLite's copy of the v2 schema, and economy reports stored in it, for read tests. + +The simulation service stores these rows in staging and production; here a +test stores them directly and reads them through ``V2StoredReportService``. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.models import ( + V2_METADATA, + Report, + Simulation, + SimulationType, +) +from policyengine_api.services.v2.simulation_service.types import PackageVersions +from policyengine_api.services.v2.stored_reports.database_session import ( + StoredReportDatabaseSession, +) +from policyengine_api.services.v2.stored_reports.services import V2StoredReportService + +if TYPE_CHECKING: + from collections.abc import Iterator + +STORED_AT = datetime(2026, 11, 14, 9, tzinfo=UTC) +CURRENT_VERSIONS = PackageVersions( + country_package_version="2.1.0", policyengine_version="6.17.1" +) + + +def sqlite_sessions() -> Iterator[sessionmaker[Session]]: + """Yield sessions of a new in-memory v2 schema, and dispose of it after.""" + engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + V2_METADATA.create_all(engine) + try: + yield sessionmaker(engine, class_=Session, expire_on_commit=False) + finally: + engine.dispose() + + +def report_service(sessions: sessionmaker[Session]) -> V2StoredReportService: + """Return the report service reading through ``sessions``.""" + return V2StoredReportService( + StoredReportDatabaseSession(sessions), + current_versions=lambda country_id: CURRENT_VERSIONS, + ) + + +@dataclass(frozen=True) +class ReportPlace: + """Where a report calculates: its model, catalog version, dataset and region.""" + + country: str + model_id: UUID + version_id: UUID + dataset_id: UUID + region_id: UUID + + +@dataclass(frozen=True) +class ReportToStore: + """The report columns a test chooses; the rest are fixed. + + A report with a ``dataset_independent_key`` also gets a ``key_version`` + and a definition key of its own, as the simulation service stores them; + without one it has no keys, as a report stored before them. + """ + + dataset_independent_key: str | None + created_at: datetime = STORED_AT + report_id: UUID | None = None + budget_window_size: int | None = None + + +def store_economy_report( + session: Session, place: ReportPlace, report: ReportToStore +) -> UUID: + """Store an economy report, or a budget window, of current law against itself.""" + simulations = [ + Simulation( + simulation_type=SimulationType.ECONOMY, + dataset_id=place.dataset_id, + region_id=place.region_id, + tax_benefit_model_version_id=place.version_id, + year=2026, + ) + for _ in range(2) + ] + session.add_all(simulations) + session.flush() + keys = ( + {} + if report.dataset_independent_key is None + else { + "key_version": 1, + "definition_key": uuid4().hex * 2, + "dataset_independent_key": report.dataset_independent_key, + } + ) + stored = Report( + id=uuid4() if report.report_id is None else report.report_id, + label="A stored report", + country=place.country, + tax_benefit_model_id=place.model_id, + baseline_simulation_id=simulations[0].id, + reform_simulation_id=simulations[1].id, + dataset_id=place.dataset_id, + region_id=place.region_id, + year=2026, + budget_window_size=report.budget_window_size, + inputs={}, + created_at=report.created_at, + **keys, + ) + session.add(stored) + session.commit() + return stored.id diff --git a/tests/unit/v2/stored_report_support.py b/tests/unit/v2/stored_report_support.py index d660ac7c4..b566275f2 100644 --- a/tests/unit/v2/stored_report_support.py +++ b/tests/unit/v2/stored_report_support.py @@ -83,9 +83,11 @@ def report_summary_fields(**changes: Any) -> dict[str, Any]: "kind": "economy", "label": "Expanded child tax credit", "description": None, + "dataset_independent_key": None, "year": 2026, "region_code": "state/ca", "dataset_name": None, + "data_source": None, "baseline_simulation": ReportSimulationRead( id=BASELINE_SIMULATION_ID, policy_id=None, dynamic_id=None ), @@ -149,6 +151,7 @@ class FakeReportService: report: ReportRead = field(default_factory=report_read) listed: tuple[ReportSummaryRead, ...] = () + found: tuple[ReportSummaryRead, ...] = () error: Exception | None = None calls: list[tuple[str, dict[str, Any]]] = field(default_factory=list) # The user each read, create or re-run named, None for a request naming @@ -185,6 +188,21 @@ def list_reports( self._answer("list_reports", country_id=country_id, report_ids=report_ids) return self.listed + def search_reports( + self, + *, + country_id: str, + dataset_independent_key: str, + requester: AccountPrincipal | None = None, + ) -> tuple[ReportSummaryRead, ...]: + self.requesters.append(requester) + self._answer( + "search_reports", + country_id=country_id, + dataset_independent_key=dataset_independent_key, + ) + return self.found + def create_report( self, report_input: EconomyReportCreationInput, diff --git a/tests/unit/v2/test_economy_result_rebuild.py b/tests/unit/v2/test_economy_result_rebuild.py index 30eb29082..bc42fb37f 100644 --- a/tests/unit/v2/test_economy_result_rebuild.py +++ b/tests/unit/v2/test_economy_result_rebuild.py @@ -31,6 +31,7 @@ LocalAuthorityImpact, Poverty, ProgramStatistics, + WithheldAreaCode, ) from policyengine_api.services.v2.stored_reports.economy_results import ( transformations as result_transformations, @@ -45,6 +46,7 @@ RunIds, economy_result_rows, uk_economy_result, + uk_local_area_economy_result, us_economy_result, ) @@ -66,16 +68,23 @@ "congressional_district_impacts": CongressionalDistrictImpact, "constituency_impacts": ConstituencyImpact, "local_authority_impacts": LocalAuthorityImpact, + "withheld_area_codes": WithheldAreaCode, } def stored_rows( - output: dict[str, Any], *, country: str, reverse_rows: bool = False + output: dict[str, Any], + *, + country: str, + reverse_rows: bool = False, + on_local_area_dataset: bool = False, ) -> EconomyResultRows: """Store ``output`` as the writer does and load its rows as table models. With ``reverse_rows``, each table's rows come back in reverse order, as a - query without an order may return them. + query without an order may return them. With ``on_local_area_dataset``, + the rows are a local-area report's, read with its withheld area codes; + otherwise those are not read, as for every other report. """ run_ids = RunIds( report_run_id=uuid4(), @@ -89,6 +98,12 @@ def stored_rows( if reverse_rows: loaded.reverse() tables[table] = tuple(loaded) + withheld_area_codes = tables.pop("withheld_area_codes") + local_area_rows: dict[str, Any] = ( + {"on_local_area_dataset": True, "withheld_area_codes": withheld_area_codes} + if on_local_area_dataset + else {} + ) return EconomyResultRows( country_id=country, report_run=ReportRunResultColumns( @@ -98,6 +113,7 @@ def stored_rows( baseline_simulation_run_id=run_ids.baseline_simulation_run_id, reform_simulation_run_id=run_ids.reform_simulation_run_id, **tables, + **local_area_rows, ) @@ -359,3 +375,56 @@ def test_a_country_without_poverty_measures_is_unreadable() -> None: with pytest.raises(EconomyResultUnreadableError, match="'ca'"): result_transformations.economy_result(rows) + + +@pytest.mark.parametrize("row_order", ["as-stored", "reversed"]) +def test_a_local_area_reports_rows_rebuild_its_areas_and_sorted_withheld_codes( + row_order: str, +) -> None: + output = uk_local_area_economy_result() + + rebuilt = result_transformations.economy_result( + stored_rows( + output, + country="uk", + reverse_rows=row_order == "reversed", + on_local_area_dataset=True, + ) + ) + + assert rebuilt.model_dump(mode="json", by_alias=True) == as_json(output) + + +def test_a_local_area_report_with_every_area_of_a_level_withheld_reads_it_empty() -> ( + None +): + output = { + **uk_local_area_economy_result(), + "constituency_impact": [], + "constituency_withheld_codes": ["E14001063", "W07000081"], + "local_authority_withheld_codes": [], + } + + rebuilt = result_transformations.economy_result( + stored_rows(output, country="uk", on_local_area_dataset=True) + ) + + # The run calculated both levels, so a level with no published area is + # empty rather than missing, and one with none withheld lists none. + assert rebuilt.constituency_impact == [] + assert rebuilt.constituency_withheld_codes == ["E14001063", "W07000081"] + assert rebuilt.local_authority_withheld_codes == [] + assert rebuilt.model_dump(mode="json", by_alias=True) == as_json(output) + + +def test_another_reports_result_reports_no_withheld_codes() -> None: + rebuilt = result_transformations.economy_result( + stored_rows(uk_economy_result(), country="uk") + ) + + assert rebuilt.constituency_withheld_codes is None + assert rebuilt.local_authority_withheld_codes is None + # As the simulation service's result, it leaves the two lists out. + dumped = rebuilt.model_dump(mode="json", by_alias=True) + assert "constituency_withheld_codes" not in dumped + assert "local_authority_withheld_codes" not in dumped diff --git a/tests/unit/v2/test_household_report_routes.py b/tests/unit/v2/test_household_report_routes.py index 6b6af2afb..c78a7bd1c 100644 --- a/tests/unit/v2/test_household_report_routes.py +++ b/tests/unit/v2/test_household_report_routes.py @@ -58,6 +58,7 @@ def household_summary_fields(**changes: Any) -> dict[str, Any]: "kind": "household", "label": "My household under the reform", "description": None, + "dataset_independent_key": None, "year": 2026, "household_id": HOUSEHOLD_ID, "baseline_simulation": ReportSimulationRead( diff --git a/tests/unit/v2/test_local_area_report_reads.py b/tests/unit/v2/test_local_area_report_reads.py new file mode 100644 index 000000000..b2d3f2893 --- /dev/null +++ b/tests/unit/v2/test_local_area_report_reads.py @@ -0,0 +1,221 @@ +"""Read which v2 reports are local-area reports, from their catalog rows. + +The tables are SQLite's copy of the v2 schema. A report created with +``data_source: "local_area_dataset"`` is stored as an ordinary report on the +dataset the catalog version gives local areas, and nothing stores the +request's field. So a UK-wide or nation report is a local-area report when +its dataset is the default dataset of both the constituency and the +local-authority regions, as the simulation service decides it. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +import pytest + +from policyengine_api.data.v2.models import ( + Dataset, + Region, + RegionType, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from tests.unit.v2.sqlite_report_rows import ( + ReportPlace, + ReportToStore, + report_service, + sqlite_sessions, + store_economy_report, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.orm import sessionmaker + from sqlmodel import Session + + from policyengine_api.services.v2.stored_reports.services import ( + V2StoredReportService, + ) + +DATASET_INDEPENDENT_KEY = "bcabc98b4e0d3eff" * 4 + + +@pytest.fixture +def sessions() -> Iterator[sessionmaker[Session]]: + yield from sqlite_sessions() + + +@pytest.fixture +def reports(sessions: sessionmaker[Session]) -> V2StoredReportService: + return report_service(sessions) + + +@dataclass(frozen=True) +class UKCatalog: + """A UK catalog version: its datasets and regions by code.""" + + model_id: UUID + version_id: UUID + datasets: dict[str, UUID] + regions: dict[str, UUID] + + +def store_uk_catalog( + session: Session, *, local_authority_dataset: str = "microcosm_uk_2024_25_dense" +) -> UKCatalog: + """Store a UK catalog version as .py 6.17.1's bundle describes it. + + UK-wide and the nations use the national dataset; constituencies use the + local-area dataset, and local authorities ``local_authority_dataset``, + the local-area dataset unless a test names another. + """ + model = TaxBenefitModel(name="policyengine-uk") + version = TaxBenefitModelVersion( + model=model, + version=f"local-areas-{uuid4().hex[:8]}", + current_law_id=1, + metadata_time_periods=[2026], + ) + session.add_all([model, version]) + session.flush() + datasets = { + name: Dataset(name=name, year=2026, tax_benefit_model_version_id=version.id) + for name in { + "microcosm_uk_2024_25", + "microcosm_uk_2024_25_dense", + local_authority_dataset, + } + } + session.add_all(datasets.values()) + session.flush() + region_rows = [ + ("uk", RegionType.NATIONAL, "microcosm_uk_2024_25"), + ("country/wales", RegionType.COUNTRY, "microcosm_uk_2024_25"), + ( + "constituency/W07000081", + RegionType.CONSTITUENCY, + "microcosm_uk_2024_25_dense", + ), + ( + "local_authority/W06000015", + RegionType.LOCAL_AUTHORITY, + local_authority_dataset, + ), + ] + regions = { + code: Region( + code=code, + label=code, + region_type=region_type, + tax_benefit_model_version_id=version.id, + default_dataset_id=datasets[dataset_name].id, + ) + for code, region_type, dataset_name in region_rows + } + session.add_all(regions.values()) + session.commit() + return UKCatalog( + model_id=model.id, + version_id=version.id, + datasets={name: dataset.id for name, dataset in datasets.items()}, + regions={code: region.id for code, region in regions.items()}, + ) + + +def store_report( + session: Session, catalog: UKCatalog, *, region_code: str, dataset_name: str +) -> UUID: + """Store a UK economy report of current law on the region and dataset named.""" + return store_economy_report( + session, + ReportPlace( + country="uk", + model_id=catalog.model_id, + version_id=catalog.version_id, + dataset_id=catalog.datasets[dataset_name], + region_id=catalog.regions[region_code], + ), + ReportToStore(dataset_independent_key=DATASET_INDEPENDENT_KEY), + ) + + +@pytest.mark.parametrize("region_code", ["uk", "country/wales"], ids=["uk", "nation"]) +def test_a_uk_wide_or_nation_report_on_the_local_area_dataset_is_a_local_area_report( + sessions: sessionmaker[Session], reports: V2StoredReportService, region_code: str +) -> None: + with sessions() as session: + catalog = store_uk_catalog(session) + report_id = store_report( + session, + catalog, + region_code=region_code, + dataset_name="microcosm_uk_2024_25_dense", + ) + + read = reports.get_report(country_id="uk", report_id=report_id) + (listed,) = reports.list_reports(country_id="uk", report_ids=[report_id]) + (found,) = reports.search_reports( + country_id="uk", dataset_independent_key=DATASET_INDEPENDENT_KEY + ) + + assert read.data_source == "local_area_dataset" + assert read.dataset_name == "microcosm_uk_2024_25_dense" + assert listed.data_source == "local_area_dataset" + assert found.data_source == "local_area_dataset" + + +@pytest.mark.parametrize("region_code", ["uk", "country/wales"], ids=["uk", "nation"]) +def test_a_report_on_its_regions_own_dataset_is_not_a_local_area_report( + sessions: sessionmaker[Session], reports: V2StoredReportService, region_code: str +) -> None: + with sessions() as session: + report_id = store_report( + session, + store_uk_catalog(session), + region_code=region_code, + dataset_name="microcosm_uk_2024_25", + ) + + read = reports.get_report(country_id="uk", report_id=report_id) + + assert read.data_source is None + + +def test_a_local_area_regions_own_report_is_not_a_local_area_report( + sessions: sessionmaker[Session], reports: V2StoredReportService +) -> None: + # A constituency's own report runs on the local-area dataset filtered to + # that constituency, so it has no other areas to list. + with sessions() as session: + report_id = store_report( + session, + store_uk_catalog(session), + region_code="constituency/W07000081", + dataset_name="microcosm_uk_2024_25_dense", + ) + + read = reports.get_report(country_id="uk", report_id=report_id) + + assert read.data_source is None + + +def test_a_dataset_only_one_level_of_area_uses_is_not_the_local_area_dataset( + sessions: sessionmaker[Session], reports: V2StoredReportService +) -> None: + # A release whose two area levels name different datasets doesn't route + # areas to their own dataset, so it has no local-area dataset. + with sessions() as session: + report_id = store_report( + session, + store_uk_catalog(session, local_authority_dataset="microcosm_uk_la_only"), + region_code="uk", + dataset_name="microcosm_uk_2024_25_dense", + ) + + read = reports.get_report(country_id="uk", report_id=report_id) + + assert read.data_source is None diff --git a/tests/unit/v2/test_report_create_routes.py b/tests/unit/v2/test_report_create_routes.py index 3c7cd6b55..ba9cd6ca6 100644 --- a/tests/unit/v2/test_report_create_routes.py +++ b/tests/unit/v2/test_report_create_routes.py @@ -1050,3 +1050,79 @@ def test_a_code_that_is_a_fixed_identifier_is_passed_on(record: str, code: str) "status": "error", "message": f"The simulation service is unavailable (503 {code}).", } + + +def test_an_economy_report_passes_its_data_source_on() -> None: + service = FakeReportService() + + response = report_route_client(reports=service).post( + "/v2/reports?country_id=uk", + json=report_body( + country_id="uk", region_code="uk", data_source="local_area_dataset" + ), + ) + + assert response.status_code == 201, response.text + ((operation, arguments),) = service.calls + assert operation == "create_report" + assert arguments["report_input"].data_source == "local_area_dataset" + + +def test_an_economy_report_without_a_data_source_names_none() -> None: + service = FakeReportService() + + response = report_route_client(reports=service).post( + "/v2/reports?country_id=us", json=report_body() + ) + + assert response.status_code == 201, response.text + ((_, arguments),) = service.calls + assert arguments["report_input"].data_source is None + + +def test_the_simulation_service_checks_a_data_sources_region() -> None: + # The Public API passes data_source on for any region; the simulation + # service refuses a region or country it doesn't apply to. + service = FakeReportService() + + response = report_route_client(reports=service).post( + "/v2/reports?country_id=us", + json=report_body(region_code="state/ca", data_source="local_area_dataset"), + ) + + assert response.status_code == 201, response.text + ((_, arguments),) = service.calls + assert arguments["report_input"].data_source == "local_area_dataset" + + +@pytest.mark.parametrize( + "body", + [ + report_body(data_source=None), + report_body(data_source="national_dataset"), + report_body(data_source=["local_area_dataset"]), + report_body( + kind="budget_window", budget_window_size=3, data_source="local_area_dataset" + ), + { + "country_id": "us", + "kind": "household", + "label": "My household", + "household_id": str(REPORT_ID), + "baseline": {}, + "data_source": "local_area_dataset", + }, + ], + ids=["null", "another-value", "a-list", "budget-window", "household-report"], +) +def test_a_data_source_the_public_api_does_not_take_is_refused_before_the_service( + body: dict[str, Any], +) -> None: + service = FakeReportService() + + response = report_route_client(reports=service).post( + "/v2/reports?country_id=us", json=body + ) + + assert response.status_code == 422, response.text + assert service.calls == [] diff --git a/tests/unit/v2/test_report_dataset_independent_keys.py b/tests/unit/v2/test_report_dataset_independent_keys.py new file mode 100644 index 000000000..9cac9c32b --- /dev/null +++ b/tests/unit/v2/test_report_dataset_independent_keys.py @@ -0,0 +1,276 @@ +"""Read v2 reports' dataset-independent keys, and find reports by one. + +The tables are SQLite's copy of the v2 schema. The simulation service stores +each report's ``dataset_independent_key``: its definition without the +dataset, which a national report shares with its local-area report. Reports +created before the keys have none. +""" + +from __future__ import annotations + +from dataclasses import replace +from datetime import timedelta +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +import pytest + +from policyengine_api.data.v2.models import ( + Dataset, + Household, + Region, + RegionType, + Report, + Simulation, + SimulationType, + TaxBenefitModel, + TaxBenefitModelVersion, +) +from policyengine_api.query_parameters import MAXIMUM_LISTED_REPORTS +from tests.unit.v2.sqlite_report_rows import ( + STORED_AT, + ReportPlace, + ReportToStore, + report_service, + sqlite_sessions, + store_economy_report, +) + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.orm import sessionmaker + from sqlmodel import Session + + from policyengine_api.services.v2.stored_reports.services import ( + V2StoredReportService, + ) + +# The shared key of the proposal's worked example, and another report's. +DATASET_INDEPENDENT_KEY = "bcabc98b4e0d3eff" * 4 +OTHER_DATASET_INDEPENDENT_KEY = "4bb1299756cb0a89" * 4 +KEYED = ReportToStore(dataset_independent_key=DATASET_INDEPENDENT_KEY) + + +@pytest.fixture +def sessions() -> Iterator[sessionmaker[Session]]: + yield from sqlite_sessions() + + +@pytest.fixture +def reports(sessions: sessionmaker[Session]) -> V2StoredReportService: + return report_service(sessions) + + +def store_catalog(session: Session, country: str) -> ReportPlace: + """Store a catalog version of the country's model with its national region.""" + model = TaxBenefitModel(name=f"policyengine-{country}") + version = TaxBenefitModelVersion( + model=model, + version=f"keys-{uuid4().hex[:8]}", + current_law_id=1, + metadata_time_periods=[2026], + ) + dataset = Dataset( + name=f"national_{country}_2026", + year=2026, + tax_benefit_model_version=version, + ) + session.add_all([model, version, dataset]) + session.flush() + region = Region( + code=country, + label=country.upper(), + region_type=RegionType.NATIONAL, + tax_benefit_model_version_id=version.id, + default_dataset_id=dataset.id, + ) + session.add(region) + session.commit() + return ReportPlace( + country=country, + model_id=model.id, + version_id=version.id, + dataset_id=dataset.id, + region_id=region.id, + ) + + +def store_household_report( + session: Session, catalog: ReportPlace, *, dataset_independent_key: str +) -> UUID: + """Store a household report of current law against itself.""" + household = Household( + country_id=catalog.country, + default_year=2026, + household_data={"people": {"you": {"age": {"2026": 40}}}}, + canonicalization_version=1, + content_hash=uuid4().hex * 2, + ) + session.add(household) + session.flush() + simulations = [ + Simulation( + simulation_type=SimulationType.HOUSEHOLD, + household_id=household.id, + tax_benefit_model_version_id=catalog.version_id, + year=2026, + ) + for _ in range(2) + ] + session.add_all(simulations) + session.flush() + report = Report( + label="My household", + country=catalog.country, + tax_benefit_model_id=catalog.model_id, + baseline_simulation_id=simulations[0].id, + reform_simulation_id=simulations[1].id, + household_id=household.id, + year=2026, + inputs={}, + created_at=STORED_AT, + key_version=1, + definition_key=uuid4().hex * 2, + dataset_independent_key=dataset_independent_key, + ) + session.add(report) + session.commit() + return report.id + + +def test_every_kind_of_report_read_answers_its_dataset_independent_key( + sessions: sessionmaker[Session], reports: V2StoredReportService +) -> None: + with sessions() as session: + catalog = store_catalog(session, "uk") + economy_id = store_economy_report(session, catalog, KEYED) + window_id = store_economy_report( + session, + catalog, + ReportToStore( + dataset_independent_key=OTHER_DATASET_INDEPENDENT_KEY, + budget_window_size=3, + ), + ) + household_id = store_household_report( + session, catalog, dataset_independent_key=DATASET_INDEPENDENT_KEY + ) + + reads = { + report_id: reports.get_report(country_id="uk", report_id=report_id) + for report_id in (economy_id, window_id, household_id) + } + listed = reports.list_reports( + country_id="uk", report_ids=[economy_id, window_id, household_id] + ) + + assert { + report_id: read.dataset_independent_key for report_id, read in reads.items() + } == { + economy_id: DATASET_INDEPENDENT_KEY, + window_id: OTHER_DATASET_INDEPENDENT_KEY, + household_id: DATASET_INDEPENDENT_KEY, + } + assert [(item.kind, item.dataset_independent_key) for item in listed] == [ + ("economy", DATASET_INDEPENDENT_KEY), + ("budget_window", OTHER_DATASET_INDEPENDENT_KEY), + ("household", DATASET_INDEPENDENT_KEY), + ] + + +def test_a_report_created_before_the_keys_reads_no_dataset_independent_key( + sessions: sessionmaker[Session], reports: V2StoredReportService +) -> None: + with sessions() as session: + report_id = store_economy_report( + session, + store_catalog(session, "uk"), + ReportToStore(dataset_independent_key=None), + ) + + read = reports.get_report(country_id="uk", report_id=report_id) + + assert read.dataset_independent_key is None + + +def test_a_search_finds_the_countrys_reports_with_the_key_newest_first( + sessions: sessionmaker[Session], reports: V2StoredReportService +) -> None: + with sessions() as session: + uk = store_catalog(session, "uk") + us = store_catalog(session, "us") + oldest, newest, middle = ( + store_economy_report( + session, + uk, + replace(KEYED, created_at=STORED_AT + timedelta(minutes=minutes)), + ) + for minutes in (0, 20, 10) + ) + for other_key in (OTHER_DATASET_INDEPENDENT_KEY, None): + store_economy_report( + session, uk, ReportToStore(dataset_independent_key=other_key) + ) + store_economy_report(session, us, KEYED) + + found = reports.search_reports( + country_id="uk", dataset_independent_key=DATASET_INDEPENDENT_KEY + ) + + assert [read.id for read in found] == [newest, middle, oldest] + assert all( + read.dataset_independent_key == DATASET_INDEPENDENT_KEY for read in found + ) + assert all(not hasattr(read, "result") for read in found) + + +def test_a_search_puts_the_larger_id_first_among_reports_created_together( + sessions: sessionmaker[Session], reports: V2StoredReportService +) -> None: + smaller_id = UUID("00000000-0000-4000-8000-000000000001") + larger_id = UUID("ffffffff-0000-4000-8000-000000000001") + with sessions() as session: + catalog = store_catalog(session, "uk") + for report_id in (smaller_id, larger_id): + store_economy_report(session, catalog, replace(KEYED, report_id=report_id)) + + found = reports.search_reports( + country_id="uk", dataset_independent_key=DATASET_INDEPENDENT_KEY + ) + + assert [read.id for read in found] == [larger_id, smaller_id] + + +def test_a_search_answers_at_most_the_newest_twenty_five_reports( + sessions: sessionmaker[Session], reports: V2StoredReportService +) -> None: + with sessions() as session: + catalog = store_catalog(session, "uk") + stored = [ + store_economy_report( + session, + catalog, + replace(KEYED, created_at=STORED_AT + timedelta(minutes=minutes)), + ) + for minutes in range(MAXIMUM_LISTED_REPORTS + 1) + ] + + found = reports.search_reports( + country_id="uk", dataset_independent_key=DATASET_INDEPENDENT_KEY + ) + + assert [read.id for read in found] == stored[:0:-1] + + +def test_a_search_with_a_key_no_report_has_finds_none( + sessions: sessionmaker[Session], reports: V2StoredReportService +) -> None: + with sessions() as session: + store_economy_report(session, store_catalog(session, "uk"), KEYED) + + found = reports.search_reports( + country_id="uk", dataset_independent_key=OTHER_DATASET_INDEPENDENT_KEY + ) + + assert found == () diff --git a/tests/unit/v2/test_report_read_routes.py b/tests/unit/v2/test_report_read_routes.py index 1d220199a..f9a935d96 100644 --- a/tests/unit/v2/test_report_read_routes.py +++ b/tests/unit/v2/test_report_read_routes.py @@ -234,3 +234,103 @@ def test_simulation_read_failures_answer_typed_errors( assert response.status_code == status_code assert response.json()["status"] == "error" + + +DATASET_INDEPENDENT_KEY = "bcabc98b4e0d3eff" * 4 + + +def test_a_report_and_its_listing_answer_its_dataset_independent_key() -> None: + service = FakeReportService( + listed=(report_summary_read(dataset_independent_key=DATASET_INDEPENDENT_KEY),) + ) + service.report = replace( + service.report, dataset_independent_key=DATASET_INDEPENDENT_KEY + ) + client = report_route_client(reports=service) + + item = client.get(f"/v2/reports/{REPORT_ID}?country_id=us").json()["result"]["item"] + items = client.get(f"/v2/reports?country_id=us&report_id={REPORT_ID}").json()[ + "result" + ]["items"] + + assert item["dataset_independent_key"] == DATASET_INDEPENDENT_KEY + assert [listed["dataset_independent_key"] for listed in items] == [ + DATASET_INDEPENDENT_KEY + ] + + +def test_searching_by_the_dataset_independent_key_answers_the_reports_found() -> None: + newer, older = uuid4(), uuid4() + service = FakeReportService( + found=tuple( + report_summary_read( + id=report_id, dataset_independent_key=DATASET_INDEPENDENT_KEY + ) + for report_id in (newer, older) + ) + ) + + response = report_route_client(reports=service).get( + f"/v2/reports?country_id=uk&dataset_independent_key={DATASET_INDEPENDENT_KEY}" + ) + + assert response.status_code == 200 + items = response.json()["result"]["items"] + assert [item["id"] for item in items] == [str(newer), str(older)] + assert all("result" not in item for item in items) + assert service.calls == [ + ( + "search_reports", + {"country_id": "uk", "dataset_independent_key": DATASET_INDEPENDENT_KEY}, + ) + ] + + +@pytest.mark.parametrize( + "query", + [ + f"country_id=uk&report_id={uuid4()}&dataset_independent_key={DATASET_INDEPENDENT_KEY}", + f"country_id=uk&dataset_independent_key={DATASET_INDEPENDENT_KEY.upper()}", + f"country_id=uk&dataset_independent_key={DATASET_INDEPENDENT_KEY[:-1]}", + f"country_id=uk&dataset_independent_key={DATASET_INDEPENDENT_KEY[:-1]}g", + ( + f"country_id=uk&dataset_independent_key={DATASET_INDEPENDENT_KEY}" + f"&dataset_independent_key={DATASET_INDEPENDENT_KEY}" + ), + f"dataset_independent_key={DATASET_INDEPENDENT_KEY}", + ], + ids=[ + "both-filters", + "uppercase", + "too-short", + "not-hexadecimal", + "repeated", + "no-country", + ], +) +def test_searching_refuses_a_malformed_query(query: str) -> None: + service = FakeReportService() + + response = report_route_client(reports=service).get(f"/v2/reports?{query}") + + assert response.status_code == 422 + assert service.calls == [] + + +def test_the_openapi_document_gives_the_list_either_filter() -> None: + document = report_route_client().get("/v2/openapi.json").json() + + parameters = { + parameter["name"]: parameter + for parameter in document["paths"]["/v2/reports"]["get"]["parameters"] + } + + assert parameters["country_id"]["required"] is True + assert parameters["report_id"]["required"] is False + assert parameters["dataset_independent_key"]["required"] is False + key_schema = next( + schema + for schema in parameters["dataset_independent_key"]["schema"]["anyOf"] + if schema.get("type") == "string" + ) + assert key_schema["pattern"] == "^[0-9a-f]{64}$" diff --git a/tests/unit/v2/test_report_requester_routes.py b/tests/unit/v2/test_report_requester_routes.py index 301ad3a2e..39d7be3b2 100644 --- a/tests/unit/v2/test_report_requester_routes.py +++ b/tests/unit/v2/test_report_requester_routes.py @@ -86,8 +86,9 @@ def test_a_malformed_browser_credential_is_refused_before_the_service() -> None: ("POST", f"/v2/reports/{REPORT_ID}/runs?country_id=us", {}), ("GET", f"/v2/reports/{REPORT_ID}?country_id=us", None), ("GET", f"/v2/reports?country_id=us&report_id={REPORT_ID}", None), + ("GET", f"/v2/reports?country_id=us&dataset_independent_key={'a' * 64}", None), ], - ids=["create", "rerun", "one-report", "report-list"], + ids=["create", "rerun", "one-report", "report-list", "report-search"], ) def test_a_browser_credential_naming_a_signed_in_user_answers_401( method: str, path: str, body: dict[str, object] | None diff --git a/tests/unit/v2/test_report_rerun_service.py b/tests/unit/v2/test_report_rerun_service.py index ea14728d7..bd948a040 100644 --- a/tests/unit/v2/test_report_rerun_service.py +++ b/tests/unit/v2/test_report_rerun_service.py @@ -85,6 +85,7 @@ def stored_rows(**report_columns: object) -> StoredReportRows: report=report, region_code="us", dataset_name=None, + on_local_area_dataset=False, baseline_simulation=None, reform_simulation=None, run=None,