Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
a5cc4ce
Merge C1 · 29's branch: send no dataset_name to the simulation service
anth-volk Nov 13, 2026
1dd03b2
Test reading a report's dataset-independent key and finding reports b…
anth-volk Nov 13, 2026
1840e89
Answer each report's dataset-independent key, and find reports by it
anth-volk Nov 13, 2026
130bf85
Test passing an economy report's data_source to the simulation servic…
anth-volk Nov 13, 2026
77636a8
Pass an economy report's data_source to the simulation service
anth-volk Nov 13, 2026
153425a
Merge remote-tracking branch 'origin/v2plan/c1-25-shared-report-keys'…
anth-volk Nov 13, 2026
a50c195
Test a local-area report's read: its areas and withheld codes (failing)
anth-volk Nov 13, 2026
a21ae3e
Read a local-area report's data_source, areas and withheld codes
anth-volk Nov 13, 2026
6df4a7e
Merge #4303's nullable per-user columns into C1 · 30
anth-volk Nov 13, 2026
b480ebd
Fix review round 1's findings: one data_source alias, docs and tests
anth-volk Nov 13, 2026
8f9771d
Fix review round 2's findings: UK requests in the refusal stand-ins
anth-volk Nov 13, 2026
ec5a83c
Merge #4303's credential test change into C1 · 30
anth-volk Nov 16, 2026
8c1dfa7
Merge master, with #4303 merged, into C1 · 30
anth-volk Nov 16, 2026
5ac9e96
Merge master's release of #4303 into C1 · 30
anth-volk Nov 16, 2026
d5bd228
Merge master, with #4297's invitations, into C1 · 30
anth-volk Nov 16, 2026
fa2b8be
Merge master, with #4319's column-grant fix, into C1 · 30
anth-volk Nov 17, 2026
4bb1588
Merge master, with C1 · 28's #4313, into C1 · 30
anth-volk Nov 17, 2026
ff3a294
Merge master, with C1 · 29's #4307, into C1 · 30
anth-volk Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/c1-30-data-source-and-keys.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`POST /v2/reports` takes `data_source: "local_area_dataset"` on an economy report and passes it to the simulation service, which calculates a UK-wide or nation report on the dataset its catalog gives local areas. Every report read answers its `dataset_independent_key`, `GET /v2/reports?dataset_independent_key=` finds the country's reports that share one, and a local-area report's read says `data_source` and lists the constituency and local-authority codes disclosure control withheld.
14 changes: 14 additions & 0 deletions policyengine_api/fastapi_routes/resource_service_protocols.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,20 @@ def list_reports(
"""
...

def search_reports(
self,
*,
country_id: str,
dataset_independent_key: str,
requester: AccountPrincipal | None = None,
) -> tuple[StoredReportSummaryRead, ...]:
"""Return the country's reports with the dataset-independent key, newest first.

At most ``MAXIMUM_LISTED_REPORTS``, the ones created last, each read
as ``list_reports`` reads it.
"""
...

def create_report(
self,
report_input: ReportCreationInput,
Expand Down
35 changes: 27 additions & 8 deletions policyengine_api/fastapi_routes/v2/reports/response_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from policyengine_api.fastapi_routes.v2.simulation_resources_support import (
error_response,
)
from policyengine_api.services.v2.simulation_service.types import ReportDataSource
from policyengine_api.services.v2.stored_reports.budget_window_results.types import (
BudgetWindowResult,
)
Expand Down Expand Up @@ -90,13 +91,18 @@ class ReportDefinitionItem(StrictReportAPIModel):
a report the simulation service shares between requests records no one
caller's. ``current_versions`` are the versions the Public API runs now,
so a run on other versions is out of date. ``options`` are the options
the report was created with.
the report was created with. ``dataset_independent_key`` is the
simulation service's key for the report's definition without its
dataset, which a national report shares with its local-area report: find
reports by it with ``GET /v2/reports?dataset_independent_key=``. It is
null for a report stored before the keys.
"""

id: UUID
country_id: str
label: str | None
description: str | None
dataset_independent_key: str | None
year: int | None
baseline_simulation: ReportSimulationItem | None
reform_simulation: ReportSimulationItem | None
Expand All @@ -108,16 +114,23 @@ class ReportDefinitionItem(StrictReportAPIModel):
class ReportSummaryItem(ReportDefinitionItem):
"""An economy report comparing a baseline and a reform simulation.

Both simulations use the region and dataset named. ``run`` is the run
the read answers: the one it names, else the requesting user's newest
recorded run, else the newest succeeded run that wasn't forced, else the
newest run that wasn't forced, whatever its versions. ``status`` is that
run's, or ``not_run`` without one.
Both simulations use the region and dataset named. ``data_source`` is
``local_area_dataset`` for a local-area report: a UK-wide or nation
report on the dataset the catalog version gives local areas, whose
result also has each constituency's and local authority's impact and the
codes of those withheld. A request with that data_source creates one,
but the request's field isn't stored: the read derives it from the
report's region and dataset. It is null for every other report.
``run`` is the run the read answers: the one it names, else the
requesting user's newest recorded run, else the newest succeeded run
that wasn't forced, else the newest run that wasn't forced, whatever its
versions. ``status`` is that run's, or ``not_run`` without one.
"""

kind: Literal["economy"]
region_code: str | None
dataset_name: str | None
data_source: ReportDataSource | None
status: Literal["not_run", "pending", "running", "succeeded", "failed"]
run: ReportRunItem | None

Expand Down Expand Up @@ -272,14 +285,16 @@ class BudgetWindowReportSummaryItem(StrictReportAPIModel):
``current_versions`` are the versions the Public API runs now, so a run
on other versions is out of date. ``options`` are the options the report
was created with. ``label`` and ``description`` are the requesting user's
saved report's, null otherwise.
saved report's, null otherwise. ``dataset_independent_key`` is as
``ReportDefinitionItem`` describes it.
"""

id: UUID
country_id: str
kind: Literal["budget_window"]
label: str | None
description: str | None
dataset_independent_key: str | None
year: int | None
budget_window_size: int
region_code: str | None
Expand Down Expand Up @@ -348,7 +363,11 @@ class ReportDetailResult(StrictReportAPIModel):


class ReportListResult(StrictReportAPIModel):
"""The listed reports in ``items``, in the order named, without results."""
"""The listed reports in ``items``, without results.

Reports chosen by ``report_id`` are in the order named; reports found by
``dataset_independent_key`` are newest created first.
"""

items: list[
Annotated[
Expand Down
65 changes: 49 additions & 16 deletions policyengine_api/fastapi_routes/v2/reports/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,38 @@ def _requester_refused(
)


def _list_or_search(
service: V2StoredReportResourceService,
query: ReportCollectionQuery,
requester: AccountPrincipal | None,
) -> ReportListResponse | JSONResponse:
"""List the reports the query names, or find those with its key.

Each is read as the requester sees it; a browser credential naming a
signed-in user is refused with a 401.
"""
try:
if query.dataset_independent_key is None:
reports = service.list_reports(
country_id=query.country_id,
report_ids=query.report_id,
requester=requester,
)
else:
reports = service.search_reports(
country_id=query.country_id,
dataset_independent_key=query.dataset_independent_key,
requester=requester,
)
except StandardUserRequiresSignInError as error:
return _requester_refused(error)
return ReportListResponse(
result=ReportListResult(
items=[report_summary_item(report) for report in reports]
)
)


# The two routes that record a request take its user's credential, but
# optionally: FastAPI lists the schemes a credential may use, and an empty
# requirement says a request may send none.
Expand Down Expand Up @@ -437,7 +469,14 @@ def build_v2_report_router(dependencies: NativeRouteDependencies) -> APIRouter:
"those policies set, then has the simulation service create the "
"report, its two "
"simulations and their runs on its default worker's PolicyEngine.py "
"version, and start the run. A household report (kind household) "
"version, and start the run. With data_source local_area_dataset, "
"an economy report calculates on the dataset the catalog version "
"gives local areas, and its result also has each constituency's and "
"local authority's impact and the codes of those withheld; the "
"simulation service takes it for uk and its four nations only, on a "
"release that gives local areas a dataset of their own, and the "
"Public API answers its refusal with 422. A household report (kind "
"household) "
"is checked the same way, except that the country must have "
"household calculations instead (a household-only country does), "
"and the household it names must be stored in the country, with a "
Expand Down Expand Up @@ -479,7 +518,7 @@ def create() -> ReportDetailResponse | JSONResponse:
"/reports",
response_model=ReportListResponse,
responses={401: READ_RESPONSES[401], 403: READ_RESPONSES[403]},
summary="List chosen reports",
summary="List chosen reports, or find reports by their dataset-independent key",
description=(
"Lists the country's reports among the report_id values (at most "
"25), in the order named and once each, each with the run a read of "
Expand All @@ -488,7 +527,13 @@ def create() -> ReportDetailResponse | JSONResponse:
"description are that user's saved report's, and a report the user "
"asked to run shows the run of their newest request; without one, "
"the label is null. Unknown reports and other countries' are left "
"out."
"out. With dataset_independent_key in place of report_id, lists the "
"same way the country's reports whose dataset_independent_key it "
"is, newest created first, at most the 25 created last: reports "
"that differ only in their dataset, such as a national report and "
"its local-area report (data_source local_area_dataset), and the "
"same reports on another data release. Give one of the two, never "
"both."
),
openapi_extra=OPTIONAL_USER_CREDENTIAL,
)
Expand All @@ -497,19 +542,7 @@ def list_reports(
requester: Annotated[AccountPrincipal | None, Depends(optional_user)],
) -> ReportListResponse | JSONResponse:
def read() -> ReportListResponse | JSONResponse:
try:
reports = service_factory().list_reports(
country_id=query.country_id,
report_ids=query.report_id,
requester=requester,
)
except StandardUserRequiresSignInError as error:
return _requester_refused(error)
return ReportListResponse(
result=ReportListResult(
items=[report_summary_item(report) for report in reports]
)
)
return _list_or_search(service_factory(), query, requester)

return _report_operation(read)

Expand Down
33 changes: 19 additions & 14 deletions policyengine_api/libs/simulation_reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,20 @@
current law). A report request names the country, ``report_type``
(``economy``), the year and region its two simulations share, each
simulation's policy and dynamic, ``include_cliffs``, and the SPM settings
only when the caller gave them. No request names a dataset: the simulation
service calculates on the dataset its catalog version gives the region. A
budget window goes to the same ``POST /reports`` as ``report_type``
``budget_window``, naming its first year and ``budget_window_size`` in place
of the year, and no ``include_cliffs``. A household report goes there too,
as ``report_type`` ``household``, naming the stored household, each
simulation's policy (the reform null for a report of the baseline alone),
but no year: the simulation service calculates the household's default
year, adding every variable the household lacks. None names a
PolicyEngine.py version: the simulation service runs its default worker's.
None names a label or a description, which belong to users' saved reports.
The simulation service answers:
and ``data_source`` only when the caller gave them. No request names a
dataset: the simulation service calculates on the dataset its catalog
version gives the region, or, for ``data_source`` ``local_area_dataset``,
the one it gives local areas. A budget window goes to the same
``POST /reports`` as ``report_type`` ``budget_window``, naming its first
year and ``budget_window_size`` in place of the year, and no
``include_cliffs``. A household report goes there too, as ``report_type``
``household``, naming the stored household, each simulation's policy (the
reform null for a report of the baseline alone), but no year: the
simulation service calculates the household's default year, adding every
variable the household lacks. None names a PolicyEngine.py version: the
simulation service runs its default worker's. None names a label or a
description, which belong to users' saved reports. The simulation service
answers:

- 201 or 202 with the created record's ID and its first run's ID: for a
budget window, its budget-window run's ID (``budget_window_run_id``). One
Expand Down Expand Up @@ -253,9 +255,12 @@ def _report_definition(
def report_request(report: EconomyReportCreationInput) -> dict[str, Any]:
"""Return the simulation service's request body for the report.

``spm`` is sent only when the caller gave it.
``spm`` and ``data_source`` are each sent only when the caller gave them.
"""
return {**_report_definition(report), "include_cliffs": report.include_cliffs}
body = {**_report_definition(report), "include_cliffs": report.include_cliffs}
if report.data_source is not None:
body["data_source"] = report.data_source
return body


def budget_window_request(report: BudgetWindowReportCreationInput) -> dict[str, Any]:
Expand Down
40 changes: 36 additions & 4 deletions policyengine_api/query_parameters.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,17 @@ def validate_legacy_user_id(value: str) -> str:
),
]
ResourceId = Annotated[UUID, Field(description="Exact resource UUID")]
# A report's dataset-independent key, as the simulation service stores it: the
# SHA-256 of its definition without the dataset, in lowercase hexadecimal.
DatasetIndependentKey = Annotated[
str,
Field(
pattern=r"^[0-9a-f]{64}$",
description=(
"A report's dataset_independent_key: 64 lowercase hexadecimal digits"
),
),
]
DefaultYear = Annotated[
int,
Field(
Expand Down Expand Up @@ -459,16 +470,37 @@ class ReportRunsQuery(CountryQuery):


class ReportCollectionQuery(CountryQuery):
"""Query contract for listing chosen reports of a country.
"""Query contract for listing chosen reports of a country, or finding them.

``report_id`` is repeated once per report, at most 25 times.
Give either ``report_id``, repeated once per report and at most 25
times, or one ``dataset_independent_key``, which finds the country's
reports that share it; never both.
"""

report_id: list[ResourceId] = Field(
min_length=1,
default_factory=list,
max_length=MAXIMUM_LISTED_REPORTS,
description="A report to list; repeat the key for each report",
description=(
"A report to list; repeat the key for each report. Not with "
"dataset_independent_key"
),
)
dataset_independent_key: DatasetIndependentKey | None = Field(
default=None,
description=(
"Find the country's reports with this dataset_independent_key, which "
"a national report shares with its local-area report. Not with "
"report_id"
),
)

@model_validator(mode="after")
def require_one_filter(self) -> ReportCollectionQuery:
"""Refuse a query naming both report_id and a key, or neither."""
if bool(self.report_id) == (self.dataset_independent_key is not None):
msg = "Give either report_id or dataset_independent_key"
raise ValueError(msg)
return self


class SimulationCreateQuery(CountryQuery):
Expand Down
37 changes: 33 additions & 4 deletions policyengine_api/services/v2/simulation_service/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,8 +202,9 @@ class EconomySimulationCreationInput(EconomySimulationFields):
country_id: CountryId


def _omit_spm_default(schema: dict[str, Any]) -> None:
# Omitting spm sends no SPM settings; an explicit null is not a selection.
def _omit_null_default(schema: dict[str, Any]) -> None:
# Omitting the field (spm, data_source) sends nothing; an explicit null is
# refused, so the schema documents no null default.
schema.pop("default", None)


Expand Down Expand Up @@ -233,7 +234,7 @@ class ReportDefinitionFields(StrictSubmissionModel):
baseline: PolicyChoice
reform: PolicyChoice
spm: SPMSelection | SkipJsonSchema[None] = Field(
default=None, json_schema_extra=_omit_spm_default
default=None, json_schema_extra=_omit_null_default
)

@field_validator("spm", mode="before")
Expand All @@ -246,16 +247,44 @@ def require_spm_object(cls, value: object) -> object:
return value


# The dataset an economy report may ask for in place of its region's: the one
# the catalog version gives local areas. A report read names it for a UK-wide
# or nation report on that dataset (``stored_reports.types.ReportSummaryRead``),
# never for a local area's own report, which is on it too.
ReportDataSource = Literal["local_area_dataset"]
LOCAL_AREA_DATASET: ReportDataSource = "local_area_dataset"


class EconomyReportFields(ReportDefinitionFields):
"""An economy report: a baseline and a reform simulation of one region's data.

``include_cliffs`` also calculates the baseline's and reform's cliff
impact; the other fields are as ``ReportDefinitionFields`` describes
impact. ``data_source`` ``local_area_dataset`` calculates both
simulations on the dataset the catalog version gives local areas
instead of the region's, so the result also has each constituency's and
local authority's impact and the codes of those disclosure control
withheld; omit it, rather than sending null, for the region's dataset.
The simulation service takes it only for ``uk`` and the four
``country/*`` nations, on a release that gives local areas a dataset of
their own, and refuses it with 422 otherwise: the Public API checks only
its value. The other fields are as ``ReportDefinitionFields`` describes
them.
"""

kind: Literal["economy"]
include_cliffs: StrictBool = False
data_source: ReportDataSource | SkipJsonSchema[None] = Field(
default=None, json_schema_extra=_omit_null_default
)

@field_validator("data_source", mode="before")
@classmethod
def require_data_source_value(cls, value: object) -> object:
"""Refuse an explicit null ``data_source``; omit the field instead."""
if value is None:
msg = "data_source must be local_area_dataset when supplied"
raise ValueError(msg)
return value


class EconomyReportCreationInput(EconomyReportFields):
Expand Down
Loading
Loading