From c6e9426e43f4ff0a6a1e7e5ef14647b76d0d6be6 Mon Sep 17 00:00:00 2001 From: saileshwar-skyflow Date: Mon, 9 Nov 2026 14:11:05 +0530 Subject: [PATCH 1/3] SK-3289: address Detect docs and sample parity feedback (Python) Fix the Detect README and sample gaps raised in the final Detect feedback, for both the flowvault and skyvault Python SDKs. flowvault README: - drop the unsupported SKYFLOW_ID file source (the enum is BASE64/PRESIGNED_URL only) - document Entity/CustomType include_prefix and prefix_override, and CustomType deidentification_type and destination - add where-to-create pointers for the configuration id, token groups and redaction patterns - add response examples and a common-errors section - make the re-identify example a round trip; branch re-identify file on output_type skyvault README: - make the re-identify example a round trip that stores tokens in a table.column; branch re-identify file on output_type - add a bold warning that deidentify_file no longer waits by default - note the server's default toxicity check; fix the stale v1 EOL banner samples (both SDKs): print request_id in Detect error handlers. Nomenclature: drop the "Detect V2" label from both READMEs and the samples README; rename the migration guide to docs/migrate_detect.md, retitle it, cite skyflow 3.0.0, and promote the deidentify_file behaviour change. setup.py version is managed by release automation and left unchanged; the guide now names 3.0.0 as the target. Co-Authored-By: Claude Opus 4.8 --- ...e_detect_v1_to_v2.md => migrate_detect.md} | 36 ++++--- flowvault/README.md | 100 +++++++++++++++--- flowvault/samples/README.md | 2 +- .../samples/detect_api/check_guardrails.py | 1 + .../samples/detect_api/deidentify_file.py | 3 + .../samples/detect_api/deidentify_string.py | 2 + flowvault/samples/detect_api/get_run.py | 1 + .../samples/detect_api/reidentify_file.py | 1 + .../samples/detect_api/reidentify_string.py | 1 + skyvault/README.md | 46 ++++++-- .../samples/detect_api/check_guardrails.py | 1 + .../samples/detect_api/deidentify_file.py | 1 + .../samples/detect_api/deidentify_string.py | 2 + skyvault/samples/detect_api/get_run.py | 1 + .../samples/detect_api/reidentify_file.py | 1 + .../samples/detect_api/reidentify_string.py | 1 + 16 files changed, 159 insertions(+), 41 deletions(-) rename docs/{migrate_detect_v1_to_v2.md => migrate_detect.md} (91%) diff --git a/docs/migrate_detect_v1_to_v2.md b/docs/migrate_detect.md similarity index 91% rename from docs/migrate_detect_v1_to_v2.md rename to docs/migrate_detect.md index 28e39524..12b3d45e 100644 --- a/docs/migrate_detect_v1_to_v2.md +++ b/docs/migrate_detect.md @@ -1,14 +1,22 @@ -# Migrate Detect from V1 to V2 (skyvault Python SDK) - -The skyvault Python SDK replaces the Detect V1 methods with the V2 methods, using the -same method and class names as the flowvault Detect SDK. There is no `V2` suffix -anywhere. `deidentify_file` keeps its name and takes the V2 `DeidentifyFileRequest`, -returning the V2 `DeidentifyFileResponse`. Because the V1 methods and classes are -removed, there is no name conflict. All V2 classes live in `skyflow.vault.detect` and -the enums in `skyflow.utils.enums`. - -V1 callers must migrate when they upgrade, so this ships as a major release. Client -creation and the `skyflow_client.detect()` accessor do not change. +# Migrate to the new Detect methods (skyflow 3.0.0) + +The skyvault Python SDK replaces the previous Detect methods with the new Detect methods, +using the same method and class names as the flowvault Detect SDK — there is no version +suffix on any of them. `deidentify_file` keeps its name and now takes the new +`DeidentifyFileRequest`, returning the new `DeidentifyFileResponse`. Because the previous +methods and classes are removed, there is no name conflict. All Detect classes live in +`skyflow.vault.detect` and the enums in `skyflow.utils.enums`. + +Callers of the previous Detect methods must migrate when they upgrade, so this ships in the +next major release, **skyflow 3.0.0** — pin or upgrade accordingly (`pip install "skyflow==3.0.0"`). +Client creation and the `skyflow_client.detect()` accessor do not change. + +> **⚠️ Biggest behaviour change:** `deidentify_file` **no longer waits for the result by +> default.** It used to always poll; now, without `poll_options`, it submits and returns +> immediately — `status` is `None`, `output` is empty, and nothing is written to disk. Pass +> `poll_options` to block until the run finishes, or retrieve the result later with `get_run`. +> Code ported unchanged runs without error but produces no outputs, so the failure surfaces +> downstream rather than at the call site. ## What changes @@ -355,7 +363,7 @@ fields as the V2 `DeidentifyFileResponse` (`run_id`, `status`, `output_type`, `o ## Timeline -| Release | V1 methods | V2 methods | +| Release | Previous Detect methods | New Detect methods | | --- | --- | --- | -| skyvault 2.1.x | available | not available | -| next major | removed | available under the flowvault names | +| skyflow 2.1.x | available | not available | +| skyflow 3.0.0 | removed | available under the flowvault names | diff --git a/flowvault/README.md b/flowvault/README.md index 3e1d2773..faf17b12 100644 --- a/flowvault/README.md +++ b/flowvault/README.md @@ -983,7 +983,7 @@ The same pattern applies to every operation, via its corresponding options class # Detect -Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The flowvault SDK exposes the Detect V2 API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. +Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The flowvault SDK exposes the Detect API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. Detect configurations are created in Studio or through the management API — the SDK does not create, update or delete them. Import request/response models from `skyflow.vault.detect` and enums from `skyflow.utils.enums`. @@ -1003,8 +1003,8 @@ detect = skyflow_client.detect() ## Before you start - **Role.** The service account needs the Detect Invoker, Detect Manager or Vault Owner role on the vault. Detect Invoker can de-identify data and write vault tokens into the vault; Detect Manager adds the ability to create, update and delete Detect configurations. Re-identifying values stored as `VAULT_TOKEN` is governed like detokenization: the caller also needs a policy granting the Detokenization action on the destination, unless it is a Vault Owner. -- **Configuration.** Pass `configuration_id` (the id of a stored Detect configuration) or an inline `configuration` (a `DetectConfiguration`). Both are optional: with neither set the SDK sends only the vault id and the run uses the vault's default Detect settings; sending both is rejected by the API with HTTP 400. -- **Destination for vault tokens.** An entity de-identified with `VAULT_TOKEN` stores its value in the token group named by `destination`. That token group must already exist in the vault; Detect writes into it but does not create it. +- **Configuration.** Pass `configuration_id` (the id of a stored Detect configuration) or an inline `configuration` (a `DetectConfiguration`). Both are optional: with neither set the SDK sends only the vault id and the run uses the vault's default Detect settings; sending both is rejected by the API with HTTP 400. Create and manage stored configurations — and copy their ids — in Studio (vault → **Detect**) or through the management API; the SDK does not create them. +- **Destination for vault tokens.** An entity de-identified with `VAULT_TOKEN` stores its value in the token group named by `destination`. That token group must already exist in the vault; Detect writes into it but does not create it. Create token groups, and the named redaction patterns that re-identification applies, in the vault schema in Studio. - **API reference.** Every Detect request, response and configuration class is listed in the [API Reference](../docs/api_reference.md). ## deidentify_string @@ -1060,9 +1060,9 @@ Returns a `DeidentifyStringResponse` with `processed_text`, `entities` (each a ` | Class | Fields | Notes | |-------|--------|-------| | `Detect` | `entities`, `object_entities`, `restrict`, `skip`, `custom_types`, `return_entities` | `restrict` / `skip` are regexes always / never treated as sensitive. `return_entities` is `EXCLUDE_SENSITIVE_DATA` (server default), `ALL` or `NONE`. | -| `Entity` | `entity_type`, `deidentification_type`, `destination`, `transformation` | `entity_type` defaults to `ALL`. `deidentification_type` is `ENTITY_UNIQUE_COUNTER` (server default), `ENTITY_ONLY`, `MASK` or `VAULT_TOKEN`. `destination` (a token group name) is used only with `VAULT_TOKEN`. | +| `Entity` | `entity_type`, `deidentification_type`, `destination`, `transformation`, `include_prefix`, `prefix_override` | `entity_type` defaults to `ALL`. `deidentification_type` is `ENTITY_UNIQUE_COUNTER` (server default), `ENTITY_ONLY`, `MASK` or `VAULT_TOKEN`. `destination` (a token group name) is used only with `VAULT_TOKEN`. `include_prefix` (`IncludePrefix.YES` / `IncludePrefix.NO`) and `prefix_override` also apply only to `VAULT_TOKEN`: `include_prefix=IncludePrefix.NO` stores a raw token that **cannot be re-identified** (a data-loss trap), and `prefix_override` replaces the entity-type label in the token with a custom one. | | `Transformation` / `ShiftDates` | `shift_dates`; `min_days`, `max_days` | Shifts detected dates by a random number of days within the range. | -| `CustomType` | `label`, `match` | A regex-defined entity type; `label` is reported as the entity type of its matches. | +| `CustomType` | `label`, `match`, `deidentification_type`, `destination`, `include_prefix`, `prefix_override` | A regex-defined entity type; `label` is reported as the entity type of its matches. `deidentification_type`, `destination`, `include_prefix` and `prefix_override` behave exactly as on `Entity`. | | `ObjectEntity` | `entity_type`, `deidentification_type` | Visual objects in images: `FACE`, `LICENSE_PLATE`, `LOGO`, `SIGNATURE` or `ALL`, handled with `REDACT` or `UNREDACT`. File inputs only. | | `FileMapping` | `source`, `destination`, `output_file_name_suffix`, `entities_destination`, `object_entities_destination` | Where the processed file and entity files are written. File inputs only. | | `Media` | `audio`, `document`, `image` | Media-type options. File inputs only. | @@ -1108,7 +1108,7 @@ configuration = DetectConfiguration( De-identifies a file. The API is asynchronous: the submit call returns a run id and the result is retrieved from the run. Build the request with `DeidentifyFileRequest`. -Supply the file in exactly one of three ways: `file_path`, `file`, or `value` plus a `DeidentifySourceType` `data_source` (`BASE64`, the `SKYFLOW_ID` of a file record in the vault, or `PRESIGNED_URL`). A local `file` / `file_path` is read when the request is sent, base64-encoded, and its `data_format` inferred from the extension. Passing more than one of them, or none, fails client-side validation before any network call. `configuration_id` / `configuration` follow the same rules as for strings (see [Before you start](#before-you-start)). +Supply the file in exactly one of three ways: `file_path`, `file`, or `value` plus a `DeidentifySourceType` `data_source` (`BASE64` or `PRESIGNED_URL`). flowvault does **not** support a Skyflow ID as a file source — its `DeidentifySourceType` has only `BASE64` and `PRESIGNED_URL`. A local `file` / `file_path` is read when the request is sent, base64-encoded, and its `data_format` inferred from the extension. Passing more than one of them, or none, fails client-side validation before any network call. `configuration_id` / `configuration` follow the same rules as for strings (see [Before you start](#before-you-start)). - **Submit only** (default): one call, returns `run_id`. `status` is `None` and nothing is written to disk — retrieve the result later with [`get_run`](#get_run). - **Submit and poll**: pass `poll_options`. The SDK polls with exponential backoff (1, 2, 4, 8, 16, then 32 seconds) until the run is `SUCCESS`, `FAILED` or `UNKNOWN`, or until `wait_time` (default 60 s, max 300) or `max_attempts` (default and max 23) is used up. If the budget runs out the response carries the run id with `IN_PROGRESS` or `QUEUED`, so you can resume with `get_run`. `output_directory` is valid only with `poll_options`; setting it without them fails client-side validation. @@ -1149,7 +1149,7 @@ else: # IN_PROGRESS or QUEUED: budget ran out, resume - `/data/processed/processed-entities-patient-notes.json` — the detected entities - `/data/processed/processed-object-entities-patient-notes.json` — object entities, when the run produced them -`output_directory` is created if it does not exist. **If you poll a local file and omit `output_directory`, these files are written to the current working directory** — often your project root or a server's working directory — so set `output_directory` explicitly. A request built from a `SKYFLOW_ID` or `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in those cases read `response.output` and write the files yourself. +`output_directory` is created if it does not exist. **If you poll a local file and omit `output_directory`, these files are written to the current working directory** — often your project root or a server's working directory — so set `output_directory` explicitly. A request built from a `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in that case read `response.output` and write the files yourself. **Supported formats and limits.** `data_format` accepts every `FileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. @@ -1200,18 +1200,37 @@ Synchronous. Re-identification looks tokens up in the vault, so it applies only Build the request with `ReidentifyStringRequest`: the tokenized text and an optional list of `RedactionLevel` rules. On flowvault vaults each rule targets a `token_group_name` and applies a named `redaction_pattern` configured in the vault. Without rules every token is replaced by its original value. +Only text that actually contains vault tokens can be re-identified, so the example below is a round trip: first de-identify with `VAULT_TOKEN` to mint tokens, then re-identify the `processed_text` that comes back. A vault token is an opaque value like `a1b2c3d4-...` (optionally prefixed, e.g. `[NAME_a1b2c3d4-...]`), not the `[NAME_1]` counter placeholder produced by `ENTITY_UNIQUE_COUNTER`. + ```python -from skyflow.vault.detect import ReidentifyStringRequest, RedactionLevel +from skyflow.vault.detect import ( + DeidentifyStringRequest, DetectConfiguration, Detect, Entity, + ReidentifyStringRequest, RedactionLevel, +) +from skyflow.utils.enums import EntityType, DeidentificationType + +# 1. De-identify with VAULT_TOKEN so the value is stored and can be re-identified later. +deidentified = skyflow_client.detect().deidentify_string( + DeidentifyStringRequest( + text='My name is John Doe.', + configuration=DetectConfiguration(detect=Detect(entities=[ + Entity(entity_type=EntityType.NAME, deidentification_type=DeidentificationType.VAULT_TOKEN, destination=''), + ])), + ) +) +# deidentified.processed_text now holds a vault token, e.g. 'My name is a1b2c3d4-...' +# 2. Re-identify those tokens. response = skyflow_client.detect().reidentify_string( ReidentifyStringRequest( - text='My name is [NAME_1].', + text=deidentified.processed_text, redaction_level=[ RedactionLevel(token_group_name='', redaction_pattern=''), ], ) ) print(response.processed_text) +# My name is John Doe. ``` > The flowvault `RedactionLevel` has only `token_group_name` and `redaction_pattern`; the skyvault SDK's has `entity_name` and `redaction_type` instead. Token groups and redaction patterns are configured in the vault. @@ -1227,6 +1246,7 @@ import base64 from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, DetectRunStatus +data_format = FileDataFormat.TXT with open('/data/redacted/notes.txt', 'rb') as handle: encoded = base64.b64encode(handle.read()).decode('utf-8') @@ -1234,23 +1254,26 @@ response = skyflow_client.detect().reidentify_file( ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=data_format, redaction_level=[RedactionLevel(token_group_name='', redaction_pattern='')], ) ) if response.status == DetectRunStatus.SUCCESS: for index, output in enumerate(response.output): - # output.processed_file is base64 or a presigned URL, per response.output_type - with open(f'reidentified-{index}.txt', 'wb') as out: - out.write(base64.b64decode(output.processed_file)) + # Branch on output_type: base64 bytes are decoded, a presigned URL is downloaded. + if response.output_type == ReidentifySourceType.BASE64: + with open(f'reidentified-{index}.{data_format.value}', 'wb') as out: + out.write(base64.b64decode(output.processed_file)) + else: # ReidentifySourceType.PRESIGNED_URL — processed_file is a URL to fetch + print('download re-identified file from', output.processed_file) ``` -Returns a `ReidentifyFileResponse` with `status`, `output_type`, `output` (a list of `FileOutput`) and `metrics`. +Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `FileOutput`) and `metrics`. ## check_guardrails -Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. The response is passed through as the API returns it. +Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. **When `check_toxicity` is omitted, the server still checks toxicity by default.** The response is passed through as the API returns it. ```python from skyflow.vault.detect import CheckGuardrailsRequest @@ -1268,6 +1291,53 @@ Returns a `CheckGuardrailsResponse` with `text`, `validation` (`GuardrailsValida All Detect methods raise `skyflow.error.SkyflowError` on validation failure, credential failure, or an API error — see [Error Handling](#error-handling). Client-side validation fails before any network call. +## Response examples and common errors + +Enum-typed fields (`status`, `output_type`, `validation`, …) are shown below by their string values. + +`deidentify_string`: + +```json +{ + "processed_text": "My name is [NAME_1], and my email is [EMAIL_ADDRESS_1].", + "entities": [ + { "token": "[NAME_1]", "value": "", "location": { "start_index": 11, "end_index": 19, "start_index_processed": 11, "end_index_processed": 19 }, "entity_type": "NAME", "entity_scores": { "NAME": 0.99 } } + ], + "metrics": { "word_count": 10, "character_count": 54 } +} +``` + +`deidentify_file` (polled) / `get_run`: + +```json +{ + "run_id": "run_a1b2c3d4", + "status": "SUCCESS", + "output_type": "BASE64", + "output": [ { "processed_file": "", "processed_file_type": "application/pdf", "processed_file_extension": "pdf" } ], + "metrics": { "pages": 3 }, + "message": null +} +``` + +`reidentify_string`: + +```json +{ "processed_text": "My name is John Doe.", "metrics": { "word_count": 5 } } +``` + +`check_guardrails`: + +```json +{ "text": "", "validation": "FAILED", "toxic": true, "denied_topic": false } +``` + +Common Detect errors (all raised as `SkyflowError`): + +- **HTTP 400 — both configuration sources set.** Passing `configuration_id` and `configuration` in the same request is rejected; pass at most one. +- **Missing token group.** A `VAULT_TOKEN` entity whose `destination` token group does not exist in the vault fails — create the token group first. +- **Unsupported format with `VAULT_TOKEN`.** `VAULT_TOKEN` is only valid for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`; other `data_format` values are rejected. + # Error Handling ## Two layers of errors diff --git a/flowvault/samples/README.md b/flowvault/samples/README.md index 0737bc10..f5b4fc79 100644 --- a/flowvault/samples/README.md +++ b/flowvault/samples/README.md @@ -44,7 +44,7 @@ python flowvault/samples/vault_api/insert_records.py | [upload_files.py](vault_api/upload_files.py) | Upload files to file columns (`file_path` / `base64` / `file_object`) | | [delete_files.py](vault_api/delete_files.py) | Delete files from file columns (by `skyflow_id` or `unique_values`) | -## Detect V2 +## Detect | Sample | Demonstrates | |---|---| diff --git a/flowvault/samples/detect_api/check_guardrails.py b/flowvault/samples/detect_api/check_guardrails.py index 54082337..5e338281 100644 --- a/flowvault/samples/detect_api/check_guardrails.py +++ b/flowvault/samples/detect_api/check_guardrails.py @@ -36,6 +36,7 @@ def perform_check_guardrails(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/deidentify_file.py b/flowvault/samples/detect_api/deidentify_file.py index 77375643..5edcb528 100644 --- a/flowvault/samples/detect_api/deidentify_file.py +++ b/flowvault/samples/detect_api/deidentify_file.py @@ -34,6 +34,7 @@ def perform_deidentify_file_submit_only(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -75,6 +76,7 @@ def perform_deidentify_file_poll(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -110,6 +112,7 @@ def perform_deidentify_file_presigned(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/deidentify_string.py b/flowvault/samples/detect_api/deidentify_string.py index c057b358..84165c6d 100644 --- a/flowvault/samples/detect_api/deidentify_string.py +++ b/flowvault/samples/detect_api/deidentify_string.py @@ -36,6 +36,7 @@ def perform_deidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -85,6 +86,7 @@ def perform_deidentify_string_inline(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/get_run.py b/flowvault/samples/detect_api/get_run.py index 10a39c2f..f68ae924 100644 --- a/flowvault/samples/detect_api/get_run.py +++ b/flowvault/samples/detect_api/get_run.py @@ -35,6 +35,7 @@ def perform_get_run(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/reidentify_file.py b/flowvault/samples/detect_api/reidentify_file.py index 01f4b994..08dc16c1 100644 --- a/flowvault/samples/detect_api/reidentify_file.py +++ b/flowvault/samples/detect_api/reidentify_file.py @@ -49,6 +49,7 @@ def perform_reidentify_file(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/reidentify_string.py b/flowvault/samples/detect_api/reidentify_string.py index 045e35e1..716e5ad4 100644 --- a/flowvault/samples/detect_api/reidentify_string.py +++ b/flowvault/samples/detect_api/reidentify_string.py @@ -59,6 +59,7 @@ def perform_reidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/README.md b/skyvault/README.md index 9ce5a7e3..e58fc21e 100644 --- a/skyvault/README.md +++ b/skyvault/README.md @@ -2,7 +2,7 @@ > **This is the current, recommended version of the Skyflow SDK.** V2.1.0 brings flexible auth, multi-vault support, native data types, and rich error diagnostics. > -> Migrating from v1? See the **[Migration Guide](https://github.com/skyflowapi/skyflow-python/blob/main/docs/migrate_to_v2.md)** for step-by-step instructions. V1 is in maintenance mode and will reach End of Life on October 31, 2026. +> Migrating from v1? See the **[Migration Guide](https://github.com/skyflowapi/skyflow-python/blob/main/docs/migrate_to_v2.md)** for step-by-step instructions. V1 reached End of Life on October 31, 2026 and is no longer supported. The Skyflow Python SDK is designed to help with integrating Skyflow into a Python backend. @@ -552,7 +552,7 @@ Tokenization result: TokenizeResponse(tokenized_fields=[{'token': 'a1b2c3d4-...' ## Detect -Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The SDK exposes the Detect V2 API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. +Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The SDK exposes the Detect API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. Detect configurations are created in Studio or through the management API — the SDK does not create, update or delete them. Import request/response models from `skyflow.vault.detect` and enums from `skyflow.utils.enums`. @@ -677,6 +677,8 @@ configuration = DetectConfiguration( De-identifies a file. The API is asynchronous: the submit call returns a run id and the result is retrieved from the run. Build the request with `DeidentifyFileRequest`. +> **⚠️ Behaviour change from Detect V1:** `deidentify_file` **no longer waits for the result by default.** Without `poll_options` it submits and returns immediately — `status` is `None`, `output` is empty, and nothing is written to disk. Pass `poll_options` to block until the run finishes, or retrieve the result later with [`get_run`](#get-run-get_runrequest). Code ported from the old blocking behaviour runs without error but produces no outputs, so the failure surfaces downstream rather than at the call site. + Supply the file in exactly one of three ways: `file_path`, `file`, or `value` plus a `DeidentifySourceType` `data_source` (`BASE64`, the `SKYFLOW_ID` of a file record in the vault, or `PRESIGNED_URL`). A local `file` / `file_path` is read when the request is sent, base64-encoded, and its `data_format` inferred from the extension. Passing more than one of them, or none, fails client-side validation before any network call. `configuration_id` / `configuration` follow the same rules as for strings (see [Before you start](#before-you-start)). - **Submit only** (default): one call, returns `run_id`. `status` is `None` and nothing is written to disk — retrieve the result later with [`get_run`](#get-run-get_runrequest). @@ -769,13 +771,31 @@ Synchronous. Re-identification looks tokens up in the vault, so it applies only Build the request with `ReidentifyStringRequest`: the tokenized text and an optional list of `RedactionLevel` rules. On skyvault vaults each rule names an `entity_name` and a `redaction_type` — `PLAIN_TEXT` reveals the original value, `MASKED` masks it, `REDACTED` keeps the token, `DEFAULT` applies the vault's default. Without rules every token is replaced by its original value. +Only text that contains vault tokens can be re-identified, so the example is a round trip: de-identify with `VAULT_TOKEN` (which stores each value in a `table.column`), then re-identify the `processed_text` that comes back. A vault token is an opaque value like `a1b2c3d4-...`, not the `[NAME_1]` counter placeholder produced by `ENTITY_UNIQUE_COUNTER`. + ```python -from skyflow.vault.detect import ReidentifyStringRequest, RedactionLevel -from skyflow.utils.enums import EntityType, RedactionType +from skyflow.vault.detect import ( + DeidentifyStringRequest, DetectConfiguration, Detect, Entity, + ReidentifyStringRequest, RedactionLevel, +) +from skyflow.utils.enums import EntityType, DeidentificationType, RedactionType + +# 1. De-identify with VAULT_TOKEN so each value is stored in a table.column and can be re-identified. +deidentified = skyflow_client.detect().deidentify_string( + DeidentifyStringRequest( + text='My name is John Doe and my email is johndoe@acme.com.', + configuration=DetectConfiguration(detect=Detect(entities=[ + Entity(entity_type=EntityType.NAME, deidentification_type=DeidentificationType.VAULT_TOKEN, destination='persons.name'), + Entity(entity_type=EntityType.EMAIL_ADDRESS, deidentification_type=DeidentificationType.VAULT_TOKEN, destination='persons.email'), + ])), + ) +) +# deidentified.processed_text now holds vault tokens, e.g. 'My name is a1b2c3d4-... and my email is e5f6a7b8-...' +# 2. Re-identify those tokens; on skyvault each rule names an entity_name + redaction_type. response = skyflow_client.detect().reidentify_string( ReidentifyStringRequest( - text='My name is [NAME_1] and my email is [EMAIL_ADDRESS_1].', + text=deidentified.processed_text, redaction_level=[ RedactionLevel(entity_name=EntityType.NAME, redaction_type=RedactionType.PLAIN_TEXT), RedactionLevel(entity_name=EntityType.EMAIL_ADDRESS, redaction_type=RedactionType.MASKED), @@ -799,6 +819,7 @@ import base64 from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, EntityType, RedactionType, DetectRunStatus +data_format = FileDataFormat.TXT with open('/data/redacted/notes.txt', 'rb') as handle: encoded = base64.b64encode(handle.read()).decode('utf-8') @@ -806,23 +827,26 @@ response = skyflow_client.detect().reidentify_file( ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=data_format, redaction_level=[RedactionLevel(entity_name=EntityType.NAME, redaction_type=RedactionType.PLAIN_TEXT)], ) ) if response.status == DetectRunStatus.SUCCESS: for index, output in enumerate(response.output): - # output.processed_file is base64 or a presigned URL, per response.output_type - with open(f'reidentified-{index}.txt', 'wb') as out: - out.write(base64.b64decode(output.processed_file)) + # Branch on output_type: base64 bytes are decoded, a presigned URL is downloaded. + if response.output_type == ReidentifySourceType.BASE64: + with open(f'reidentified-{index}.{data_format.value}', 'wb') as out: + out.write(base64.b64decode(output.processed_file)) + else: # ReidentifySourceType.PRESIGNED_URL — processed_file is a URL to fetch + print('download re-identified file from', output.processed_file) ``` -Returns a `ReidentifyFileResponse` with `status`, `output_type`, `output` (a list of `FileOutput`) and `metrics`. +Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `FileOutput`) and `metrics`. ### Check Guardrails: `.check_guardrails(request)` -Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. The response is passed through as the API returns it. +Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. **When `check_toxicity` is omitted, the server still checks toxicity by default.** The response is passed through as the API returns it. ```python from skyflow.vault.detect import CheckGuardrailsRequest diff --git a/skyvault/samples/detect_api/check_guardrails.py b/skyvault/samples/detect_api/check_guardrails.py index 070d69fa..ce4b5db5 100644 --- a/skyvault/samples/detect_api/check_guardrails.py +++ b/skyvault/samples/detect_api/check_guardrails.py @@ -36,6 +36,7 @@ def perform_check_guardrails(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/deidentify_file.py b/skyvault/samples/detect_api/deidentify_file.py index 6940345f..3aad5a1d 100644 --- a/skyvault/samples/detect_api/deidentify_file.py +++ b/skyvault/samples/detect_api/deidentify_file.py @@ -51,6 +51,7 @@ def perform_deidentify_file(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/deidentify_string.py b/skyvault/samples/detect_api/deidentify_string.py index fbbbdd0f..def6c3dd 100644 --- a/skyvault/samples/detect_api/deidentify_string.py +++ b/skyvault/samples/detect_api/deidentify_string.py @@ -36,6 +36,7 @@ def perform_deidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -78,6 +79,7 @@ def perform_deidentify_string_inline(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/get_run.py b/skyvault/samples/detect_api/get_run.py index 9ce02d67..b4f69810 100644 --- a/skyvault/samples/detect_api/get_run.py +++ b/skyvault/samples/detect_api/get_run.py @@ -37,6 +37,7 @@ def perform_get_run(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/reidentify_file.py b/skyvault/samples/detect_api/reidentify_file.py index 9bc71148..5ba7942f 100644 --- a/skyvault/samples/detect_api/reidentify_file.py +++ b/skyvault/samples/detect_api/reidentify_file.py @@ -46,6 +46,7 @@ def perform_reidentify_file(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/reidentify_string.py b/skyvault/samples/detect_api/reidentify_string.py index b685da1f..23479ec4 100644 --- a/skyvault/samples/detect_api/reidentify_string.py +++ b/skyvault/samples/detect_api/reidentify_string.py @@ -37,6 +37,7 @@ def perform_reidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) From a15d88b04340fdb652c53543e47934fbcc872585 Mon Sep 17 00:00:00 2001 From: saileshwar-skyflow Date: Tue, 10 Nov 2026 17:37:28 +0530 Subject: [PATCH 2/3] SK-3289: fix internal parity-gap issues in Python SDK flowvault: - Disable the generated client's hidden retry layer so the SDK's configured max_retries (default 0) is honored and POSTs are not silently auto-retried. - Pass the bearer token as None when absent so no empty "Authorization: Bearer " header is sent. - Build insert/get wire objects inside the try block so pydantic ValidationError surfaces as a SkyflowError. - Rewind a caller-supplied file_object to the start before upload (without closing the caller's handle). - Guard client re-init and close with a lock. common (shared by flowvault and skyvault): - Validate env with isinstance instead of enum membership to avoid a TypeError on Python 3.9-3.11 for non-Env values. - Attach the skyflow-python log handler once instead of clearing and re-adding it on every log call. Co-Authored-By: Claude Opus 4.8 --- common/utils/_utils.py | 2 +- common/utils/logger/_logger.py | 23 ++++----- common/utils/validations/_validations.py | 4 +- flowvault/skyflow/vault/client/client.py | 54 +++++++++++++------- flowvault/skyflow/vault/controller/_vault.py | 48 ++++++++++------- skyvault/tests/utils/logger/test__logger.py | 8 +++ 6 files changed, 87 insertions(+), 52 deletions(-) diff --git a/common/utils/_utils.py b/common/utils/_utils.py index 4a1c0637..04c63332 100644 --- a/common/utils/_utils.py +++ b/common/utils/_utils.py @@ -41,7 +41,7 @@ def get_vault_url(cluster_id, env, vault_id, logger=None): if not cluster_id or not isinstance(cluster_id, str) or not cluster_id.strip(): raise SkyflowError(SkyflowMessages.Error.INVALID_CLUSTER_ID.value.format(vault_id), invalid_input_error_code) - if env not in Env: + if not isinstance(env, Env): raise SkyflowError(SkyflowMessages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code) base_url = EnvUrls[env.name].value diff --git a/common/utils/logger/_logger.py b/common/utils/logger/_logger.py index 45519fb1..d2e77547 100644 --- a/common/utils/logger/_logger.py +++ b/common/utils/logger/_logger.py @@ -1,27 +1,26 @@ import logging from ..enums.log_level import LogLevel +LOGGER_NAME = 'skyflow-python' +LOG_MESSAGE_FORMAT = '%(message)s' +_SKYFLOW_HANDLER_FLAG = '_skyflow_handler' + class Logger: def __init__(self, level=LogLevel.ERROR): self.current_level = level - self.logger = logging.getLogger('skyflow-python') - self.logger.propagate = False # Prevent logs from being handled by parent loggers + self.logger = logging.getLogger(LOGGER_NAME) + self.logger.propagate = False - # Remove any existing handlers to avoid duplicates or inherited handlers - if self.logger.hasHandlers(): + if not any(getattr(handler, _SKYFLOW_HANDLER_FLAG, False) for handler in self.logger.handlers): self.logger.handlers.clear() + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter(LOG_MESSAGE_FORMAT)) + setattr(handler, _SKYFLOW_HANDLER_FLAG, True) + self.logger.addHandler(handler) self.set_log_level(level) - handler = logging.StreamHandler() - - # Create a formatter that only includes the message without any prefixes - formatter = logging.Formatter('%(message)s') - handler.setFormatter(formatter) - - self.logger.addHandler(handler) - def set_log_level(self, level): self.current_level = level log_level_mapping = { diff --git a/common/utils/validations/_validations.py b/common/utils/validations/_validations.py index 1d45b41d..9b3070a1 100644 --- a/common/utils/validations/_validations.py +++ b/common/utils/validations/_validations.py @@ -232,7 +232,7 @@ def validate_vault_config(logger, config, messages=None, allowed_keys=None): if ConfigField.CREDENTIALS in config and config.get(ConfigField.CREDENTIALS): validate_credentials(logger, config.get(ConfigField.CREDENTIALS), "vault", vault_id, messages=messages) - if ConfigField.ENV in config and config.get(ConfigField.ENV) not in Env: + if ConfigField.ENV in config and not isinstance(config.get(ConfigField.ENV), Env): log_error_log(messages.ErrorLogs.ENV_IS_REQUIRED.value, logger) raise SkyflowError(messages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code) @@ -256,7 +256,7 @@ def validate_update_vault_config(logger, config, messages=None, allowed_keys=Non if ConfigField.CLUSTER_ID in config and not config.get(ConfigField.CLUSTER_ID): raise SkyflowError(messages.Error.INVALID_CLUSTER_ID.value.format(vault_id), invalid_input_error_code) - if ConfigField.ENV in config and config.get(ConfigField.ENV) not in Env: + if ConfigField.ENV in config and not isinstance(config.get(ConfigField.ENV), Env): raise SkyflowError(messages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code) if ConfigField.CREDENTIALS not in config: diff --git a/flowvault/skyflow/vault/client/client.py b/flowvault/skyflow/vault/client/client.py index 4154c2ff..2c6fa325 100644 --- a/flowvault/skyflow/vault/client/client.py +++ b/flowvault/skyflow/vault/client/client.py @@ -1,3 +1,5 @@ +import threading + import httpx from common.vault.base_vault_client import BaseVaultClient @@ -28,6 +30,7 @@ from skyflow.utils._retry import RetryTransport CONTENT_TYPE_HEADER = "content-type" +GENERATED_CLIENT_MAX_RETRIES = 0 class VaultClient(BaseVaultClient): @@ -37,6 +40,7 @@ def __init__(self, config): self._sync_httpx_client = None self._sync_upload_client = None self._detect_api_client = None + self._client_lock = threading.Lock() def set_common_http_config(self, common_http_config): self._common_http_config = common_http_config or {} @@ -75,27 +79,39 @@ def _build_limits(self): ) def initialize_api_client(self, vault_url, bearer_token): - self.__close_httpx_clients() - timeout = self._build_timeout() - limits = self._build_limits() - max_retries, initial_millis, max_millis, call_timeout = self._retry_params() - sync_client = httpx.Client( - timeout=timeout, - follow_redirects=True, - transport=RetryTransport(httpx.HTTPTransport(limits=limits), max_retries, initial_millis, max_millis, call_timeout), - ) - self._sync_httpx_client = sync_client - self._sync_upload_client = httpx.Client( - timeout=timeout, - follow_redirects=True, - transport=httpx.HTTPTransport(limits=limits), - ) - self._api_client = SkyflowAuth(base_url=vault_url, token=bearer_token or "", httpx_client=sync_client) - self._detect_api_client = DetectSkyflowAuth(base_url=vault_url, token=bearer_token or "", httpx_client=sync_client) + with self._client_lock: + self.__close_httpx_clients() + timeout = self._build_timeout() + limits = self._build_limits() + max_retries, initial_millis, max_millis, call_timeout = self._retry_params() + sync_client = httpx.Client( + timeout=timeout, + follow_redirects=True, + transport=RetryTransport(httpx.HTTPTransport(limits=limits), max_retries, initial_millis, max_millis, call_timeout), + ) + self._sync_httpx_client = sync_client + self._sync_upload_client = httpx.Client( + timeout=timeout, + follow_redirects=True, + transport=httpx.HTTPTransport(limits=limits), + ) + self._api_client = SkyflowAuth( + base_url=vault_url, + token=bearer_token or None, + httpx_client=sync_client, + max_retries=GENERATED_CLIENT_MAX_RETRIES, + ) + self._detect_api_client = DetectSkyflowAuth( + base_url=vault_url, + token=bearer_token or None, + httpx_client=sync_client, + max_retries=GENERATED_CLIENT_MAX_RETRIES, + ) def close(self): - self.__close_httpx_clients() - self._api_client = None + with self._client_lock: + self.__close_httpx_clients() + self._api_client = None def __close_httpx_clients(self): for attribute in ("_sync_httpx_client", "_sync_upload_client"): diff --git a/flowvault/skyflow/vault/controller/_vault.py b/flowvault/skyflow/vault/controller/_vault.py index 68962817..14b7cda9 100644 --- a/flowvault/skyflow/vault/controller/_vault.py +++ b/flowvault/skyflow/vault/controller/_vault.py @@ -124,13 +124,12 @@ def insert(self, request: InsertRequest, options: InsertOptions = None) -> Inser needs_per_record_table = any(r.table_name is not None for r in request.records) needs_per_record_upsert = any(r.upsert is not None for r in request.records) - wire_records = [ - self.__build_wire_record(record, request, needs_per_record_table, needs_per_record_upsert) - for record in request.records - ] - try: log_info(SkyflowMessages.Info.INSERT_TRIGGERED.value, self._vault_client.get_logger()) + wire_records = [ + self.__build_wire_record(record, request, needs_per_record_table, needs_per_record_upsert) + for record in request.records + ] upsert_kwargs = self.__omit_none( upsert=None if needs_per_record_upsert else self.__to_upsert(request.upsert), ) @@ -168,21 +167,20 @@ def get(self, request: GetRequest, options: GetOptions = None) -> GetResponse: records_api = self._vault_client.get_records_api() - if request.records is not None: - call_kwargs = {'records': self.__to_get_request_data(request.records)} - else: - call_kwargs = { - 'table_name': request.table_name, - 'skyflow_i_ds': request.skyflow_ids, - 'unique_values': self.__to_unique_values(request.unique_values), - 'columns': request.columns, - 'column_redactions': self.__to_column_redactions(request.column_redactions), - 'limit': request.limit, - 'offset': request.offset, - } - try: log_info(SkyflowMessages.Info.GET_TRIGGERED.value, self._vault_client.get_logger()) + if request.records is not None: + call_kwargs = {'records': self.__to_get_request_data(request.records)} + else: + call_kwargs = { + 'table_name': request.table_name, + 'skyflow_i_ds': request.skyflow_ids, + 'unique_values': self.__to_unique_values(request.unique_values), + 'columns': request.columns, + 'column_redactions': self.__to_column_redactions(request.column_redactions), + 'limit': request.limit, + 'offset': request.offset, + } raw_response = records_api.with_raw_response.get_records( vault_id=self._vault_client.get_vault_id(), request_options=self.__unary_request_options(OPERATION_GET, options), @@ -785,8 +783,22 @@ def __put_file_source(self, signed_url, source, content_type): if kind == SOURCE_KIND_PATH: with open(payload, 'rb') as handle: return self._vault_client.put_signed_url(signed_url, handle, content_type).status_code + if kind == SOURCE_KIND_FILE_OBJECT and payload is not None: + self.__rewind_file_object(payload) return self._vault_client.put_signed_url(signed_url, payload, content_type).status_code + def __rewind_file_object(self, file_object): + seek = getattr(file_object, 'seek', None) + if not callable(seek): + return + seekable = getattr(file_object, 'seekable', None) + if callable(seekable) and not seekable(): + return + try: + seek(0) + except (OSError, ValueError): + pass + def __to_delete_files_request_data(self, records): return [ FileDeleteRecord( diff --git a/skyvault/tests/utils/logger/test__logger.py b/skyvault/tests/utils/logger/test__logger.py index cdfcf13f..aeabf781 100644 --- a/skyvault/tests/utils/logger/test__logger.py +++ b/skyvault/tests/utils/logger/test__logger.py @@ -11,6 +11,7 @@ class TestLogger(unittest.TestCase): def test_logger_initialization_with_default_level(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger() @@ -21,6 +22,7 @@ def test_logger_initialization_with_default_level(self, mock_get_logger): def test_logger_initialization_with_custom_level(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.INFO) @@ -31,6 +33,7 @@ def test_logger_initialization_with_custom_level(self, mock_get_logger): def test_set_log_level(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger() logger.set_log_level(LogLevel.DEBUG) @@ -42,6 +45,7 @@ def test_set_log_level(self, mock_get_logger): def test_debug_logging(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.DEBUG) logger.debug("Debug message") @@ -52,6 +56,7 @@ def test_debug_logging(self, mock_get_logger): def test_info_logging(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.INFO) logger.info("Info message") @@ -63,6 +68,7 @@ def test_info_logging(self, mock_get_logger): def test_warn_logging(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.WARN) logger.warn("Warn message") @@ -75,6 +81,7 @@ def test_warn_logging(self, mock_get_logger): def test_error_logging(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.ERROR) logger.error("Error message") @@ -88,6 +95,7 @@ def test_error_logging(self, mock_get_logger): def test_logging_with_level_off(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.OFF) logger.debug("Debug message") From ce43cf32d7bff690a0d075cd7b1c3b21e20f903b Mon Sep 17 00:00:00 2001 From: saileshwar-skyflow Date: Wed, 11 Nov 2026 01:17:49 +0530 Subject: [PATCH 3/3] SK-3289: split file data format into deidentify and reidentify (Python) Mirror the Java flowvault change: reidentify_file only supports the file formats that carry VAULT_TOKEN, so the single FileDataFormat enum is split into separate types. - enums: rename FileDataFormat -> DeidentifyFileDataFormat, add ReidentifyFileDataFormat (csv, docx, json, jsonl, pptx, txt, xlsx, xml) - models: FileOutput uses DeidentifyFileDataFormat; add ReidentifyFileOutput; ReidentifyFileResponse.output is now a list of ReidentifyFileOutput - mappers: separate deidentify/reidentify format resolvers; an unknown reidentify format maps to None - re-exports, samples, READMEs and tests updated across flowvault and skyvault - regenerate the public API contract baselines Co-Authored-By: Claude Opus 4.8 --- common/tests/vault/detect/test_detect.py | 23 +++++--- common/utils/enums/detect.py | 13 ++++- common/vault/detect/__init__.py | 2 + common/vault/detect/_mappers.py | 31 ++++++++-- common/vault/detect/_models.py | 17 ++++-- flowvault/README.md | 14 ++--- flowvault/api-report/skyflow.api.json | 58 +++++++++++-------- .../samples/detect_api/deidentify_file.py | 4 +- .../samples/detect_api/reidentify_file.py | 4 +- flowvault/skyflow/utils/enums/__init__.py | 3 +- flowvault/skyflow/vault/detect/__init__.py | 1 + .../tests/vault/detect/test_reexports.py | 6 +- skyvault/README.md | 14 ++--- skyvault/api-report/skyflow.api.json | 58 +++++++++++-------- .../samples/detect_api/reidentify_file.py | 4 +- skyvault/skyflow/utils/enums/__init__.py | 3 +- skyvault/skyflow/vault/detect/__init__.py | 1 + skyvault/tests/vault/detect/test_reexports.py | 6 +- 18 files changed, 169 insertions(+), 93 deletions(-) diff --git a/common/tests/vault/detect/test_detect.py b/common/tests/vault/detect/test_detect.py index b24540c9..fde39b44 100644 --- a/common/tests/vault/detect/test_detect.py +++ b/common/tests/vault/detect/test_detect.py @@ -10,11 +10,12 @@ from common.utils import SkyflowMessages from common.utils.enums.detect import ( DeidentificationType, + DeidentifyFileDataFormat, DetectRunStatus, EntityType, - FileDataFormat, GuardrailsValidation, IncludePrefix, + ReidentifyFileDataFormat, ) @@ -274,7 +275,7 @@ def test_reidentify_file_value_only(self): controller = DetectController(_vault_client(files=files)) response = controller.reidentify_file(ReidentifyFileRequest( - data_source=ReidentifySourceType.BASE64, value="b64", data_format=FileDataFormat.TXT, + data_source=ReidentifySourceType.BASE64, value="b64", data_format=ReidentifyFileDataFormat.TXT, )) _name, _args, kwargs = files.calls[0] @@ -328,7 +329,7 @@ def test_file_output_writer_names(self): tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) outputs = [ - mappers.FileOutput(processed_file=base64.b64encode(b"a").decode(), processed_file_type="REDACTED_FILE", processed_file_extension=FileDataFormat.PDF), + mappers.FileOutput(processed_file=base64.b64encode(b"a").decode(), processed_file_type="REDACTED_FILE", processed_file_extension=DeidentifyFileDataFormat.PDF), mappers.FileOutput(processed_file=base64.b64encode(b"b").decode(), processed_file_type="ENTITIES", processed_file_extension=None), ] written = writer.write_outputs(outputs, DeidentifySourceType.BASE64, "invoice.pdf", tmp) @@ -336,7 +337,7 @@ def test_file_output_writer_names(self): self.assertEqual(names, ["processed-entities-invoice.json", "processed-invoice.pdf"]) def test_file_output_writer_skips_presigned(self): - outputs = [mappers.FileOutput(processed_file="http://x", processed_file_type="REDACTED_FILE", processed_file_extension=FileDataFormat.PDF)] + outputs = [mappers.FileOutput(processed_file="http://x", processed_file_type="REDACTED_FILE", processed_file_extension=DeidentifyFileDataFormat.PDF)] self.assertEqual(writer.write_outputs(outputs, DeidentifySourceType.PRESIGNED_URL, "a.pdf", None), []) def test_validations(self): @@ -368,10 +369,16 @@ def test_deidentify_file_source_rules(self): self.assertEqual(no_data_source.exception.message, SkyflowMessages.Error.INVALID_DATA_SOURCE_IN_DEIDENTIFY_FILE.value) def test_file_data_format_is_case_insensitive(self): - self.assertEqual(mappers._file_data_format("PDF"), FileDataFormat.PDF) - self.assertEqual(mappers._file_data_format("pdf"), FileDataFormat.PDF) - self.assertIsNone(mappers._file_data_format("bogus")) - self.assertIsNone(mappers._file_data_format(None)) + self.assertEqual(mappers._deidentify_file_data_format("PDF"), DeidentifyFileDataFormat.PDF) + self.assertEqual(mappers._deidentify_file_data_format("pdf"), DeidentifyFileDataFormat.PDF) + self.assertIsNone(mappers._deidentify_file_data_format("bogus")) + self.assertIsNone(mappers._deidentify_file_data_format(None)) + + def test_reidentify_file_data_format_narrower_set(self): + self.assertEqual(mappers._reidentify_file_data_format("TXT"), ReidentifyFileDataFormat.TXT) + self.assertEqual(mappers._reidentify_file_data_format("txt"), ReidentifyFileDataFormat.TXT) + self.assertIsNone(mappers._reidentify_file_data_format("pdf")) + self.assertIsNone(mappers._reidentify_file_data_format(None)) def test_entity_prefix_fields_serialize(self): wire = mappers.to_entity(Entity( diff --git a/common/utils/enums/detect.py b/common/utils/enums/detect.py index 6f307ef7..91ca3e6f 100644 --- a/common/utils/enums/detect.py +++ b/common/utils/enums/detect.py @@ -94,7 +94,7 @@ class ReturnEntitiesType(Enum): NONE = "NONE" -class FileDataFormat(Enum): +class DeidentifyFileDataFormat(Enum): BMP = "bmp" CSV = "csv" DCM = "dcm" @@ -120,6 +120,17 @@ class FileDataFormat(Enum): ZIP = "zip" +class ReidentifyFileDataFormat(Enum): + CSV = "csv" + DOCX = "docx" + JSON = "json" + JSONL = "jsonl" + PPTX = "pptx" + TXT = "txt" + XLSX = "xlsx" + XML = "xml" + + class DetectRunStatus(Enum): UNKNOWN = "UNKNOWN" QUEUED = "QUEUED" diff --git a/common/vault/detect/__init__.py b/common/vault/detect/__init__.py index ae8b5470..bd8c0a5f 100644 --- a/common/vault/detect/__init__.py +++ b/common/vault/detect/__init__.py @@ -26,6 +26,7 @@ Pdf, PollOptions, RedactionRule, + ReidentifyFileOutput, ReidentifyFileRequest, ReidentifyFileResponse, ReidentifyStringRequest, @@ -62,6 +63,7 @@ "Pdf", "PollOptions", "RedactionRule", + "ReidentifyFileOutput", "ReidentifyFileRequest", "ReidentifyFileResponse", "ReidentifyStringRequest", diff --git a/common/vault/detect/_mappers.py b/common/vault/detect/_mappers.py index 14e078ff..0e76006f 100644 --- a/common/vault/detect/_mappers.py +++ b/common/vault/detect/_mappers.py @@ -16,9 +16,10 @@ RedactionLevel as WireRedactionLevel, ) from common.utils.enums.detect import ( + DeidentifyFileDataFormat, DetectRunStatus, - FileDataFormat, GuardrailsValidation, + ReidentifyFileDataFormat, ) from ._constants import EXTENSION_SEPARATOR @@ -31,6 +32,7 @@ FileOutput, GetRunResponse, Metrics, + ReidentifyFileOutput, ReidentifyFileResponse, ReidentifyStringResponse, ) @@ -53,11 +55,20 @@ def _enum_or_default(enum_cls, value, fallback): return fallback -def _file_data_format(value): +def _deidentify_file_data_format(value): if value is None: return None try: - return FileDataFormat(str(value).lower()) + return DeidentifyFileDataFormat(str(value).lower()) + except ValueError: + return None + + +def _reidentify_file_data_format(value): + if value is None: + return None + try: + return ReidentifyFileDataFormat(str(value).lower()) except ValueError: return None @@ -69,7 +80,7 @@ def infer_format(file_name): if index < 0 or index == len(file_name) - 1: return None try: - return FileDataFormat[file_name[index + 1:].upper()] + return DeidentifyFileDataFormat[file_name[index + 1:].upper()] except KeyError: return None @@ -226,7 +237,15 @@ def to_file_output(api_output): return FileOutput( processed_file=api_output.processed_file, processed_file_type=api_output.processed_file_type, - processed_file_extension=_file_data_format(api_output.processed_file_extension), + processed_file_extension=_deidentify_file_data_format(api_output.processed_file_extension), + ) + + +def to_reidentify_file_output(api_output): + return ReidentifyFileOutput( + processed_file=api_output.processed_file, + processed_file_type=api_output.processed_file_type, + processed_file_extension=_reidentify_file_data_format(api_output.processed_file_extension), ) @@ -303,7 +322,7 @@ def to_deidentify_file_response(run_id, run): def to_reidentify_file_response(response, source_type_cls): - outputs = [to_file_output(output) for output in (response.output or []) if output is not None] + outputs = [to_reidentify_file_output(output) for output in (response.output or []) if output is not None] return ReidentifyFileResponse( status=_enum_or_default(DetectRunStatus, response.status, DetectRunStatus.UNKNOWN), output_type=_enum_or_default(source_type_cls, response.output_type, None), diff --git a/common/vault/detect/_models.py b/common/vault/detect/_models.py index d78ea8cf..1ecd2938 100644 --- a/common/vault/detect/_models.py +++ b/common/vault/detect/_models.py @@ -18,7 +18,16 @@ def __init__(self, size: float = None, word_count: int = None, class FileOutput: def __init__(self, processed_file: str = None, processed_file_type: str = None, - processed_file_extension: "FileDataFormat" = None): + processed_file_extension: "DeidentifyFileDataFormat" = None): + self.processed_file = processed_file + self.processed_file_type = processed_file_type + self.processed_file_extension = processed_file_extension + + +class ReidentifyFileOutput: + def __init__(self, processed_file: str = None, + processed_file_type: str = None, + processed_file_extension: "ReidentifyFileDataFormat" = None): self.processed_file = processed_file self.processed_file_type = processed_file_type self.processed_file_extension = processed_file_extension @@ -220,7 +229,7 @@ def __init__(self, processed_text: str = None, class DeidentifyFileRequest: def __init__(self, data_source: "DeidentifySourceType" = None, value: str = None, - data_format: "FileDataFormat" = None, + data_format: "DeidentifyFileDataFormat" = None, configuration_id: str = None, configuration: DetectConfiguration = None, file: BinaryIO = None, file_path: str = None, @@ -288,7 +297,7 @@ def __init__(self, processed_text: str = None, metrics: Metrics = None): class ReidentifyFileRequest: def __init__(self, data_source: "ReidentifySourceType" = None, value: str = None, - data_format: "FileDataFormat" = None, + data_format: "ReidentifyFileDataFormat" = None, redaction_level: List[RedactionRule] = None): self.data_source = data_source self.value = value @@ -299,7 +308,7 @@ def __init__(self, data_source: "ReidentifySourceType" = None, class ReidentifyFileResponse: def __init__(self, status: "DetectRunStatus" = None, output_type: "ReidentifySourceType" = None, - output: List[FileOutput] = None, + output: List[ReidentifyFileOutput] = None, metrics: Metrics = None): self.status = status self.output_type = output_type diff --git a/flowvault/README.md b/flowvault/README.md index faf17b12..ffcbb670 100644 --- a/flowvault/README.md +++ b/flowvault/README.md @@ -1151,7 +1151,7 @@ else: # IN_PROGRESS or QUEUED: budget ran out, resume `output_directory` is created if it does not exist. **If you poll a local file and omit `output_directory`, these files are written to the current working directory** — often your project root or a server's working directory — so set `output_directory` explicitly. A request built from a `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in that case read `response.output` and write the files yourself. -**Supported formats and limits.** `data_format` accepts every `FileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. +**Supported formats and limits.** `data_format` accepts every `DeidentifyFileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. **Polling blocks the calling thread** — each lookup is a synchronous `get_run` followed by a sleep. For concurrency, submit without `poll_options` and poll `get_run` yourself on a scheduler, or run the blocking call on a worker thread you own. @@ -1159,13 +1159,13 @@ A file at a presigned URL is referenced through `data_source` + `value`: ```python from skyflow.vault.detect import DeidentifyFileRequest -from skyflow.utils.enums import DeidentifySourceType, FileDataFormat +from skyflow.utils.enums import DeidentifySourceType, DeidentifyFileDataFormat skyflow_client.detect().deidentify_file( DeidentifyFileRequest( data_source=DeidentifySourceType.PRESIGNED_URL, value='', - data_format=FileDataFormat.PDF, + data_format=DeidentifyFileDataFormat.PDF, configuration_id='', ) ) @@ -1239,14 +1239,14 @@ Returns a `ReidentifyStringResponse` with `processed_text` and `metrics`. ## reidentify_file -Synchronous — the re-identified file comes back in the response, not through a run id. Build the request with `ReidentifyFileRequest`: a `data_source` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `data_format`, and the same `RedactionLevel` rules as `reidentify_string`. `data_format` is limited to the formats that support `VAULT_TOKEN`: `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. +Synchronous — the re-identified file comes back in the response, not through a run id. Build the request with `ReidentifyFileRequest`: a `data_source` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `data_format` (a `ReidentifyFileDataFormat`), and the same `RedactionLevel` rules as `reidentify_string`. `data_format` is limited to the formats that support `VAULT_TOKEN`: `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. ```python import base64 from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel -from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, DetectRunStatus +from skyflow.utils.enums import ReidentifySourceType, ReidentifyFileDataFormat, DetectRunStatus -data_format = FileDataFormat.TXT +data_format = ReidentifyFileDataFormat.TXT with open('/data/redacted/notes.txt', 'rb') as handle: encoded = base64.b64encode(handle.read()).decode('utf-8') @@ -1269,7 +1269,7 @@ if response.status == DetectRunStatus.SUCCESS: print('download re-identified file from', output.processed_file) ``` -Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `FileOutput`) and `metrics`. +Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `ReidentifyFileOutput`) and `metrics`. ## check_guardrails diff --git a/flowvault/api-report/skyflow.api.json b/flowvault/api-report/skyflow.api.json index 60eaa0de..335f827a 100644 --- a/flowvault/api-report/skyflow.api.json +++ b/flowvault/api-report/skyflow.api.json @@ -35,6 +35,30 @@ "skyflow.utils.enums.DeidentificationType.MASK": "attr = 'MASK'", "skyflow.utils.enums.DeidentificationType.UNKNOWN": "attr = 'UNKNOWN'", "skyflow.utils.enums.DeidentificationType.VAULT_TOKEN": "attr = 'VAULT_TOKEN'", + "skyflow.utils.enums.DeidentifyFileDataFormat": "class (Enum)", + "skyflow.utils.enums.DeidentifyFileDataFormat.BMP": "attr = 'bmp'", + "skyflow.utils.enums.DeidentifyFileDataFormat.CSV": "attr = 'csv'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DCM": "attr = 'dcm'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DOC": "attr = 'doc'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DOCX": "attr = 'docx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.GIF": "attr = 'gif'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JPEG": "attr = 'jpeg'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JPG": "attr = 'jpg'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JSON": "attr = 'json'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JSONL": "attr = 'jsonl'", + "skyflow.utils.enums.DeidentifyFileDataFormat.MP3": "attr = 'mp3'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PDF": "attr = 'pdf'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PNG": "attr = 'png'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PPT": "attr = 'ppt'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PPTX": "attr = 'pptx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TIF": "attr = 'tif'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TIFF": "attr = 'tiff'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TXT": "attr = 'txt'", + "skyflow.utils.enums.DeidentifyFileDataFormat.WAV": "attr = 'wav'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XLS": "attr = 'xls'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XLSX": "attr = 'xlsx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XML": "attr = 'xml'", + "skyflow.utils.enums.DeidentifyFileDataFormat.ZIP": "attr = 'zip'", "skyflow.utils.enums.DeidentifySourceType": "class (Enum)", "skyflow.utils.enums.DeidentifySourceType.BASE64": "attr = 'BASE64'", "skyflow.utils.enums.DeidentifySourceType.PRESIGNED_URL": "attr = 'PRESIGNED_URL'", @@ -121,30 +145,6 @@ "skyflow.utils.enums.EnvUrls.PROD": "attr = 'skyvault.skyflowapis.com'", "skyflow.utils.enums.EnvUrls.SANDBOX": "attr = 'skyvault.skyflowapis-preview.com'", "skyflow.utils.enums.EnvUrls.STAGE": "attr = 'skyvault.skyflowapis.tech'", - "skyflow.utils.enums.FileDataFormat": "class (Enum)", - "skyflow.utils.enums.FileDataFormat.BMP": "attr = 'bmp'", - "skyflow.utils.enums.FileDataFormat.CSV": "attr = 'csv'", - "skyflow.utils.enums.FileDataFormat.DCM": "attr = 'dcm'", - "skyflow.utils.enums.FileDataFormat.DOC": "attr = 'doc'", - "skyflow.utils.enums.FileDataFormat.DOCX": "attr = 'docx'", - "skyflow.utils.enums.FileDataFormat.GIF": "attr = 'gif'", - "skyflow.utils.enums.FileDataFormat.JPEG": "attr = 'jpeg'", - "skyflow.utils.enums.FileDataFormat.JPG": "attr = 'jpg'", - "skyflow.utils.enums.FileDataFormat.JSON": "attr = 'json'", - "skyflow.utils.enums.FileDataFormat.JSONL": "attr = 'jsonl'", - "skyflow.utils.enums.FileDataFormat.MP3": "attr = 'mp3'", - "skyflow.utils.enums.FileDataFormat.PDF": "attr = 'pdf'", - "skyflow.utils.enums.FileDataFormat.PNG": "attr = 'png'", - "skyflow.utils.enums.FileDataFormat.PPT": "attr = 'ppt'", - "skyflow.utils.enums.FileDataFormat.PPTX": "attr = 'pptx'", - "skyflow.utils.enums.FileDataFormat.TIF": "attr = 'tif'", - "skyflow.utils.enums.FileDataFormat.TIFF": "attr = 'tiff'", - "skyflow.utils.enums.FileDataFormat.TXT": "attr = 'txt'", - "skyflow.utils.enums.FileDataFormat.WAV": "attr = 'wav'", - "skyflow.utils.enums.FileDataFormat.XLS": "attr = 'xls'", - "skyflow.utils.enums.FileDataFormat.XLSX": "attr = 'xlsx'", - "skyflow.utils.enums.FileDataFormat.XML": "attr = 'xml'", - "skyflow.utils.enums.FileDataFormat.ZIP": "attr = 'zip'", "skyflow.utils.enums.FileUploadStatus": "class (Enum)", "skyflow.utils.enums.FileUploadStatus.FAILED": "attr = 'FAILED'", "skyflow.utils.enums.FileUploadStatus.SKIPPED": "attr = 'SKIPPED'", @@ -186,6 +186,15 @@ "skyflow.utils.enums.RedactionType.MASKED": "attr = 'MASKED'", "skyflow.utils.enums.RedactionType.PLAIN_TEXT": "attr = 'PLAIN_TEXT'", "skyflow.utils.enums.RedactionType.REDACTED": "attr = 'REDACTED'", + "skyflow.utils.enums.ReidentifyFileDataFormat": "class (Enum)", + "skyflow.utils.enums.ReidentifyFileDataFormat.CSV": "attr = 'csv'", + "skyflow.utils.enums.ReidentifyFileDataFormat.DOCX": "attr = 'docx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.JSON": "attr = 'json'", + "skyflow.utils.enums.ReidentifyFileDataFormat.JSONL": "attr = 'jsonl'", + "skyflow.utils.enums.ReidentifyFileDataFormat.PPTX": "attr = 'pptx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.TXT": "attr = 'txt'", + "skyflow.utils.enums.ReidentifyFileDataFormat.XLSX": "attr = 'xlsx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.XML": "attr = 'xml'", "skyflow.utils.enums.ReidentifySourceType": "class (Enum)", "skyflow.utils.enums.ReidentifySourceType.BASE64": "attr = 'BASE64'", "skyflow.utils.enums.ReidentifySourceType.PRESIGNED_URL": "attr = 'PRESIGNED_URL'", @@ -484,6 +493,7 @@ "skyflow.vault.detect.RedactionLevel.__init__": "def (self, token_group_name: str = None, redaction_pattern: str = None)", "skyflow.vault.detect.RedactionLevel.redaction_pattern": "attr", "skyflow.vault.detect.RedactionLevel.token_group_name": "attr", + "skyflow.vault.detect.ReidentifyFileOutput": "alias -> common.vault.detect.ReidentifyFileOutput", "skyflow.vault.detect.ReidentifyFileRequest": "alias -> common.vault.detect.ReidentifyFileRequest", "skyflow.vault.detect.ReidentifyFileResponse": "alias -> common.vault.detect.ReidentifyFileResponse", "skyflow.vault.detect.ReidentifyStringRequest": "alias -> common.vault.detect.ReidentifyStringRequest", diff --git a/flowvault/samples/detect_api/deidentify_file.py b/flowvault/samples/detect_api/deidentify_file.py index 5edcb528..c687305c 100644 --- a/flowvault/samples/detect_api/deidentify_file.py +++ b/flowvault/samples/detect_api/deidentify_file.py @@ -1,7 +1,7 @@ from skyflow.error import SkyflowError from skyflow import Env, Skyflow, LogLevel from skyflow.vault.detect import DeidentifyFileRequest, PollOptions -from skyflow.utils.enums import DetectRunStatus, DeidentifySourceType, FileDataFormat +from skyflow.utils.enums import DetectRunStatus, DeidentifySourceType, DeidentifyFileDataFormat def perform_deidentify_file_submit_only(): @@ -99,7 +99,7 @@ def perform_deidentify_file_presigned(): request = DeidentifyFileRequest( data_source=DeidentifySourceType.PRESIGNED_URL, value='', - data_format=FileDataFormat.PDF, + data_format=DeidentifyFileDataFormat.PDF, configuration_id='', ) diff --git a/flowvault/samples/detect_api/reidentify_file.py b/flowvault/samples/detect_api/reidentify_file.py index 08dc16c1..48b12efc 100644 --- a/flowvault/samples/detect_api/reidentify_file.py +++ b/flowvault/samples/detect_api/reidentify_file.py @@ -3,7 +3,7 @@ from skyflow.error import SkyflowError from skyflow import Env, Skyflow, LogLevel from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel -from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, DetectRunStatus +from skyflow.utils.enums import ReidentifySourceType, ReidentifyFileDataFormat, DetectRunStatus def perform_reidentify_file(): @@ -28,7 +28,7 @@ def perform_reidentify_file(): request = ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=ReidentifyFileDataFormat.TXT, redaction_level=[ RedactionLevel( token_group_name='', diff --git a/flowvault/skyflow/utils/enums/__init__.py b/flowvault/skyflow/utils/enums/__init__.py index 1c567e0e..bba26acf 100644 --- a/flowvault/skyflow/utils/enums/__init__.py +++ b/flowvault/skyflow/utils/enums/__init__.py @@ -7,10 +7,11 @@ from common.utils.enums.redaction_type import RedactionType from common.utils.enums.detect import ( DeidentificationType, + DeidentifyFileDataFormat, DetectRunStatus, EntityType, - FileDataFormat, GuardrailsValidation, + ReidentifyFileDataFormat, IncludePrefix, MaskingMethod, ObjectDeidentificationType, diff --git a/flowvault/skyflow/vault/detect/__init__.py b/flowvault/skyflow/vault/detect/__init__.py index 15517bb2..0d030ab8 100644 --- a/flowvault/skyflow/vault/detect/__init__.py +++ b/flowvault/skyflow/vault/detect/__init__.py @@ -25,6 +25,7 @@ ObjectEntity, Pdf, PollOptions, + ReidentifyFileOutput, ReidentifyFileRequest, ReidentifyFileResponse, ReidentifyStringRequest, diff --git a/flowvault/tests/vault/detect/test_reexports.py b/flowvault/tests/vault/detect/test_reexports.py index ddff0965..ba2233fc 100644 --- a/flowvault/tests/vault/detect/test_reexports.py +++ b/flowvault/tests/vault/detect/test_reexports.py @@ -29,7 +29,8 @@ def test_enums_import_from_utils_enums(self): RedactionType, DeidentifySourceType, ReidentifySourceType, - FileDataFormat, + DeidentifyFileDataFormat, + ReidentifyFileDataFormat, DetectRunStatus, ObjectEntityType, ObjectDeidentificationType, @@ -40,7 +41,8 @@ def test_enums_import_from_utils_enums(self): ) self.assertEqual(EntityType.NAME.value, "NAME") - self.assertEqual(FileDataFormat.PDF.value, "pdf") + self.assertEqual(DeidentifyFileDataFormat.PDF.value, "pdf") + self.assertEqual(ReidentifyFileDataFormat.TXT.value, "txt") self.assertEqual(DetectRunStatus.SUCCESS.value, "SUCCESS") diff --git a/skyvault/README.md b/skyvault/README.md index e58fc21e..42bded78 100644 --- a/skyvault/README.md +++ b/skyvault/README.md @@ -722,7 +722,7 @@ else: # IN_PROGRESS or QUEUED: budget ran out, resume `output_directory` is created if it does not exist. **If you poll a local file and omit `output_directory`, these files are written to the current working directory** — often your project root or a server's working directory — so set `output_directory` explicitly. A request built from a `SKYFLOW_ID` or `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in those cases read `response.output` and write the files yourself. -**Supported formats and limits.** `data_format` accepts every `FileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. +**Supported formats and limits.** `data_format` accepts every `DeidentifyFileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. **Polling blocks the calling thread** — each lookup is a synchronous `get_run` followed by a sleep. For concurrency, submit without `poll_options` and poll `get_run` yourself on a scheduler, or run the blocking call on a worker thread you own. @@ -730,13 +730,13 @@ A file at a presigned URL is referenced through `data_source` + `value`: ```python from skyflow.vault.detect import DeidentifyFileRequest -from skyflow.utils.enums import DeidentifySourceType, FileDataFormat +from skyflow.utils.enums import DeidentifySourceType, DeidentifyFileDataFormat skyflow_client.detect().deidentify_file( DeidentifyFileRequest( data_source=DeidentifySourceType.PRESIGNED_URL, value='', - data_format=FileDataFormat.PDF, + data_format=DeidentifyFileDataFormat.PDF, configuration_id='', ) ) @@ -812,14 +812,14 @@ Returns a `ReidentifyStringResponse` with `processed_text` and `metrics`. ### Re-identify File: `.reidentify_file(request)` -Synchronous — the re-identified file comes back in the response, not through a run id. Build the request with `ReidentifyFileRequest`: a `data_source` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `data_format`, and the same `RedactionLevel` rules as `reidentify_string`. `data_format` is limited to the formats that support `VAULT_TOKEN`: `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. +Synchronous — the re-identified file comes back in the response, not through a run id. Build the request with `ReidentifyFileRequest`: a `data_source` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `data_format` (a `ReidentifyFileDataFormat`), and the same `RedactionLevel` rules as `reidentify_string`. `data_format` is limited to the formats that support `VAULT_TOKEN`: `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. ```python import base64 from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel -from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, EntityType, RedactionType, DetectRunStatus +from skyflow.utils.enums import ReidentifySourceType, ReidentifyFileDataFormat, EntityType, RedactionType, DetectRunStatus -data_format = FileDataFormat.TXT +data_format = ReidentifyFileDataFormat.TXT with open('/data/redacted/notes.txt', 'rb') as handle: encoded = base64.b64encode(handle.read()).decode('utf-8') @@ -842,7 +842,7 @@ if response.status == DetectRunStatus.SUCCESS: print('download re-identified file from', output.processed_file) ``` -Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `FileOutput`) and `metrics`. +Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `ReidentifyFileOutput`) and `metrics`. ### Check Guardrails: `.check_guardrails(request)` diff --git a/skyvault/api-report/skyflow.api.json b/skyvault/api-report/skyflow.api.json index fe322ff1..99d845c7 100644 --- a/skyvault/api-report/skyflow.api.json +++ b/skyvault/api-report/skyflow.api.json @@ -38,6 +38,30 @@ "skyflow.utils.enums.DeidentificationType.MASK": "attr = 'MASK'", "skyflow.utils.enums.DeidentificationType.UNKNOWN": "attr = 'UNKNOWN'", "skyflow.utils.enums.DeidentificationType.VAULT_TOKEN": "attr = 'VAULT_TOKEN'", + "skyflow.utils.enums.DeidentifyFileDataFormat": "class (Enum)", + "skyflow.utils.enums.DeidentifyFileDataFormat.BMP": "attr = 'bmp'", + "skyflow.utils.enums.DeidentifyFileDataFormat.CSV": "attr = 'csv'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DCM": "attr = 'dcm'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DOC": "attr = 'doc'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DOCX": "attr = 'docx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.GIF": "attr = 'gif'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JPEG": "attr = 'jpeg'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JPG": "attr = 'jpg'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JSON": "attr = 'json'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JSONL": "attr = 'jsonl'", + "skyflow.utils.enums.DeidentifyFileDataFormat.MP3": "attr = 'mp3'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PDF": "attr = 'pdf'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PNG": "attr = 'png'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PPT": "attr = 'ppt'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PPTX": "attr = 'pptx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TIF": "attr = 'tif'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TIFF": "attr = 'tiff'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TXT": "attr = 'txt'", + "skyflow.utils.enums.DeidentifyFileDataFormat.WAV": "attr = 'wav'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XLS": "attr = 'xls'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XLSX": "attr = 'xlsx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XML": "attr = 'xml'", + "skyflow.utils.enums.DeidentifyFileDataFormat.ZIP": "attr = 'zip'", "skyflow.utils.enums.DeidentifySourceType": "class (Enum)", "skyflow.utils.enums.DeidentifySourceType.BASE64": "attr = 'BASE64'", "skyflow.utils.enums.DeidentifySourceType.PRESIGNED_URL": "attr = 'PRESIGNED_URL'", @@ -207,30 +231,6 @@ "skyflow.utils.enums.EnvUrls.PROD": "attr = 'vault.skyflowapis.com'", "skyflow.utils.enums.EnvUrls.SANDBOX": "attr = 'vault.skyflowapis-preview.com'", "skyflow.utils.enums.EnvUrls.STAGE": "attr = 'vault.skyflowapis.tech'", - "skyflow.utils.enums.FileDataFormat": "class (Enum)", - "skyflow.utils.enums.FileDataFormat.BMP": "attr = 'bmp'", - "skyflow.utils.enums.FileDataFormat.CSV": "attr = 'csv'", - "skyflow.utils.enums.FileDataFormat.DCM": "attr = 'dcm'", - "skyflow.utils.enums.FileDataFormat.DOC": "attr = 'doc'", - "skyflow.utils.enums.FileDataFormat.DOCX": "attr = 'docx'", - "skyflow.utils.enums.FileDataFormat.GIF": "attr = 'gif'", - "skyflow.utils.enums.FileDataFormat.JPEG": "attr = 'jpeg'", - "skyflow.utils.enums.FileDataFormat.JPG": "attr = 'jpg'", - "skyflow.utils.enums.FileDataFormat.JSON": "attr = 'json'", - "skyflow.utils.enums.FileDataFormat.JSONL": "attr = 'jsonl'", - "skyflow.utils.enums.FileDataFormat.MP3": "attr = 'mp3'", - "skyflow.utils.enums.FileDataFormat.PDF": "attr = 'pdf'", - "skyflow.utils.enums.FileDataFormat.PNG": "attr = 'png'", - "skyflow.utils.enums.FileDataFormat.PPT": "attr = 'ppt'", - "skyflow.utils.enums.FileDataFormat.PPTX": "attr = 'pptx'", - "skyflow.utils.enums.FileDataFormat.TIF": "attr = 'tif'", - "skyflow.utils.enums.FileDataFormat.TIFF": "attr = 'tiff'", - "skyflow.utils.enums.FileDataFormat.TXT": "attr = 'txt'", - "skyflow.utils.enums.FileDataFormat.WAV": "attr = 'wav'", - "skyflow.utils.enums.FileDataFormat.XLS": "attr = 'xls'", - "skyflow.utils.enums.FileDataFormat.XLSX": "attr = 'xlsx'", - "skyflow.utils.enums.FileDataFormat.XML": "attr = 'xml'", - "skyflow.utils.enums.FileDataFormat.ZIP": "attr = 'zip'", "skyflow.utils.enums.GuardrailsValidation": "class (Enum)", "skyflow.utils.enums.GuardrailsValidation.FAILED": "attr = 'FAILED'", "skyflow.utils.enums.GuardrailsValidation.PASSED": "attr = 'PASSED'", @@ -274,6 +274,15 @@ "skyflow.utils.enums.RedactionType.MASKED": "attr = 'MASKED'", "skyflow.utils.enums.RedactionType.PLAIN_TEXT": "attr = 'PLAIN_TEXT'", "skyflow.utils.enums.RedactionType.REDACTED": "attr = 'REDACTED'", + "skyflow.utils.enums.ReidentifyFileDataFormat": "class (Enum)", + "skyflow.utils.enums.ReidentifyFileDataFormat.CSV": "attr = 'csv'", + "skyflow.utils.enums.ReidentifyFileDataFormat.DOCX": "attr = 'docx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.JSON": "attr = 'json'", + "skyflow.utils.enums.ReidentifyFileDataFormat.JSONL": "attr = 'jsonl'", + "skyflow.utils.enums.ReidentifyFileDataFormat.PPTX": "attr = 'pptx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.TXT": "attr = 'txt'", + "skyflow.utils.enums.ReidentifyFileDataFormat.XLSX": "attr = 'xlsx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.XML": "attr = 'xml'", "skyflow.utils.enums.ReidentifySourceType": "class (Enum)", "skyflow.utils.enums.ReidentifySourceType.BASE64": "attr = 'BASE64'", "skyflow.utils.enums.ReidentifySourceType.PRESIGNED_URL": "attr = 'PRESIGNED_URL'", @@ -416,6 +425,7 @@ "skyflow.vault.detect.RedactionLevel.__init__": "def (self, entity_name: EntityType = None, redaction_type: RedactionType = None)", "skyflow.vault.detect.RedactionLevel.entity_name": "attr", "skyflow.vault.detect.RedactionLevel.redaction_type": "attr", + "skyflow.vault.detect.ReidentifyFileOutput": "alias -> common.vault.detect.ReidentifyFileOutput", "skyflow.vault.detect.ReidentifyFileRequest": "alias -> common.vault.detect.ReidentifyFileRequest", "skyflow.vault.detect.ReidentifyFileResponse": "alias -> common.vault.detect.ReidentifyFileResponse", "skyflow.vault.detect.ReidentifyStringRequest": "alias -> common.vault.detect.ReidentifyStringRequest", diff --git a/skyvault/samples/detect_api/reidentify_file.py b/skyvault/samples/detect_api/reidentify_file.py index 5ba7942f..b1f36042 100644 --- a/skyvault/samples/detect_api/reidentify_file.py +++ b/skyvault/samples/detect_api/reidentify_file.py @@ -3,7 +3,7 @@ from skyflow.error import SkyflowError from skyflow import Env, Skyflow, LogLevel from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel -from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, EntityType, RedactionType, DetectRunStatus +from skyflow.utils.enums import ReidentifySourceType, ReidentifyFileDataFormat, EntityType, RedactionType, DetectRunStatus def perform_reidentify_file(): @@ -28,7 +28,7 @@ def perform_reidentify_file(): request = ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=ReidentifyFileDataFormat.TXT, redaction_level=[ RedactionLevel(entity_name=EntityType.NAME, redaction_type=RedactionType.PLAIN_TEXT), ], diff --git a/skyvault/skyflow/utils/enums/__init__.py b/skyvault/skyflow/utils/enums/__init__.py index 8cc888c0..a7b4cf96 100644 --- a/skyvault/skyflow/utils/enums/__init__.py +++ b/skyvault/skyflow/utils/enums/__init__.py @@ -11,10 +11,11 @@ from ._reidentify_source_type import ReidentifySourceType from common.utils.enums.detect import ( DeidentificationType, + DeidentifyFileDataFormat, DetectRunStatus, EntityType, - FileDataFormat, GuardrailsValidation, + ReidentifyFileDataFormat, IncludePrefix, MaskingMethod, ObjectDeidentificationType, diff --git a/skyvault/skyflow/vault/detect/__init__.py b/skyvault/skyflow/vault/detect/__init__.py index 15517bb2..0d030ab8 100644 --- a/skyvault/skyflow/vault/detect/__init__.py +++ b/skyvault/skyflow/vault/detect/__init__.py @@ -25,6 +25,7 @@ ObjectEntity, Pdf, PollOptions, + ReidentifyFileOutput, ReidentifyFileRequest, ReidentifyFileResponse, ReidentifyStringRequest, diff --git a/skyvault/tests/vault/detect/test_reexports.py b/skyvault/tests/vault/detect/test_reexports.py index fcaa3561..c5cd95bb 100644 --- a/skyvault/tests/vault/detect/test_reexports.py +++ b/skyvault/tests/vault/detect/test_reexports.py @@ -29,7 +29,8 @@ def test_enums_import_from_utils_enums(self): RedactionType, DeidentifySourceType, ReidentifySourceType, - FileDataFormat, + DeidentifyFileDataFormat, + ReidentifyFileDataFormat, DetectRunStatus, ObjectEntityType, ObjectDeidentificationType, @@ -40,7 +41,8 @@ def test_enums_import_from_utils_enums(self): ) self.assertEqual(EntityType.NAME.value, "NAME") - self.assertEqual(FileDataFormat.PDF.value, "pdf") + self.assertEqual(DeidentifyFileDataFormat.PDF.value, "pdf") + self.assertEqual(ReidentifyFileDataFormat.TXT.value, "txt") self.assertEqual(MaskingMethod.BLUR.value, "BLUR") def test_detect_accessor(self):