diff --git a/common/tests/vault/detect/test_detect.py b/common/tests/vault/detect/test_detect.py index b24540c9..fde39b44 100644 --- a/common/tests/vault/detect/test_detect.py +++ b/common/tests/vault/detect/test_detect.py @@ -10,11 +10,12 @@ from common.utils import SkyflowMessages from common.utils.enums.detect import ( DeidentificationType, + DeidentifyFileDataFormat, DetectRunStatus, EntityType, - FileDataFormat, GuardrailsValidation, IncludePrefix, + ReidentifyFileDataFormat, ) @@ -274,7 +275,7 @@ def test_reidentify_file_value_only(self): controller = DetectController(_vault_client(files=files)) response = controller.reidentify_file(ReidentifyFileRequest( - data_source=ReidentifySourceType.BASE64, value="b64", data_format=FileDataFormat.TXT, + data_source=ReidentifySourceType.BASE64, value="b64", data_format=ReidentifyFileDataFormat.TXT, )) _name, _args, kwargs = files.calls[0] @@ -328,7 +329,7 @@ def test_file_output_writer_names(self): tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, tmp, ignore_errors=True) outputs = [ - mappers.FileOutput(processed_file=base64.b64encode(b"a").decode(), processed_file_type="REDACTED_FILE", processed_file_extension=FileDataFormat.PDF), + mappers.FileOutput(processed_file=base64.b64encode(b"a").decode(), processed_file_type="REDACTED_FILE", processed_file_extension=DeidentifyFileDataFormat.PDF), mappers.FileOutput(processed_file=base64.b64encode(b"b").decode(), processed_file_type="ENTITIES", processed_file_extension=None), ] written = writer.write_outputs(outputs, DeidentifySourceType.BASE64, "invoice.pdf", tmp) @@ -336,7 +337,7 @@ def test_file_output_writer_names(self): self.assertEqual(names, ["processed-entities-invoice.json", "processed-invoice.pdf"]) def test_file_output_writer_skips_presigned(self): - outputs = [mappers.FileOutput(processed_file="http://x", processed_file_type="REDACTED_FILE", processed_file_extension=FileDataFormat.PDF)] + outputs = [mappers.FileOutput(processed_file="http://x", processed_file_type="REDACTED_FILE", processed_file_extension=DeidentifyFileDataFormat.PDF)] self.assertEqual(writer.write_outputs(outputs, DeidentifySourceType.PRESIGNED_URL, "a.pdf", None), []) def test_validations(self): @@ -368,10 +369,16 @@ def test_deidentify_file_source_rules(self): self.assertEqual(no_data_source.exception.message, SkyflowMessages.Error.INVALID_DATA_SOURCE_IN_DEIDENTIFY_FILE.value) def test_file_data_format_is_case_insensitive(self): - self.assertEqual(mappers._file_data_format("PDF"), FileDataFormat.PDF) - self.assertEqual(mappers._file_data_format("pdf"), FileDataFormat.PDF) - self.assertIsNone(mappers._file_data_format("bogus")) - self.assertIsNone(mappers._file_data_format(None)) + self.assertEqual(mappers._deidentify_file_data_format("PDF"), DeidentifyFileDataFormat.PDF) + self.assertEqual(mappers._deidentify_file_data_format("pdf"), DeidentifyFileDataFormat.PDF) + self.assertIsNone(mappers._deidentify_file_data_format("bogus")) + self.assertIsNone(mappers._deidentify_file_data_format(None)) + + def test_reidentify_file_data_format_narrower_set(self): + self.assertEqual(mappers._reidentify_file_data_format("TXT"), ReidentifyFileDataFormat.TXT) + self.assertEqual(mappers._reidentify_file_data_format("txt"), ReidentifyFileDataFormat.TXT) + self.assertIsNone(mappers._reidentify_file_data_format("pdf")) + self.assertIsNone(mappers._reidentify_file_data_format(None)) def test_entity_prefix_fields_serialize(self): wire = mappers.to_entity(Entity( diff --git a/common/utils/_utils.py b/common/utils/_utils.py index 4a1c0637..04c63332 100644 --- a/common/utils/_utils.py +++ b/common/utils/_utils.py @@ -41,7 +41,7 @@ def get_vault_url(cluster_id, env, vault_id, logger=None): if not cluster_id or not isinstance(cluster_id, str) or not cluster_id.strip(): raise SkyflowError(SkyflowMessages.Error.INVALID_CLUSTER_ID.value.format(vault_id), invalid_input_error_code) - if env not in Env: + if not isinstance(env, Env): raise SkyflowError(SkyflowMessages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code) base_url = EnvUrls[env.name].value diff --git a/common/utils/enums/detect.py b/common/utils/enums/detect.py index 6f307ef7..91ca3e6f 100644 --- a/common/utils/enums/detect.py +++ b/common/utils/enums/detect.py @@ -94,7 +94,7 @@ class ReturnEntitiesType(Enum): NONE = "NONE" -class FileDataFormat(Enum): +class DeidentifyFileDataFormat(Enum): BMP = "bmp" CSV = "csv" DCM = "dcm" @@ -120,6 +120,17 @@ class FileDataFormat(Enum): ZIP = "zip" +class ReidentifyFileDataFormat(Enum): + CSV = "csv" + DOCX = "docx" + JSON = "json" + JSONL = "jsonl" + PPTX = "pptx" + TXT = "txt" + XLSX = "xlsx" + XML = "xml" + + class DetectRunStatus(Enum): UNKNOWN = "UNKNOWN" QUEUED = "QUEUED" diff --git a/common/utils/logger/_logger.py b/common/utils/logger/_logger.py index 45519fb1..d2e77547 100644 --- a/common/utils/logger/_logger.py +++ b/common/utils/logger/_logger.py @@ -1,27 +1,26 @@ import logging from ..enums.log_level import LogLevel +LOGGER_NAME = 'skyflow-python' +LOG_MESSAGE_FORMAT = '%(message)s' +_SKYFLOW_HANDLER_FLAG = '_skyflow_handler' + class Logger: def __init__(self, level=LogLevel.ERROR): self.current_level = level - self.logger = logging.getLogger('skyflow-python') - self.logger.propagate = False # Prevent logs from being handled by parent loggers + self.logger = logging.getLogger(LOGGER_NAME) + self.logger.propagate = False - # Remove any existing handlers to avoid duplicates or inherited handlers - if self.logger.hasHandlers(): + if not any(getattr(handler, _SKYFLOW_HANDLER_FLAG, False) for handler in self.logger.handlers): self.logger.handlers.clear() + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter(LOG_MESSAGE_FORMAT)) + setattr(handler, _SKYFLOW_HANDLER_FLAG, True) + self.logger.addHandler(handler) self.set_log_level(level) - handler = logging.StreamHandler() - - # Create a formatter that only includes the message without any prefixes - formatter = logging.Formatter('%(message)s') - handler.setFormatter(formatter) - - self.logger.addHandler(handler) - def set_log_level(self, level): self.current_level = level log_level_mapping = { diff --git a/common/utils/validations/_validations.py b/common/utils/validations/_validations.py index 1d45b41d..9b3070a1 100644 --- a/common/utils/validations/_validations.py +++ b/common/utils/validations/_validations.py @@ -232,7 +232,7 @@ def validate_vault_config(logger, config, messages=None, allowed_keys=None): if ConfigField.CREDENTIALS in config and config.get(ConfigField.CREDENTIALS): validate_credentials(logger, config.get(ConfigField.CREDENTIALS), "vault", vault_id, messages=messages) - if ConfigField.ENV in config and config.get(ConfigField.ENV) not in Env: + if ConfigField.ENV in config and not isinstance(config.get(ConfigField.ENV), Env): log_error_log(messages.ErrorLogs.ENV_IS_REQUIRED.value, logger) raise SkyflowError(messages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code) @@ -256,7 +256,7 @@ def validate_update_vault_config(logger, config, messages=None, allowed_keys=Non if ConfigField.CLUSTER_ID in config and not config.get(ConfigField.CLUSTER_ID): raise SkyflowError(messages.Error.INVALID_CLUSTER_ID.value.format(vault_id), invalid_input_error_code) - if ConfigField.ENV in config and config.get(ConfigField.ENV) not in Env: + if ConfigField.ENV in config and not isinstance(config.get(ConfigField.ENV), Env): raise SkyflowError(messages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code) if ConfigField.CREDENTIALS not in config: diff --git a/common/vault/detect/__init__.py b/common/vault/detect/__init__.py index ae8b5470..bd8c0a5f 100644 --- a/common/vault/detect/__init__.py +++ b/common/vault/detect/__init__.py @@ -26,6 +26,7 @@ Pdf, PollOptions, RedactionRule, + ReidentifyFileOutput, ReidentifyFileRequest, ReidentifyFileResponse, ReidentifyStringRequest, @@ -62,6 +63,7 @@ "Pdf", "PollOptions", "RedactionRule", + "ReidentifyFileOutput", "ReidentifyFileRequest", "ReidentifyFileResponse", "ReidentifyStringRequest", diff --git a/common/vault/detect/_mappers.py b/common/vault/detect/_mappers.py index 14e078ff..0e76006f 100644 --- a/common/vault/detect/_mappers.py +++ b/common/vault/detect/_mappers.py @@ -16,9 +16,10 @@ RedactionLevel as WireRedactionLevel, ) from common.utils.enums.detect import ( + DeidentifyFileDataFormat, DetectRunStatus, - FileDataFormat, GuardrailsValidation, + ReidentifyFileDataFormat, ) from ._constants import EXTENSION_SEPARATOR @@ -31,6 +32,7 @@ FileOutput, GetRunResponse, Metrics, + ReidentifyFileOutput, ReidentifyFileResponse, ReidentifyStringResponse, ) @@ -53,11 +55,20 @@ def _enum_or_default(enum_cls, value, fallback): return fallback -def _file_data_format(value): +def _deidentify_file_data_format(value): if value is None: return None try: - return FileDataFormat(str(value).lower()) + return DeidentifyFileDataFormat(str(value).lower()) + except ValueError: + return None + + +def _reidentify_file_data_format(value): + if value is None: + return None + try: + return ReidentifyFileDataFormat(str(value).lower()) except ValueError: return None @@ -69,7 +80,7 @@ def infer_format(file_name): if index < 0 or index == len(file_name) - 1: return None try: - return FileDataFormat[file_name[index + 1:].upper()] + return DeidentifyFileDataFormat[file_name[index + 1:].upper()] except KeyError: return None @@ -226,7 +237,15 @@ def to_file_output(api_output): return FileOutput( processed_file=api_output.processed_file, processed_file_type=api_output.processed_file_type, - processed_file_extension=_file_data_format(api_output.processed_file_extension), + processed_file_extension=_deidentify_file_data_format(api_output.processed_file_extension), + ) + + +def to_reidentify_file_output(api_output): + return ReidentifyFileOutput( + processed_file=api_output.processed_file, + processed_file_type=api_output.processed_file_type, + processed_file_extension=_reidentify_file_data_format(api_output.processed_file_extension), ) @@ -303,7 +322,7 @@ def to_deidentify_file_response(run_id, run): def to_reidentify_file_response(response, source_type_cls): - outputs = [to_file_output(output) for output in (response.output or []) if output is not None] + outputs = [to_reidentify_file_output(output) for output in (response.output or []) if output is not None] return ReidentifyFileResponse( status=_enum_or_default(DetectRunStatus, response.status, DetectRunStatus.UNKNOWN), output_type=_enum_or_default(source_type_cls, response.output_type, None), diff --git a/common/vault/detect/_models.py b/common/vault/detect/_models.py index d78ea8cf..1ecd2938 100644 --- a/common/vault/detect/_models.py +++ b/common/vault/detect/_models.py @@ -18,7 +18,16 @@ def __init__(self, size: float = None, word_count: int = None, class FileOutput: def __init__(self, processed_file: str = None, processed_file_type: str = None, - processed_file_extension: "FileDataFormat" = None): + processed_file_extension: "DeidentifyFileDataFormat" = None): + self.processed_file = processed_file + self.processed_file_type = processed_file_type + self.processed_file_extension = processed_file_extension + + +class ReidentifyFileOutput: + def __init__(self, processed_file: str = None, + processed_file_type: str = None, + processed_file_extension: "ReidentifyFileDataFormat" = None): self.processed_file = processed_file self.processed_file_type = processed_file_type self.processed_file_extension = processed_file_extension @@ -220,7 +229,7 @@ def __init__(self, processed_text: str = None, class DeidentifyFileRequest: def __init__(self, data_source: "DeidentifySourceType" = None, value: str = None, - data_format: "FileDataFormat" = None, + data_format: "DeidentifyFileDataFormat" = None, configuration_id: str = None, configuration: DetectConfiguration = None, file: BinaryIO = None, file_path: str = None, @@ -288,7 +297,7 @@ def __init__(self, processed_text: str = None, metrics: Metrics = None): class ReidentifyFileRequest: def __init__(self, data_source: "ReidentifySourceType" = None, value: str = None, - data_format: "FileDataFormat" = None, + data_format: "ReidentifyFileDataFormat" = None, redaction_level: List[RedactionRule] = None): self.data_source = data_source self.value = value @@ -299,7 +308,7 @@ def __init__(self, data_source: "ReidentifySourceType" = None, class ReidentifyFileResponse: def __init__(self, status: "DetectRunStatus" = None, output_type: "ReidentifySourceType" = None, - output: List[FileOutput] = None, + output: List[ReidentifyFileOutput] = None, metrics: Metrics = None): self.status = status self.output_type = output_type diff --git a/docs/migrate_detect_v1_to_v2.md b/docs/migrate_detect.md similarity index 91% rename from docs/migrate_detect_v1_to_v2.md rename to docs/migrate_detect.md index 28e39524..12b3d45e 100644 --- a/docs/migrate_detect_v1_to_v2.md +++ b/docs/migrate_detect.md @@ -1,14 +1,22 @@ -# Migrate Detect from V1 to V2 (skyvault Python SDK) - -The skyvault Python SDK replaces the Detect V1 methods with the V2 methods, using the -same method and class names as the flowvault Detect SDK. There is no `V2` suffix -anywhere. `deidentify_file` keeps its name and takes the V2 `DeidentifyFileRequest`, -returning the V2 `DeidentifyFileResponse`. Because the V1 methods and classes are -removed, there is no name conflict. All V2 classes live in `skyflow.vault.detect` and -the enums in `skyflow.utils.enums`. - -V1 callers must migrate when they upgrade, so this ships as a major release. Client -creation and the `skyflow_client.detect()` accessor do not change. +# Migrate to the new Detect methods (skyflow 3.0.0) + +The skyvault Python SDK replaces the previous Detect methods with the new Detect methods, +using the same method and class names as the flowvault Detect SDK — there is no version +suffix on any of them. `deidentify_file` keeps its name and now takes the new +`DeidentifyFileRequest`, returning the new `DeidentifyFileResponse`. Because the previous +methods and classes are removed, there is no name conflict. All Detect classes live in +`skyflow.vault.detect` and the enums in `skyflow.utils.enums`. + +Callers of the previous Detect methods must migrate when they upgrade, so this ships in the +next major release, **skyflow 3.0.0** — pin or upgrade accordingly (`pip install "skyflow==3.0.0"`). +Client creation and the `skyflow_client.detect()` accessor do not change. + +> **⚠️ Biggest behaviour change:** `deidentify_file` **no longer waits for the result by +> default.** It used to always poll; now, without `poll_options`, it submits and returns +> immediately — `status` is `None`, `output` is empty, and nothing is written to disk. Pass +> `poll_options` to block until the run finishes, or retrieve the result later with `get_run`. +> Code ported unchanged runs without error but produces no outputs, so the failure surfaces +> downstream rather than at the call site. ## What changes @@ -355,7 +363,7 @@ fields as the V2 `DeidentifyFileResponse` (`run_id`, `status`, `output_type`, `o ## Timeline -| Release | V1 methods | V2 methods | +| Release | Previous Detect methods | New Detect methods | | --- | --- | --- | -| skyvault 2.1.x | available | not available | -| next major | removed | available under the flowvault names | +| skyflow 2.1.x | available | not available | +| skyflow 3.0.0 | removed | available under the flowvault names | diff --git a/flowvault/README.md b/flowvault/README.md index 3e1d2773..ffcbb670 100644 --- a/flowvault/README.md +++ b/flowvault/README.md @@ -983,7 +983,7 @@ The same pattern applies to every operation, via its corresponding options class # Detect -Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The flowvault SDK exposes the Detect V2 API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. +Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The flowvault SDK exposes the Detect API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. Detect configurations are created in Studio or through the management API — the SDK does not create, update or delete them. Import request/response models from `skyflow.vault.detect` and enums from `skyflow.utils.enums`. @@ -1003,8 +1003,8 @@ detect = skyflow_client.detect() ## Before you start - **Role.** The service account needs the Detect Invoker, Detect Manager or Vault Owner role on the vault. Detect Invoker can de-identify data and write vault tokens into the vault; Detect Manager adds the ability to create, update and delete Detect configurations. Re-identifying values stored as `VAULT_TOKEN` is governed like detokenization: the caller also needs a policy granting the Detokenization action on the destination, unless it is a Vault Owner. -- **Configuration.** Pass `configuration_id` (the id of a stored Detect configuration) or an inline `configuration` (a `DetectConfiguration`). Both are optional: with neither set the SDK sends only the vault id and the run uses the vault's default Detect settings; sending both is rejected by the API with HTTP 400. -- **Destination for vault tokens.** An entity de-identified with `VAULT_TOKEN` stores its value in the token group named by `destination`. That token group must already exist in the vault; Detect writes into it but does not create it. +- **Configuration.** Pass `configuration_id` (the id of a stored Detect configuration) or an inline `configuration` (a `DetectConfiguration`). Both are optional: with neither set the SDK sends only the vault id and the run uses the vault's default Detect settings; sending both is rejected by the API with HTTP 400. Create and manage stored configurations — and copy their ids — in Studio (vault → **Detect**) or through the management API; the SDK does not create them. +- **Destination for vault tokens.** An entity de-identified with `VAULT_TOKEN` stores its value in the token group named by `destination`. That token group must already exist in the vault; Detect writes into it but does not create it. Create token groups, and the named redaction patterns that re-identification applies, in the vault schema in Studio. - **API reference.** Every Detect request, response and configuration class is listed in the [API Reference](../docs/api_reference.md). ## deidentify_string @@ -1060,9 +1060,9 @@ Returns a `DeidentifyStringResponse` with `processed_text`, `entities` (each a ` | Class | Fields | Notes | |-------|--------|-------| | `Detect` | `entities`, `object_entities`, `restrict`, `skip`, `custom_types`, `return_entities` | `restrict` / `skip` are regexes always / never treated as sensitive. `return_entities` is `EXCLUDE_SENSITIVE_DATA` (server default), `ALL` or `NONE`. | -| `Entity` | `entity_type`, `deidentification_type`, `destination`, `transformation` | `entity_type` defaults to `ALL`. `deidentification_type` is `ENTITY_UNIQUE_COUNTER` (server default), `ENTITY_ONLY`, `MASK` or `VAULT_TOKEN`. `destination` (a token group name) is used only with `VAULT_TOKEN`. | +| `Entity` | `entity_type`, `deidentification_type`, `destination`, `transformation`, `include_prefix`, `prefix_override` | `entity_type` defaults to `ALL`. `deidentification_type` is `ENTITY_UNIQUE_COUNTER` (server default), `ENTITY_ONLY`, `MASK` or `VAULT_TOKEN`. `destination` (a token group name) is used only with `VAULT_TOKEN`. `include_prefix` (`IncludePrefix.YES` / `IncludePrefix.NO`) and `prefix_override` also apply only to `VAULT_TOKEN`: `include_prefix=IncludePrefix.NO` stores a raw token that **cannot be re-identified** (a data-loss trap), and `prefix_override` replaces the entity-type label in the token with a custom one. | | `Transformation` / `ShiftDates` | `shift_dates`; `min_days`, `max_days` | Shifts detected dates by a random number of days within the range. | -| `CustomType` | `label`, `match` | A regex-defined entity type; `label` is reported as the entity type of its matches. | +| `CustomType` | `label`, `match`, `deidentification_type`, `destination`, `include_prefix`, `prefix_override` | A regex-defined entity type; `label` is reported as the entity type of its matches. `deidentification_type`, `destination`, `include_prefix` and `prefix_override` behave exactly as on `Entity`. | | `ObjectEntity` | `entity_type`, `deidentification_type` | Visual objects in images: `FACE`, `LICENSE_PLATE`, `LOGO`, `SIGNATURE` or `ALL`, handled with `REDACT` or `UNREDACT`. File inputs only. | | `FileMapping` | `source`, `destination`, `output_file_name_suffix`, `entities_destination`, `object_entities_destination` | Where the processed file and entity files are written. File inputs only. | | `Media` | `audio`, `document`, `image` | Media-type options. File inputs only. | @@ -1108,7 +1108,7 @@ configuration = DetectConfiguration( De-identifies a file. The API is asynchronous: the submit call returns a run id and the result is retrieved from the run. Build the request with `DeidentifyFileRequest`. -Supply the file in exactly one of three ways: `file_path`, `file`, or `value` plus a `DeidentifySourceType` `data_source` (`BASE64`, the `SKYFLOW_ID` of a file record in the vault, or `PRESIGNED_URL`). A local `file` / `file_path` is read when the request is sent, base64-encoded, and its `data_format` inferred from the extension. Passing more than one of them, or none, fails client-side validation before any network call. `configuration_id` / `configuration` follow the same rules as for strings (see [Before you start](#before-you-start)). +Supply the file in exactly one of three ways: `file_path`, `file`, or `value` plus a `DeidentifySourceType` `data_source` (`BASE64` or `PRESIGNED_URL`). flowvault does **not** support a Skyflow ID as a file source — its `DeidentifySourceType` has only `BASE64` and `PRESIGNED_URL`. A local `file` / `file_path` is read when the request is sent, base64-encoded, and its `data_format` inferred from the extension. Passing more than one of them, or none, fails client-side validation before any network call. `configuration_id` / `configuration` follow the same rules as for strings (see [Before you start](#before-you-start)). - **Submit only** (default): one call, returns `run_id`. `status` is `None` and nothing is written to disk — retrieve the result later with [`get_run`](#get_run). - **Submit and poll**: pass `poll_options`. The SDK polls with exponential backoff (1, 2, 4, 8, 16, then 32 seconds) until the run is `SUCCESS`, `FAILED` or `UNKNOWN`, or until `wait_time` (default 60 s, max 300) or `max_attempts` (default and max 23) is used up. If the budget runs out the response carries the run id with `IN_PROGRESS` or `QUEUED`, so you can resume with `get_run`. `output_directory` is valid only with `poll_options`; setting it without them fails client-side validation. @@ -1149,9 +1149,9 @@ else: # IN_PROGRESS or QUEUED: budget ran out, resume - `/data/processed/processed-entities-patient-notes.json` — the detected entities - `/data/processed/processed-object-entities-patient-notes.json` — object entities, when the run produced them -`output_directory` is created if it does not exist. **If you poll a local file and omit `output_directory`, these files are written to the current working directory** — often your project root or a server's working directory — so set `output_directory` explicitly. A request built from a `SKYFLOW_ID` or `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in those cases read `response.output` and write the files yourself. +`output_directory` is created if it does not exist. **If you poll a local file and omit `output_directory`, these files are written to the current working directory** — often your project root or a server's working directory — so set `output_directory` explicitly. A request built from a `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in that case read `response.output` and write the files yourself. -**Supported formats and limits.** `data_format` accepts every `FileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. +**Supported formats and limits.** `data_format` accepts every `DeidentifyFileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. **Polling blocks the calling thread** — each lookup is a synchronous `get_run` followed by a sleep. For concurrency, submit without `poll_options` and poll `get_run` yourself on a scheduler, or run the blocking call on a worker thread you own. @@ -1159,13 +1159,13 @@ A file at a presigned URL is referenced through `data_source` + `value`: ```python from skyflow.vault.detect import DeidentifyFileRequest -from skyflow.utils.enums import DeidentifySourceType, FileDataFormat +from skyflow.utils.enums import DeidentifySourceType, DeidentifyFileDataFormat skyflow_client.detect().deidentify_file( DeidentifyFileRequest( data_source=DeidentifySourceType.PRESIGNED_URL, value='', - data_format=FileDataFormat.PDF, + data_format=DeidentifyFileDataFormat.PDF, configuration_id='', ) ) @@ -1200,18 +1200,37 @@ Synchronous. Re-identification looks tokens up in the vault, so it applies only Build the request with `ReidentifyStringRequest`: the tokenized text and an optional list of `RedactionLevel` rules. On flowvault vaults each rule targets a `token_group_name` and applies a named `redaction_pattern` configured in the vault. Without rules every token is replaced by its original value. +Only text that actually contains vault tokens can be re-identified, so the example below is a round trip: first de-identify with `VAULT_TOKEN` to mint tokens, then re-identify the `processed_text` that comes back. A vault token is an opaque value like `a1b2c3d4-...` (optionally prefixed, e.g. `[NAME_a1b2c3d4-...]`), not the `[NAME_1]` counter placeholder produced by `ENTITY_UNIQUE_COUNTER`. + ```python -from skyflow.vault.detect import ReidentifyStringRequest, RedactionLevel +from skyflow.vault.detect import ( + DeidentifyStringRequest, DetectConfiguration, Detect, Entity, + ReidentifyStringRequest, RedactionLevel, +) +from skyflow.utils.enums import EntityType, DeidentificationType + +# 1. De-identify with VAULT_TOKEN so the value is stored and can be re-identified later. +deidentified = skyflow_client.detect().deidentify_string( + DeidentifyStringRequest( + text='My name is John Doe.', + configuration=DetectConfiguration(detect=Detect(entities=[ + Entity(entity_type=EntityType.NAME, deidentification_type=DeidentificationType.VAULT_TOKEN, destination=''), + ])), + ) +) +# deidentified.processed_text now holds a vault token, e.g. 'My name is a1b2c3d4-...' +# 2. Re-identify those tokens. response = skyflow_client.detect().reidentify_string( ReidentifyStringRequest( - text='My name is [NAME_1].', + text=deidentified.processed_text, redaction_level=[ RedactionLevel(token_group_name='', redaction_pattern=''), ], ) ) print(response.processed_text) +# My name is John Doe. ``` > The flowvault `RedactionLevel` has only `token_group_name` and `redaction_pattern`; the skyvault SDK's has `entity_name` and `redaction_type` instead. Token groups and redaction patterns are configured in the vault. @@ -1220,13 +1239,14 @@ Returns a `ReidentifyStringResponse` with `processed_text` and `metrics`. ## reidentify_file -Synchronous — the re-identified file comes back in the response, not through a run id. Build the request with `ReidentifyFileRequest`: a `data_source` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `data_format`, and the same `RedactionLevel` rules as `reidentify_string`. `data_format` is limited to the formats that support `VAULT_TOKEN`: `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. +Synchronous — the re-identified file comes back in the response, not through a run id. Build the request with `ReidentifyFileRequest`: a `data_source` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `data_format` (a `ReidentifyFileDataFormat`), and the same `RedactionLevel` rules as `reidentify_string`. `data_format` is limited to the formats that support `VAULT_TOKEN`: `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. ```python import base64 from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel -from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, DetectRunStatus +from skyflow.utils.enums import ReidentifySourceType, ReidentifyFileDataFormat, DetectRunStatus +data_format = ReidentifyFileDataFormat.TXT with open('/data/redacted/notes.txt', 'rb') as handle: encoded = base64.b64encode(handle.read()).decode('utf-8') @@ -1234,23 +1254,26 @@ response = skyflow_client.detect().reidentify_file( ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=data_format, redaction_level=[RedactionLevel(token_group_name='', redaction_pattern='')], ) ) if response.status == DetectRunStatus.SUCCESS: for index, output in enumerate(response.output): - # output.processed_file is base64 or a presigned URL, per response.output_type - with open(f'reidentified-{index}.txt', 'wb') as out: - out.write(base64.b64decode(output.processed_file)) + # Branch on output_type: base64 bytes are decoded, a presigned URL is downloaded. + if response.output_type == ReidentifySourceType.BASE64: + with open(f'reidentified-{index}.{data_format.value}', 'wb') as out: + out.write(base64.b64decode(output.processed_file)) + else: # ReidentifySourceType.PRESIGNED_URL — processed_file is a URL to fetch + print('download re-identified file from', output.processed_file) ``` -Returns a `ReidentifyFileResponse` with `status`, `output_type`, `output` (a list of `FileOutput`) and `metrics`. +Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `ReidentifyFileOutput`) and `metrics`. ## check_guardrails -Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. The response is passed through as the API returns it. +Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. **When `check_toxicity` is omitted, the server still checks toxicity by default.** The response is passed through as the API returns it. ```python from skyflow.vault.detect import CheckGuardrailsRequest @@ -1268,6 +1291,53 @@ Returns a `CheckGuardrailsResponse` with `text`, `validation` (`GuardrailsValida All Detect methods raise `skyflow.error.SkyflowError` on validation failure, credential failure, or an API error — see [Error Handling](#error-handling). Client-side validation fails before any network call. +## Response examples and common errors + +Enum-typed fields (`status`, `output_type`, `validation`, …) are shown below by their string values. + +`deidentify_string`: + +```json +{ + "processed_text": "My name is [NAME_1], and my email is [EMAIL_ADDRESS_1].", + "entities": [ + { "token": "[NAME_1]", "value": "", "location": { "start_index": 11, "end_index": 19, "start_index_processed": 11, "end_index_processed": 19 }, "entity_type": "NAME", "entity_scores": { "NAME": 0.99 } } + ], + "metrics": { "word_count": 10, "character_count": 54 } +} +``` + +`deidentify_file` (polled) / `get_run`: + +```json +{ + "run_id": "run_a1b2c3d4", + "status": "SUCCESS", + "output_type": "BASE64", + "output": [ { "processed_file": "", "processed_file_type": "application/pdf", "processed_file_extension": "pdf" } ], + "metrics": { "pages": 3 }, + "message": null +} +``` + +`reidentify_string`: + +```json +{ "processed_text": "My name is John Doe.", "metrics": { "word_count": 5 } } +``` + +`check_guardrails`: + +```json +{ "text": "", "validation": "FAILED", "toxic": true, "denied_topic": false } +``` + +Common Detect errors (all raised as `SkyflowError`): + +- **HTTP 400 — both configuration sources set.** Passing `configuration_id` and `configuration` in the same request is rejected; pass at most one. +- **Missing token group.** A `VAULT_TOKEN` entity whose `destination` token group does not exist in the vault fails — create the token group first. +- **Unsupported format with `VAULT_TOKEN`.** `VAULT_TOKEN` is only valid for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`; other `data_format` values are rejected. + # Error Handling ## Two layers of errors diff --git a/flowvault/api-report/skyflow.api.json b/flowvault/api-report/skyflow.api.json index 60eaa0de..335f827a 100644 --- a/flowvault/api-report/skyflow.api.json +++ b/flowvault/api-report/skyflow.api.json @@ -35,6 +35,30 @@ "skyflow.utils.enums.DeidentificationType.MASK": "attr = 'MASK'", "skyflow.utils.enums.DeidentificationType.UNKNOWN": "attr = 'UNKNOWN'", "skyflow.utils.enums.DeidentificationType.VAULT_TOKEN": "attr = 'VAULT_TOKEN'", + "skyflow.utils.enums.DeidentifyFileDataFormat": "class (Enum)", + "skyflow.utils.enums.DeidentifyFileDataFormat.BMP": "attr = 'bmp'", + "skyflow.utils.enums.DeidentifyFileDataFormat.CSV": "attr = 'csv'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DCM": "attr = 'dcm'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DOC": "attr = 'doc'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DOCX": "attr = 'docx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.GIF": "attr = 'gif'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JPEG": "attr = 'jpeg'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JPG": "attr = 'jpg'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JSON": "attr = 'json'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JSONL": "attr = 'jsonl'", + "skyflow.utils.enums.DeidentifyFileDataFormat.MP3": "attr = 'mp3'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PDF": "attr = 'pdf'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PNG": "attr = 'png'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PPT": "attr = 'ppt'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PPTX": "attr = 'pptx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TIF": "attr = 'tif'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TIFF": "attr = 'tiff'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TXT": "attr = 'txt'", + "skyflow.utils.enums.DeidentifyFileDataFormat.WAV": "attr = 'wav'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XLS": "attr = 'xls'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XLSX": "attr = 'xlsx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XML": "attr = 'xml'", + "skyflow.utils.enums.DeidentifyFileDataFormat.ZIP": "attr = 'zip'", "skyflow.utils.enums.DeidentifySourceType": "class (Enum)", "skyflow.utils.enums.DeidentifySourceType.BASE64": "attr = 'BASE64'", "skyflow.utils.enums.DeidentifySourceType.PRESIGNED_URL": "attr = 'PRESIGNED_URL'", @@ -121,30 +145,6 @@ "skyflow.utils.enums.EnvUrls.PROD": "attr = 'skyvault.skyflowapis.com'", "skyflow.utils.enums.EnvUrls.SANDBOX": "attr = 'skyvault.skyflowapis-preview.com'", "skyflow.utils.enums.EnvUrls.STAGE": "attr = 'skyvault.skyflowapis.tech'", - "skyflow.utils.enums.FileDataFormat": "class (Enum)", - "skyflow.utils.enums.FileDataFormat.BMP": "attr = 'bmp'", - "skyflow.utils.enums.FileDataFormat.CSV": "attr = 'csv'", - "skyflow.utils.enums.FileDataFormat.DCM": "attr = 'dcm'", - "skyflow.utils.enums.FileDataFormat.DOC": "attr = 'doc'", - "skyflow.utils.enums.FileDataFormat.DOCX": "attr = 'docx'", - "skyflow.utils.enums.FileDataFormat.GIF": "attr = 'gif'", - "skyflow.utils.enums.FileDataFormat.JPEG": "attr = 'jpeg'", - "skyflow.utils.enums.FileDataFormat.JPG": "attr = 'jpg'", - "skyflow.utils.enums.FileDataFormat.JSON": "attr = 'json'", - "skyflow.utils.enums.FileDataFormat.JSONL": "attr = 'jsonl'", - "skyflow.utils.enums.FileDataFormat.MP3": "attr = 'mp3'", - "skyflow.utils.enums.FileDataFormat.PDF": "attr = 'pdf'", - "skyflow.utils.enums.FileDataFormat.PNG": "attr = 'png'", - "skyflow.utils.enums.FileDataFormat.PPT": "attr = 'ppt'", - "skyflow.utils.enums.FileDataFormat.PPTX": "attr = 'pptx'", - "skyflow.utils.enums.FileDataFormat.TIF": "attr = 'tif'", - "skyflow.utils.enums.FileDataFormat.TIFF": "attr = 'tiff'", - "skyflow.utils.enums.FileDataFormat.TXT": "attr = 'txt'", - "skyflow.utils.enums.FileDataFormat.WAV": "attr = 'wav'", - "skyflow.utils.enums.FileDataFormat.XLS": "attr = 'xls'", - "skyflow.utils.enums.FileDataFormat.XLSX": "attr = 'xlsx'", - "skyflow.utils.enums.FileDataFormat.XML": "attr = 'xml'", - "skyflow.utils.enums.FileDataFormat.ZIP": "attr = 'zip'", "skyflow.utils.enums.FileUploadStatus": "class (Enum)", "skyflow.utils.enums.FileUploadStatus.FAILED": "attr = 'FAILED'", "skyflow.utils.enums.FileUploadStatus.SKIPPED": "attr = 'SKIPPED'", @@ -186,6 +186,15 @@ "skyflow.utils.enums.RedactionType.MASKED": "attr = 'MASKED'", "skyflow.utils.enums.RedactionType.PLAIN_TEXT": "attr = 'PLAIN_TEXT'", "skyflow.utils.enums.RedactionType.REDACTED": "attr = 'REDACTED'", + "skyflow.utils.enums.ReidentifyFileDataFormat": "class (Enum)", + "skyflow.utils.enums.ReidentifyFileDataFormat.CSV": "attr = 'csv'", + "skyflow.utils.enums.ReidentifyFileDataFormat.DOCX": "attr = 'docx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.JSON": "attr = 'json'", + "skyflow.utils.enums.ReidentifyFileDataFormat.JSONL": "attr = 'jsonl'", + "skyflow.utils.enums.ReidentifyFileDataFormat.PPTX": "attr = 'pptx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.TXT": "attr = 'txt'", + "skyflow.utils.enums.ReidentifyFileDataFormat.XLSX": "attr = 'xlsx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.XML": "attr = 'xml'", "skyflow.utils.enums.ReidentifySourceType": "class (Enum)", "skyflow.utils.enums.ReidentifySourceType.BASE64": "attr = 'BASE64'", "skyflow.utils.enums.ReidentifySourceType.PRESIGNED_URL": "attr = 'PRESIGNED_URL'", @@ -484,6 +493,7 @@ "skyflow.vault.detect.RedactionLevel.__init__": "def (self, token_group_name: str = None, redaction_pattern: str = None)", "skyflow.vault.detect.RedactionLevel.redaction_pattern": "attr", "skyflow.vault.detect.RedactionLevel.token_group_name": "attr", + "skyflow.vault.detect.ReidentifyFileOutput": "alias -> common.vault.detect.ReidentifyFileOutput", "skyflow.vault.detect.ReidentifyFileRequest": "alias -> common.vault.detect.ReidentifyFileRequest", "skyflow.vault.detect.ReidentifyFileResponse": "alias -> common.vault.detect.ReidentifyFileResponse", "skyflow.vault.detect.ReidentifyStringRequest": "alias -> common.vault.detect.ReidentifyStringRequest", diff --git a/flowvault/samples/README.md b/flowvault/samples/README.md index 0737bc10..f5b4fc79 100644 --- a/flowvault/samples/README.md +++ b/flowvault/samples/README.md @@ -44,7 +44,7 @@ python flowvault/samples/vault_api/insert_records.py | [upload_files.py](vault_api/upload_files.py) | Upload files to file columns (`file_path` / `base64` / `file_object`) | | [delete_files.py](vault_api/delete_files.py) | Delete files from file columns (by `skyflow_id` or `unique_values`) | -## Detect V2 +## Detect | Sample | Demonstrates | |---|---| diff --git a/flowvault/samples/detect_api/check_guardrails.py b/flowvault/samples/detect_api/check_guardrails.py index 54082337..5e338281 100644 --- a/flowvault/samples/detect_api/check_guardrails.py +++ b/flowvault/samples/detect_api/check_guardrails.py @@ -36,6 +36,7 @@ def perform_check_guardrails(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/deidentify_file.py b/flowvault/samples/detect_api/deidentify_file.py index 77375643..c687305c 100644 --- a/flowvault/samples/detect_api/deidentify_file.py +++ b/flowvault/samples/detect_api/deidentify_file.py @@ -1,7 +1,7 @@ from skyflow.error import SkyflowError from skyflow import Env, Skyflow, LogLevel from skyflow.vault.detect import DeidentifyFileRequest, PollOptions -from skyflow.utils.enums import DetectRunStatus, DeidentifySourceType, FileDataFormat +from skyflow.utils.enums import DetectRunStatus, DeidentifySourceType, DeidentifyFileDataFormat def perform_deidentify_file_submit_only(): @@ -34,6 +34,7 @@ def perform_deidentify_file_submit_only(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -75,6 +76,7 @@ def perform_deidentify_file_poll(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -97,7 +99,7 @@ def perform_deidentify_file_presigned(): request = DeidentifyFileRequest( data_source=DeidentifySourceType.PRESIGNED_URL, value='', - data_format=FileDataFormat.PDF, + data_format=DeidentifyFileDataFormat.PDF, configuration_id='', ) @@ -110,6 +112,7 @@ def perform_deidentify_file_presigned(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/deidentify_string.py b/flowvault/samples/detect_api/deidentify_string.py index c057b358..84165c6d 100644 --- a/flowvault/samples/detect_api/deidentify_string.py +++ b/flowvault/samples/detect_api/deidentify_string.py @@ -36,6 +36,7 @@ def perform_deidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -85,6 +86,7 @@ def perform_deidentify_string_inline(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/get_run.py b/flowvault/samples/detect_api/get_run.py index 10a39c2f..f68ae924 100644 --- a/flowvault/samples/detect_api/get_run.py +++ b/flowvault/samples/detect_api/get_run.py @@ -35,6 +35,7 @@ def perform_get_run(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/reidentify_file.py b/flowvault/samples/detect_api/reidentify_file.py index 01f4b994..48b12efc 100644 --- a/flowvault/samples/detect_api/reidentify_file.py +++ b/flowvault/samples/detect_api/reidentify_file.py @@ -3,7 +3,7 @@ from skyflow.error import SkyflowError from skyflow import Env, Skyflow, LogLevel from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel -from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, DetectRunStatus +from skyflow.utils.enums import ReidentifySourceType, ReidentifyFileDataFormat, DetectRunStatus def perform_reidentify_file(): @@ -28,7 +28,7 @@ def perform_reidentify_file(): request = ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=ReidentifyFileDataFormat.TXT, redaction_level=[ RedactionLevel( token_group_name='', @@ -49,6 +49,7 @@ def perform_reidentify_file(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/samples/detect_api/reidentify_string.py b/flowvault/samples/detect_api/reidentify_string.py index 045e35e1..716e5ad4 100644 --- a/flowvault/samples/detect_api/reidentify_string.py +++ b/flowvault/samples/detect_api/reidentify_string.py @@ -59,6 +59,7 @@ def perform_reidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/flowvault/skyflow/utils/enums/__init__.py b/flowvault/skyflow/utils/enums/__init__.py index 1c567e0e..bba26acf 100644 --- a/flowvault/skyflow/utils/enums/__init__.py +++ b/flowvault/skyflow/utils/enums/__init__.py @@ -7,10 +7,11 @@ from common.utils.enums.redaction_type import RedactionType from common.utils.enums.detect import ( DeidentificationType, + DeidentifyFileDataFormat, DetectRunStatus, EntityType, - FileDataFormat, GuardrailsValidation, + ReidentifyFileDataFormat, IncludePrefix, MaskingMethod, ObjectDeidentificationType, diff --git a/flowvault/skyflow/vault/client/client.py b/flowvault/skyflow/vault/client/client.py index 4154c2ff..2c6fa325 100644 --- a/flowvault/skyflow/vault/client/client.py +++ b/flowvault/skyflow/vault/client/client.py @@ -1,3 +1,5 @@ +import threading + import httpx from common.vault.base_vault_client import BaseVaultClient @@ -28,6 +30,7 @@ from skyflow.utils._retry import RetryTransport CONTENT_TYPE_HEADER = "content-type" +GENERATED_CLIENT_MAX_RETRIES = 0 class VaultClient(BaseVaultClient): @@ -37,6 +40,7 @@ def __init__(self, config): self._sync_httpx_client = None self._sync_upload_client = None self._detect_api_client = None + self._client_lock = threading.Lock() def set_common_http_config(self, common_http_config): self._common_http_config = common_http_config or {} @@ -75,27 +79,39 @@ def _build_limits(self): ) def initialize_api_client(self, vault_url, bearer_token): - self.__close_httpx_clients() - timeout = self._build_timeout() - limits = self._build_limits() - max_retries, initial_millis, max_millis, call_timeout = self._retry_params() - sync_client = httpx.Client( - timeout=timeout, - follow_redirects=True, - transport=RetryTransport(httpx.HTTPTransport(limits=limits), max_retries, initial_millis, max_millis, call_timeout), - ) - self._sync_httpx_client = sync_client - self._sync_upload_client = httpx.Client( - timeout=timeout, - follow_redirects=True, - transport=httpx.HTTPTransport(limits=limits), - ) - self._api_client = SkyflowAuth(base_url=vault_url, token=bearer_token or "", httpx_client=sync_client) - self._detect_api_client = DetectSkyflowAuth(base_url=vault_url, token=bearer_token or "", httpx_client=sync_client) + with self._client_lock: + self.__close_httpx_clients() + timeout = self._build_timeout() + limits = self._build_limits() + max_retries, initial_millis, max_millis, call_timeout = self._retry_params() + sync_client = httpx.Client( + timeout=timeout, + follow_redirects=True, + transport=RetryTransport(httpx.HTTPTransport(limits=limits), max_retries, initial_millis, max_millis, call_timeout), + ) + self._sync_httpx_client = sync_client + self._sync_upload_client = httpx.Client( + timeout=timeout, + follow_redirects=True, + transport=httpx.HTTPTransport(limits=limits), + ) + self._api_client = SkyflowAuth( + base_url=vault_url, + token=bearer_token or None, + httpx_client=sync_client, + max_retries=GENERATED_CLIENT_MAX_RETRIES, + ) + self._detect_api_client = DetectSkyflowAuth( + base_url=vault_url, + token=bearer_token or None, + httpx_client=sync_client, + max_retries=GENERATED_CLIENT_MAX_RETRIES, + ) def close(self): - self.__close_httpx_clients() - self._api_client = None + with self._client_lock: + self.__close_httpx_clients() + self._api_client = None def __close_httpx_clients(self): for attribute in ("_sync_httpx_client", "_sync_upload_client"): diff --git a/flowvault/skyflow/vault/controller/_vault.py b/flowvault/skyflow/vault/controller/_vault.py index 68962817..14b7cda9 100644 --- a/flowvault/skyflow/vault/controller/_vault.py +++ b/flowvault/skyflow/vault/controller/_vault.py @@ -124,13 +124,12 @@ def insert(self, request: InsertRequest, options: InsertOptions = None) -> Inser needs_per_record_table = any(r.table_name is not None for r in request.records) needs_per_record_upsert = any(r.upsert is not None for r in request.records) - wire_records = [ - self.__build_wire_record(record, request, needs_per_record_table, needs_per_record_upsert) - for record in request.records - ] - try: log_info(SkyflowMessages.Info.INSERT_TRIGGERED.value, self._vault_client.get_logger()) + wire_records = [ + self.__build_wire_record(record, request, needs_per_record_table, needs_per_record_upsert) + for record in request.records + ] upsert_kwargs = self.__omit_none( upsert=None if needs_per_record_upsert else self.__to_upsert(request.upsert), ) @@ -168,21 +167,20 @@ def get(self, request: GetRequest, options: GetOptions = None) -> GetResponse: records_api = self._vault_client.get_records_api() - if request.records is not None: - call_kwargs = {'records': self.__to_get_request_data(request.records)} - else: - call_kwargs = { - 'table_name': request.table_name, - 'skyflow_i_ds': request.skyflow_ids, - 'unique_values': self.__to_unique_values(request.unique_values), - 'columns': request.columns, - 'column_redactions': self.__to_column_redactions(request.column_redactions), - 'limit': request.limit, - 'offset': request.offset, - } - try: log_info(SkyflowMessages.Info.GET_TRIGGERED.value, self._vault_client.get_logger()) + if request.records is not None: + call_kwargs = {'records': self.__to_get_request_data(request.records)} + else: + call_kwargs = { + 'table_name': request.table_name, + 'skyflow_i_ds': request.skyflow_ids, + 'unique_values': self.__to_unique_values(request.unique_values), + 'columns': request.columns, + 'column_redactions': self.__to_column_redactions(request.column_redactions), + 'limit': request.limit, + 'offset': request.offset, + } raw_response = records_api.with_raw_response.get_records( vault_id=self._vault_client.get_vault_id(), request_options=self.__unary_request_options(OPERATION_GET, options), @@ -785,8 +783,22 @@ def __put_file_source(self, signed_url, source, content_type): if kind == SOURCE_KIND_PATH: with open(payload, 'rb') as handle: return self._vault_client.put_signed_url(signed_url, handle, content_type).status_code + if kind == SOURCE_KIND_FILE_OBJECT and payload is not None: + self.__rewind_file_object(payload) return self._vault_client.put_signed_url(signed_url, payload, content_type).status_code + def __rewind_file_object(self, file_object): + seek = getattr(file_object, 'seek', None) + if not callable(seek): + return + seekable = getattr(file_object, 'seekable', None) + if callable(seekable) and not seekable(): + return + try: + seek(0) + except (OSError, ValueError): + pass + def __to_delete_files_request_data(self, records): return [ FileDeleteRecord( diff --git a/flowvault/skyflow/vault/detect/__init__.py b/flowvault/skyflow/vault/detect/__init__.py index 15517bb2..0d030ab8 100644 --- a/flowvault/skyflow/vault/detect/__init__.py +++ b/flowvault/skyflow/vault/detect/__init__.py @@ -25,6 +25,7 @@ ObjectEntity, Pdf, PollOptions, + ReidentifyFileOutput, ReidentifyFileRequest, ReidentifyFileResponse, ReidentifyStringRequest, diff --git a/flowvault/tests/vault/detect/test_reexports.py b/flowvault/tests/vault/detect/test_reexports.py index ddff0965..ba2233fc 100644 --- a/flowvault/tests/vault/detect/test_reexports.py +++ b/flowvault/tests/vault/detect/test_reexports.py @@ -29,7 +29,8 @@ def test_enums_import_from_utils_enums(self): RedactionType, DeidentifySourceType, ReidentifySourceType, - FileDataFormat, + DeidentifyFileDataFormat, + ReidentifyFileDataFormat, DetectRunStatus, ObjectEntityType, ObjectDeidentificationType, @@ -40,7 +41,8 @@ def test_enums_import_from_utils_enums(self): ) self.assertEqual(EntityType.NAME.value, "NAME") - self.assertEqual(FileDataFormat.PDF.value, "pdf") + self.assertEqual(DeidentifyFileDataFormat.PDF.value, "pdf") + self.assertEqual(ReidentifyFileDataFormat.TXT.value, "txt") self.assertEqual(DetectRunStatus.SUCCESS.value, "SUCCESS") diff --git a/skyvault/README.md b/skyvault/README.md index 9ce5a7e3..42bded78 100644 --- a/skyvault/README.md +++ b/skyvault/README.md @@ -2,7 +2,7 @@ > **This is the current, recommended version of the Skyflow SDK.** V2.1.0 brings flexible auth, multi-vault support, native data types, and rich error diagnostics. > -> Migrating from v1? See the **[Migration Guide](https://github.com/skyflowapi/skyflow-python/blob/main/docs/migrate_to_v2.md)** for step-by-step instructions. V1 is in maintenance mode and will reach End of Life on October 31, 2026. +> Migrating from v1? See the **[Migration Guide](https://github.com/skyflowapi/skyflow-python/blob/main/docs/migrate_to_v2.md)** for step-by-step instructions. V1 reached End of Life on October 31, 2026 and is no longer supported. The Skyflow Python SDK is designed to help with integrating Skyflow into a Python backend. @@ -552,7 +552,7 @@ Tokenization result: TokenizeResponse(tokenized_fields=[{'token': 'a1b2c3d4-...' ## Detect -Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The SDK exposes the Detect V2 API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. +Skyflow Detect finds and de-identifies sensitive data in strings and files, re-identifies tokens on demand, and screens text against guardrails. The SDK exposes the Detect API through `DetectController`, returned by `skyflow_client.detect()` and `skyflow_client.detect('')`. Every request is scoped to that vault, and the controller shares the same credentials, timeouts and retries as `vault()`; the `vault_id` is filled in from the vault config for every method. Detect configurations are created in Studio or through the management API — the SDK does not create, update or delete them. Import request/response models from `skyflow.vault.detect` and enums from `skyflow.utils.enums`. @@ -677,6 +677,8 @@ configuration = DetectConfiguration( De-identifies a file. The API is asynchronous: the submit call returns a run id and the result is retrieved from the run. Build the request with `DeidentifyFileRequest`. +> **⚠️ Behaviour change from Detect V1:** `deidentify_file` **no longer waits for the result by default.** Without `poll_options` it submits and returns immediately — `status` is `None`, `output` is empty, and nothing is written to disk. Pass `poll_options` to block until the run finishes, or retrieve the result later with [`get_run`](#get-run-get_runrequest). Code ported from the old blocking behaviour runs without error but produces no outputs, so the failure surfaces downstream rather than at the call site. + Supply the file in exactly one of three ways: `file_path`, `file`, or `value` plus a `DeidentifySourceType` `data_source` (`BASE64`, the `SKYFLOW_ID` of a file record in the vault, or `PRESIGNED_URL`). A local `file` / `file_path` is read when the request is sent, base64-encoded, and its `data_format` inferred from the extension. Passing more than one of them, or none, fails client-side validation before any network call. `configuration_id` / `configuration` follow the same rules as for strings (see [Before you start](#before-you-start)). - **Submit only** (default): one call, returns `run_id`. `status` is `None` and nothing is written to disk — retrieve the result later with [`get_run`](#get-run-get_runrequest). @@ -720,7 +722,7 @@ else: # IN_PROGRESS or QUEUED: budget ran out, resume `output_directory` is created if it does not exist. **If you poll a local file and omit `output_directory`, these files are written to the current working directory** — often your project root or a server's working directory — so set `output_directory` explicitly. A request built from a `SKYFLOW_ID` or `PRESIGNED_URL` value has no local name to derive from, and presigned-URL outputs are not fetched, so in those cases read `response.output` and write the files yourself. -**Supported formats and limits.** `data_format` accepts every `FileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. +**Supported formats and limits.** `data_format` accepts every `DeidentifyFileDataFormat`: `bmp`, `csv`, `dcm`, `doc`, `docx`, `gif`, `jpeg`, `jpg`, `json`, `jsonl`, `mp3`, `pdf`, `png`, `ppt`, `pptx`, `tif`, `tiff`, `txt`, `wav`, `xls`, `xlsx`, `xml`, `zip`. The `VAULT_TOKEN` de-identification type is supported only for `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. **Polling blocks the calling thread** — each lookup is a synchronous `get_run` followed by a sleep. For concurrency, submit without `poll_options` and poll `get_run` yourself on a scheduler, or run the blocking call on a worker thread you own. @@ -728,13 +730,13 @@ A file at a presigned URL is referenced through `data_source` + `value`: ```python from skyflow.vault.detect import DeidentifyFileRequest -from skyflow.utils.enums import DeidentifySourceType, FileDataFormat +from skyflow.utils.enums import DeidentifySourceType, DeidentifyFileDataFormat skyflow_client.detect().deidentify_file( DeidentifyFileRequest( data_source=DeidentifySourceType.PRESIGNED_URL, value='', - data_format=FileDataFormat.PDF, + data_format=DeidentifyFileDataFormat.PDF, configuration_id='', ) ) @@ -769,13 +771,31 @@ Synchronous. Re-identification looks tokens up in the vault, so it applies only Build the request with `ReidentifyStringRequest`: the tokenized text and an optional list of `RedactionLevel` rules. On skyvault vaults each rule names an `entity_name` and a `redaction_type` — `PLAIN_TEXT` reveals the original value, `MASKED` masks it, `REDACTED` keeps the token, `DEFAULT` applies the vault's default. Without rules every token is replaced by its original value. +Only text that contains vault tokens can be re-identified, so the example is a round trip: de-identify with `VAULT_TOKEN` (which stores each value in a `table.column`), then re-identify the `processed_text` that comes back. A vault token is an opaque value like `a1b2c3d4-...`, not the `[NAME_1]` counter placeholder produced by `ENTITY_UNIQUE_COUNTER`. + ```python -from skyflow.vault.detect import ReidentifyStringRequest, RedactionLevel -from skyflow.utils.enums import EntityType, RedactionType +from skyflow.vault.detect import ( + DeidentifyStringRequest, DetectConfiguration, Detect, Entity, + ReidentifyStringRequest, RedactionLevel, +) +from skyflow.utils.enums import EntityType, DeidentificationType, RedactionType + +# 1. De-identify with VAULT_TOKEN so each value is stored in a table.column and can be re-identified. +deidentified = skyflow_client.detect().deidentify_string( + DeidentifyStringRequest( + text='My name is John Doe and my email is johndoe@acme.com.', + configuration=DetectConfiguration(detect=Detect(entities=[ + Entity(entity_type=EntityType.NAME, deidentification_type=DeidentificationType.VAULT_TOKEN, destination='persons.name'), + Entity(entity_type=EntityType.EMAIL_ADDRESS, deidentification_type=DeidentificationType.VAULT_TOKEN, destination='persons.email'), + ])), + ) +) +# deidentified.processed_text now holds vault tokens, e.g. 'My name is a1b2c3d4-... and my email is e5f6a7b8-...' +# 2. Re-identify those tokens; on skyvault each rule names an entity_name + redaction_type. response = skyflow_client.detect().reidentify_string( ReidentifyStringRequest( - text='My name is [NAME_1] and my email is [EMAIL_ADDRESS_1].', + text=deidentified.processed_text, redaction_level=[ RedactionLevel(entity_name=EntityType.NAME, redaction_type=RedactionType.PLAIN_TEXT), RedactionLevel(entity_name=EntityType.EMAIL_ADDRESS, redaction_type=RedactionType.MASKED), @@ -792,13 +812,14 @@ Returns a `ReidentifyStringResponse` with `processed_text` and `metrics`. ### Re-identify File: `.reidentify_file(request)` -Synchronous — the re-identified file comes back in the response, not through a run id. Build the request with `ReidentifyFileRequest`: a `data_source` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `data_format`, and the same `RedactionLevel` rules as `reidentify_string`. `data_format` is limited to the formats that support `VAULT_TOKEN`: `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. +Synchronous — the re-identified file comes back in the response, not through a run id. Build the request with `ReidentifyFileRequest`: a `data_source` (`ReidentifySourceType.BASE64` or `PRESIGNED_URL`; re-identification does not accept a skyflow id as the source), the matching `value`, an optional `data_format` (a `ReidentifyFileDataFormat`), and the same `RedactionLevel` rules as `reidentify_string`. `data_format` is limited to the formats that support `VAULT_TOKEN`: `txt`, `csv`, `json`, `jsonl`, `xml`, `docx`, `xlsx` and `pptx`. ```python import base64 from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel -from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, EntityType, RedactionType, DetectRunStatus +from skyflow.utils.enums import ReidentifySourceType, ReidentifyFileDataFormat, EntityType, RedactionType, DetectRunStatus +data_format = ReidentifyFileDataFormat.TXT with open('/data/redacted/notes.txt', 'rb') as handle: encoded = base64.b64encode(handle.read()).decode('utf-8') @@ -806,23 +827,26 @@ response = skyflow_client.detect().reidentify_file( ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=data_format, redaction_level=[RedactionLevel(entity_name=EntityType.NAME, redaction_type=RedactionType.PLAIN_TEXT)], ) ) if response.status == DetectRunStatus.SUCCESS: for index, output in enumerate(response.output): - # output.processed_file is base64 or a presigned URL, per response.output_type - with open(f'reidentified-{index}.txt', 'wb') as out: - out.write(base64.b64decode(output.processed_file)) + # Branch on output_type: base64 bytes are decoded, a presigned URL is downloaded. + if response.output_type == ReidentifySourceType.BASE64: + with open(f'reidentified-{index}.{data_format.value}', 'wb') as out: + out.write(base64.b64decode(output.processed_file)) + else: # ReidentifySourceType.PRESIGNED_URL — processed_file is a URL to fetch + print('download re-identified file from', output.processed_file) ``` -Returns a `ReidentifyFileResponse` with `status`, `output_type`, `output` (a list of `FileOutput`) and `metrics`. +Returns a `ReidentifyFileResponse` with `status`, `output_type` (a `ReidentifySourceType`), `output` (a list of `ReidentifyFileOutput`) and `metrics`. ### Check Guardrails: `.check_guardrails(request)` -Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. The response is passed through as the API returns it. +Synchronous. Screens text for toxicity and a list of denied topics — a natural pre-LLM check. Build the request with `CheckGuardrailsRequest`. `check_toxicity` and `deny_topics` are optional; the SDK sends each only when set and fills in no default. **When `check_toxicity` is omitted, the server still checks toxicity by default.** The response is passed through as the API returns it. ```python from skyflow.vault.detect import CheckGuardrailsRequest diff --git a/skyvault/api-report/skyflow.api.json b/skyvault/api-report/skyflow.api.json index fe322ff1..99d845c7 100644 --- a/skyvault/api-report/skyflow.api.json +++ b/skyvault/api-report/skyflow.api.json @@ -38,6 +38,30 @@ "skyflow.utils.enums.DeidentificationType.MASK": "attr = 'MASK'", "skyflow.utils.enums.DeidentificationType.UNKNOWN": "attr = 'UNKNOWN'", "skyflow.utils.enums.DeidentificationType.VAULT_TOKEN": "attr = 'VAULT_TOKEN'", + "skyflow.utils.enums.DeidentifyFileDataFormat": "class (Enum)", + "skyflow.utils.enums.DeidentifyFileDataFormat.BMP": "attr = 'bmp'", + "skyflow.utils.enums.DeidentifyFileDataFormat.CSV": "attr = 'csv'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DCM": "attr = 'dcm'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DOC": "attr = 'doc'", + "skyflow.utils.enums.DeidentifyFileDataFormat.DOCX": "attr = 'docx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.GIF": "attr = 'gif'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JPEG": "attr = 'jpeg'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JPG": "attr = 'jpg'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JSON": "attr = 'json'", + "skyflow.utils.enums.DeidentifyFileDataFormat.JSONL": "attr = 'jsonl'", + "skyflow.utils.enums.DeidentifyFileDataFormat.MP3": "attr = 'mp3'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PDF": "attr = 'pdf'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PNG": "attr = 'png'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PPT": "attr = 'ppt'", + "skyflow.utils.enums.DeidentifyFileDataFormat.PPTX": "attr = 'pptx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TIF": "attr = 'tif'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TIFF": "attr = 'tiff'", + "skyflow.utils.enums.DeidentifyFileDataFormat.TXT": "attr = 'txt'", + "skyflow.utils.enums.DeidentifyFileDataFormat.WAV": "attr = 'wav'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XLS": "attr = 'xls'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XLSX": "attr = 'xlsx'", + "skyflow.utils.enums.DeidentifyFileDataFormat.XML": "attr = 'xml'", + "skyflow.utils.enums.DeidentifyFileDataFormat.ZIP": "attr = 'zip'", "skyflow.utils.enums.DeidentifySourceType": "class (Enum)", "skyflow.utils.enums.DeidentifySourceType.BASE64": "attr = 'BASE64'", "skyflow.utils.enums.DeidentifySourceType.PRESIGNED_URL": "attr = 'PRESIGNED_URL'", @@ -207,30 +231,6 @@ "skyflow.utils.enums.EnvUrls.PROD": "attr = 'vault.skyflowapis.com'", "skyflow.utils.enums.EnvUrls.SANDBOX": "attr = 'vault.skyflowapis-preview.com'", "skyflow.utils.enums.EnvUrls.STAGE": "attr = 'vault.skyflowapis.tech'", - "skyflow.utils.enums.FileDataFormat": "class (Enum)", - "skyflow.utils.enums.FileDataFormat.BMP": "attr = 'bmp'", - "skyflow.utils.enums.FileDataFormat.CSV": "attr = 'csv'", - "skyflow.utils.enums.FileDataFormat.DCM": "attr = 'dcm'", - "skyflow.utils.enums.FileDataFormat.DOC": "attr = 'doc'", - "skyflow.utils.enums.FileDataFormat.DOCX": "attr = 'docx'", - "skyflow.utils.enums.FileDataFormat.GIF": "attr = 'gif'", - "skyflow.utils.enums.FileDataFormat.JPEG": "attr = 'jpeg'", - "skyflow.utils.enums.FileDataFormat.JPG": "attr = 'jpg'", - "skyflow.utils.enums.FileDataFormat.JSON": "attr = 'json'", - "skyflow.utils.enums.FileDataFormat.JSONL": "attr = 'jsonl'", - "skyflow.utils.enums.FileDataFormat.MP3": "attr = 'mp3'", - "skyflow.utils.enums.FileDataFormat.PDF": "attr = 'pdf'", - "skyflow.utils.enums.FileDataFormat.PNG": "attr = 'png'", - "skyflow.utils.enums.FileDataFormat.PPT": "attr = 'ppt'", - "skyflow.utils.enums.FileDataFormat.PPTX": "attr = 'pptx'", - "skyflow.utils.enums.FileDataFormat.TIF": "attr = 'tif'", - "skyflow.utils.enums.FileDataFormat.TIFF": "attr = 'tiff'", - "skyflow.utils.enums.FileDataFormat.TXT": "attr = 'txt'", - "skyflow.utils.enums.FileDataFormat.WAV": "attr = 'wav'", - "skyflow.utils.enums.FileDataFormat.XLS": "attr = 'xls'", - "skyflow.utils.enums.FileDataFormat.XLSX": "attr = 'xlsx'", - "skyflow.utils.enums.FileDataFormat.XML": "attr = 'xml'", - "skyflow.utils.enums.FileDataFormat.ZIP": "attr = 'zip'", "skyflow.utils.enums.GuardrailsValidation": "class (Enum)", "skyflow.utils.enums.GuardrailsValidation.FAILED": "attr = 'FAILED'", "skyflow.utils.enums.GuardrailsValidation.PASSED": "attr = 'PASSED'", @@ -274,6 +274,15 @@ "skyflow.utils.enums.RedactionType.MASKED": "attr = 'MASKED'", "skyflow.utils.enums.RedactionType.PLAIN_TEXT": "attr = 'PLAIN_TEXT'", "skyflow.utils.enums.RedactionType.REDACTED": "attr = 'REDACTED'", + "skyflow.utils.enums.ReidentifyFileDataFormat": "class (Enum)", + "skyflow.utils.enums.ReidentifyFileDataFormat.CSV": "attr = 'csv'", + "skyflow.utils.enums.ReidentifyFileDataFormat.DOCX": "attr = 'docx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.JSON": "attr = 'json'", + "skyflow.utils.enums.ReidentifyFileDataFormat.JSONL": "attr = 'jsonl'", + "skyflow.utils.enums.ReidentifyFileDataFormat.PPTX": "attr = 'pptx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.TXT": "attr = 'txt'", + "skyflow.utils.enums.ReidentifyFileDataFormat.XLSX": "attr = 'xlsx'", + "skyflow.utils.enums.ReidentifyFileDataFormat.XML": "attr = 'xml'", "skyflow.utils.enums.ReidentifySourceType": "class (Enum)", "skyflow.utils.enums.ReidentifySourceType.BASE64": "attr = 'BASE64'", "skyflow.utils.enums.ReidentifySourceType.PRESIGNED_URL": "attr = 'PRESIGNED_URL'", @@ -416,6 +425,7 @@ "skyflow.vault.detect.RedactionLevel.__init__": "def (self, entity_name: EntityType = None, redaction_type: RedactionType = None)", "skyflow.vault.detect.RedactionLevel.entity_name": "attr", "skyflow.vault.detect.RedactionLevel.redaction_type": "attr", + "skyflow.vault.detect.ReidentifyFileOutput": "alias -> common.vault.detect.ReidentifyFileOutput", "skyflow.vault.detect.ReidentifyFileRequest": "alias -> common.vault.detect.ReidentifyFileRequest", "skyflow.vault.detect.ReidentifyFileResponse": "alias -> common.vault.detect.ReidentifyFileResponse", "skyflow.vault.detect.ReidentifyStringRequest": "alias -> common.vault.detect.ReidentifyStringRequest", diff --git a/skyvault/samples/detect_api/check_guardrails.py b/skyvault/samples/detect_api/check_guardrails.py index 070d69fa..ce4b5db5 100644 --- a/skyvault/samples/detect_api/check_guardrails.py +++ b/skyvault/samples/detect_api/check_guardrails.py @@ -36,6 +36,7 @@ def perform_check_guardrails(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/deidentify_file.py b/skyvault/samples/detect_api/deidentify_file.py index 6940345f..3aad5a1d 100644 --- a/skyvault/samples/detect_api/deidentify_file.py +++ b/skyvault/samples/detect_api/deidentify_file.py @@ -51,6 +51,7 @@ def perform_deidentify_file(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/deidentify_string.py b/skyvault/samples/detect_api/deidentify_string.py index fbbbdd0f..def6c3dd 100644 --- a/skyvault/samples/detect_api/deidentify_string.py +++ b/skyvault/samples/detect_api/deidentify_string.py @@ -36,6 +36,7 @@ def perform_deidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) @@ -78,6 +79,7 @@ def perform_deidentify_string_inline(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/get_run.py b/skyvault/samples/detect_api/get_run.py index 9ce02d67..b4f69810 100644 --- a/skyvault/samples/detect_api/get_run.py +++ b/skyvault/samples/detect_api/get_run.py @@ -37,6 +37,7 @@ def perform_get_run(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/reidentify_file.py b/skyvault/samples/detect_api/reidentify_file.py index 9bc71148..b1f36042 100644 --- a/skyvault/samples/detect_api/reidentify_file.py +++ b/skyvault/samples/detect_api/reidentify_file.py @@ -3,7 +3,7 @@ from skyflow.error import SkyflowError from skyflow import Env, Skyflow, LogLevel from skyflow.vault.detect import ReidentifyFileRequest, RedactionLevel -from skyflow.utils.enums import ReidentifySourceType, FileDataFormat, EntityType, RedactionType, DetectRunStatus +from skyflow.utils.enums import ReidentifySourceType, ReidentifyFileDataFormat, EntityType, RedactionType, DetectRunStatus def perform_reidentify_file(): @@ -28,7 +28,7 @@ def perform_reidentify_file(): request = ReidentifyFileRequest( data_source=ReidentifySourceType.BASE64, value=encoded, - data_format=FileDataFormat.TXT, + data_format=ReidentifyFileDataFormat.TXT, redaction_level=[ RedactionLevel(entity_name=EntityType.NAME, redaction_type=RedactionType.PLAIN_TEXT), ], @@ -46,6 +46,7 @@ def perform_reidentify_file(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/samples/detect_api/reidentify_string.py b/skyvault/samples/detect_api/reidentify_string.py index b685da1f..23479ec4 100644 --- a/skyvault/samples/detect_api/reidentify_string.py +++ b/skyvault/samples/detect_api/reidentify_string.py @@ -37,6 +37,7 @@ def perform_reidentify_string(): 'code': error.http_code, 'message': error.message, 'details': error.details, + 'request_id': error.request_id, }) diff --git a/skyvault/skyflow/utils/enums/__init__.py b/skyvault/skyflow/utils/enums/__init__.py index 8cc888c0..a7b4cf96 100644 --- a/skyvault/skyflow/utils/enums/__init__.py +++ b/skyvault/skyflow/utils/enums/__init__.py @@ -11,10 +11,11 @@ from ._reidentify_source_type import ReidentifySourceType from common.utils.enums.detect import ( DeidentificationType, + DeidentifyFileDataFormat, DetectRunStatus, EntityType, - FileDataFormat, GuardrailsValidation, + ReidentifyFileDataFormat, IncludePrefix, MaskingMethod, ObjectDeidentificationType, diff --git a/skyvault/skyflow/vault/detect/__init__.py b/skyvault/skyflow/vault/detect/__init__.py index 15517bb2..0d030ab8 100644 --- a/skyvault/skyflow/vault/detect/__init__.py +++ b/skyvault/skyflow/vault/detect/__init__.py @@ -25,6 +25,7 @@ ObjectEntity, Pdf, PollOptions, + ReidentifyFileOutput, ReidentifyFileRequest, ReidentifyFileResponse, ReidentifyStringRequest, diff --git a/skyvault/tests/utils/logger/test__logger.py b/skyvault/tests/utils/logger/test__logger.py index cdfcf13f..aeabf781 100644 --- a/skyvault/tests/utils/logger/test__logger.py +++ b/skyvault/tests/utils/logger/test__logger.py @@ -11,6 +11,7 @@ class TestLogger(unittest.TestCase): def test_logger_initialization_with_default_level(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger() @@ -21,6 +22,7 @@ def test_logger_initialization_with_default_level(self, mock_get_logger): def test_logger_initialization_with_custom_level(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.INFO) @@ -31,6 +33,7 @@ def test_logger_initialization_with_custom_level(self, mock_get_logger): def test_set_log_level(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger() logger.set_log_level(LogLevel.DEBUG) @@ -42,6 +45,7 @@ def test_set_log_level(self, mock_get_logger): def test_debug_logging(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.DEBUG) logger.debug("Debug message") @@ -52,6 +56,7 @@ def test_debug_logging(self, mock_get_logger): def test_info_logging(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.INFO) logger.info("Info message") @@ -63,6 +68,7 @@ def test_info_logging(self, mock_get_logger): def test_warn_logging(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.WARN) logger.warn("Warn message") @@ -75,6 +81,7 @@ def test_warn_logging(self, mock_get_logger): def test_error_logging(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.ERROR) logger.error("Error message") @@ -88,6 +95,7 @@ def test_error_logging(self, mock_get_logger): def test_logging_with_level_off(self, mock_get_logger): mock_logger_instance = Mock() mock_get_logger.return_value = mock_logger_instance + mock_logger_instance.handlers = [] logger = Logger(LogLevel.OFF) logger.debug("Debug message") diff --git a/skyvault/tests/vault/detect/test_reexports.py b/skyvault/tests/vault/detect/test_reexports.py index fcaa3561..c5cd95bb 100644 --- a/skyvault/tests/vault/detect/test_reexports.py +++ b/skyvault/tests/vault/detect/test_reexports.py @@ -29,7 +29,8 @@ def test_enums_import_from_utils_enums(self): RedactionType, DeidentifySourceType, ReidentifySourceType, - FileDataFormat, + DeidentifyFileDataFormat, + ReidentifyFileDataFormat, DetectRunStatus, ObjectEntityType, ObjectDeidentificationType, @@ -40,7 +41,8 @@ def test_enums_import_from_utils_enums(self): ) self.assertEqual(EntityType.NAME.value, "NAME") - self.assertEqual(FileDataFormat.PDF.value, "pdf") + self.assertEqual(DeidentifyFileDataFormat.PDF.value, "pdf") + self.assertEqual(ReidentifyFileDataFormat.TXT.value, "txt") self.assertEqual(MaskingMethod.BLUR.value, "BLUR") def test_detect_accessor(self):