SK-3289: address Detect docs and sample parity feedback (Python)

common/tests/vault/detect/test_detect.py

@@ -10,11 +10,12 @@
 from common.utils import SkyflowMessages
 from common.utils.enums.detect import (
     DeidentificationType,
+    DeidentifyFileDataFormat,
     DetectRunStatus,
     EntityType,
-    FileDataFormat,
     GuardrailsValidation,
     IncludePrefix,
+    ReidentifyFileDataFormat,
 )
 
 
@@ -274,7 +275,7 @@ def test_reidentify_file_value_only(self):
         controller = DetectController(_vault_client(files=files))
 
         response = controller.reidentify_file(ReidentifyFileRequest(
-            data_source=ReidentifySourceType.BASE64, value="b64", data_format=FileDataFormat.TXT,
+            data_source=ReidentifySourceType.BASE64, value="b64", data_format=ReidentifyFileDataFormat.TXT,
         ))
 
         _name, _args, kwargs = files.calls[0]
@@ -328,15 +329,15 @@ def test_file_output_writer_names(self):
         tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, tmp, ignore_errors=True)
         outputs = [
-            mappers.FileOutput(processed_file=base64.b64encode(b"a").decode(), processed_file_type="REDACTED_FILE", processed_file_extension=FileDataFormat.PDF),
+            mappers.FileOutput(processed_file=base64.b64encode(b"a").decode(), processed_file_type="REDACTED_FILE", processed_file_extension=DeidentifyFileDataFormat.PDF),
             mappers.FileOutput(processed_file=base64.b64encode(b"b").decode(), processed_file_type="ENTITIES", processed_file_extension=None),
         ]
         written = writer.write_outputs(outputs, DeidentifySourceType.BASE64, "invoice.pdf", tmp)
         names = sorted(os.path.basename(path) for path in written)
         self.assertEqual(names, ["processed-entities-invoice.json", "processed-invoice.pdf"])
 
     def test_file_output_writer_skips_presigned(self):
-        outputs = [mappers.FileOutput(processed_file="http://x", processed_file_type="REDACTED_FILE", processed_file_extension=FileDataFormat.PDF)]
+        outputs = [mappers.FileOutput(processed_file="http://x", processed_file_type="REDACTED_FILE", processed_file_extension=DeidentifyFileDataFormat.PDF)]
         self.assertEqual(writer.write_outputs(outputs, DeidentifySourceType.PRESIGNED_URL, "a.pdf", None), [])
 
     def test_validations(self):
@@ -368,10 +369,16 @@ def test_deidentify_file_source_rules(self):
         self.assertEqual(no_data_source.exception.message, SkyflowMessages.Error.INVALID_DATA_SOURCE_IN_DEIDENTIFY_FILE.value)
 
     def test_file_data_format_is_case_insensitive(self):
-        self.assertEqual(mappers._file_data_format("PDF"), FileDataFormat.PDF)
-        self.assertEqual(mappers._file_data_format("pdf"), FileDataFormat.PDF)
-        self.assertIsNone(mappers._file_data_format("bogus"))
-        self.assertIsNone(mappers._file_data_format(None))
+        self.assertEqual(mappers._deidentify_file_data_format("PDF"), DeidentifyFileDataFormat.PDF)
+        self.assertEqual(mappers._deidentify_file_data_format("pdf"), DeidentifyFileDataFormat.PDF)
+        self.assertIsNone(mappers._deidentify_file_data_format("bogus"))
+        self.assertIsNone(mappers._deidentify_file_data_format(None))
+
+    def test_reidentify_file_data_format_narrower_set(self):
+        self.assertEqual(mappers._reidentify_file_data_format("TXT"), ReidentifyFileDataFormat.TXT)
+        self.assertEqual(mappers._reidentify_file_data_format("txt"), ReidentifyFileDataFormat.TXT)
+        self.assertIsNone(mappers._reidentify_file_data_format("pdf"))
+        self.assertIsNone(mappers._reidentify_file_data_format(None))
 
     def test_entity_prefix_fields_serialize(self):
         wire = mappers.to_entity(Entity(

common/utils/_utils.py

@@ -41,7 +41,7 @@ def get_vault_url(cluster_id, env, vault_id, logger=None):
     if not cluster_id or not isinstance(cluster_id, str) or not cluster_id.strip():
         raise SkyflowError(SkyflowMessages.Error.INVALID_CLUSTER_ID.value.format(vault_id), invalid_input_error_code)
 
-    if env not in Env:
+    if not isinstance(env, Env):
         raise SkyflowError(SkyflowMessages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code)
 
     base_url = EnvUrls[env.name].value

common/utils/enums/detect.py

@@ -94,7 +94,7 @@ class ReturnEntitiesType(Enum):
     NONE = "NONE"
 
 
-class FileDataFormat(Enum):
+class DeidentifyFileDataFormat(Enum):
     BMP = "bmp"
     CSV = "csv"
     DCM = "dcm"
@@ -120,6 +120,17 @@ class FileDataFormat(Enum):
     ZIP = "zip"
 
 
+class ReidentifyFileDataFormat(Enum):
+    CSV = "csv"
+    DOCX = "docx"
+    JSON = "json"
+    JSONL = "jsonl"
+    PPTX = "pptx"
+    TXT = "txt"
+    XLSX = "xlsx"
+    XML = "xml"
+
+
 class DetectRunStatus(Enum):
     UNKNOWN = "UNKNOWN"
     QUEUED = "QUEUED"

common/utils/logger/_logger.py

@@ -1,27 +1,26 @@
 import logging
 from ..enums.log_level import LogLevel
 
+LOGGER_NAME = 'skyflow-python'
+LOG_MESSAGE_FORMAT = '%(message)s'
+_SKYFLOW_HANDLER_FLAG = '_skyflow_handler'
+
 
 class Logger:
     def __init__(self, level=LogLevel.ERROR):
         self.current_level = level
-        self.logger = logging.getLogger('skyflow-python')
-        self.logger.propagate = False  # Prevent logs from being handled by parent loggers
+        self.logger = logging.getLogger(LOGGER_NAME)
+        self.logger.propagate = False
 
-        # Remove any existing handlers to avoid duplicates or inherited handlers
-        if self.logger.hasHandlers():
+        if not any(getattr(handler, _SKYFLOW_HANDLER_FLAG, False) for handler in self.logger.handlers):
             self.logger.handlers.clear()
+            handler = logging.StreamHandler()
+            handler.setFormatter(logging.Formatter(LOG_MESSAGE_FORMAT))
+            setattr(handler, _SKYFLOW_HANDLER_FLAG, True)
+            self.logger.addHandler(handler)
 
         self.set_log_level(level)
 
-        handler = logging.StreamHandler()
-
-        # Create a formatter that only includes the message without any prefixes
-        formatter = logging.Formatter('%(message)s')
-        handler.setFormatter(formatter)
-
-        self.logger.addHandler(handler)
-
     def set_log_level(self, level):
         self.current_level = level
         log_level_mapping = {

common/utils/validations/_validations.py

@@ -232,7 +232,7 @@ def validate_vault_config(logger, config, messages=None, allowed_keys=None):
     if ConfigField.CREDENTIALS in config and config.get(ConfigField.CREDENTIALS):
         validate_credentials(logger, config.get(ConfigField.CREDENTIALS), "vault", vault_id, messages=messages)
 
-    if ConfigField.ENV in config and config.get(ConfigField.ENV) not in Env:
+    if ConfigField.ENV in config and not isinstance(config.get(ConfigField.ENV), Env):
         log_error_log(messages.ErrorLogs.ENV_IS_REQUIRED.value, logger)
         raise SkyflowError(messages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code)
 
@@ -256,7 +256,7 @@ def validate_update_vault_config(logger, config, messages=None, allowed_keys=Non
     if ConfigField.CLUSTER_ID in config and not config.get(ConfigField.CLUSTER_ID):
         raise SkyflowError(messages.Error.INVALID_CLUSTER_ID.value.format(vault_id), invalid_input_error_code)
 
-    if ConfigField.ENV in config and config.get(ConfigField.ENV) not in Env:
+    if ConfigField.ENV in config and not isinstance(config.get(ConfigField.ENV), Env):
         raise SkyflowError(messages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code)
 
     if ConfigField.CREDENTIALS not in config:

common/vault/detect/__init__.py

@@ -26,6 +26,7 @@
     Pdf,
     PollOptions,
     RedactionRule,
+    ReidentifyFileOutput,
     ReidentifyFileRequest,
     ReidentifyFileResponse,
     ReidentifyStringRequest,
@@ -62,6 +63,7 @@
     "Pdf",
     "PollOptions",
     "RedactionRule",
+    "ReidentifyFileOutput",
     "ReidentifyFileRequest",
     "ReidentifyFileResponse",
     "ReidentifyStringRequest",

common/vault/detect/_mappers.py

@@ -16,9 +16,10 @@
     RedactionLevel as WireRedactionLevel,
 )
 from common.utils.enums.detect import (
+    DeidentifyFileDataFormat,
     DetectRunStatus,
-    FileDataFormat,
     GuardrailsValidation,
+    ReidentifyFileDataFormat,
 )
 
 from ._constants import EXTENSION_SEPARATOR
@@ -31,6 +32,7 @@
     FileOutput,
     GetRunResponse,
     Metrics,
+    ReidentifyFileOutput,
     ReidentifyFileResponse,
     ReidentifyStringResponse,
 )
@@ -53,11 +55,20 @@ def _enum_or_default(enum_cls, value, fallback):
         return fallback
 
 
-def _file_data_format(value):
+def _deidentify_file_data_format(value):
     if value is None:
         return None
     try:
-        return FileDataFormat(str(value).lower())
+        return DeidentifyFileDataFormat(str(value).lower())
+    except ValueError:
+        return None
+
+
+def _reidentify_file_data_format(value):
+    if value is None:
+        return None
+    try:
+        return ReidentifyFileDataFormat(str(value).lower())
     except ValueError:
         return None
 
@@ -69,7 +80,7 @@ def infer_format(file_name):
     if index < 0 or index == len(file_name) - 1:
         return None
     try:
-        return FileDataFormat[file_name[index + 1:].upper()]
+        return DeidentifyFileDataFormat[file_name[index + 1:].upper()]
     except KeyError:
         return None
 
@@ -226,7 +237,15 @@ def to_file_output(api_output):
     return FileOutput(
         processed_file=api_output.processed_file,
         processed_file_type=api_output.processed_file_type,
-        processed_file_extension=_file_data_format(api_output.processed_file_extension),
+        processed_file_extension=_deidentify_file_data_format(api_output.processed_file_extension),
+    )
+
+
+def to_reidentify_file_output(api_output):
+    return ReidentifyFileOutput(
+        processed_file=api_output.processed_file,
+        processed_file_type=api_output.processed_file_type,
+        processed_file_extension=_reidentify_file_data_format(api_output.processed_file_extension),
     )
 
 
@@ -303,7 +322,7 @@ def to_deidentify_file_response(run_id, run):
 
 
 def to_reidentify_file_response(response, source_type_cls):
-    outputs = [to_file_output(output) for output in (response.output or []) if output is not None]
+    outputs = [to_reidentify_file_output(output) for output in (response.output or []) if output is not None]
     return ReidentifyFileResponse(
         status=_enum_or_default(DetectRunStatus, response.status, DetectRunStatus.UNKNOWN),
         output_type=_enum_or_default(source_type_cls, response.output_type, None),

common/vault/detect/_models.py

@@ -18,7 +18,16 @@ def __init__(self, size: float = None, word_count: int = None,
 class FileOutput:
     def __init__(self, processed_file: str = None,
                  processed_file_type: str = None,
-                 processed_file_extension: "FileDataFormat" = None):
+                 processed_file_extension: "DeidentifyFileDataFormat" = None):
+        self.processed_file = processed_file
+        self.processed_file_type = processed_file_type
+        self.processed_file_extension = processed_file_extension
+
+
+class ReidentifyFileOutput:
+    def __init__(self, processed_file: str = None,
+                 processed_file_type: str = None,
+                 processed_file_extension: "ReidentifyFileDataFormat" = None):
         self.processed_file = processed_file
         self.processed_file_type = processed_file_type
         self.processed_file_extension = processed_file_extension
@@ -220,7 +229,7 @@ def __init__(self, processed_text: str = None,
 class DeidentifyFileRequest:
     def __init__(self, data_source: "DeidentifySourceType" = None,
                  value: str = None,
-                 data_format: "FileDataFormat" = None,
+                 data_format: "DeidentifyFileDataFormat" = None,
                  configuration_id: str = None,
                  configuration: DetectConfiguration = None,
                  file: BinaryIO = None, file_path: str = None,
@@ -288,7 +297,7 @@ def __init__(self, processed_text: str = None, metrics: Metrics = None):
 class ReidentifyFileRequest:
     def __init__(self, data_source: "ReidentifySourceType" = None,
                  value: str = None,
-                 data_format: "FileDataFormat" = None,
+                 data_format: "ReidentifyFileDataFormat" = None,
                  redaction_level: List[RedactionRule] = None):
         self.data_source = data_source
         self.value = value
@@ -299,7 +308,7 @@ def __init__(self, data_source: "ReidentifySourceType" = None,
 class ReidentifyFileResponse:
     def __init__(self, status: "DetectRunStatus" = None,
                  output_type: "ReidentifySourceType" = None,
-                 output: List[FileOutput] = None,
+                 output: List[ReidentifyFileOutput] = None,
                  metrics: Metrics = None):
         self.status = status
         self.output_type = output_type

docs/migrate_detect_v1_to_v2.md → docs/migrate_detect.md

@@ -1,14 +1,22 @@
-# Migrate Detect from V1 to V2 (skyvault Python SDK)
-
-The skyvault Python SDK replaces the Detect V1 methods with the V2 methods, using the
-same method and class names as the flowvault Detect SDK. There is no `V2` suffix
-anywhere. `deidentify_file` keeps its name and takes the V2 `DeidentifyFileRequest`,
-returning the V2 `DeidentifyFileResponse`. Because the V1 methods and classes are
-removed, there is no name conflict. All V2 classes live in `skyflow.vault.detect` and
-the enums in `skyflow.utils.enums`.
-
-V1 callers must migrate when they upgrade, so this ships as a major release. Client
-creation and the `skyflow_client.detect()` accessor do not change.
+# Migrate to the new Detect methods (skyflow 3.0.0)
+
+The skyvault Python SDK replaces the previous Detect methods with the new Detect methods,
+using the same method and class names as the flowvault Detect SDK — there is no version
+suffix on any of them. `deidentify_file` keeps its name and now takes the new
+`DeidentifyFileRequest`, returning the new `DeidentifyFileResponse`. Because the previous
+methods and classes are removed, there is no name conflict. All Detect classes live in
+`skyflow.vault.detect` and the enums in `skyflow.utils.enums`.
+
+Callers of the previous Detect methods must migrate when they upgrade, so this ships in the
+next major release, **skyflow 3.0.0** — pin or upgrade accordingly (`pip install "skyflow==3.0.0"`).
+Client creation and the `skyflow_client.detect()` accessor do not change.
+
+> **⚠️ Biggest behaviour change:** `deidentify_file` **no longer waits for the result by
+> default.** It used to always poll; now, without `poll_options`, it submits and returns
+> immediately — `status` is `None`, `output` is empty, and nothing is written to disk. Pass
+> `poll_options` to block until the run finishes, or retrieve the result later with `get_run`.
+> Code ported unchanged runs without error but produces no outputs, so the failure surfaces
+> downstream rather than at the call site.
 
 ## What changes
 
@@ -355,7 +363,7 @@ fields as the V2 `DeidentifyFileResponse` (`run_id`, `status`, `output_type`, `o
 
 ## Timeline
 
-| Release | V1 methods | V2 methods |
+| Release | Previous Detect methods | New Detect methods |
 | --- | --- | --- |
-| skyvault 2.1.x | available | not available |
-| next major | removed | available under the flowvault names |
+| skyflow 2.1.x | available | not available |
+| skyflow 3.0.0 | removed | available under the flowvault names |