From 44c541781c9568aa72c04c46f8597127376df7a6 Mon Sep 17 00:00:00 2001 From: Ben Shaw Date: Thu, 17 Sep 2026 21:56:40 +1200 Subject: [PATCH] feat!: DM-3761: drop the Snowflake external-stage fields DataMasque 3.26.18 stages Snowflake masked data inside Snowflake, so the server no longer exposes the S3 bucket, Azure container, storage integration or Snowflake IAM role fields. Remove them from SnowflakeConnectionConfig. The server detects a Snowflake SPCS deployment from its own environment (is_spcs_env()), so the client no longer states it: snowflake_stage_location and the SnowflakeStageLocation enum are removed. Connections listed from an older server still parse: the removed keys and any snowflake_stage_location value (aws_s3, azure_blob_storage, local, spcs) are dropped on the way in so they are not sent back on create_or_update_connection. Release as 1.4.0; HISTORY.rst calls the breaking change out. BREAKING CHANGE: SnowflakeConnectionConfig loses s3_bucket_name, iam_role_arn, snowflake_azure_container_name, snowflake_azure_connection_string, snowflake_azure_connection_string_encrypted, snowflake_storage_integration_name and snowflake_stage_location; the SnowflakeStageLocation enum is removed from datamasque.client. --- HISTORY.rst | 11 + datamasque/client/__init__.py | 2 - datamasque/client/models/connection.py | 42 ++-- tests/helpers.py | 37 +--- tests/test_connections.py | 293 ++++--------------------- 5 files changed, 79 insertions(+), 306 deletions(-) diff --git a/HISTORY.rst b/HISTORY.rst index 14cffa2..1a419f5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,17 @@ History 1.4.0 (2026-09-25) ------------------ +* **Breaking:** Snowflake masking now stages inside Snowflake, so ``SnowflakeConnectionConfig`` + no longer has the external staging fields ``s3_bucket_name``, ``iam_role_arn``, + ``snowflake_azure_container_name``, ``snowflake_azure_connection_string``, + ``snowflake_azure_connection_string_encrypted`` and ``snowflake_storage_integration_name``. + ``snowflake_stage_location`` and the ``SnowflakeStageLocation`` enum are removed: the server + detects a Snowflake SPCS deployment from its own environment, so the client no longer states it. + +* Snowflake connections listed from an older server still parse: the removed fields and + ``snowflake_stage_location`` are dropped rather than raising a validation error, and are not sent + back on ``create_or_update_connection``. + * Added ``LicenseLock`` (``locked_at`` and ``editable_from``) and a read-only ``license_lock`` field on connection configs. The server reports it only on instances whose license caps the number of connections and counts that connection's type against the cap, diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index c2c3b29..709dc79 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -44,7 +44,6 @@ MssqlLinkedServerConnectionConfig, S3ConnectionConfig, SnowflakeConnectionConfig, - SnowflakeStageLocation, SseConfig, SseSelection, ) @@ -298,7 +297,6 @@ "SelectedFileData", "SnowflakeConnectionConfig", "SnowflakeKeyFile", - "SnowflakeStageLocation", "SseConfig", "SseSelection", "SslZipFile", diff --git a/datamasque/client/models/connection.py b/datamasque/client/models/connection.py index d0e4d34..ae15821 100644 --- a/datamasque/client/models/connection.py +++ b/datamasque/client/models/connection.py @@ -53,15 +53,6 @@ class DatabaseType(Enum): saphana = "saphana" -class SnowflakeStageLocation(str, Enum): - """Storage backend for a Snowflake connection's external stage.""" - - local = "local" # Not supported for production use - aws_s3 = "aws_s3" - azure_blob_storage = "azure_blob_storage" - spcs = "spcs" # DataMasque running inside Snowflake SPCS; staged on the container's own storage - - class SseSelection(Enum): """Mirrors the available options in the AWS console for DynamoDB Server-Side Encryption.""" @@ -285,6 +276,19 @@ def database_type(self) -> DatabaseType: return DatabaseType.cosmosdb +# Servers before 3.26.18 still return these on Snowflake connections. With `extra="allow"` +# they would be sent back on create/update unless dropped here. +_SNOWFLAKE_EXTERNAL_STAGE_KEYS = ( + "s3_bucket_name", + "iam_role_arn", + "snowflake_azure_container_name", + "snowflake_azure_connection_string", + "snowflake_azure_connection_string_encrypted", + "snowflake_storage_integration_name", + "snowflake_stage_location", +) + + class SnowflakeConnectionConfig(ConnectionConfig): """ Connection configuration for a Snowflake database. @@ -294,14 +298,14 @@ class SnowflakeConnectionConfig(ConnectionConfig): """ database: str - # Optional because DataMasque-in-SPCS connections leave these unset: the agent uses the - # container's OAuth token + SNOWFLAKE_HOST/SNOWFLAKE_ACCOUNT env and the app-owned QUERY_WAREHOUSE, - # so user/account/storage-integration/warehouse are null for stage_location=spcs. Mirrors the app's - # canonical model (agent .../schemas/connection/connection.py), which types these `| None = None`. + # Optional because connections leave these unset when DataMasque runs inside Snowflake SPCS: + # the server detects SPCS from its environment and the agent uses the container's OAuth token, + # the SNOWFLAKE_HOST/SNOWFLAKE_ACCOUNT env and the app-owned QUERY_WAREHOUSE, so user/account/ + # warehouse are None. Mirrors the app's canonical model + # (agent .../schemas/connection/connection.py), which types these `| None = None`. user: Optional[str] = None snowflake_account_id: Optional[str] = None snowflake_warehouse: Optional[str] = None - snowflake_storage_integration_name: Optional[str] = None host: str = "" port: Optional[int] = None db_schema: Optional[str] = Field(default=None, alias="schema") @@ -310,12 +314,6 @@ class SnowflakeConnectionConfig(ConnectionConfig): password: Optional[str] = None snowflake_private_key: Optional[FileId] = None snowflake_private_key_passphrase: Optional[str] = None - snowflake_stage_location: Optional[SnowflakeStageLocation] = None - s3_bucket_name: Optional[str] = None - iam_role_arn: Optional[str] = None - snowflake_azure_container_name: Optional[str] = None - snowflake_azure_connection_string: Optional[str] = None - snowflake_azure_connection_string_encrypted: Optional[str] = None mask_type: Literal["database"] = "database" db_type: Literal["snowflake"] = "snowflake" @@ -338,9 +336,9 @@ def _serialize(self, handler: Callable) -> dict: @model_validator(mode="before") @classmethod - def _strip_encrypted_password(cls, data: dict) -> dict: + def _strip_server_only_fields(cls, data: dict) -> dict: if isinstance(data, dict): - for key in ("password_encrypted", "dbpassword"): + for key in ("password_encrypted", "dbpassword", *_SNOWFLAKE_EXTERNAL_STAGE_KEYS): data.pop(key, None) return data diff --git a/tests/helpers.py b/tests/helpers.py index 6a075b4..2303cfc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -8,7 +8,6 @@ DatabaseType, S3ConnectionConfig, SnowflakeConnectionConfig, - SnowflakeStageLocation, ) fake = Faker() @@ -117,44 +116,12 @@ def make_ok_response() -> Response: return response -def snowflake_connection_config_s3(): +def snowflake_connection_config(): return SnowflakeConnectionConfig( - name="snowflake_s3", + name="snowflake", database="test_db", user="snowflake_user", snowflake_account_id="ACCOUNT-123", snowflake_warehouse="test_warehouse", - snowflake_storage_integration_name="test_integration", password="test_password", - snowflake_stage_location=SnowflakeStageLocation.aws_s3, - s3_bucket_name="test-bucket", - iam_role_arn="arn:aws:iam::123456789012:role/test-role", - ) - - -def snowflake_connection_config_azure(): - return SnowflakeConnectionConfig( - name="snowflake_azure", - database="test_db", - user="snowflake_user", - snowflake_account_id="ACCOUNT-456", - snowflake_warehouse="test_warehouse", - snowflake_storage_integration_name="test_integration", - password="test_password", - snowflake_stage_location=SnowflakeStageLocation.azure_blob_storage, - snowflake_azure_container_name="test-container", - snowflake_azure_connection_string="DefaultEndpointsProtocol=https;AccountName=test;AccountKey=test", - ) - - -def snowflake_connection_config_local(): - return SnowflakeConnectionConfig( - name="snowflake_local", - database="test_db", - user="snowflake_user", - snowflake_account_id="ACCOUNT-789", - snowflake_warehouse="test_warehouse", - snowflake_storage_integration_name="test_integration", - password="test_password", - snowflake_stage_location=SnowflakeStageLocation.local, ) diff --git a/tests/test_connections.py b/tests/test_connections.py index bd05606..c76c920 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -22,16 +22,13 @@ MssqlLinkedServerConnectionConfig, S3ConnectionConfig, SnowflakeConnectionConfig, - SnowflakeStageLocation, SseConfig, SseSelection, validate_connection, ) from tests.helpers import ( sample_mounted_share_connection_json, - snowflake_connection_config_azure, - snowflake_connection_config_local, - snowflake_connection_config_s3, + snowflake_connection_config, ) @@ -235,11 +232,8 @@ def test_list_connnections(client): "snowflake_role": "snowballs do indeed roll", "snowflake_account_id": "ABCDEF-123456", "snowflake_warehouse": "warehouse1", - "snowflake_storage_integration_name": "mysi", "host": "snowflake.com", "port": 443, - "s3_bucket_name": "ice-bucket", - "iam_role_arn": "swiss roll", "is_read_only": False, "password_encrypted": "some_base64_here", "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", @@ -257,8 +251,6 @@ def test_list_connnections(client): "name": "snowflake_minimal_with_key", "snowflake_account_id": "ACCOUNT-1234", "snowflake_warehouse": "clothing_store", - "snowflake_storage_integration_name": "kennards", - "s3_bucket_name": "champagne-bucket", "snowflake_private_key": "2831289a-4398-abcd-4112-fe09a1239f89", "snowflake_private_key_passphrase_encrypted": "some base64 here", "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", @@ -339,9 +331,6 @@ def test_list_connnections(client): assert snowflake_connection.snowflake_role == "snowballs do indeed roll" assert snowflake_connection.snowflake_account_id == "ABCDEF-123456" assert snowflake_connection.snowflake_warehouse == "warehouse1" - assert snowflake_connection.snowflake_storage_integration_name == "mysi" - assert snowflake_connection.s3_bucket_name == "ice-bucket" - assert snowflake_connection.iam_role_arn == "swiss roll" assert snowflake_connection.is_read_only is False minimal_snowflake_connection = connections[8] @@ -355,9 +344,6 @@ def test_list_connnections(client): assert minimal_snowflake_connection.snowflake_role == "" assert minimal_snowflake_connection.snowflake_account_id == "ACCOUNT-1234" assert minimal_snowflake_connection.snowflake_warehouse == "clothing_store" - assert minimal_snowflake_connection.snowflake_storage_integration_name == "kennards" - assert minimal_snowflake_connection.s3_bucket_name == "champagne-bucket" - assert minimal_snowflake_connection.iam_role_arn is None assert minimal_snowflake_connection.is_read_only is False assert minimal_snowflake_connection.snowflake_private_key == "2831289a-4398-abcd-4112-fe09a1239f89" @@ -421,39 +407,20 @@ def test_delete_connection_that_does_not_exist(client): assert m.request_history[0].method == "GET" -@pytest.mark.parametrize( - "config_func,expected_stage_location,expected_fields,unexpected_fields", - [ - ( - snowflake_connection_config_s3, - "aws_s3", - ["s3_bucket_name", "iam_role_arn"], - ["snowflake_azure_container_name", "snowflake_azure_connection_string"], - ), - ( - snowflake_connection_config_azure, - "azure_blob_storage", - ["snowflake_azure_container_name", "snowflake_azure_connection_string"], - ["s3_bucket_name", "iam_role_arn"], - ), - ( - snowflake_connection_config_local, - "local", - [], - [ - "s3_bucket_name", - "iam_role_arn", - "snowflake_azure_container_name", - "snowflake_azure_connection_string", - ], - ), - ], +# Removed from Snowflake connections in server 3.26.18. +SNOWFLAKE_EXTERNAL_STAGE_FIELDS = ( + "s3_bucket_name", + "iam_role_arn", + "snowflake_azure_container_name", + "snowflake_azure_connection_string", + "snowflake_azure_connection_string_encrypted", + "snowflake_storage_integration_name", ) -def test_create_snowflake_connection_with_staging_platform( - client, config_func, expected_stage_location, expected_fields, unexpected_fields -): - """Test creating Snowflake connections with different staging platforms.""" - config = config_func() + + +def test_create_snowflake_connection(client): + """Creating a Snowflake connection sends none of the removed fields.""" + config = snowflake_connection_config() with requests_mock.Mocker() as m: m.get("http://test-server/api/connections/", json=[], status_code=200) @@ -462,71 +429,30 @@ def test_create_snowflake_connection_with_staging_platform( result = client.create_or_update_connection(config) assert result.id == ConnectionId("2") - # Verify the correct data was sent request_data = m.last_request.json() - assert request_data["snowflake_stage_location"] == expected_stage_location - - # Check expected fields are present - for field in expected_fields: - assert field in request_data - - # Check unexpected fields are not present - for field in unexpected_fields: + assert request_data["dbpassword"] == "test_password" + assert request_data["schema"] == "" + assert "snowflake_stage_location" not in request_data + for field in SNOWFLAKE_EXTERNAL_STAGE_FIELDS: assert field not in request_data -@pytest.mark.parametrize( - "config_func,expected_stage_location,expected_fields,unexpected_fields", - [ - ( - snowflake_connection_config_s3, - SnowflakeStageLocation.aws_s3, - { - "s3_bucket_name": "test-bucket", - "iam_role_arn": "arn:aws:iam::123456789012:role/test-role", - }, - ["snowflake_azure_container_name", "snowflake_azure_connection_string"], - ), - ( - snowflake_connection_config_azure, - SnowflakeStageLocation.azure_blob_storage, - { - "snowflake_azure_container_name": "test-container", - "snowflake_azure_connection_string": "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=test", - }, - ["s3_bucket_name", "iam_role_arn"], - ), - ( - snowflake_connection_config_local, - SnowflakeStageLocation.local, - {}, - [ - "s3_bucket_name", - "iam_role_arn", - "snowflake_azure_container_name", - "snowflake_azure_connection_string", - ], - ), - ], -) -def test_snowflake_connection_model_dump(config_func, expected_stage_location, expected_fields, unexpected_fields): - """Test that Snowflake connections serialize correctly for each staging platform.""" - config = config_func() +def test_snowflake_connection_model_dump_omits_external_stage_fields(): + config = snowflake_connection_config() api_dict = config.model_dump(exclude_none=True, by_alias=True, mode="json") - assert api_dict["snowflake_stage_location"] == expected_stage_location - - # Check expected fields and their values - for field, value in expected_fields.items(): - assert api_dict[field] == value - - # Check unexpected fields are not present - for field in unexpected_fields: + assert "snowflake_stage_location" not in api_dict + for field in SNOWFLAKE_EXTERNAL_STAGE_FIELDS: assert field not in api_dict -def test_list_snowflake_connections_with_different_platforms(client): - """Test listing Snowflake connections returns correct staging platform information.""" +def test_list_snowflake_connection_from_older_server_drops_external_stage(client): + """ + A connection listed by a server older than 3.26.18 still parses. + + Those servers return the external stage location and its storage settings. + The SDK drops them and does not send them back on update. + """ with requests_mock.Mocker() as m: m.get( "http://test-server/api/connections/", @@ -547,118 +473,17 @@ def test_list_snowflake_connections_with_different_platforms(client): "id": "s3-connection-id", "mask_type": "database", }, - { - "version": "1.0", - "user": "azure_user", - "db_type": "snowflake", - "database": "azure_db", - "name": "snowflake_azure", - "snowflake_account_id": "AZURE-ACCOUNT", - "snowflake_warehouse": "azure_warehouse", - "snowflake_storage_integration_name": "azure_integration", - "snowflake_azure_container_name": "azure-container", - "snowflake_azure_connection_string_encrypted": "encrypted_azure_string", - "snowflake_stage_location": "azure_blob_storage", - "password_encrypted": "encrypted", - "id": "azure-connection-id", - "mask_type": "database", - }, - { - "version": "1.0", - "user": "local_user", - "db_type": "snowflake", - "database": "local_db", - "name": "snowflake_local", - "snowflake_account_id": "LOCAL-ACCOUNT", - "snowflake_warehouse": "local_warehouse", - "snowflake_storage_integration_name": "local_integration", - "snowflake_stage_location": "local", - "password_encrypted": "encrypted", - "id": "local-connection-id", - "mask_type": "database", - }, ], status_code=200, ) - connections = client.list_connections() - snowflake_connections = [c for c in connections if isinstance(c, SnowflakeConnectionConfig)] - assert len(snowflake_connections) == 3 - - # Check S3 connection - s3_conn = next(c for c in snowflake_connections if c.name == "snowflake_s3") - assert s3_conn.snowflake_stage_location is SnowflakeStageLocation.aws_s3 - assert s3_conn.s3_bucket_name == "s3-bucket" - assert s3_conn.iam_role_arn == "arn:aws:iam::123456789012:role/s3-role" - assert s3_conn.snowflake_azure_container_name is None - assert s3_conn.snowflake_azure_connection_string is None - - # Check Azure connection - azure_conn = next(c for c in snowflake_connections if c.name == "snowflake_azure") - assert azure_conn.snowflake_stage_location is SnowflakeStageLocation.azure_blob_storage - assert azure_conn.snowflake_azure_container_name == "azure-container" - assert azure_conn.snowflake_azure_connection_string is None # Encrypted, so empty - assert azure_conn.s3_bucket_name is None - assert azure_conn.iam_role_arn is None - - # Check local connection - local_conn = next(c for c in snowflake_connections if c.name == "snowflake_local") - assert local_conn.snowflake_stage_location is SnowflakeStageLocation.local - assert local_conn.s3_bucket_name is None - assert local_conn.iam_role_arn is None - assert local_conn.snowflake_azure_container_name is None - assert local_conn.snowflake_azure_connection_string is None - + (conn,) = client.list_connections() -@pytest.mark.parametrize( - "stage_location,missing_fields,error_message", - [ - ( - SnowflakeStageLocation.azure_blob_storage, - { - "snowflake_azure_container_name": None, - "snowflake_azure_connection_string": None, - }, - "Missing Azure fields", - ), - ( - SnowflakeStageLocation.aws_s3, - { - "s3_bucket_name": None, - "iam_role_arn": None, # IAM role is optional, so only s3_bucket_name is truly required - }, - "Missing S3 bucket", - ), - ], -) -def test_create_snowflake_connection_missing_required_fields(client, stage_location, missing_fields, error_message): - """Test that creating a Snowflake connection with missing required fields fails appropriately.""" - config_dict = { - "name": f"invalid_{stage_location.value}", - "database": "test_db", - "user": "snowflake_user", - "snowflake_account_id": "ACCOUNT-123", - "snowflake_warehouse": "test_warehouse", - "snowflake_storage_integration_name": "test_integration", - "password": "test_password", - "snowflake_stage_location": stage_location, - } - - # Add the missing fields - config_dict.update(missing_fields) - - config = SnowflakeConnectionConfig(**config_dict) - - with requests_mock.Mocker() as m: - m.get("http://test-server/api/connections/", json=[], status_code=200) - m.post( - "http://test-server/api/connections/", - json={"error": error_message}, - status_code=400, - ) - - with pytest.raises(DataMasqueApiError): - client.create_or_update_connection(config) + assert isinstance(conn, SnowflakeConnectionConfig) + api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") + assert "snowflake_stage_location" not in api_dict + for field in SNOWFLAKE_EXTERNAL_STAGE_FIELDS: + assert field not in api_dict def test_s3_connection_model_validate(): @@ -1122,7 +947,9 @@ def test_dynamo_connection_model_validate_without_sse_uses_default(): assert conn.dynamo_default_sse == SseConfig(selection=SseSelection.dynamodb_owned, kms_key_id=None) -def test_snowflake_connection_model_validate_with_stage_location(): +@pytest.mark.parametrize("stage_location", ["aws_s3", "azure_blob_storage", "local", "spcs"]) +def test_snowflake_connection_model_validate_drops_external_stage_location(stage_location): + """Stage locations from servers older than 3.26.18 are dropped and not sent back.""" payload = { "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", "name": "snowflake", @@ -1139,49 +966,23 @@ def test_snowflake_connection_model_validate_with_stage_location(): "port": 443, "s3_bucket_name": "ice-bucket", "iam_role_arn": "swiss roll", - "snowflake_stage_location": "aws_s3", + "snowflake_azure_container_name": "ice-container", + "snowflake_azure_connection_string_encrypted": "encrypted", + "snowflake_stage_location": stage_location, "is_read_only": False, } conn = SnowflakeConnectionConfig.model_validate(payload) assert isinstance(conn, SnowflakeConnectionConfig) - assert conn.snowflake_stage_location is SnowflakeStageLocation.aws_s3 - assert conn.iam_role_arn == "swiss roll" assert conn.password is None + api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") + assert "snowflake_stage_location" not in api_dict + for field in SNOWFLAKE_EXTERNAL_STAGE_FIELDS: + assert field not in api_dict -def test_snowflake_connection_model_validate_with_spcs_stage_location(): - """ - A Snowflake connection staged in SPCS must deserialise (regression for ui-testing MR !185). - - When DataMasque runs inside Snowflake SPCS it saves connections with - `snowflake_stage_location=spcs`. Listing connections deserialises every - one, so an unknown stage value used to raise `ValidationError` and break - `create_or_update_connection` for unrelated connections on a shared instance. - """ - payload = { - "id": "a1b2c3d4-0000-0000-0000-000000000000", - "name": "snowflake_spcs", - "mask_type": "database", - "db_type": "snowflake", - "user": "snowman", - "database": "icicle", - "snowflake_account_id": "ABCDEF-123456", - "snowflake_warehouse": "warehouse1", - "snowflake_storage_integration_name": "mysi", - "snowflake_stage_location": "spcs", - } - - conn = SnowflakeConnectionConfig.model_validate(payload) - - assert conn.snowflake_stage_location is SnowflakeStageLocation.spcs - # SPCS staging carries no external-storage fields. - assert conn.s3_bucket_name is None - assert conn.snowflake_azure_container_name is None - - -def test_snowflake_connection_model_validate_without_stage_location(): +def test_snowflake_connection_model_validate_minimal_payload(): payload = { "id": "id-3", "name": "snowflake", @@ -1191,12 +992,10 @@ def test_snowflake_connection_model_validate_without_stage_location(): "database": "igloo", "snowflake_account_id": "ACCOUNT-1234", "snowflake_warehouse": "clothing_store", - "snowflake_storage_integration_name": "kennards", } conn = SnowflakeConnectionConfig.model_validate(payload) - assert conn.snowflake_stage_location is None assert conn.host == "" assert conn.port is None assert conn.db_schema is None