diff --git a/HISTORY.rst b/HISTORY.rst index 14cffa2..1a419f5 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -5,6 +5,17 @@ History 1.4.0 (2026-09-25) ------------------ +* **Breaking:** Snowflake masking now stages inside Snowflake, so ``SnowflakeConnectionConfig`` + no longer has the external staging fields ``s3_bucket_name``, ``iam_role_arn``, + ``snowflake_azure_container_name``, ``snowflake_azure_connection_string``, + ``snowflake_azure_connection_string_encrypted`` and ``snowflake_storage_integration_name``. + ``snowflake_stage_location`` and the ``SnowflakeStageLocation`` enum are removed: the server + detects a Snowflake SPCS deployment from its own environment, so the client no longer states it. + +* Snowflake connections listed from an older server still parse: the removed fields and + ``snowflake_stage_location`` are dropped rather than raising a validation error, and are not sent + back on ``create_or_update_connection``. + * Added ``LicenseLock`` (``locked_at`` and ``editable_from``) and a read-only ``license_lock`` field on connection configs. The server reports it only on instances whose license caps the number of connections and counts that connection's type against the cap, diff --git a/datamasque/client/__init__.py b/datamasque/client/__init__.py index c2c3b29..709dc79 100644 --- a/datamasque/client/__init__.py +++ b/datamasque/client/__init__.py @@ -44,7 +44,6 @@ MssqlLinkedServerConnectionConfig, S3ConnectionConfig, SnowflakeConnectionConfig, - SnowflakeStageLocation, SseConfig, SseSelection, ) @@ -298,7 +297,6 @@ "SelectedFileData", "SnowflakeConnectionConfig", "SnowflakeKeyFile", - "SnowflakeStageLocation", "SseConfig", "SseSelection", "SslZipFile", diff --git a/datamasque/client/models/connection.py b/datamasque/client/models/connection.py index d0e4d34..ae15821 100644 --- a/datamasque/client/models/connection.py +++ b/datamasque/client/models/connection.py @@ -53,15 +53,6 @@ class DatabaseType(Enum): saphana = "saphana" -class SnowflakeStageLocation(str, Enum): - """Storage backend for a Snowflake connection's external stage.""" - - local = "local" # Not supported for production use - aws_s3 = "aws_s3" - azure_blob_storage = "azure_blob_storage" - spcs = "spcs" # DataMasque running inside Snowflake SPCS; staged on the container's own storage - - class SseSelection(Enum): """Mirrors the available options in the AWS console for DynamoDB Server-Side Encryption.""" @@ -285,6 +276,19 @@ def database_type(self) -> DatabaseType: return DatabaseType.cosmosdb +# Servers before 3.26.18 still return these on Snowflake connections. With `extra="allow"` +# they would be sent back on create/update unless dropped here. +_SNOWFLAKE_EXTERNAL_STAGE_KEYS = ( + "s3_bucket_name", + "iam_role_arn", + "snowflake_azure_container_name", + "snowflake_azure_connection_string", + "snowflake_azure_connection_string_encrypted", + "snowflake_storage_integration_name", + "snowflake_stage_location", +) + + class SnowflakeConnectionConfig(ConnectionConfig): """ Connection configuration for a Snowflake database. @@ -294,14 +298,14 @@ class SnowflakeConnectionConfig(ConnectionConfig): """ database: str - # Optional because DataMasque-in-SPCS connections leave these unset: the agent uses the - # container's OAuth token + SNOWFLAKE_HOST/SNOWFLAKE_ACCOUNT env and the app-owned QUERY_WAREHOUSE, - # so user/account/storage-integration/warehouse are null for stage_location=spcs. Mirrors the app's - # canonical model (agent .../schemas/connection/connection.py), which types these `| None = None`. + # Optional because connections leave these unset when DataMasque runs inside Snowflake SPCS: + # the server detects SPCS from its environment and the agent uses the container's OAuth token, + # the SNOWFLAKE_HOST/SNOWFLAKE_ACCOUNT env and the app-owned QUERY_WAREHOUSE, so user/account/ + # warehouse are None. Mirrors the app's canonical model + # (agent .../schemas/connection/connection.py), which types these `| None = None`. user: Optional[str] = None snowflake_account_id: Optional[str] = None snowflake_warehouse: Optional[str] = None - snowflake_storage_integration_name: Optional[str] = None host: str = "" port: Optional[int] = None db_schema: Optional[str] = Field(default=None, alias="schema") @@ -310,12 +314,6 @@ class SnowflakeConnectionConfig(ConnectionConfig): password: Optional[str] = None snowflake_private_key: Optional[FileId] = None snowflake_private_key_passphrase: Optional[str] = None - snowflake_stage_location: Optional[SnowflakeStageLocation] = None - s3_bucket_name: Optional[str] = None - iam_role_arn: Optional[str] = None - snowflake_azure_container_name: Optional[str] = None - snowflake_azure_connection_string: Optional[str] = None - snowflake_azure_connection_string_encrypted: Optional[str] = None mask_type: Literal["database"] = "database" db_type: Literal["snowflake"] = "snowflake" @@ -338,9 +336,9 @@ def _serialize(self, handler: Callable) -> dict: @model_validator(mode="before") @classmethod - def _strip_encrypted_password(cls, data: dict) -> dict: + def _strip_server_only_fields(cls, data: dict) -> dict: if isinstance(data, dict): - for key in ("password_encrypted", "dbpassword"): + for key in ("password_encrypted", "dbpassword", *_SNOWFLAKE_EXTERNAL_STAGE_KEYS): data.pop(key, None) return data diff --git a/tests/helpers.py b/tests/helpers.py index 6a075b4..2303cfc 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -8,7 +8,6 @@ DatabaseType, S3ConnectionConfig, SnowflakeConnectionConfig, - SnowflakeStageLocation, ) fake = Faker() @@ -117,44 +116,12 @@ def make_ok_response() -> Response: return response -def snowflake_connection_config_s3(): +def snowflake_connection_config(): return SnowflakeConnectionConfig( - name="snowflake_s3", + name="snowflake", database="test_db", user="snowflake_user", snowflake_account_id="ACCOUNT-123", snowflake_warehouse="test_warehouse", - snowflake_storage_integration_name="test_integration", password="test_password", - snowflake_stage_location=SnowflakeStageLocation.aws_s3, - s3_bucket_name="test-bucket", - iam_role_arn="arn:aws:iam::123456789012:role/test-role", - ) - - -def snowflake_connection_config_azure(): - return SnowflakeConnectionConfig( - name="snowflake_azure", - database="test_db", - user="snowflake_user", - snowflake_account_id="ACCOUNT-456", - snowflake_warehouse="test_warehouse", - snowflake_storage_integration_name="test_integration", - password="test_password", - snowflake_stage_location=SnowflakeStageLocation.azure_blob_storage, - snowflake_azure_container_name="test-container", - snowflake_azure_connection_string="DefaultEndpointsProtocol=https;AccountName=test;AccountKey=test", - ) - - -def snowflake_connection_config_local(): - return SnowflakeConnectionConfig( - name="snowflake_local", - database="test_db", - user="snowflake_user", - snowflake_account_id="ACCOUNT-789", - snowflake_warehouse="test_warehouse", - snowflake_storage_integration_name="test_integration", - password="test_password", - snowflake_stage_location=SnowflakeStageLocation.local, ) diff --git a/tests/test_connections.py b/tests/test_connections.py index bd05606..c76c920 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -22,16 +22,13 @@ MssqlLinkedServerConnectionConfig, S3ConnectionConfig, SnowflakeConnectionConfig, - SnowflakeStageLocation, SseConfig, SseSelection, validate_connection, ) from tests.helpers import ( sample_mounted_share_connection_json, - snowflake_connection_config_azure, - snowflake_connection_config_local, - snowflake_connection_config_s3, + snowflake_connection_config, ) @@ -235,11 +232,8 @@ def test_list_connnections(client): "snowflake_role": "snowballs do indeed roll", "snowflake_account_id": "ABCDEF-123456", "snowflake_warehouse": "warehouse1", - "snowflake_storage_integration_name": "mysi", "host": "snowflake.com", "port": 443, - "s3_bucket_name": "ice-bucket", - "iam_role_arn": "swiss roll", "is_read_only": False, "password_encrypted": "some_base64_here", "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", @@ -257,8 +251,6 @@ def test_list_connnections(client): "name": "snowflake_minimal_with_key", "snowflake_account_id": "ACCOUNT-1234", "snowflake_warehouse": "clothing_store", - "snowflake_storage_integration_name": "kennards", - "s3_bucket_name": "champagne-bucket", "snowflake_private_key": "2831289a-4398-abcd-4112-fe09a1239f89", "snowflake_private_key_passphrase_encrypted": "some base64 here", "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", @@ -339,9 +331,6 @@ def test_list_connnections(client): assert snowflake_connection.snowflake_role == "snowballs do indeed roll" assert snowflake_connection.snowflake_account_id == "ABCDEF-123456" assert snowflake_connection.snowflake_warehouse == "warehouse1" - assert snowflake_connection.snowflake_storage_integration_name == "mysi" - assert snowflake_connection.s3_bucket_name == "ice-bucket" - assert snowflake_connection.iam_role_arn == "swiss roll" assert snowflake_connection.is_read_only is False minimal_snowflake_connection = connections[8] @@ -355,9 +344,6 @@ def test_list_connnections(client): assert minimal_snowflake_connection.snowflake_role == "" assert minimal_snowflake_connection.snowflake_account_id == "ACCOUNT-1234" assert minimal_snowflake_connection.snowflake_warehouse == "clothing_store" - assert minimal_snowflake_connection.snowflake_storage_integration_name == "kennards" - assert minimal_snowflake_connection.s3_bucket_name == "champagne-bucket" - assert minimal_snowflake_connection.iam_role_arn is None assert minimal_snowflake_connection.is_read_only is False assert minimal_snowflake_connection.snowflake_private_key == "2831289a-4398-abcd-4112-fe09a1239f89" @@ -421,39 +407,20 @@ def test_delete_connection_that_does_not_exist(client): assert m.request_history[0].method == "GET" -@pytest.mark.parametrize( - "config_func,expected_stage_location,expected_fields,unexpected_fields", - [ - ( - snowflake_connection_config_s3, - "aws_s3", - ["s3_bucket_name", "iam_role_arn"], - ["snowflake_azure_container_name", "snowflake_azure_connection_string"], - ), - ( - snowflake_connection_config_azure, - "azure_blob_storage", - ["snowflake_azure_container_name", "snowflake_azure_connection_string"], - ["s3_bucket_name", "iam_role_arn"], - ), - ( - snowflake_connection_config_local, - "local", - [], - [ - "s3_bucket_name", - "iam_role_arn", - "snowflake_azure_container_name", - "snowflake_azure_connection_string", - ], - ), - ], +# Removed from Snowflake connections in server 3.26.18. +SNOWFLAKE_EXTERNAL_STAGE_FIELDS = ( + "s3_bucket_name", + "iam_role_arn", + "snowflake_azure_container_name", + "snowflake_azure_connection_string", + "snowflake_azure_connection_string_encrypted", + "snowflake_storage_integration_name", ) -def test_create_snowflake_connection_with_staging_platform( - client, config_func, expected_stage_location, expected_fields, unexpected_fields -): - """Test creating Snowflake connections with different staging platforms.""" - config = config_func() + + +def test_create_snowflake_connection(client): + """Creating a Snowflake connection sends none of the removed fields.""" + config = snowflake_connection_config() with requests_mock.Mocker() as m: m.get("http://test-server/api/connections/", json=[], status_code=200) @@ -462,71 +429,30 @@ def test_create_snowflake_connection_with_staging_platform( result = client.create_or_update_connection(config) assert result.id == ConnectionId("2") - # Verify the correct data was sent request_data = m.last_request.json() - assert request_data["snowflake_stage_location"] == expected_stage_location - - # Check expected fields are present - for field in expected_fields: - assert field in request_data - - # Check unexpected fields are not present - for field in unexpected_fields: + assert request_data["dbpassword"] == "test_password" + assert request_data["schema"] == "" + assert "snowflake_stage_location" not in request_data + for field in SNOWFLAKE_EXTERNAL_STAGE_FIELDS: assert field not in request_data -@pytest.mark.parametrize( - "config_func,expected_stage_location,expected_fields,unexpected_fields", - [ - ( - snowflake_connection_config_s3, - SnowflakeStageLocation.aws_s3, - { - "s3_bucket_name": "test-bucket", - "iam_role_arn": "arn:aws:iam::123456789012:role/test-role", - }, - ["snowflake_azure_container_name", "snowflake_azure_connection_string"], - ), - ( - snowflake_connection_config_azure, - SnowflakeStageLocation.azure_blob_storage, - { - "snowflake_azure_container_name": "test-container", - "snowflake_azure_connection_string": "DefaultEndpointsProtocol=https;AccountName=test;AccountKey=test", - }, - ["s3_bucket_name", "iam_role_arn"], - ), - ( - snowflake_connection_config_local, - SnowflakeStageLocation.local, - {}, - [ - "s3_bucket_name", - "iam_role_arn", - "snowflake_azure_container_name", - "snowflake_azure_connection_string", - ], - ), - ], -) -def test_snowflake_connection_model_dump(config_func, expected_stage_location, expected_fields, unexpected_fields): - """Test that Snowflake connections serialize correctly for each staging platform.""" - config = config_func() +def test_snowflake_connection_model_dump_omits_external_stage_fields(): + config = snowflake_connection_config() api_dict = config.model_dump(exclude_none=True, by_alias=True, mode="json") - assert api_dict["snowflake_stage_location"] == expected_stage_location - - # Check expected fields and their values - for field, value in expected_fields.items(): - assert api_dict[field] == value - - # Check unexpected fields are not present - for field in unexpected_fields: + assert "snowflake_stage_location" not in api_dict + for field in SNOWFLAKE_EXTERNAL_STAGE_FIELDS: assert field not in api_dict -def test_list_snowflake_connections_with_different_platforms(client): - """Test listing Snowflake connections returns correct staging platform information.""" +def test_list_snowflake_connection_from_older_server_drops_external_stage(client): + """ + A connection listed by a server older than 3.26.18 still parses. + + Those servers return the external stage location and its storage settings. + The SDK drops them and does not send them back on update. + """ with requests_mock.Mocker() as m: m.get( "http://test-server/api/connections/", @@ -547,118 +473,17 @@ def test_list_snowflake_connections_with_different_platforms(client): "id": "s3-connection-id", "mask_type": "database", }, - { - "version": "1.0", - "user": "azure_user", - "db_type": "snowflake", - "database": "azure_db", - "name": "snowflake_azure", - "snowflake_account_id": "AZURE-ACCOUNT", - "snowflake_warehouse": "azure_warehouse", - "snowflake_storage_integration_name": "azure_integration", - "snowflake_azure_container_name": "azure-container", - "snowflake_azure_connection_string_encrypted": "encrypted_azure_string", - "snowflake_stage_location": "azure_blob_storage", - "password_encrypted": "encrypted", - "id": "azure-connection-id", - "mask_type": "database", - }, - { - "version": "1.0", - "user": "local_user", - "db_type": "snowflake", - "database": "local_db", - "name": "snowflake_local", - "snowflake_account_id": "LOCAL-ACCOUNT", - "snowflake_warehouse": "local_warehouse", - "snowflake_storage_integration_name": "local_integration", - "snowflake_stage_location": "local", - "password_encrypted": "encrypted", - "id": "local-connection-id", - "mask_type": "database", - }, ], status_code=200, ) - connections = client.list_connections() - snowflake_connections = [c for c in connections if isinstance(c, SnowflakeConnectionConfig)] - assert len(snowflake_connections) == 3 - - # Check S3 connection - s3_conn = next(c for c in snowflake_connections if c.name == "snowflake_s3") - assert s3_conn.snowflake_stage_location is SnowflakeStageLocation.aws_s3 - assert s3_conn.s3_bucket_name == "s3-bucket" - assert s3_conn.iam_role_arn == "arn:aws:iam::123456789012:role/s3-role" - assert s3_conn.snowflake_azure_container_name is None - assert s3_conn.snowflake_azure_connection_string is None - - # Check Azure connection - azure_conn = next(c for c in snowflake_connections if c.name == "snowflake_azure") - assert azure_conn.snowflake_stage_location is SnowflakeStageLocation.azure_blob_storage - assert azure_conn.snowflake_azure_container_name == "azure-container" - assert azure_conn.snowflake_azure_connection_string is None # Encrypted, so empty - assert azure_conn.s3_bucket_name is None - assert azure_conn.iam_role_arn is None - - # Check local connection - local_conn = next(c for c in snowflake_connections if c.name == "snowflake_local") - assert local_conn.snowflake_stage_location is SnowflakeStageLocation.local - assert local_conn.s3_bucket_name is None - assert local_conn.iam_role_arn is None - assert local_conn.snowflake_azure_container_name is None - assert local_conn.snowflake_azure_connection_string is None - + (conn,) = client.list_connections() -@pytest.mark.parametrize( - "stage_location,missing_fields,error_message", - [ - ( - SnowflakeStageLocation.azure_blob_storage, - { - "snowflake_azure_container_name": None, - "snowflake_azure_connection_string": None, - }, - "Missing Azure fields", - ), - ( - SnowflakeStageLocation.aws_s3, - { - "s3_bucket_name": None, - "iam_role_arn": None, # IAM role is optional, so only s3_bucket_name is truly required - }, - "Missing S3 bucket", - ), - ], -) -def test_create_snowflake_connection_missing_required_fields(client, stage_location, missing_fields, error_message): - """Test that creating a Snowflake connection with missing required fields fails appropriately.""" - config_dict = { - "name": f"invalid_{stage_location.value}", - "database": "test_db", - "user": "snowflake_user", - "snowflake_account_id": "ACCOUNT-123", - "snowflake_warehouse": "test_warehouse", - "snowflake_storage_integration_name": "test_integration", - "password": "test_password", - "snowflake_stage_location": stage_location, - } - - # Add the missing fields - config_dict.update(missing_fields) - - config = SnowflakeConnectionConfig(**config_dict) - - with requests_mock.Mocker() as m: - m.get("http://test-server/api/connections/", json=[], status_code=200) - m.post( - "http://test-server/api/connections/", - json={"error": error_message}, - status_code=400, - ) - - with pytest.raises(DataMasqueApiError): - client.create_or_update_connection(config) + assert isinstance(conn, SnowflakeConnectionConfig) + api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") + assert "snowflake_stage_location" not in api_dict + for field in SNOWFLAKE_EXTERNAL_STAGE_FIELDS: + assert field not in api_dict def test_s3_connection_model_validate(): @@ -1122,7 +947,9 @@ def test_dynamo_connection_model_validate_without_sse_uses_default(): assert conn.dynamo_default_sse == SseConfig(selection=SseSelection.dynamodb_owned, kms_key_id=None) -def test_snowflake_connection_model_validate_with_stage_location(): +@pytest.mark.parametrize("stage_location", ["aws_s3", "azure_blob_storage", "local", "spcs"]) +def test_snowflake_connection_model_validate_drops_external_stage_location(stage_location): + """Stage locations from servers older than 3.26.18 are dropped and not sent back.""" payload = { "id": "f0557fb3-1c9a-4cb1-bcf4-9699cf496bf7", "name": "snowflake", @@ -1139,49 +966,23 @@ def test_snowflake_connection_model_validate_with_stage_location(): "port": 443, "s3_bucket_name": "ice-bucket", "iam_role_arn": "swiss roll", - "snowflake_stage_location": "aws_s3", + "snowflake_azure_container_name": "ice-container", + "snowflake_azure_connection_string_encrypted": "encrypted", + "snowflake_stage_location": stage_location, "is_read_only": False, } conn = SnowflakeConnectionConfig.model_validate(payload) assert isinstance(conn, SnowflakeConnectionConfig) - assert conn.snowflake_stage_location is SnowflakeStageLocation.aws_s3 - assert conn.iam_role_arn == "swiss roll" assert conn.password is None + api_dict = conn.model_dump(exclude_none=True, by_alias=True, mode="json") + assert "snowflake_stage_location" not in api_dict + for field in SNOWFLAKE_EXTERNAL_STAGE_FIELDS: + assert field not in api_dict -def test_snowflake_connection_model_validate_with_spcs_stage_location(): - """ - A Snowflake connection staged in SPCS must deserialise (regression for ui-testing MR !185). - - When DataMasque runs inside Snowflake SPCS it saves connections with - `snowflake_stage_location=spcs`. Listing connections deserialises every - one, so an unknown stage value used to raise `ValidationError` and break - `create_or_update_connection` for unrelated connections on a shared instance. - """ - payload = { - "id": "a1b2c3d4-0000-0000-0000-000000000000", - "name": "snowflake_spcs", - "mask_type": "database", - "db_type": "snowflake", - "user": "snowman", - "database": "icicle", - "snowflake_account_id": "ABCDEF-123456", - "snowflake_warehouse": "warehouse1", - "snowflake_storage_integration_name": "mysi", - "snowflake_stage_location": "spcs", - } - - conn = SnowflakeConnectionConfig.model_validate(payload) - - assert conn.snowflake_stage_location is SnowflakeStageLocation.spcs - # SPCS staging carries no external-storage fields. - assert conn.s3_bucket_name is None - assert conn.snowflake_azure_container_name is None - - -def test_snowflake_connection_model_validate_without_stage_location(): +def test_snowflake_connection_model_validate_minimal_payload(): payload = { "id": "id-3", "name": "snowflake", @@ -1191,12 +992,10 @@ def test_snowflake_connection_model_validate_without_stage_location(): "database": "igloo", "snowflake_account_id": "ACCOUNT-1234", "snowflake_warehouse": "clothing_store", - "snowflake_storage_integration_name": "kennards", } conn = SnowflakeConnectionConfig.model_validate(payload) - assert conn.snowflake_stage_location is None assert conn.host == "" assert conn.port is None assert conn.db_schema is None