From 75350b70d12aa73ee6ee4e43f42aef4674956e49 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 21 Sep 2026 11:23:33 -0700 Subject: [PATCH 1/7] feat(SOF-8051): samples, measurements and files endpoints The lab-data intake needs three surfaces the client did not have. Samples and measurements are entity-plus-set endpoints, so they are materials with a different name; measurements add files, the n+1 of jobs/:_id/files, which returns a key and a signed url per file. Files is not an entity endpoint: the web-app serves POST files, DELETE files and POST files/signed-urls and nothing addressed by an id, so it derives from BaseEndpoint directly. put asks for one putObject url, reads the file once and sends those bytes, and answers with the key, the size and the sha256 the caller records against what it uploaded. update_set sits on the set mixin next to create_set and move_to_set; samples and measurements register the route, materials does not. Co-Authored-By: Claude Opus 5 (1M context) --- src/py/mat3ra/api_client/client.py | 6 ++ src/py/mat3ra/api_client/endpoints/files.py | 77 +++++++++++++++++++ .../api_client/endpoints/measurements.py | 38 +++++++++ .../mat3ra/api_client/endpoints/mixins/set.py | 14 ++++ src/py/mat3ra/api_client/endpoints/samples.py | 26 +++++++ tests/py/unit/test_files.py | 59 ++++++++++++++ tests/py/unit/test_measurements.py | 39 ++++++++++ tests/py/unit/test_samples.py | 41 ++++++++++ 8 files changed, 300 insertions(+) create mode 100644 src/py/mat3ra/api_client/endpoints/files.py create mode 100644 src/py/mat3ra/api_client/endpoints/measurements.py create mode 100644 src/py/mat3ra/api_client/endpoints/samples.py create mode 100644 tests/py/unit/test_files.py create mode 100644 tests/py/unit/test_measurements.py create mode 100644 tests/py/unit/test_samples.py diff --git a/src/py/mat3ra/api_client/client.py b/src/py/mat3ra/api_client/client.py index 1299e27..bdaac17 100644 --- a/src/py/mat3ra/api_client/client.py +++ b/src/py/mat3ra/api_client/client.py @@ -9,11 +9,14 @@ from .endpoints.bank_materials import BankMaterialEndpoints from .endpoints.bank_workflows import BankWorkflowEndpoints from .endpoints.clusters import ClustersEndpoint +from .endpoints.files import FileEndpoints from .endpoints.jobs import JobEndpoints from .endpoints.materials import MaterialEndpoints +from .endpoints.measurements import MeasurementEndpoints from .endpoints.metaproperties import MetaPropertiesEndpoints from .endpoints.projects import ProjectEndpoints from .endpoints.properties import PropertiesEndpoints +from .endpoints.samples import SampleEndpoints from .endpoints.workflows import WorkflowEndpoints from .models import Account, APIEnv, AuthContext, AuthEnv @@ -53,6 +56,9 @@ def _init_endpoints(self, timeout_seconds: int) -> None: base_kwargs = {"version": self.version, "secure": self.secure, "timeout": timeout_seconds, "auth": self.auth} self.materials = MaterialEndpoints(*base_args, **base_kwargs) + self.samples = SampleEndpoints(*base_args, **base_kwargs) + self.measurements = MeasurementEndpoints(*base_args, **base_kwargs) + self.files = FileEndpoints(*base_args, **base_kwargs) self.workflows = WorkflowEndpoints(*base_args, **base_kwargs) self.jobs = JobEndpoints(*base_args, **base_kwargs) self.projects = ProjectEndpoints(*base_args, **base_kwargs) diff --git a/src/py/mat3ra/api_client/endpoints/files.py b/src/py/mat3ra/api_client/endpoints/files.py new file mode 100644 index 0000000..98a7137 --- /dev/null +++ b/src/py/mat3ra/api_client/endpoints/files.py @@ -0,0 +1,77 @@ +import hashlib +import json + +import requests + +from . import BaseEndpoint +from .enums import DEFAULT_API_VERSION, SECURE + + +class FileEndpoints(BaseEndpoint): + """ + File endpoints. + + Args: + host (str): API hostname. + port (int): API port number. + account_id (str): account ID. + auth_token (str): authentication token. + version (str): API version. + secure (bool): whether to use secure http protocol (https vs http). + kwargs (dict): a dictionary of HTTP session options. + timeout (int): session timeout in seconds. + + Attributes: + name (str): endpoint name. + headers (dict): default HTTP headers. + """ + + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): + super(FileEndpoints, self).__init__(host, port, version, secure, **kwargs) + self.name = "files" + self.headers = self.get_headers(account_id, auth_token) + + def create(self, name, body): + """ + Creates a file in the account's folder. + + Args: + name (str): file name, relative to the account's folder. + body (str): file content: text, or a data URL (data:;base64,...) for a file that is not text. + + Returns: + dict: new file. + """ + data = {"name": name, "body": body} + return self.request("POST", self.name, data=json.dumps(data), headers=self.headers) + + def signed_urls(self, names, operation="getObject"): + """ + Returns pre-signed URLs for the given files. + + Args: + names (list[str]): file names, relative to the account's folder. + operation (str): getObject to download a file, putObject to upload one. + + Returns: + list: [{"key": str, "signedUrl": str, "bucket": str, "region": str, "provider": str}] + """ + data = {"names": names, "operation": operation} + return self.request("POST", "/".join((self.name, "signed-urls")), data=json.dumps(data), headers=self.headers) + + def put(self, path, key): + """ + Uploads a given file to the account's folder through a pre-signed URL. + + Args: + path (str): path to the file to upload. + key (str): file name, relative to the account's folder. + + Returns: + dict: {"key": str, "bytes": int, "sha256": str} + """ + url = self.signed_urls([key], "putObject")[0]["signedUrl"] + with open(path, "rb") as file_: + content = file_.read() + requests.put(url, data=content) + return {"key": key, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()} diff --git a/src/py/mat3ra/api_client/endpoints/measurements.py b/src/py/mat3ra/api_client/endpoints/measurements.py new file mode 100644 index 0000000..01a686e --- /dev/null +++ b/src/py/mat3ra/api_client/endpoints/measurements.py @@ -0,0 +1,38 @@ +from .entity import EntityEndpoint +from .enums import DEFAULT_API_VERSION, SECURE +from .mixins.set import EntitySetEndpointsMixin + + +class MeasurementEndpoints(EntitySetEndpointsMixin, EntityEndpoint): + """ + Measurement endpoints. + + Args: + host (str): API hostname. + port (int): API port number. + account_id (str): account ID. + auth_token (str): authentication token. + version (str): API version. + secure (bool): whether to use secure http protocol (https vs http). + kwargs (dict): a dictionary of HTTP session options. + timeout (int): session timeout in seconds. + + Attributes: + name (str): endpoint name. + """ + + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): + super(MeasurementEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) + self.name = "measurements" + + def files(self, id_): + """ + Returns a list of measurement files. + + Args: + id_ (str): measurement ID. + + Returns: + list: [{"key": str, "signedUrl": str}] + """ + return self.request("GET", "/".join((self.name, id_, "files")), headers=self.headers) diff --git a/src/py/mat3ra/api_client/endpoints/mixins/set.py b/src/py/mat3ra/api_client/endpoints/mixins/set.py index b55c30c..62c03e3 100644 --- a/src/py/mat3ra/api_client/endpoints/mixins/set.py +++ b/src/py/mat3ra/api_client/endpoints/mixins/set.py @@ -30,3 +30,17 @@ def move_to_set(self, _id, old_set_id, new_set_id): """ params = {"oldSetId": old_set_id, "newSetId": new_set_id} self.request("POST", "/".join((self.name, _id, "move-to-set")), params=params, headers=self.headers) + + def update_set(self, _id, config): + """ + Updates a entity set with given ID. + + Args: + _id (str): entity set ID. + config (dict): entity set config. + + Returns: + dict: updated entity set. + """ + path_ = "/".join((self.name, _id, "update-set")) + return self.request("PUT", path_, data=json.dumps(config), headers=self.headers) diff --git a/src/py/mat3ra/api_client/endpoints/samples.py b/src/py/mat3ra/api_client/endpoints/samples.py new file mode 100644 index 0000000..52234f5 --- /dev/null +++ b/src/py/mat3ra/api_client/endpoints/samples.py @@ -0,0 +1,26 @@ +from .entity import EntityEndpoint +from .enums import DEFAULT_API_VERSION, SECURE +from .mixins.set import EntitySetEndpointsMixin + + +class SampleEndpoints(EntitySetEndpointsMixin, EntityEndpoint): + """ + Sample endpoints. + + Args: + host (str): API hostname. + port (int): API port number. + account_id (str): account ID. + auth_token (str): authentication token. + version (str): API version. + secure (bool): whether to use secure http protocol (https vs http). + kwargs (dict): a dictionary of HTTP session options. + timeout (int): session timeout in seconds. + + Attributes: + name (str): endpoint name. + """ + + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): + super(SampleEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) + self.name = "samples" diff --git a/tests/py/unit/test_files.py b/tests/py/unit/test_files.py new file mode 100644 index 0000000..4b2b70b --- /dev/null +++ b/tests/py/unit/test_files.py @@ -0,0 +1,59 @@ +import json +import os +import tempfile +from unittest import mock + +from mat3ra.api_client.endpoints.files import FileEndpoints +from tests.py.unit import EndpointBaseUnitTest + +FILE_NAME = "loops/site-1.npy" +FILE_CONTENT = b"loop data" +FILE_SHA256 = "61719738f7cfbd0bb8e7fc91cbd2febe3b90732b4f31a994babf549e44537de8" +SIGNED_URL = "https://test-bucket.s3.amazonaws.com/user-rvuo7pgiyu/loops/site-1.npy?X-Amz-Signature=test" + +MOCK_CREATED_FILE = {"name": "record.json", "key": "user-rvuo7pgiyu/record.json", "size": 2} +MOCK_CREATE_RESPONSE = json.dumps({"status": "success", "data": MOCK_CREATED_FILE}) +MOCK_SIGNED_URLS_RESPONSE = json.dumps( + {"status": "success", "data": [{"key": "user-rvuo7pgiyu/loops/site-1.npy", "signedUrl": SIGNED_URL}]} +) + + +class EndpointFilesUnitTest(EndpointBaseUnitTest): + """ + Class for testing files endpoint. + """ + + def __init__(self, *args, **kwargs): + super(EndpointFilesUnitTest, self).__init__(*args, **kwargs) + self.base_url = f"https://{self.host}:{self.port}/api/{self.version}/files" + self.endpoints = FileEndpoints(self.host, self.port, self.account_id, self.auth_token) + + @mock.patch("requests.sessions.Session.request") + def test_create(self, mock_request): + mock_request.return_value = self.mock_response(MOCK_CREATE_RESPONSE) + self.assertEqual(self.endpoints.create("record.json", "{}"), MOCK_CREATED_FILE) + self.assertEqual(mock_request.call_args[1]["url"], self.base_url) + self.assertEqual(json.loads(mock_request.call_args[1]["data"]), {"name": "record.json", "body": "{}"}) + + @mock.patch("requests.sessions.Session.request") + def test_signed_urls(self, mock_request): + mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE) + self.assertEqual(self.endpoints.signed_urls([FILE_NAME])[0]["signedUrl"], SIGNED_URL) + self.assertEqual(mock_request.call_args[1]["url"], f"{self.base_url}/signed-urls") + self.assertEqual( + json.loads(mock_request.call_args[1]["data"]), {"names": [FILE_NAME], "operation": "getObject"} + ) + + @mock.patch("requests.put") + @mock.patch("requests.sessions.Session.request") + def test_put(self, mock_request, mock_put): + mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE) + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "site-1.npy") + with open(path, "wb") as file_: + file_.write(FILE_CONTENT) + result = self.endpoints.put(path, FILE_NAME) + self.assertEqual(result, {"key": FILE_NAME, "bytes": len(FILE_CONTENT), "sha256": FILE_SHA256}) + self.assertEqual(json.loads(mock_request.call_args[1]["data"])["operation"], "putObject") + self.assertEqual(mock_put.call_args[0][0], SIGNED_URL) + self.assertEqual(mock_put.call_args[1]["data"], FILE_CONTENT) diff --git a/tests/py/unit/test_measurements.py b/tests/py/unit/test_measurements.py new file mode 100644 index 0000000..0cc4e4a --- /dev/null +++ b/tests/py/unit/test_measurements.py @@ -0,0 +1,39 @@ +import json +from unittest import mock + +from mat3ra.api_client.endpoints.measurements import MeasurementEndpoints +from tests.py.unit.entity import TEST_ENTITY_ID, EntityEndpointsUnitTest + +ENDPOINT_NAME = "measurements" + +MOCK_FILE = {"key": "user-rvuo7pgiyu/loops/site-1.npy", "signedUrl": "https://test-bucket.s3.amazonaws.com/site-1.npy"} +MOCK_FILES_RESPONSE = json.dumps({"status": "success", "data": [MOCK_FILE]}) + + +class EndpointMeasurementsUnitTest(EntityEndpointsUnitTest): + """ + Class for testing measurements endpoint. + """ + + def __init__(self, *args, **kwargs): + super(EndpointMeasurementsUnitTest, self).__init__(*args, **kwargs) + self.endpoint_name = ENDPOINT_NAME + self.endpoints = MeasurementEndpoints(self.host, self.port, self.account_id, self.auth_token) + + @mock.patch("requests.sessions.Session.request") + def test_list(self, mock_request): + self.list(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_get(self, mock_request): + self.get(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_create(self, mock_request): + self.create(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_files(self, mock_request): + mock_request.return_value = self.mock_response(MOCK_FILES_RESPONSE) + self.assertEqual(self.endpoints.files(TEST_ENTITY_ID), [MOCK_FILE]) + self.assertEqual(mock_request.call_args[1]["url"], f"{self.base_url}/{TEST_ENTITY_ID}/files") diff --git a/tests/py/unit/test_samples.py b/tests/py/unit/test_samples.py new file mode 100644 index 0000000..2c26631 --- /dev/null +++ b/tests/py/unit/test_samples.py @@ -0,0 +1,41 @@ +import json +from unittest import mock + +from mat3ra.api_client.endpoints.samples import SampleEndpoints +from tests.py.unit.entity import MOCK_SUCCESS_RESPONSE_OBJECT, TEST_ENTITY_ID, EntityEndpointsUnitTest + +ENDPOINT_NAME = "samples" + +SET_CONFIG = {"name": "WAFER-1", "metadata": {"label": "WAFER-1"}} +HTTP_METHOD_PUT = "put" + + +class EndpointSamplesUnitTest(EntityEndpointsUnitTest): + """ + Class for testing samples endpoint. + """ + + def __init__(self, *args, **kwargs): + super(EndpointSamplesUnitTest, self).__init__(*args, **kwargs) + self.endpoint_name = ENDPOINT_NAME + self.endpoints = SampleEndpoints(self.host, self.port, self.account_id, self.auth_token) + + @mock.patch("requests.sessions.Session.request") + def test_list(self, mock_request): + self.list(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_get(self, mock_request): + self.get(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_create(self, mock_request): + self.create(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_update_set(self, mock_request): + mock_request.return_value = self.mock_response(MOCK_SUCCESS_RESPONSE_OBJECT) + self.endpoints.update_set(TEST_ENTITY_ID, SET_CONFIG) + self.assertEqual(mock_request.call_args[1]["method"], HTTP_METHOD_PUT) + self.assertEqual(mock_request.call_args[1]["url"], f"{self.base_url}/{TEST_ENTITY_ID}/update-set") + self.assertEqual(json.loads(mock_request.call_args[1]["data"]), SET_CONFIG) From 6551ab734b4a075ae6089da9acc9ba5696f1afb0 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 21 Sep 2026 11:23:33 -0700 Subject: [PATCH 2/7] chore(SOF-8051): WIP release [release] Co-Authored-By: Claude Opus 5 (1M context) From 697d76dd635ce4b10b4eaf3f4592c76bf32c326f Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 21 Sep 2026 11:26:44 -0700 Subject: [PATCH 3/7] fix(SOF-8051): files.put raises when the PUT is refused A pre-signed URL can be refused - an expired signature, a key outside the account's folder - and the object storage answers 403 to the PUT itself, not to the call that issued the URL. Without the check put returned a key, a size and a digest for a file that was never written, and an uploader would report success for a lost file. Every other call this client makes raises on an HTTP error through BaseConnection; this one now does too. Co-Authored-By: Claude Opus 5 (1M context) --- src/py/mat3ra/api_client/endpoints/files.py | 3 ++- tests/py/unit/test_files.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/py/mat3ra/api_client/endpoints/files.py b/src/py/mat3ra/api_client/endpoints/files.py index 98a7137..b0098f9 100644 --- a/src/py/mat3ra/api_client/endpoints/files.py +++ b/src/py/mat3ra/api_client/endpoints/files.py @@ -73,5 +73,6 @@ def put(self, path, key): url = self.signed_urls([key], "putObject")[0]["signedUrl"] with open(path, "rb") as file_: content = file_.read() - requests.put(url, data=content) + response = requests.put(url, data=content) + response.raise_for_status() return {"key": key, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()} diff --git a/tests/py/unit/test_files.py b/tests/py/unit/test_files.py index 4b2b70b..3cea31c 100644 --- a/tests/py/unit/test_files.py +++ b/tests/py/unit/test_files.py @@ -3,6 +3,9 @@ import tempfile from unittest import mock +import pytest +import requests + from mat3ra.api_client.endpoints.files import FileEndpoints from tests.py.unit import EndpointBaseUnitTest @@ -16,6 +19,7 @@ MOCK_SIGNED_URLS_RESPONSE = json.dumps( {"status": "success", "data": [{"key": "user-rvuo7pgiyu/loops/site-1.npy", "signedUrl": SIGNED_URL}]} ) +MOCK_REFUSED_RESPONSE = "AccessDenied" class EndpointFilesUnitTest(EndpointBaseUnitTest): @@ -57,3 +61,15 @@ def test_put(self, mock_request, mock_put): self.assertEqual(json.loads(mock_request.call_args[1]["data"])["operation"], "putObject") self.assertEqual(mock_put.call_args[0][0], SIGNED_URL) self.assertEqual(mock_put.call_args[1]["data"], FILE_CONTENT) + + @mock.patch("requests.put") + @mock.patch("requests.sessions.Session.request") + def test_put_refused(self, mock_request, mock_put): + mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE) + mock_put.return_value = self.mock_response(MOCK_REFUSED_RESPONSE, 403, "Forbidden") + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "site-1.npy") + with open(path, "wb") as file_: + file_.write(FILE_CONTENT) + with pytest.raises(requests.HTTPError): + self.endpoints.put(path, FILE_NAME) From 0d313654979d1bf3377c9a32fbe17cd8fc16467b Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 21 Sep 2026 11:26:44 -0700 Subject: [PATCH 4/7] chore(SOF-8051): WIP release [release] Co-Authored-By: Claude Opus 5 (1M context) From bc1e6ad12d1c7a12c2c789193eec3a5cab3fea38 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 21 Sep 2026 11:33:38 -0700 Subject: [PATCH 5/7] feat(SOF-8051): list_accounts exposes the account slug The slug is the name people see in a platform URL and the one the uploader's --account takes, and the account document carries it: AccountDAO writes it at creation and users/me serialises the account as it is stored (serializeMyAccount returns the document). The projection dropped it, so a caller resolving a slug had to guess it from the name. Co-Authored-By: Claude Opus 5 (1M context) --- src/py/mat3ra/api_client/client.py | 1 + tests/py/unit/test_client.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/py/mat3ra/api_client/client.py b/src/py/mat3ra/api_client/client.py index bdaac17..a1387f2 100644 --- a/src/py/mat3ra/api_client/client.py +++ b/src/py/mat3ra/api_client/client.py @@ -152,6 +152,7 @@ def list_accounts(self) -> List[dict]: { "_id": account["entity"]["_id"], "name": account["entity"].get("name", ""), + "slug": account["entity"].get("slug", ""), "type": account["entity"].get("type", "personal"), "isDefault": account.get("isDefault", False), } diff --git a/tests/py/unit/test_client.py b/tests/py/unit/test_client.py index b19aa41..3e00f93 100644 --- a/tests/py/unit/test_client.py +++ b/tests/py/unit/test_client.py @@ -24,15 +24,20 @@ }, "accounts": [ { - "entity": {"_id": "user-acc-1", "name": "John Doe", "type": "personal"}, + "entity": {"_id": "user-acc-1", "name": "John Doe", "slug": "john-doe", "type": "personal"}, "isDefault": True, }, { - "entity": {"_id": "org-acc-1", "name": "Acme Corp", "type": "enterprise"}, + "entity": {"_id": "org-acc-1", "name": "Acme Corp", "slug": "acme-corp", "type": "enterprise"}, "isDefault": True, }, { - "entity": {"_id": "org-acc-2", "name": "Beta Industries", "type": "organization"}, + "entity": { + "_id": "org-acc-2", + "name": "Beta Industries", + "slug": "beta-industries", + "type": "organization", + }, "isDefault": False, }, ], @@ -113,10 +118,12 @@ def test_list_accounts(self, mock_get): self.assertEqual(len(accounts), 3) self.assertEqual(accounts[0]["_id"], "user-acc-1") self.assertEqual(accounts[0]["name"], "John Doe") + self.assertEqual(accounts[0]["slug"], "john-doe") self.assertEqual(accounts[0]["type"], "personal") self.assertTrue(accounts[0]["isDefault"]) self.assertEqual(accounts[1]["_id"], "org-acc-1") self.assertEqual(accounts[1]["name"], "Acme Corp") + self.assertEqual(accounts[1]["slug"], "acme-corp") self.assertEqual(accounts[1]["type"], "enterprise") @mock.patch("requests.get") From f69c700fa89da910422978ceb4e51d56a7e08c34 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 21 Sep 2026 11:36:30 -0700 Subject: [PATCH 6/7] feat(SOF-8051): files endpoints accept an explicit account An uploader running with --account writes its documents into the named account, and the files have to follow them. FilesEndpoints.getAccountId reads accountId from the body and only falls back to the caller's default account when it is absent, so passing it is the whole fix; put hands it to signed_urls, since the prefix is resolved when the URL is signed. Co-Authored-By: Claude Opus 5 (1M context) --- src/py/mat3ra/api_client/endpoints/files.py | 15 ++++++++--- tests/py/unit/test_files.py | 28 ++++++++++++++++++++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/py/mat3ra/api_client/endpoints/files.py b/src/py/mat3ra/api_client/endpoints/files.py index b0098f9..10f1d4b 100644 --- a/src/py/mat3ra/api_client/endpoints/files.py +++ b/src/py/mat3ra/api_client/endpoints/files.py @@ -31,46 +31,53 @@ def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSI self.name = "files" self.headers = self.get_headers(account_id, auth_token) - def create(self, name, body): + def create(self, name, body, account_id=None): """ Creates a file in the account's folder. Args: name (str): file name, relative to the account's folder. body (str): file content: text, or a data URL (data:;base64,...) for a file that is not text. + account_id (str): account to act under. The caller's default account is used if not specified. Returns: dict: new file. """ data = {"name": name, "body": body} + if account_id: + data["accountId"] = account_id return self.request("POST", self.name, data=json.dumps(data), headers=self.headers) - def signed_urls(self, names, operation="getObject"): + def signed_urls(self, names, operation="getObject", account_id=None): """ Returns pre-signed URLs for the given files. Args: names (list[str]): file names, relative to the account's folder. operation (str): getObject to download a file, putObject to upload one. + account_id (str): account to act under. The caller's default account is used if not specified. Returns: list: [{"key": str, "signedUrl": str, "bucket": str, "region": str, "provider": str}] """ data = {"names": names, "operation": operation} + if account_id: + data["accountId"] = account_id return self.request("POST", "/".join((self.name, "signed-urls")), data=json.dumps(data), headers=self.headers) - def put(self, path, key): + def put(self, path, key, account_id=None): """ Uploads a given file to the account's folder through a pre-signed URL. Args: path (str): path to the file to upload. key (str): file name, relative to the account's folder. + account_id (str): account to act under. The caller's default account is used if not specified. Returns: dict: {"key": str, "bytes": int, "sha256": str} """ - url = self.signed_urls([key], "putObject")[0]["signedUrl"] + url = self.signed_urls([key], "putObject", account_id)[0]["signedUrl"] with open(path, "rb") as file_: content = file_.read() response = requests.put(url, data=content) diff --git a/tests/py/unit/test_files.py b/tests/py/unit/test_files.py index 3cea31c..eb10c02 100644 --- a/tests/py/unit/test_files.py +++ b/tests/py/unit/test_files.py @@ -13,6 +13,7 @@ FILE_CONTENT = b"loop data" FILE_SHA256 = "61719738f7cfbd0bb8e7fc91cbd2febe3b90732b4f31a994babf549e44537de8" SIGNED_URL = "https://test-bucket.s3.amazonaws.com/user-rvuo7pgiyu/loops/site-1.npy?X-Amz-Signature=test" +OTHER_ACCOUNT_ID = "5dJXaqqhjPZrA5Qyw" MOCK_CREATED_FILE = {"name": "record.json", "key": "user-rvuo7pgiyu/record.json", "size": 2} MOCK_CREATE_RESPONSE = json.dumps({"status": "success", "data": MOCK_CREATED_FILE}) @@ -58,10 +59,35 @@ def test_put(self, mock_request, mock_put): file_.write(FILE_CONTENT) result = self.endpoints.put(path, FILE_NAME) self.assertEqual(result, {"key": FILE_NAME, "bytes": len(FILE_CONTENT), "sha256": FILE_SHA256}) - self.assertEqual(json.loads(mock_request.call_args[1]["data"])["operation"], "putObject") + self.assertEqual( + json.loads(mock_request.call_args[1]["data"]), {"names": [FILE_NAME], "operation": "putObject"} + ) self.assertEqual(mock_put.call_args[0][0], SIGNED_URL) self.assertEqual(mock_put.call_args[1]["data"], FILE_CONTENT) + @mock.patch("requests.sessions.Session.request") + def test_create_for_account(self, mock_request): + mock_request.return_value = self.mock_response(MOCK_CREATE_RESPONSE) + self.endpoints.create("record.json", "{}", OTHER_ACCOUNT_ID) + self.assertEqual(json.loads(mock_request.call_args[1]["data"])["accountId"], OTHER_ACCOUNT_ID) + + @mock.patch("requests.sessions.Session.request") + def test_signed_urls_for_account(self, mock_request): + mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE) + self.endpoints.signed_urls([FILE_NAME], "getObject", OTHER_ACCOUNT_ID) + self.assertEqual(json.loads(mock_request.call_args[1]["data"])["accountId"], OTHER_ACCOUNT_ID) + + @mock.patch("requests.put") + @mock.patch("requests.sessions.Session.request") + def test_put_for_account(self, mock_request, mock_put): + mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE) + with tempfile.TemporaryDirectory() as directory: + path = os.path.join(directory, "site-1.npy") + with open(path, "wb") as file_: + file_.write(FILE_CONTENT) + self.endpoints.put(path, FILE_NAME, OTHER_ACCOUNT_ID) + self.assertEqual(json.loads(mock_request.call_args[1]["data"])["accountId"], OTHER_ACCOUNT_ID) + @mock.patch("requests.put") @mock.patch("requests.sessions.Session.request") def test_put_refused(self, mock_request, mock_put): From e2edf19cf76397ce0435b64c51e82e4e2ab4dc6f Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 21 Sep 2026 11:43:16 -0700 Subject: [PATCH 7/7] fix(SOF-8051): update_set only where the server has the route; put returns the stored key Review, should-fix 1: the server registers update-set from samples and measurements alone, and set_routes.ts says so where it defines the route, so jobs and materials inheriting update_set could only ever have produced a 404. It moves to its own mixin beside the shared one and is composed into the two endpoints that can serve it, the way a material alone is defaultable. Review, should-fix 3: signed_urls already answers with the key the object is stored under, which is the account's prefix plus the name. put echoed the caller's relative name back instead, so a caller naming the uploaded object had to rebuild the prefix - the one thing put exists to hide. Review, should-fix 4, declined with a reason: the presigned PUT stays on requests.put rather than BaseConnection. The URL is object storage, not the API, and BaseConnection rewrites every failure through _extract_server_message, which returns "" for an XML body - a refusal would read "Error 403: HTTP Error." instead of naming the URL that was refused. Co-Authored-By: Claude Opus 5 (1M context) --- src/py/mat3ra/api_client/endpoints/files.py | 8 ++++---- src/py/mat3ra/api_client/endpoints/measurements.py | 4 ++-- src/py/mat3ra/api_client/endpoints/mixins/set.py | 6 ++++++ src/py/mat3ra/api_client/endpoints/samples.py | 4 ++-- tests/py/unit/test_files.py | 10 +++++----- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/py/mat3ra/api_client/endpoints/files.py b/src/py/mat3ra/api_client/endpoints/files.py index 10f1d4b..6eefd2a 100644 --- a/src/py/mat3ra/api_client/endpoints/files.py +++ b/src/py/mat3ra/api_client/endpoints/files.py @@ -75,11 +75,11 @@ def put(self, path, key, account_id=None): account_id (str): account to act under. The caller's default account is used if not specified. Returns: - dict: {"key": str, "bytes": int, "sha256": str} + dict: {"key": str, "bytes": int, "sha256": str} with the key the file is stored under. """ - url = self.signed_urls([key], "putObject", account_id)[0]["signedUrl"] + signed_file = self.signed_urls([key], "putObject", account_id)[0] with open(path, "rb") as file_: content = file_.read() - response = requests.put(url, data=content) + response = requests.put(signed_file["signedUrl"], data=content) response.raise_for_status() - return {"key": key, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()} + return {"key": signed_file["key"], "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()} diff --git a/src/py/mat3ra/api_client/endpoints/measurements.py b/src/py/mat3ra/api_client/endpoints/measurements.py index 01a686e..245c92b 100644 --- a/src/py/mat3ra/api_client/endpoints/measurements.py +++ b/src/py/mat3ra/api_client/endpoints/measurements.py @@ -1,9 +1,9 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE -from .mixins.set import EntitySetEndpointsMixin +from .mixins.set import EntitySetEndpointsMixin, EntitySetUpdateEndpointsMixin -class MeasurementEndpoints(EntitySetEndpointsMixin, EntityEndpoint): +class MeasurementEndpoints(EntitySetEndpointsMixin, EntitySetUpdateEndpointsMixin, EntityEndpoint): """ Measurement endpoints. diff --git a/src/py/mat3ra/api_client/endpoints/mixins/set.py b/src/py/mat3ra/api_client/endpoints/mixins/set.py index 62c03e3..60c12ec 100644 --- a/src/py/mat3ra/api_client/endpoints/mixins/set.py +++ b/src/py/mat3ra/api_client/endpoints/mixins/set.py @@ -31,6 +31,12 @@ def move_to_set(self, _id, old_set_id, new_set_id): params = {"oldSetId": old_set_id, "newSetId": new_set_id} self.request("POST", "/".join((self.name, _id, "move-to-set")), params=params, headers=self.headers) + +class EntitySetUpdateEndpointsMixin(object): + """ + Entity Set update endpoints mixin. + """ + def update_set(self, _id, config): """ Updates a entity set with given ID. diff --git a/src/py/mat3ra/api_client/endpoints/samples.py b/src/py/mat3ra/api_client/endpoints/samples.py index 52234f5..add493e 100644 --- a/src/py/mat3ra/api_client/endpoints/samples.py +++ b/src/py/mat3ra/api_client/endpoints/samples.py @@ -1,9 +1,9 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE -from .mixins.set import EntitySetEndpointsMixin +from .mixins.set import EntitySetEndpointsMixin, EntitySetUpdateEndpointsMixin -class SampleEndpoints(EntitySetEndpointsMixin, EntityEndpoint): +class SampleEndpoints(EntitySetEndpointsMixin, EntitySetUpdateEndpointsMixin, EntityEndpoint): """ Sample endpoints. diff --git a/tests/py/unit/test_files.py b/tests/py/unit/test_files.py index eb10c02..c1d72ba 100644 --- a/tests/py/unit/test_files.py +++ b/tests/py/unit/test_files.py @@ -10,6 +10,7 @@ from tests.py.unit import EndpointBaseUnitTest FILE_NAME = "loops/site-1.npy" +STORED_KEY = "user-rvuo7pgiyu/loops/site-1.npy" FILE_CONTENT = b"loop data" FILE_SHA256 = "61719738f7cfbd0bb8e7fc91cbd2febe3b90732b4f31a994babf549e44537de8" SIGNED_URL = "https://test-bucket.s3.amazonaws.com/user-rvuo7pgiyu/loops/site-1.npy?X-Amz-Signature=test" @@ -17,9 +18,7 @@ MOCK_CREATED_FILE = {"name": "record.json", "key": "user-rvuo7pgiyu/record.json", "size": 2} MOCK_CREATE_RESPONSE = json.dumps({"status": "success", "data": MOCK_CREATED_FILE}) -MOCK_SIGNED_URLS_RESPONSE = json.dumps( - {"status": "success", "data": [{"key": "user-rvuo7pgiyu/loops/site-1.npy", "signedUrl": SIGNED_URL}]} -) +MOCK_SIGNED_URLS_RESPONSE = json.dumps({"status": "success", "data": [{"key": STORED_KEY, "signedUrl": SIGNED_URL}]}) MOCK_REFUSED_RESPONSE = "AccessDenied" @@ -58,7 +57,7 @@ def test_put(self, mock_request, mock_put): with open(path, "wb") as file_: file_.write(FILE_CONTENT) result = self.endpoints.put(path, FILE_NAME) - self.assertEqual(result, {"key": FILE_NAME, "bytes": len(FILE_CONTENT), "sha256": FILE_SHA256}) + self.assertEqual(result, {"key": STORED_KEY, "bytes": len(FILE_CONTENT), "sha256": FILE_SHA256}) self.assertEqual( json.loads(mock_request.call_args[1]["data"]), {"names": [FILE_NAME], "operation": "putObject"} ) @@ -85,7 +84,8 @@ def test_put_for_account(self, mock_request, mock_put): path = os.path.join(directory, "site-1.npy") with open(path, "wb") as file_: file_.write(FILE_CONTENT) - self.endpoints.put(path, FILE_NAME, OTHER_ACCOUNT_ID) + result = self.endpoints.put(path, FILE_NAME, OTHER_ACCOUNT_ID) + self.assertEqual(result["key"], STORED_KEY) self.assertEqual(json.loads(mock_request.call_args[1]["data"])["accountId"], OTHER_ACCOUNT_ID) @mock.patch("requests.put")