SOF-8051: samples, measurements and files endpoints

src/py/mat3ra/api_client/client.py

@@ -9,11 +9,14 @@
 from .endpoints.bank_materials import BankMaterialEndpoints
 from .endpoints.bank_workflows import BankWorkflowEndpoints
 from .endpoints.clusters import ClustersEndpoint
+from .endpoints.files import FileEndpoints
 from .endpoints.jobs import JobEndpoints
 from .endpoints.materials import MaterialEndpoints
+from .endpoints.measurements import MeasurementEndpoints
 from .endpoints.metaproperties import MetaPropertiesEndpoints
 from .endpoints.projects import ProjectEndpoints
 from .endpoints.properties import PropertiesEndpoints
+from .endpoints.samples import SampleEndpoints
 from .endpoints.workflows import WorkflowEndpoints
 from .models import Account, APIEnv, AuthContext, AuthEnv
 
@@ -53,6 +56,9 @@ def _init_endpoints(self, timeout_seconds: int) -> None:
         base_kwargs = {"version": self.version, "secure": self.secure, "timeout": timeout_seconds, "auth": self.auth}
 
         self.materials = MaterialEndpoints(*base_args, **base_kwargs)
+        self.samples = SampleEndpoints(*base_args, **base_kwargs)
+        self.measurements = MeasurementEndpoints(*base_args, **base_kwargs)
+        self.files = FileEndpoints(*base_args, **base_kwargs)
         self.workflows = WorkflowEndpoints(*base_args, **base_kwargs)
         self.jobs = JobEndpoints(*base_args, **base_kwargs)
         self.projects = ProjectEndpoints(*base_args, **base_kwargs)
@@ -146,6 +152,7 @@ def list_accounts(self) -> List[dict]:
             {
                 "_id": account["entity"]["_id"],
                 "name": account["entity"].get("name", ""),
+                "slug": account["entity"].get("slug", ""),
                 "type": account["entity"].get("type", "personal"),
                 "isDefault": account.get("isDefault", False),
             }

src/py/mat3ra/api_client/endpoints/files.py

@@ -0,0 +1,85 @@
+import hashlib
+import json
+
+import requests
+
+from . import BaseEndpoint
+from .enums import DEFAULT_API_VERSION, SECURE
+
+
+class FileEndpoints(BaseEndpoint):
+    """
+    File endpoints.
+
+    Args:
+        host (str): API hostname.
+        port (int): API port number.
+        account_id (str): account ID.
+        auth_token (str): authentication token.
+        version (str): API version.
+        secure (bool): whether to use secure http protocol (https vs http).
+        kwargs (dict): a dictionary of HTTP session options.
+            timeout (int): session timeout in seconds.
+
+    Attributes:
+        name (str): endpoint name.
+        headers (dict): default HTTP headers.
+    """
+
+    def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs):
+        super(FileEndpoints, self).__init__(host, port, version, secure, **kwargs)
+        self.name = "files"
+        self.headers = self.get_headers(account_id, auth_token)
+
+    def create(self, name, body, account_id=None):
+        """
+        Creates a file in the account's folder.
+
+        Args:
+            name (str): file name, relative to the account's folder.
+            body (str): file content: text, or a data URL (data:<type>;base64,...) for a file that is not text.
+            account_id (str): account to act under. The caller's default account is used if not specified.
+
+        Returns:
+            dict: new file.
+        """
+        data = {"name": name, "body": body}
+        if account_id:
+            data["accountId"] = account_id
+        return self.request("POST", self.name, data=json.dumps(data), headers=self.headers)
+
+    def signed_urls(self, names, operation="getObject", account_id=None):
+        """
+        Returns pre-signed URLs for the given files.
+
+        Args:
+            names (list[str]): file names, relative to the account's folder.
+            operation (str): getObject to download a file, putObject to upload one.
+            account_id (str): account to act under. The caller's default account is used if not specified.
+
+        Returns:
+            list: [{"key": str, "signedUrl": str, "bucket": str, "region": str, "provider": str}]
+        """
+        data = {"names": names, "operation": operation}
+        if account_id:
+            data["accountId"] = account_id
+        return self.request("POST", "/".join((self.name, "signed-urls")), data=json.dumps(data), headers=self.headers)
+
+    def put(self, path, key, account_id=None):
+        """
+        Uploads a given file to the account's folder through a pre-signed URL.
+
+        Args:
+            path (str): path to the file to upload.
+            key (str): file name, relative to the account's folder.
+            account_id (str): account to act under. The caller's default account is used if not specified.
+
+        Returns:
+            dict: {"key": str, "bytes": int, "sha256": str} with the key the file is stored under.
+        """
+        signed_file = self.signed_urls([key], "putObject", account_id)[0]
+        with open(path, "rb") as file_:
+            content = file_.read()
+        response = requests.put(signed_file["signedUrl"], data=content)
+        response.raise_for_status()
+        return {"key": signed_file["key"], "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()}

src/py/mat3ra/api_client/endpoints/measurements.py

@@ -0,0 +1,38 @@
+from .entity import EntityEndpoint
+from .enums import DEFAULT_API_VERSION, SECURE
+from .mixins.set import EntitySetEndpointsMixin, EntitySetUpdateEndpointsMixin
+
+
+class MeasurementEndpoints(EntitySetEndpointsMixin, EntitySetUpdateEndpointsMixin, EntityEndpoint):
+    """
+    Measurement endpoints.
+
+    Args:
+        host (str): API hostname.
+        port (int): API port number.
+        account_id (str): account ID.
+        auth_token (str): authentication token.
+        version (str): API version.
+        secure (bool): whether to use secure http protocol (https vs http).
+        kwargs (dict): a dictionary of HTTP session options.
+            timeout (int): session timeout in seconds.
+
+    Attributes:
+        name (str): endpoint name.
+    """
+
+    def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs):
+        super(MeasurementEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs)
+        self.name = "measurements"
+
+    def files(self, id_):
+        """
+        Returns a list of measurement files.
+
+        Args:
+            id_ (str): measurement ID.
+
+        Returns:
+            list: [{"key": str, "signedUrl": str}]
+        """
+        return self.request("GET", "/".join((self.name, id_, "files")), headers=self.headers)

src/py/mat3ra/api_client/endpoints/mixins/set.py

@@ -30,3 +30,23 @@ def move_to_set(self, _id, old_set_id, new_set_id):
         """
         params = {"oldSetId": old_set_id, "newSetId": new_set_id}
         self.request("POST", "/".join((self.name, _id, "move-to-set")), params=params, headers=self.headers)
+
+
+class EntitySetUpdateEndpointsMixin(object):
+    """
+    Entity Set update endpoints mixin.
+    """
+
+    def update_set(self, _id, config):
+        """
+        Updates a entity set with given ID.
+
+        Args:
+            _id (str): entity set ID.
+            config (dict): entity set config.
+
+        Returns:
+             dict: updated entity set.
+        """
+        path_ = "/".join((self.name, _id, "update-set"))
+        return self.request("PUT", path_, data=json.dumps(config), headers=self.headers)

src/py/mat3ra/api_client/endpoints/samples.py

@@ -0,0 +1,26 @@
+from .entity import EntityEndpoint
+from .enums import DEFAULT_API_VERSION, SECURE
+from .mixins.set import EntitySetEndpointsMixin, EntitySetUpdateEndpointsMixin
+
+
+class SampleEndpoints(EntitySetEndpointsMixin, EntitySetUpdateEndpointsMixin, EntityEndpoint):
+    """
+    Sample endpoints.
+
+    Args:
+        host (str): API hostname.
+        port (int): API port number.
+        account_id (str): account ID.
+        auth_token (str): authentication token.
+        version (str): API version.
+        secure (bool): whether to use secure http protocol (https vs http).
+        kwargs (dict): a dictionary of HTTP session options.
+            timeout (int): session timeout in seconds.
+
+    Attributes:
+        name (str): endpoint name.
+    """
+
+    def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs):
+        super(SampleEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs)
+        self.name = "samples"

tests/py/unit/test_client.py

@@ -24,15 +24,20 @@
         },
         "accounts": [
             {
-                "entity": {"_id": "user-acc-1", "name": "John Doe", "type": "personal"},
+                "entity": {"_id": "user-acc-1", "name": "John Doe", "slug": "john-doe", "type": "personal"},
                 "isDefault": True,
             },
             {
-                "entity": {"_id": "org-acc-1", "name": "Acme Corp", "type": "enterprise"},
+                "entity": {"_id": "org-acc-1", "name": "Acme Corp", "slug": "acme-corp", "type": "enterprise"},
                 "isDefault": True,
             },
             {
-                "entity": {"_id": "org-acc-2", "name": "Beta Industries", "type": "organization"},
+                "entity": {
+                    "_id": "org-acc-2",
+                    "name": "Beta Industries",
+                    "slug": "beta-industries",
+                    "type": "organization",
+                },
                 "isDefault": False,
             },
         ],
@@ -113,10 +118,12 @@ def test_list_accounts(self, mock_get):
             self.assertEqual(len(accounts), 3)
             self.assertEqual(accounts[0]["_id"], "user-acc-1")
             self.assertEqual(accounts[0]["name"], "John Doe")
+            self.assertEqual(accounts[0]["slug"], "john-doe")
             self.assertEqual(accounts[0]["type"], "personal")
             self.assertTrue(accounts[0]["isDefault"])
             self.assertEqual(accounts[1]["_id"], "org-acc-1")
             self.assertEqual(accounts[1]["name"], "Acme Corp")
+            self.assertEqual(accounts[1]["slug"], "acme-corp")
             self.assertEqual(accounts[1]["type"], "enterprise")
 
     @mock.patch("requests.get")

tests/py/unit/test_files.py

@@ -0,0 +1,101 @@
+import json
+import os
+import tempfile
+from unittest import mock
+
+import pytest
+import requests
+
+from mat3ra.api_client.endpoints.files import FileEndpoints
+from tests.py.unit import EndpointBaseUnitTest
+
+FILE_NAME = "loops/site-1.npy"
+STORED_KEY = "user-rvuo7pgiyu/loops/site-1.npy"
+FILE_CONTENT = b"loop data"
+FILE_SHA256 = "61719738f7cfbd0bb8e7fc91cbd2febe3b90732b4f31a994babf549e44537de8"
+SIGNED_URL = "https://test-bucket.s3.amazonaws.com/user-rvuo7pgiyu/loops/site-1.npy?X-Amz-Signature=test"
+OTHER_ACCOUNT_ID = "5dJXaqqhjPZrA5Qyw"
+
+MOCK_CREATED_FILE = {"name": "record.json", "key": "user-rvuo7pgiyu/record.json", "size": 2}
+MOCK_CREATE_RESPONSE = json.dumps({"status": "success", "data": MOCK_CREATED_FILE})
+MOCK_SIGNED_URLS_RESPONSE = json.dumps({"status": "success", "data": [{"key": STORED_KEY, "signedUrl": SIGNED_URL}]})
+MOCK_REFUSED_RESPONSE = "<Error><Code>AccessDenied</Code></Error>"
+
+
+class EndpointFilesUnitTest(EndpointBaseUnitTest):
+    """
+    Class for testing files endpoint.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(EndpointFilesUnitTest, self).__init__(*args, **kwargs)
+        self.base_url = f"https://{self.host}:{self.port}/api/{self.version}/files"
+        self.endpoints = FileEndpoints(self.host, self.port, self.account_id, self.auth_token)
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_create(self, mock_request):
+        mock_request.return_value = self.mock_response(MOCK_CREATE_RESPONSE)
+        self.assertEqual(self.endpoints.create("record.json", "{}"), MOCK_CREATED_FILE)
+        self.assertEqual(mock_request.call_args[1]["url"], self.base_url)
+        self.assertEqual(json.loads(mock_request.call_args[1]["data"]), {"name": "record.json", "body": "{}"})
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_signed_urls(self, mock_request):
+        mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE)
+        self.assertEqual(self.endpoints.signed_urls([FILE_NAME])[0]["signedUrl"], SIGNED_URL)
+        self.assertEqual(mock_request.call_args[1]["url"], f"{self.base_url}/signed-urls")
+        self.assertEqual(
+            json.loads(mock_request.call_args[1]["data"]), {"names": [FILE_NAME], "operation": "getObject"}
+        )
+
+    @mock.patch("requests.put")
+    @mock.patch("requests.sessions.Session.request")
+    def test_put(self, mock_request, mock_put):
+        mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE)
+        with tempfile.TemporaryDirectory() as directory:
+            path = os.path.join(directory, "site-1.npy")
+            with open(path, "wb") as file_:
+                file_.write(FILE_CONTENT)
+            result = self.endpoints.put(path, FILE_NAME)
+        self.assertEqual(result, {"key": STORED_KEY, "bytes": len(FILE_CONTENT), "sha256": FILE_SHA256})
+        self.assertEqual(
+            json.loads(mock_request.call_args[1]["data"]), {"names": [FILE_NAME], "operation": "putObject"}
+        )
+        self.assertEqual(mock_put.call_args[0][0], SIGNED_URL)
+        self.assertEqual(mock_put.call_args[1]["data"], FILE_CONTENT)
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_create_for_account(self, mock_request):
+        mock_request.return_value = self.mock_response(MOCK_CREATE_RESPONSE)
+        self.endpoints.create("record.json", "{}", OTHER_ACCOUNT_ID)
+        self.assertEqual(json.loads(mock_request.call_args[1]["data"])["accountId"], OTHER_ACCOUNT_ID)
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_signed_urls_for_account(self, mock_request):
+        mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE)
+        self.endpoints.signed_urls([FILE_NAME], "getObject", OTHER_ACCOUNT_ID)
+        self.assertEqual(json.loads(mock_request.call_args[1]["data"])["accountId"], OTHER_ACCOUNT_ID)
+
+    @mock.patch("requests.put")
+    @mock.patch("requests.sessions.Session.request")
+    def test_put_for_account(self, mock_request, mock_put):
+        mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE)
+        with tempfile.TemporaryDirectory() as directory:
+            path = os.path.join(directory, "site-1.npy")
+            with open(path, "wb") as file_:
+                file_.write(FILE_CONTENT)
+            result = self.endpoints.put(path, FILE_NAME, OTHER_ACCOUNT_ID)
+        self.assertEqual(result["key"], STORED_KEY)
+        self.assertEqual(json.loads(mock_request.call_args[1]["data"])["accountId"], OTHER_ACCOUNT_ID)
+
+    @mock.patch("requests.put")
+    @mock.patch("requests.sessions.Session.request")
+    def test_put_refused(self, mock_request, mock_put):
+        mock_request.return_value = self.mock_response(MOCK_SIGNED_URLS_RESPONSE)
+        mock_put.return_value = self.mock_response(MOCK_REFUSED_RESPONSE, 403, "Forbidden")
+        with tempfile.TemporaryDirectory() as directory:
+            path = os.path.join(directory, "site-1.npy")
+            with open(path, "wb") as file_:
+                file_.write(FILE_CONTENT)
+            with pytest.raises(requests.HTTPError):
+                self.endpoints.put(path, FILE_NAME)

tests/py/unit/test_measurements.py

@@ -0,0 +1,39 @@
+import json
+from unittest import mock
+
+from mat3ra.api_client.endpoints.measurements import MeasurementEndpoints
+from tests.py.unit.entity import TEST_ENTITY_ID, EntityEndpointsUnitTest
+
+ENDPOINT_NAME = "measurements"
+
+MOCK_FILE = {"key": "user-rvuo7pgiyu/loops/site-1.npy", "signedUrl": "https://test-bucket.s3.amazonaws.com/site-1.npy"}
+MOCK_FILES_RESPONSE = json.dumps({"status": "success", "data": [MOCK_FILE]})
+
+
+class EndpointMeasurementsUnitTest(EntityEndpointsUnitTest):
+    """
+    Class for testing measurements endpoint.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(EndpointMeasurementsUnitTest, self).__init__(*args, **kwargs)
+        self.endpoint_name = ENDPOINT_NAME
+        self.endpoints = MeasurementEndpoints(self.host, self.port, self.account_id, self.auth_token)
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_list(self, mock_request):
+        self.list(mock_request)
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_get(self, mock_request):
+        self.get(mock_request)
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_create(self, mock_request):
+        self.create(mock_request)
+
+    @mock.patch("requests.sessions.Session.request")
+    def test_files(self, mock_request):
+        mock_request.return_value = self.mock_response(MOCK_FILES_RESPONSE)
+        self.assertEqual(self.endpoints.files(TEST_ENTITY_ID), [MOCK_FILE])
+        self.assertEqual(mock_request.call_args[1]["url"], f"{self.base_url}/{TEST_ENTITY_ID}/files")