From a4c73707082e22bcaa85e106c736ea15ae7277f2 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Tue, 6 Oct 2026 18:17:00 +0300 Subject: [PATCH 1/6] feat(SOF-8032): add show_default to the defaultable endpoints `client.materials`, `client.workflows` and `client.projects` can now ask the platform for an account's default entity with `show_default(account_id=None)`, which calls `GET /default` (defaulting to the user's default account). This replaces the flat-param `.request("GET", ..., {"isDefault": ...})` workaround notebooks needed after the list endpoints stopped accepting the `query=` blob. Co-Authored-By: Claude Sonnet 5.5 --- .../api_client/endpoints/mixins/default.py | 13 +++++++++ tests/py/unit/entity.py | 10 +++++++ tests/py/unit/test_materials.py | 4 +++ tests/py/unit/test_projects.py | 29 +++++++++++++++++++ tests/py/unit/test_workflows.py | 4 +++ 5 files changed, 60 insertions(+) create mode 100644 tests/py/unit/test_projects.py diff --git a/src/py/mat3ra/api_client/endpoints/mixins/default.py b/src/py/mat3ra/api_client/endpoints/mixins/default.py index 10989e0..a39d3e6 100644 --- a/src/py/mat3ra/api_client/endpoints/mixins/default.py +++ b/src/py/mat3ra/api_client/endpoints/mixins/default.py @@ -14,3 +14,16 @@ def set_default(self, id_): dict: new entity. """ self.request("POST", "/".join((self.name, id_, "set-default")), headers=self.headers) + + def show_default(self, account_id=None): + """ + Returns the default entity of the given account. + + Args: + account_id (str): account ID. The user's default account is used if not specified. + + Returns: + dict: default entity. + """ + params = {"accountId": account_id} if account_id else None + return self.request("GET", "/".join((self.name, "default")), params=params, headers=self.headers) diff --git a/tests/py/unit/entity.py b/tests/py/unit/entity.py index 56f8390..d0a246d 100644 --- a/tests/py/unit/entity.py +++ b/tests/py/unit/entity.py @@ -38,6 +38,16 @@ def create(self, mock_request): self.endpoints.create({}) self.assertEqual(mock_request.call_args[1]["headers"]["Content-Type"], CONTENT_TYPE_JSON) + def show_default(self, mock_request): + mock_request.return_value = self.mock_response(MOCK_SUCCESS_RESPONSE_OBJECT) + self.assertEqual(self.endpoints.show_default(), {}) + self.assertEqual(mock_request.call_args[1]["method"], HTTP_METHOD_GET) + self.assertEqual(mock_request.call_args[1]["url"], f"{self.base_url}/default") + self.assertIsNone(mock_request.call_args[1]["params"]) + + self.endpoints.show_default(self.account_id) + self.assertEqual(mock_request.call_args[1]["params"], {"accountId": self.account_id}) + def delete(self, mock_request): mock_request.return_value = self.mock_response(MOCK_SUCCESS_RESPONSE_OBJECT) self.assertEqual(self.endpoints.delete(TEST_ENTITY_ID), {}) diff --git a/tests/py/unit/test_materials.py b/tests/py/unit/test_materials.py index 4dcb849..1194bbe 100644 --- a/tests/py/unit/test_materials.py +++ b/tests/py/unit/test_materials.py @@ -31,3 +31,7 @@ def test_create(self, mock_request): @mock.patch("requests.sessions.Session.request") def test_delete(self, mock_request): self.create(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_show_default(self, mock_request): + self.show_default(mock_request) diff --git a/tests/py/unit/test_projects.py b/tests/py/unit/test_projects.py new file mode 100644 index 0000000..a9b6ba9 --- /dev/null +++ b/tests/py/unit/test_projects.py @@ -0,0 +1,29 @@ +from unittest import mock + +from mat3ra.api_client.endpoints.projects import ProjectEndpoints +from tests.py.unit.entity import EntityEndpointsUnitTest + +ENDPOINT_NAME = "projects" + + +class EndpointProjectsUnitTest(EntityEndpointsUnitTest): + """ + Class for testing projects endpoint. + """ + + def __init__(self, *args, **kwargs): + super(EndpointProjectsUnitTest, self).__init__(*args, **kwargs) + self.endpoint_name = ENDPOINT_NAME + self.endpoints = ProjectEndpoints(self.host, self.port, self.account_id, self.auth_token) + + @mock.patch("requests.sessions.Session.request") + def test_list(self, mock_request): + self.list(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_get(self, mock_request): + self.get(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_show_default(self, mock_request): + self.show_default(mock_request) diff --git a/tests/py/unit/test_workflows.py b/tests/py/unit/test_workflows.py index 9586a85..49ba457 100644 --- a/tests/py/unit/test_workflows.py +++ b/tests/py/unit/test_workflows.py @@ -31,3 +31,7 @@ def test_create(self, mock_request): @mock.patch("requests.sessions.Session.request") def test_delete(self, mock_request): self.delete(mock_request) + + @mock.patch("requests.sessions.Session.request") + def test_show_default(self, mock_request): + self.show_default(mock_request) From e3953bc851eb592a69de78abe90f3a0a104cf917 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Tue, 6 Oct 2026 19:42:45 +0300 Subject: [PATCH 2/6] feat(SOF-8032): list() takes the list parameters of the endpoint and translates Mongo-style queries The migrated list endpoints validate their parameters against the flat keys of their list use case and silently drop everything else, including the `query=` blob `list()` sent. `client.materials.list({...})[0]` therefore returned an arbitrary entity instead of the one asked for. - Endpoints declare `list_parameters` (the keys of their list use case, plus `limit`/`skip`/`sort`); `list({"ownerId": id, "formula": "Si"})` passes them on as they are. Booleans are sent as "true"/"false", lists as repeated parameters. - Endpoints declare `query_fields` (Mongo path -> parameter); a Mongo-style query such as `{"owner._id": id, "hash": h}` is translated (value, `$eq`, `$in`, and `$ne` on booleans; `limit`/`skip`/`sort` from the options). The `query` blob is still sent for servers that read it. Charges and metaproperties are unchanged. - A Mongo-style query that does not mention a set means "anywhere", as before (`globalSearch`); a query of list parameters means what the endpoint says. Jobs requested by id are left as they are. - An unknown field or condition raises ValueError naming the supported ones instead of being dropped, and `$in: []` returns [] without a request (an empty parameter would be dropped and match everything). - `get_for_job` and `get_property` filter on `unitId` on the server. Needs the web-app list use cases from SOF-8032: `formula` (materials) and `unitId`/`precisionValue` (properties). Older servers drop those silently. Co-Authored-By: Claude Sonnet 5.5 --- .../api_client/endpoints/bank_materials.py | 12 ++ .../api_client/endpoints/bank_workflows.py | 20 ++ src/py/mat3ra/api_client/endpoints/entity.py | 46 ++++- src/py/mat3ra/api_client/endpoints/jobs.py | 26 +++ .../mat3ra/api_client/endpoints/materials.py | 34 ++++ .../mat3ra/api_client/endpoints/projects.py | 14 ++ .../mat3ra/api_client/endpoints/properties.py | 36 ++++ .../mat3ra/api_client/endpoints/workflows.py | 25 +++ src/py/mat3ra/api_client/utils/query.py | 134 ++++++++++++ tests/py/unit/test_list_query.py | 191 ++++++++++++++++++ tests/py/unit/test_properties.py | 18 ++ 11 files changed, 554 insertions(+), 2 deletions(-) create mode 100644 src/py/mat3ra/api_client/utils/query.py create mode 100644 tests/py/unit/test_list_query.py diff --git a/src/py/mat3ra/api_client/endpoints/bank_materials.py b/src/py/mat3ra/api_client/endpoints/bank_materials.py index fb53de6..8d717eb 100644 --- a/src/py/mat3ra/api_client/endpoints/bank_materials.py +++ b/src/py/mat3ra/api_client/endpoints/bank_materials.py @@ -1,5 +1,6 @@ from .bank_entity import BankEntityEndpoints from .enums import DEFAULT_API_VERSION, SECURE +from ..utils.query import PAGINATION_PARAMETERS class BankMaterialEndpoints(BankEntityEndpoints): @@ -20,6 +21,17 @@ class BankMaterialEndpoints(BankEntityEndpoints): name (str): endpoint name. """ + list_parameters = ( + "id", + "slug", + "name", + "quickSearch", + "isCurated", + "formula", + *PAGINATION_PARAMETERS, + ) + query_fields = {"_id": "id", "name": "name", "formula": "formula", "slug": "slug", "isCurated": "isCurated"} + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(BankMaterialEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "bank-materials" diff --git a/src/py/mat3ra/api_client/endpoints/bank_workflows.py b/src/py/mat3ra/api_client/endpoints/bank_workflows.py index 2267665..b70afdf 100644 --- a/src/py/mat3ra/api_client/endpoints/bank_workflows.py +++ b/src/py/mat3ra/api_client/endpoints/bank_workflows.py @@ -1,5 +1,6 @@ from .bank_entity import BankEntityEndpoints from .enums import DEFAULT_API_VERSION, SECURE +from ..utils.query import PAGINATION_PARAMETERS class BankWorkflowEndpoints(BankEntityEndpoints): @@ -20,6 +21,25 @@ class BankWorkflowEndpoints(BankEntityEndpoints): name (str): endpoint name. """ + list_parameters = ( + "id", + "slug", + "name", + "quickSearch", + "systemName", + "isDefault", + "isCurated", + *PAGINATION_PARAMETERS, + ) + query_fields = { + "_id": "id", + "name": "name", + "systemName": "systemName", + "slug": "slug", + "isDefault": "isDefault", + "isCurated": "isCurated", + } + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(BankWorkflowEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "bank-workflows" diff --git a/src/py/mat3ra/api_client/endpoints/entity.py b/src/py/mat3ra/api_client/endpoints/entity.py index be0ab1e..72ed39a 100644 --- a/src/py/mat3ra/api_client/endpoints/entity.py +++ b/src/py/mat3ra/api_client/endpoints/entity.py @@ -1,5 +1,6 @@ import json +from ..utils.query import translate_projection, translate_query, uses_list_parameters from . import BaseEndpoint from .enums import DEFAULT_API_VERSION, SECURE @@ -21,8 +22,14 @@ class EntityEndpoint(BaseEndpoint): Attributes: name (str): endpoint name. headers (dict): default HTTP headers. + list_parameters (tuple): flat parameters of the list endpoint, accepted by `list()` as they are. + query_fields (dict): Mongo field path -> flat parameter, for endpoints whose list ignores the `query` blob. + None where the endpoint still reads it. """ + list_parameters = () + query_fields = None + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(EntityEndpoint, self).__init__(host, port, version, secure, **kwargs) self.name = None @@ -32,16 +39,51 @@ def list(self, query=None, projection=None): """ Returns a list of entities. + Endpoints that filter on flat parameters take their list parameters directly (e.g. {"ownerId": id}); a + Mongo-style query is translated into them (the `query` blob is still sent, for servers that read it). + Args: - query (dict): Mongo query. Defaults to {}. - projection (dict): Mongo projection. Defaults to {}. + query (dict): list parameters and/or a Mongo query. Defaults to {}. + projection (dict): options: limit, skip, sort. Defaults to {}. Returns: list[dict] + + Raises: + ValueError: if the endpoint cannot filter on a field or condition of the query. """ params = {"query": json.dumps(query or {}), "projection": json.dumps(projection or {})} + if self.query_fields is not None: + filters = self.build_filter_parameters(query or {}, projection or {}) + if [] in filters.values(): + return [] # an empty $in matches nothing, but an empty parameter would be dropped and match everything + params.update(filters) return self.request("GET", self.name, params=params, headers=self.headers) + def build_filter_parameters(self, query, projection): + """ + Translates a query and options into the flat parameters of the list endpoint. + + A Mongo-style query that does not mention a set means "anywhere", as it did before; set-aware endpoints would + default to top-level entities. A query of list parameters means what the endpoint says: pass `globalSearch`. + + Args: + query (dict): list parameters and/or a Mongo query. + projection (dict): options: limit, skip, sort. + + Returns: + dict + """ + parameters = { + **translate_query(query, self.query_fields, self.list_parameters), + **translate_projection(projection), + } + is_mongo_style = not uses_list_parameters(query, self.query_fields, self.list_parameters) + is_set_aware = "setId" in self.list_parameters + if is_mongo_style and is_set_aware and "setId" not in parameters and parameters.get("isEntitySet") != "true": + parameters["globalSearch"] = "true" + return parameters + def get(self, id_): """ Returns a entity with given ID. diff --git a/src/py/mat3ra/api_client/endpoints/jobs.py b/src/py/mat3ra/api_client/endpoints/jobs.py index 0001695..5913a48 100644 --- a/src/py/mat3ra/api_client/endpoints/jobs.py +++ b/src/py/mat3ra/api_client/endpoints/jobs.py @@ -3,6 +3,7 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE from .mixins.set import EntitySetEndpointsMixin +from ..utils.query import OWNED_ENTITY_QUERY_FIELDS, PAGINATION_PARAMETERS class JobEndpoints(EntitySetEndpointsMixin, EntityEndpoint): @@ -23,10 +24,35 @@ class JobEndpoints(EntitySetEndpointsMixin, EntityEndpoint): name (str): endpoint name. """ + list_parameters = ( + "id", "name", "ownerId", "ownerSlug", "teamId", "sharedWithAccountId", "sharedWithAccountSlug", "accountSlug", + "projectId", "status", "setId", "globalSearch", "quickSearch", "isDefault", "materialId", "workflowName", + *PAGINATION_PARAMETERS, + ) + query_fields = { + **OWNED_ENTITY_QUERY_FIELDS, + "inSet._id": "setId", + "status": "status", + "_material._id": "materialId", + "_project._id": "projectId", + "workflow.name": "workflowName", + } + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(JobEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "jobs" + def build_filter_parameters(self, query, projection): + """ + Translates a Mongo-style query and options into the flat parameters of the list endpoint. + + Jobs requested by id are found wherever they are, so the search is not widened (which would drop job sets). + """ + parameters = super(JobEndpoints, self).build_filter_parameters(query, projection) + if "id" in parameters: + parameters.pop("globalSearch", None) + return parameters + def submit(self, id_): """ Submits a given job. diff --git a/src/py/mat3ra/api_client/endpoints/materials.py b/src/py/mat3ra/api_client/endpoints/materials.py index 856e592..45bf634 100644 --- a/src/py/mat3ra/api_client/endpoints/materials.py +++ b/src/py/mat3ra/api_client/endpoints/materials.py @@ -5,6 +5,7 @@ from .mixins.default import DefaultableEntityEndpointsMixin from .mixins.set import EntitySetEndpointsMixin from ..utils.materials import get_materialsproject_url +from ..utils.query import OWNED_ENTITY_QUERY_FIELDS, PAGINATION_PARAMETERS class MaterialEndpoints(EntitySetEndpointsMixin, DefaultableEntityEndpointsMixin, EntityEndpoint): @@ -25,6 +26,39 @@ class MaterialEndpoints(EntitySetEndpointsMixin, DefaultableEntityEndpointsMixin name (str): endpoint name. """ + list_parameters = ( + "id", + "name", + "ownerId", + "ownerSlug", + "teamId", + "sharedWithAccountId", + "sharedWithAccountSlug", + "setId", + "notSetId", + "isDefault", + "formula", + "hashes", + "srcHashes", + "scaledHashes", + "tags", + "metadataElement", + "withProperties", + "isEntitySet", + "globalSearch", + "quickSearch", + *PAGINATION_PARAMETERS, + ) + query_fields = { + **OWNED_ENTITY_QUERY_FIELDS, + "isEntitySet": "isEntitySet", + "inSet._id": "setId", + "formula": "formula", + "hash": "hashes", + "scaledHash": "scaledHashes", + "tags": "tags", + } + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(MaterialEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "materials" diff --git a/src/py/mat3ra/api_client/endpoints/projects.py b/src/py/mat3ra/api_client/endpoints/projects.py index 91adff4..51a40e5 100644 --- a/src/py/mat3ra/api_client/endpoints/projects.py +++ b/src/py/mat3ra/api_client/endpoints/projects.py @@ -1,6 +1,7 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE from .mixins.default import DefaultableEntityEndpointsMixin +from ..utils.query import OWNED_ENTITY_QUERY_FIELDS, PAGINATION_PARAMETERS class ProjectEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): @@ -24,6 +25,19 @@ class ProjectEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): headers (dict): default HTTP headers. """ + list_parameters = ( + "id", + "ownerId", + "ownerSlug", + "teamId", + "isDefault", + "sharedWithAccountId", + "sharedWithAccountSlug", + "quickSearch", + *PAGINATION_PARAMETERS, + ) + query_fields = OWNED_ENTITY_QUERY_FIELDS + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(ProjectEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "projects" diff --git a/src/py/mat3ra/api_client/endpoints/properties.py b/src/py/mat3ra/api_client/endpoints/properties.py index 30b88c1..e0e3ead 100644 --- a/src/py/mat3ra/api_client/endpoints/properties.py +++ b/src/py/mat3ra/api_client/endpoints/properties.py @@ -1,5 +1,6 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE +from ..utils.query import PAGINATION_PARAMETERS class BasePropertiesEndpoints(EntityEndpoint): @@ -42,6 +43,41 @@ class PropertiesEndpoints(BasePropertiesEndpoints): headers (dict): default HTTP headers. """ + list_parameters = ( + "id", + "ownerId", + "ownerSlug", + "sharedWithAccountId", + "sharedWithAccountSlug", + "accountSlug", + "quickSearch", + "name", + "propertyName", + "repetition", + "slug", + "group", + "groupPrefix", + "exabyteId", + "jobId", + "unitId", + "precisionValue", + "systemTags", + *PAGINATION_PARAMETERS, + ) + query_fields = { + "_id": "id", + "owner._id": "ownerId", + "owner.slug": "ownerSlug", + "name": "name", + "slug": "slug", + "exabyteId": "exabyteId", + "group": "group", + "source.info.jobId": "jobId", + "source.info.unitId": "unitId", + "data.name": "propertyName", + "precision.value": "precisionValue", + } + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(PropertiesEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "properties" diff --git a/src/py/mat3ra/api_client/endpoints/workflows.py b/src/py/mat3ra/api_client/endpoints/workflows.py index 5e256e3..655c72e 100644 --- a/src/py/mat3ra/api_client/endpoints/workflows.py +++ b/src/py/mat3ra/api_client/endpoints/workflows.py @@ -1,6 +1,7 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE from .mixins.default import DefaultableEntityEndpointsMixin +from ..utils.query import OWNED_ENTITY_QUERY_FIELDS, PAGINATION_PARAMETERS class WorkflowEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): @@ -24,6 +25,30 @@ class WorkflowEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): headers (dict): default HTTP headers. """ + list_parameters = ( + "id", + "hash", + "name", + "applicationName", + "ownerId", + "ownerSlug", + "teamId", + "sharedWithAccountId", + "sharedWithAccountSlug", + "accountSlug", + "setId", + "globalSearch", + "quickSearch", + "isDefault", + *PAGINATION_PARAMETERS, + ) + query_fields = { + **OWNED_ENTITY_QUERY_FIELDS, + "inSet._id": "setId", + "hash": "hash", + "application.name": "applicationName", + } + def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(WorkflowEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "workflows" diff --git a/src/py/mat3ra/api_client/utils/query.py b/src/py/mat3ra/api_client/utils/query.py new file mode 100644 index 0000000..2ad95c6 --- /dev/null +++ b/src/py/mat3ra/api_client/utils/query.py @@ -0,0 +1,134 @@ +""" +Translation of Mongo-style list queries into the flat filters of the platform's list endpoints. + +The list endpoints validate their parameters against a declared set of flat keys (e.g. `ownerId`, +`hashes`) and silently drop anything else - including the `query=` blob `list()` sends. An +endpoint opts in by declaring: + list_parameters: the flat parameters of its list use case, which `list()` accepts directly; + query_fields: the Mongo field paths it can still translate, each mapped to a flat parameter. +""" + +OWNED_ENTITY_QUERY_FIELDS = { + "_id": "id", + "owner._id": "ownerId", + "owner.slug": "ownerSlug", + "name": "name", + "isDefault": "isDefault", +} + +PAGINATION_PARAMETERS = ("limit", "skip", "sort") +BOOLEAN_PARAMETERS = ("isDefault", "isEntitySet", "isCurated") +PROJECTION_OPTIONS = ("limit", "skip", "sort", "fields") + + +def _unsupported_condition(field): + return ValueError( + f'Unsupported condition on query field "{field}". Supported: a value, $eq, $in and, for boolean fields, $ne.' + ) + + +def _translate_condition(field, parameter, condition): + if not isinstance(condition, dict): + return condition + if len(condition) != 1: + raise _unsupported_condition(field) + + ((operator, operand),) = condition.items() + if operator == "$eq": + return operand + if operator == "$in" and isinstance(operand, (list, tuple)): + return list(operand) + if operator == "$ne" and isinstance(operand, bool) and parameter in BOOLEAN_PARAMETERS: + return not operand + raise _unsupported_condition(field) + + +def _to_parameter_value(value): + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, list): + return [_to_parameter_value(item) for item in value] + return value + + +def _translate_direct_value(parameter, value): + if isinstance(value, dict): + if parameter == "sort": + return _translate_sort(value) + raise ValueError(f'"{parameter}" is a list parameter: pass a value or a list, not a condition.') + return _to_parameter_value(value) + + +def uses_list_parameters(query, query_fields, list_parameters): + """ + Whether a query uses the list parameters of the endpoint (e.g. `ownerId`) rather than Mongo field paths + (e.g. `owner._id`). Such a query means exactly what the list endpoint says. + """ + return any(field in list_parameters and field not in query_fields for field in query) + + +def translate_query(query, query_fields, list_parameters=()): + """ + Translates a query into flat list parameters. + + Args: + query (dict): list parameters (passed on as they are), and/or Mongo field paths whose value may be a + value, `$eq`, `$in` or (booleans) `$ne`. + query_fields (dict): Mongo field path -> flat parameter name the endpoint accepts. + list_parameters (tuple): flat parameters the endpoint accepts as they are. + + Returns: + dict: flat parameters. + + Raises: + ValueError: for a field or condition the endpoint cannot filter on. + """ + parameters = {} + for field, condition in query.items(): + if field in query_fields: + parameter = query_fields[field] + parameters[parameter] = _to_parameter_value(_translate_condition(field, parameter, condition)) + elif field in list_parameters: + parameters[field] = _translate_direct_value(field, condition) + else: + supported = ", ".join((*list_parameters, *query_fields)) + raise ValueError(f'Unsupported query field "{field}". Supported: {supported}.') + return parameters + + +def _translate_sort(sort): + if isinstance(sort, str): + return sort + if not isinstance(sort, dict) or len(sort) != 1: + raise ValueError("Unsupported sort: a single field, as a string or {field: 1 | -1}, is supported.") + + ((field, direction),) = sort.items() + return f"-{field}" if direction in (-1, "desc") else field + + +def translate_projection(projection): + """ + Translates the Mongo-style options of a list call into flat list parameters. + + `limit`, `skip` and a single-field `sort` are honored. `fields` is ignored: the endpoints return whole + documents, which can only be more than was asked for. + + Args: + projection (dict): options, e.g. {"limit": 1, "sort": {"precision.value": -1}}. + + Returns: + dict: flat parameters. + + Raises: + ValueError: for an unsupported option or sort. + """ + unsupported = [option for option in projection if option not in PROJECTION_OPTIONS] + if unsupported: + raise ValueError( + f'Unsupported projection option "{unsupported[0]}". Supported: {", ".join(PROJECTION_OPTIONS)}.' + ) + + parameters = {option: projection[option] for option in ("limit", "skip") if option in projection} + if "sort" in projection: + parameters["sort"] = _translate_sort(projection["sort"]) + return parameters diff --git a/tests/py/unit/test_list_query.py b/tests/py/unit/test_list_query.py new file mode 100644 index 0000000..814320a --- /dev/null +++ b/tests/py/unit/test_list_query.py @@ -0,0 +1,191 @@ +import json +from unittest import mock + +import pytest +from mat3ra.api_client.endpoints.bank_workflows import BankWorkflowEndpoints +from mat3ra.api_client.endpoints.charges import ChargeEndpoints +from mat3ra.api_client.endpoints.jobs import JobEndpoints +from mat3ra.api_client.endpoints.materials import MaterialEndpoints +from mat3ra.api_client.endpoints.projects import ProjectEndpoints +from mat3ra.api_client.endpoints.properties import PropertiesEndpoints +from mat3ra.api_client.endpoints.workflows import WorkflowEndpoints +from mat3ra.api_client.utils.query import OWNED_ENTITY_QUERY_FIELDS, translate_projection, translate_query +from tests.py.unit import EndpointBaseUnitTest +from tests.py.unit.entity import MOCK_SUCCESS_RESPONSE_LIST + +OWNER_ID = "ubxMkAyx37Rjn8qK9" +SET_ID = "setIdValue" + + +def test_translates_values_and_in_conditions(): + query = {"owner._id": OWNER_ID, "_id": {"$in": ["a", "b"]}, "name": {"$eq": "Si"}} + + assert translate_query(query, OWNED_ENTITY_QUERY_FIELDS) == {"ownerId": OWNER_ID, "id": ["a", "b"], "name": "Si"} + + +def test_list_parameters_are_passed_on_as_they_are(): + query = {"ownerId": OWNER_ID, "isEntitySet": False, "id": ["a", "b"], "limit": 5, "sort": {"name": -1}} + parameters = ("ownerId", "isEntitySet", "id", "limit", "sort") + + assert translate_query(query, {}, parameters) == { + "ownerId": OWNER_ID, + "isEntitySet": "false", + "id": ["a", "b"], + "limit": 5, + "sort": "-name", + } + + +def test_a_condition_on_a_list_parameter_is_rejected(): + with pytest.raises(ValueError, match="list parameter"): + translate_query({"ownerId": {"$in": ["a"]}}, {}, ("ownerId",)) + + +def test_booleans_are_sent_as_lowercase_strings_and_ne_negates_them(): + query = {"isDefault": True, "isEntitySet": {"$ne": True}} + fields = {**OWNED_ENTITY_QUERY_FIELDS, "isEntitySet": "isEntitySet"} + + assert translate_query(query, fields) == {"isDefault": "true", "isEntitySet": "false"} + + +@pytest.mark.parametrize( + "query", + [ + {"formula": "Si"}, + {"$or": [{"name": "Si"}]}, + {"name": {"$regex": "Si", "$options": "i"}}, + {"name": {"$ne": "Si"}}, + {"name": {"$in": ["a"], "$ne": "b"}}, + ], +) +def test_rejects_what_the_endpoint_cannot_filter_on(query): + with pytest.raises(ValueError): + translate_query(query, OWNED_ENTITY_QUERY_FIELDS) + + +def test_projection_limit_skip_and_sort(): + projection = {"limit": 1, "skip": 2, "sort": {"precision.value": -1}, "fields": {"status": 1}} + + assert translate_projection(projection) == {"limit": 1, "skip": 2, "sort": "-precision.value"} + assert translate_projection({"sort": {"name": 1}}) == {"sort": "name"} + + +@pytest.mark.parametrize("projection", [{"hint": "x"}, {"sort": {"a": 1, "b": 1}}]) +def test_rejects_unsupported_projection(projection): + with pytest.raises(ValueError): + translate_projection(projection) + + +class EndpointListQueryUnitTest(EndpointBaseUnitTest): + """ + Class for testing the flat filters `list()` adds for endpoints that ignore the `query` blob. + """ + + def sent_params(self, endpoint_class, query=None, projection=None): + endpoint = endpoint_class(self.host, self.port, self.account_id, self.auth_token) + with mock.patch("requests.sessions.Session.request") as mock_request: + mock_request.return_value = self.mock_response(MOCK_SUCCESS_RESPONSE_LIST) + endpoint.list(query, projection) + return mock_request.call_args[1]["params"] + + def test_materials_send_flat_filters_next_to_the_blob(self): + query = {"hash": "abc", "owner._id": OWNER_ID} + params = self.sent_params(MaterialEndpoints, query, {"limit": 1}) + + self.assertEqual(json.loads(params["query"]), query) + self.assertEqual(json.loads(params["projection"]), {"limit": 1}) + self.assertEqual((params["hashes"], params["ownerId"], params["limit"]), ("abc", OWNER_ID, 1)) + + def test_materials_without_a_set_are_searched_everywhere(self): + self.assertEqual(self.sent_params(MaterialEndpoints, {"owner._id": OWNER_ID})["globalSearch"], "true") + self.assertEqual(self.sent_params(MaterialEndpoints)["globalSearch"], "true") + + def test_materials_filter_on_formula_and_properties_on_unit_and_precision(self): + direct = self.sent_params(MaterialEndpoints, {"ownerId": OWNER_ID, "formula": ["Si", "Ge"]}) + mongo_style = self.sent_params(MaterialEndpoints, {"owner._id": OWNER_ID, "formula": "Si"}) + properties = self.sent_params( + PropertiesEndpoints, {"source.info.unitId": "pw-nscf", "precision.value": 10, "jobId": "j"} + ) + + self.assertEqual(direct["formula"], ["Si", "Ge"]) + self.assertEqual(mongo_style["formula"], "Si") + self.assertEqual((properties["unitId"], properties["precisionValue"]), ("pw-nscf", 10)) + + def test_workflows_and_jobs_without_a_set_are_searched_everywhere(self): + self.assertEqual(self.sent_params(WorkflowEndpoints, {"hash": "h"})["globalSearch"], "true") + self.assertEqual(self.sent_params(JobEndpoints, {"status": "finished"})["globalSearch"], "true") + self.assertNotIn("globalSearch", self.sent_params(JobEndpoints, {"inSet._id": SET_ID})) + self.assertNotIn("globalSearch", self.sent_params(ProjectEndpoints, {"isDefault": True})) + + def test_jobs_requested_by_id_are_not_restricted_to_non_sets(self): + self.assertNotIn("globalSearch", self.sent_params(JobEndpoints, {"_id": {"$in": ["j1"]}})) + + def test_an_empty_in_matches_nothing_and_sends_no_request(self): + for endpoint_class in (MaterialEndpoints, JobEndpoints): + for query in ({"_id": {"$in": []}}, {"id": []}): + endpoint = endpoint_class(self.host, self.port, self.account_id, self.auth_token) + with mock.patch("requests.sessions.Session.request") as mock_request: + self.assertEqual(endpoint.list(query), []) + mock_request.assert_not_called() + + def test_materials_take_list_parameters_directly(self): + params = self.sent_params(MaterialEndpoints, {"name": "Si", "ownerId": OWNER_ID, "hashes": ["h1", "h2"]}) + + self.assertEqual((params["name"], params["ownerId"], params["hashes"]), ("Si", OWNER_ID, ["h1", "h2"])) + + def test_list_parameters_mean_what_the_endpoint_says(self): + top_level = self.sent_params(MaterialEndpoints, {"name": "Si", "ownerId": OWNER_ID}) + anywhere = self.sent_params(MaterialEndpoints, {"name": "Si", "ownerId": OWNER_ID, "globalSearch": True}) + mixed = self.sent_params(MaterialEndpoints, {"owner._id": OWNER_ID, "hashes": "h"}) + + self.assertNotIn("globalSearch", top_level) + self.assertEqual(anywhere["globalSearch"], "true") + self.assertNotIn("globalSearch", mixed) + + def test_the_unsupported_field_message_names_the_supported_ones(self): + endpoint = MaterialEndpoints(self.host, self.port, self.account_id, self.auth_token) + with self.assertRaisesRegex(ValueError, "ownerId.*owner._id"): + endpoint.list({"lattice.type": "FCC"}) + + def test_materials_in_a_set_or_sets_keep_the_set_semantics(self): + in_set = self.sent_params(MaterialEndpoints, {"inSet._id": SET_ID, "isEntitySet": {"$ne": True}}) + sets = self.sent_params(MaterialEndpoints, {"owner._id": OWNER_ID, "isEntitySet": True}) + + self.assertEqual((in_set["setId"], in_set["isEntitySet"]), (SET_ID, "false")) + self.assertNotIn("globalSearch", in_set) + self.assertEqual(sets["isEntitySet"], "true") + self.assertNotIn("globalSearch", sets) + + def test_workflows_projects_jobs_properties_and_bank(self): + self.assertEqual(self.sent_params(WorkflowEndpoints, {"hash": "h"})["hash"], "h") + self.assertEqual(self.sent_params(ProjectEndpoints, {"isDefault": True})["isDefault"], "true") + self.assertEqual(self.sent_params(BankWorkflowEndpoints, {"systemName": "s"})["systemName"], "s") + jobs = self.sent_params(JobEndpoints, {"_material._id": {"$in": ["m1", "m2"]}, "status": "finished"}) + self.assertEqual((jobs["materialId"], jobs["status"]), (["m1", "m2"], "finished")) + properties = self.sent_params(PropertiesEndpoints, {"source.info.jobId": "j", "data.name": "total_energy"}) + self.assertEqual((properties["jobId"], properties["propertyName"]), ("j", "total_energy")) + unit_template_query = { + "exabyteId": {"$in": ["e1"]}, + "slug": "total_energy", + "group": "qe:dft", + "owner.slug": {"$in": ["me", "curators"]}, + } + template = self.sent_params( + PropertiesEndpoints, unit_template_query, {"sort": {"precision.value": -1}, "limit": 1} + ) + self.assertEqual( + (template["exabyteId"], template["group"], template["ownerSlug"], template["sort"], template["limit"]), + (["e1"], "qe:dft", ["me", "curators"], "-precision.value", 1), + ) + + def test_an_unsupported_field_raises_before_any_request(self): + endpoint = MaterialEndpoints(self.host, self.port, self.account_id, self.auth_token) + with mock.patch("requests.sessions.Session.request") as mock_request: + with self.assertRaises(ValueError): + endpoint.list({"lattice.type": "FCC"}) + mock_request.assert_not_called() + + def test_endpoints_that_still_read_the_blob_are_unchanged(self): + params = self.sent_params(ChargeEndpoints, {"jid": "1"}) + + self.assertEqual(set(params), {"query", "projection"}) diff --git a/tests/py/unit/test_properties.py b/tests/py/unit/test_properties.py index 62f6ccb..2426383 100644 --- a/tests/py/unit/test_properties.py +++ b/tests/py/unit/test_properties.py @@ -84,3 +84,21 @@ def test_get_for_job_filtered_by_unit_id_and_name(self, mock_request): sent_query = json.loads(mock_request.call_args[1]["params"]["query"]) self.assertEqual(sent_query["source.info.unitId"], UNIT_ID_1) self.assertEqual(sent_query["data.name"], PROPERTY_NAME_1) + + @mock.patch("requests.sessions.Session.request") + def test_get_for_job_sends_flat_filters(self, mock_request): + mock_request.return_value = self.mock_response(MOCK_SINGLE_PROPERTY_RESPONSE) + self.endpoints.get_for_job(JOB_ID, PROPERTY_NAME_1, unit_id=UNIT_ID_1) + params = mock_request.call_args[1]["params"] + self.assertEqual( + (params["jobId"], params["propertyName"], params["unitId"]), (JOB_ID, PROPERTY_NAME_1, UNIT_ID_1) + ) + + @mock.patch("requests.sessions.Session.request") + def test_get_property_sends_flat_filters(self, mock_request): + mock_request.return_value = self.mock_response(MOCK_PROPERTIES_RESPONSE) + self.endpoints.get_property(JOB_ID, UNIT_ID_1, PROPERTY_NAME_1) + params = mock_request.call_args[1]["params"] + self.assertEqual( + (params["jobId"], params["propertyName"], params["unitId"]), (JOB_ID, PROPERTY_NAME_1, UNIT_ID_1) + ) From 591753ae56d60cb95e3c682bcfa10a0b4b0a1964 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Wed, 7 Oct 2026 01:25:50 +0300 Subject: [PATCH 3/6] feat(SOF-8032): send list() queries as advancedSearches The migrated list endpoints ignore the query= blob. list() keeps sending it for older servers and now also sends the query as advancedSearches, plus limit, skip and sort from the projection, for materials, workflows, jobs, projects, properties and the bank endpoints. Materials, workflows and jobs also get setId when the query names inSet._id, and globalSearch otherwise. An unsupported projection option raises ValueError before any request. Charges and metaproperties are unchanged. Replaces the show_default and list-parameter translation commits. Co-Authored-By: Claude Sonnet 5.5 --- .../api_client/endpoints/bank_materials.py | 12 +- .../api_client/endpoints/bank_workflows.py | 20 +- src/py/mat3ra/api_client/endpoints/entity.py | 48 ++--- src/py/mat3ra/api_client/endpoints/jobs.py | 27 +-- .../mat3ra/api_client/endpoints/materials.py | 35 +-- .../api_client/endpoints/mixins/default.py | 13 -- .../mat3ra/api_client/endpoints/projects.py | 14 +- .../mat3ra/api_client/endpoints/properties.py | 36 +--- .../mat3ra/api_client/endpoints/workflows.py | 26 +-- src/py/mat3ra/api_client/utils/query.py | 118 +++------- tests/py/unit/entity.py | 10 - tests/py/unit/test_list_query.py | 202 ++++++------------ tests/py/unit/test_materials.py | 4 - tests/py/unit/test_projects.py | 29 --- tests/py/unit/test_properties.py | 18 -- tests/py/unit/test_workflows.py | 4 - 16 files changed, 122 insertions(+), 494 deletions(-) delete mode 100644 tests/py/unit/test_projects.py diff --git a/src/py/mat3ra/api_client/endpoints/bank_materials.py b/src/py/mat3ra/api_client/endpoints/bank_materials.py index 8d717eb..7eaba05 100644 --- a/src/py/mat3ra/api_client/endpoints/bank_materials.py +++ b/src/py/mat3ra/api_client/endpoints/bank_materials.py @@ -1,6 +1,5 @@ from .bank_entity import BankEntityEndpoints from .enums import DEFAULT_API_VERSION, SECURE -from ..utils.query import PAGINATION_PARAMETERS class BankMaterialEndpoints(BankEntityEndpoints): @@ -21,16 +20,7 @@ class BankMaterialEndpoints(BankEntityEndpoints): name (str): endpoint name. """ - list_parameters = ( - "id", - "slug", - "name", - "quickSearch", - "isCurated", - "formula", - *PAGINATION_PARAMETERS, - ) - query_fields = {"_id": "id", "name": "name", "formula": "formula", "slug": "slug", "isCurated": "isCurated"} + advanced_searches = True def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(BankMaterialEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) diff --git a/src/py/mat3ra/api_client/endpoints/bank_workflows.py b/src/py/mat3ra/api_client/endpoints/bank_workflows.py index b70afdf..4ea0ebb 100644 --- a/src/py/mat3ra/api_client/endpoints/bank_workflows.py +++ b/src/py/mat3ra/api_client/endpoints/bank_workflows.py @@ -1,6 +1,5 @@ from .bank_entity import BankEntityEndpoints from .enums import DEFAULT_API_VERSION, SECURE -from ..utils.query import PAGINATION_PARAMETERS class BankWorkflowEndpoints(BankEntityEndpoints): @@ -21,24 +20,7 @@ class BankWorkflowEndpoints(BankEntityEndpoints): name (str): endpoint name. """ - list_parameters = ( - "id", - "slug", - "name", - "quickSearch", - "systemName", - "isDefault", - "isCurated", - *PAGINATION_PARAMETERS, - ) - query_fields = { - "_id": "id", - "name": "name", - "systemName": "systemName", - "slug": "slug", - "isDefault": "isDefault", - "isCurated": "isCurated", - } + advanced_searches = True def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(BankWorkflowEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) diff --git a/src/py/mat3ra/api_client/endpoints/entity.py b/src/py/mat3ra/api_client/endpoints/entity.py index 72ed39a..9f7e0c8 100644 --- a/src/py/mat3ra/api_client/endpoints/entity.py +++ b/src/py/mat3ra/api_client/endpoints/entity.py @@ -1,6 +1,6 @@ import json -from ..utils.query import translate_projection, translate_query, uses_list_parameters +from ..utils.query import set_parameters, translate_projection from . import BaseEndpoint from .enums import DEFAULT_API_VERSION, SECURE @@ -22,13 +22,12 @@ class EntityEndpoint(BaseEndpoint): Attributes: name (str): endpoint name. headers (dict): default HTTP headers. - list_parameters (tuple): flat parameters of the list endpoint, accepted by `list()` as they are. - query_fields (dict): Mongo field path -> flat parameter, for endpoints whose list ignores the `query` blob. - None where the endpoint still reads it. + advanced_searches (bool): whether the list endpoint filters on `advancedSearches` instead of the `query` blob. + searches_sets (bool): whether the entities can be in sets (list endpoints default to the top level). """ - list_parameters = () - query_fields = None + advanced_searches = False + searches_sets = False def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(EntityEndpoint, self).__init__(host, port, version, secure, **kwargs) @@ -39,49 +38,40 @@ def list(self, query=None, projection=None): """ Returns a list of entities. - Endpoints that filter on flat parameters take their list parameters directly (e.g. {"ownerId": id}); a - Mongo-style query is translated into them (the `query` blob is still sent, for servers that read it). + Endpoints that filter on `advancedSearches` get the query as such (the `query` blob is still sent, for servers + that read it). A query is answered "anywhere" unless it names a set, as it was before. Args: - query (dict): list parameters and/or a Mongo query. Defaults to {}. + query (dict): Mongo query. Defaults to {}. projection (dict): options: limit, skip, sort. Defaults to {}. Returns: list[dict] Raises: - ValueError: if the endpoint cannot filter on a field or condition of the query. + ValueError: for an option or sort the endpoint does not support. """ params = {"query": json.dumps(query or {}), "projection": json.dumps(projection or {})} - if self.query_fields is not None: - filters = self.build_filter_parameters(query or {}, projection or {}) - if [] in filters.values(): - return [] # an empty $in matches nothing, but an empty parameter would be dropped and match everything - params.update(filters) + if self.advanced_searches: + params.update(self.build_advanced_search_parameters(query or {}, projection or {})) return self.request("GET", self.name, params=params, headers=self.headers) - def build_filter_parameters(self, query, projection): + def build_advanced_search_parameters(self, query, projection): """ - Translates a query and options into the flat parameters of the list endpoint. - - A Mongo-style query that does not mention a set means "anywhere", as it did before; set-aware endpoints would - default to top-level entities. A query of list parameters means what the endpoint says: pass `globalSearch`. + Builds the parameters of an `advancedSearches` list. Args: - query (dict): list parameters and/or a Mongo query. + query (dict): Mongo query. projection (dict): options: limit, skip, sort. Returns: dict """ - parameters = { - **translate_query(query, self.query_fields, self.list_parameters), - **translate_projection(projection), - } - is_mongo_style = not uses_list_parameters(query, self.query_fields, self.list_parameters) - is_set_aware = "setId" in self.list_parameters - if is_mongo_style and is_set_aware and "setId" not in parameters and parameters.get("isEntitySet") != "true": - parameters["globalSearch"] = "true" + parameters = translate_projection(projection) + if query: + parameters["advancedSearches"] = json.dumps([query]) + if self.searches_sets: + parameters.update(set_parameters(query)) return parameters def get(self, id_): diff --git a/src/py/mat3ra/api_client/endpoints/jobs.py b/src/py/mat3ra/api_client/endpoints/jobs.py index 5913a48..5844a72 100644 --- a/src/py/mat3ra/api_client/endpoints/jobs.py +++ b/src/py/mat3ra/api_client/endpoints/jobs.py @@ -3,7 +3,6 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE from .mixins.set import EntitySetEndpointsMixin -from ..utils.query import OWNED_ENTITY_QUERY_FIELDS, PAGINATION_PARAMETERS class JobEndpoints(EntitySetEndpointsMixin, EntityEndpoint): @@ -24,35 +23,13 @@ class JobEndpoints(EntitySetEndpointsMixin, EntityEndpoint): name (str): endpoint name. """ - list_parameters = ( - "id", "name", "ownerId", "ownerSlug", "teamId", "sharedWithAccountId", "sharedWithAccountSlug", "accountSlug", - "projectId", "status", "setId", "globalSearch", "quickSearch", "isDefault", "materialId", "workflowName", - *PAGINATION_PARAMETERS, - ) - query_fields = { - **OWNED_ENTITY_QUERY_FIELDS, - "inSet._id": "setId", - "status": "status", - "_material._id": "materialId", - "_project._id": "projectId", - "workflow.name": "workflowName", - } + advanced_searches = True + searches_sets = True def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(JobEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "jobs" - def build_filter_parameters(self, query, projection): - """ - Translates a Mongo-style query and options into the flat parameters of the list endpoint. - - Jobs requested by id are found wherever they are, so the search is not widened (which would drop job sets). - """ - parameters = super(JobEndpoints, self).build_filter_parameters(query, projection) - if "id" in parameters: - parameters.pop("globalSearch", None) - return parameters - def submit(self, id_): """ Submits a given job. diff --git a/src/py/mat3ra/api_client/endpoints/materials.py b/src/py/mat3ra/api_client/endpoints/materials.py index 45bf634..9242d79 100644 --- a/src/py/mat3ra/api_client/endpoints/materials.py +++ b/src/py/mat3ra/api_client/endpoints/materials.py @@ -5,7 +5,6 @@ from .mixins.default import DefaultableEntityEndpointsMixin from .mixins.set import EntitySetEndpointsMixin from ..utils.materials import get_materialsproject_url -from ..utils.query import OWNED_ENTITY_QUERY_FIELDS, PAGINATION_PARAMETERS class MaterialEndpoints(EntitySetEndpointsMixin, DefaultableEntityEndpointsMixin, EntityEndpoint): @@ -26,38 +25,8 @@ class MaterialEndpoints(EntitySetEndpointsMixin, DefaultableEntityEndpointsMixin name (str): endpoint name. """ - list_parameters = ( - "id", - "name", - "ownerId", - "ownerSlug", - "teamId", - "sharedWithAccountId", - "sharedWithAccountSlug", - "setId", - "notSetId", - "isDefault", - "formula", - "hashes", - "srcHashes", - "scaledHashes", - "tags", - "metadataElement", - "withProperties", - "isEntitySet", - "globalSearch", - "quickSearch", - *PAGINATION_PARAMETERS, - ) - query_fields = { - **OWNED_ENTITY_QUERY_FIELDS, - "isEntitySet": "isEntitySet", - "inSet._id": "setId", - "formula": "formula", - "hash": "hashes", - "scaledHash": "scaledHashes", - "tags": "tags", - } + advanced_searches = True + searches_sets = True def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(MaterialEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) diff --git a/src/py/mat3ra/api_client/endpoints/mixins/default.py b/src/py/mat3ra/api_client/endpoints/mixins/default.py index a39d3e6..10989e0 100644 --- a/src/py/mat3ra/api_client/endpoints/mixins/default.py +++ b/src/py/mat3ra/api_client/endpoints/mixins/default.py @@ -14,16 +14,3 @@ def set_default(self, id_): dict: new entity. """ self.request("POST", "/".join((self.name, id_, "set-default")), headers=self.headers) - - def show_default(self, account_id=None): - """ - Returns the default entity of the given account. - - Args: - account_id (str): account ID. The user's default account is used if not specified. - - Returns: - dict: default entity. - """ - params = {"accountId": account_id} if account_id else None - return self.request("GET", "/".join((self.name, "default")), params=params, headers=self.headers) diff --git a/src/py/mat3ra/api_client/endpoints/projects.py b/src/py/mat3ra/api_client/endpoints/projects.py index 51a40e5..77a7a28 100644 --- a/src/py/mat3ra/api_client/endpoints/projects.py +++ b/src/py/mat3ra/api_client/endpoints/projects.py @@ -1,7 +1,6 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE from .mixins.default import DefaultableEntityEndpointsMixin -from ..utils.query import OWNED_ENTITY_QUERY_FIELDS, PAGINATION_PARAMETERS class ProjectEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): @@ -25,18 +24,7 @@ class ProjectEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): headers (dict): default HTTP headers. """ - list_parameters = ( - "id", - "ownerId", - "ownerSlug", - "teamId", - "isDefault", - "sharedWithAccountId", - "sharedWithAccountSlug", - "quickSearch", - *PAGINATION_PARAMETERS, - ) - query_fields = OWNED_ENTITY_QUERY_FIELDS + advanced_searches = True def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(ProjectEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) diff --git a/src/py/mat3ra/api_client/endpoints/properties.py b/src/py/mat3ra/api_client/endpoints/properties.py index e0e3ead..4b045eb 100644 --- a/src/py/mat3ra/api_client/endpoints/properties.py +++ b/src/py/mat3ra/api_client/endpoints/properties.py @@ -1,6 +1,5 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE -from ..utils.query import PAGINATION_PARAMETERS class BasePropertiesEndpoints(EntityEndpoint): @@ -43,40 +42,7 @@ class PropertiesEndpoints(BasePropertiesEndpoints): headers (dict): default HTTP headers. """ - list_parameters = ( - "id", - "ownerId", - "ownerSlug", - "sharedWithAccountId", - "sharedWithAccountSlug", - "accountSlug", - "quickSearch", - "name", - "propertyName", - "repetition", - "slug", - "group", - "groupPrefix", - "exabyteId", - "jobId", - "unitId", - "precisionValue", - "systemTags", - *PAGINATION_PARAMETERS, - ) - query_fields = { - "_id": "id", - "owner._id": "ownerId", - "owner.slug": "ownerSlug", - "name": "name", - "slug": "slug", - "exabyteId": "exabyteId", - "group": "group", - "source.info.jobId": "jobId", - "source.info.unitId": "unitId", - "data.name": "propertyName", - "precision.value": "precisionValue", - } + advanced_searches = True def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(PropertiesEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) diff --git a/src/py/mat3ra/api_client/endpoints/workflows.py b/src/py/mat3ra/api_client/endpoints/workflows.py index 655c72e..b24bc11 100644 --- a/src/py/mat3ra/api_client/endpoints/workflows.py +++ b/src/py/mat3ra/api_client/endpoints/workflows.py @@ -1,7 +1,6 @@ from .entity import EntityEndpoint from .enums import DEFAULT_API_VERSION, SECURE from .mixins.default import DefaultableEntityEndpointsMixin -from ..utils.query import OWNED_ENTITY_QUERY_FIELDS, PAGINATION_PARAMETERS class WorkflowEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): @@ -25,29 +24,8 @@ class WorkflowEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): headers (dict): default HTTP headers. """ - list_parameters = ( - "id", - "hash", - "name", - "applicationName", - "ownerId", - "ownerSlug", - "teamId", - "sharedWithAccountId", - "sharedWithAccountSlug", - "accountSlug", - "setId", - "globalSearch", - "quickSearch", - "isDefault", - *PAGINATION_PARAMETERS, - ) - query_fields = { - **OWNED_ENTITY_QUERY_FIELDS, - "inSet._id": "setId", - "hash": "hash", - "application.name": "applicationName", - } + advanced_searches = True + searches_sets = True def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(WorkflowEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) diff --git a/src/py/mat3ra/api_client/utils/query.py b/src/py/mat3ra/api_client/utils/query.py index 2ad95c6..dfcc408 100644 --- a/src/py/mat3ra/api_client/utils/query.py +++ b/src/py/mat3ra/api_client/utils/query.py @@ -1,101 +1,14 @@ """ -Translation of Mongo-style list queries into the flat filters of the platform's list endpoints. +Helpers for the list endpoints that filter on `advancedSearches`. -The list endpoints validate their parameters against a declared set of flat keys (e.g. `ownerId`, -`hashes`) and silently drop anything else - including the `query=` blob `list()` sends. An -endpoint opts in by declaring: - list_parameters: the flat parameters of its list use case, which `list()` accepts directly; - query_fields: the Mongo field paths it can still translate, each mapped to a flat parameter. +A migrated list endpoint validates its parameters against the flat keys of its use case and silently drops +anything else - including the `query=` blob `list()` sends. It does accept `advancedSearches`: raw Mongo +selectors, sent as a JSON string, applied next to the access scope. `list()` carries the whole query that way. """ -OWNED_ENTITY_QUERY_FIELDS = { - "_id": "id", - "owner._id": "ownerId", - "owner.slug": "ownerSlug", - "name": "name", - "isDefault": "isDefault", -} - -PAGINATION_PARAMETERS = ("limit", "skip", "sort") -BOOLEAN_PARAMETERS = ("isDefault", "isEntitySet", "isCurated") PROJECTION_OPTIONS = ("limit", "skip", "sort", "fields") -def _unsupported_condition(field): - return ValueError( - f'Unsupported condition on query field "{field}". Supported: a value, $eq, $in and, for boolean fields, $ne.' - ) - - -def _translate_condition(field, parameter, condition): - if not isinstance(condition, dict): - return condition - if len(condition) != 1: - raise _unsupported_condition(field) - - ((operator, operand),) = condition.items() - if operator == "$eq": - return operand - if operator == "$in" and isinstance(operand, (list, tuple)): - return list(operand) - if operator == "$ne" and isinstance(operand, bool) and parameter in BOOLEAN_PARAMETERS: - return not operand - raise _unsupported_condition(field) - - -def _to_parameter_value(value): - if isinstance(value, bool): - return "true" if value else "false" - if isinstance(value, list): - return [_to_parameter_value(item) for item in value] - return value - - -def _translate_direct_value(parameter, value): - if isinstance(value, dict): - if parameter == "sort": - return _translate_sort(value) - raise ValueError(f'"{parameter}" is a list parameter: pass a value or a list, not a condition.') - return _to_parameter_value(value) - - -def uses_list_parameters(query, query_fields, list_parameters): - """ - Whether a query uses the list parameters of the endpoint (e.g. `ownerId`) rather than Mongo field paths - (e.g. `owner._id`). Such a query means exactly what the list endpoint says. - """ - return any(field in list_parameters and field not in query_fields for field in query) - - -def translate_query(query, query_fields, list_parameters=()): - """ - Translates a query into flat list parameters. - - Args: - query (dict): list parameters (passed on as they are), and/or Mongo field paths whose value may be a - value, `$eq`, `$in` or (booleans) `$ne`. - query_fields (dict): Mongo field path -> flat parameter name the endpoint accepts. - list_parameters (tuple): flat parameters the endpoint accepts as they are. - - Returns: - dict: flat parameters. - - Raises: - ValueError: for a field or condition the endpoint cannot filter on. - """ - parameters = {} - for field, condition in query.items(): - if field in query_fields: - parameter = query_fields[field] - parameters[parameter] = _to_parameter_value(_translate_condition(field, parameter, condition)) - elif field in list_parameters: - parameters[field] = _translate_direct_value(field, condition) - else: - supported = ", ".join((*list_parameters, *query_fields)) - raise ValueError(f'Unsupported query field "{field}". Supported: {supported}.') - return parameters - - def _translate_sort(sort): if isinstance(sort, str): return sort @@ -108,7 +21,7 @@ def _translate_sort(sort): def translate_projection(projection): """ - Translates the Mongo-style options of a list call into flat list parameters. + Translates the Mongo-style options of a list call into the pagination parameters of the endpoint. `limit`, `skip` and a single-field `sort` are honored. `fields` is ignored: the endpoints return whole documents, which can only be more than was asked for. @@ -132,3 +45,24 @@ def translate_projection(projection): if "sort" in projection: parameters["sort"] = _translate_sort(projection["sort"]) return parameters + + +def set_parameters(query): + """ + The set parameters of a list that answers a Mongo query. + + The list endpoint defaults to top-level entities. A Mongo query means "anywhere" unless it names the set + (`setId`) or asks for the sets themselves, which sit at the top level. + + Args: + query (dict): Mongo query. + + Returns: + dict + """ + set_id = query.get("inSet._id") + if isinstance(set_id, str): + return {"setId": set_id} + if query.get("isEntitySet") is True: + return {} + return {"globalSearch": "true"} diff --git a/tests/py/unit/entity.py b/tests/py/unit/entity.py index d0a246d..56f8390 100644 --- a/tests/py/unit/entity.py +++ b/tests/py/unit/entity.py @@ -38,16 +38,6 @@ def create(self, mock_request): self.endpoints.create({}) self.assertEqual(mock_request.call_args[1]["headers"]["Content-Type"], CONTENT_TYPE_JSON) - def show_default(self, mock_request): - mock_request.return_value = self.mock_response(MOCK_SUCCESS_RESPONSE_OBJECT) - self.assertEqual(self.endpoints.show_default(), {}) - self.assertEqual(mock_request.call_args[1]["method"], HTTP_METHOD_GET) - self.assertEqual(mock_request.call_args[1]["url"], f"{self.base_url}/default") - self.assertIsNone(mock_request.call_args[1]["params"]) - - self.endpoints.show_default(self.account_id) - self.assertEqual(mock_request.call_args[1]["params"], {"accountId": self.account_id}) - def delete(self, mock_request): mock_request.return_value = self.mock_response(MOCK_SUCCESS_RESPONSE_OBJECT) self.assertEqual(self.endpoints.delete(TEST_ENTITY_ID), {}) diff --git a/tests/py/unit/test_list_query.py b/tests/py/unit/test_list_query.py index 814320a..07697a7 100644 --- a/tests/py/unit/test_list_query.py +++ b/tests/py/unit/test_list_query.py @@ -2,14 +2,16 @@ from unittest import mock import pytest +from mat3ra.api_client.endpoints.bank_materials import BankMaterialEndpoints from mat3ra.api_client.endpoints.bank_workflows import BankWorkflowEndpoints from mat3ra.api_client.endpoints.charges import ChargeEndpoints from mat3ra.api_client.endpoints.jobs import JobEndpoints from mat3ra.api_client.endpoints.materials import MaterialEndpoints +from mat3ra.api_client.endpoints.metaproperties import MetaPropertiesEndpoints from mat3ra.api_client.endpoints.projects import ProjectEndpoints from mat3ra.api_client.endpoints.properties import PropertiesEndpoints from mat3ra.api_client.endpoints.workflows import WorkflowEndpoints -from mat3ra.api_client.utils.query import OWNED_ENTITY_QUERY_FIELDS, translate_projection, translate_query +from mat3ra.api_client.utils.query import set_parameters, translate_projection from tests.py.unit import EndpointBaseUnitTest from tests.py.unit.entity import MOCK_SUCCESS_RESPONSE_LIST @@ -17,57 +19,12 @@ SET_ID = "setIdValue" -def test_translates_values_and_in_conditions(): - query = {"owner._id": OWNER_ID, "_id": {"$in": ["a", "b"]}, "name": {"$eq": "Si"}} - - assert translate_query(query, OWNED_ENTITY_QUERY_FIELDS) == {"ownerId": OWNER_ID, "id": ["a", "b"], "name": "Si"} - - -def test_list_parameters_are_passed_on_as_they_are(): - query = {"ownerId": OWNER_ID, "isEntitySet": False, "id": ["a", "b"], "limit": 5, "sort": {"name": -1}} - parameters = ("ownerId", "isEntitySet", "id", "limit", "sort") - - assert translate_query(query, {}, parameters) == { - "ownerId": OWNER_ID, - "isEntitySet": "false", - "id": ["a", "b"], - "limit": 5, - "sort": "-name", - } - - -def test_a_condition_on_a_list_parameter_is_rejected(): - with pytest.raises(ValueError, match="list parameter"): - translate_query({"ownerId": {"$in": ["a"]}}, {}, ("ownerId",)) - - -def test_booleans_are_sent_as_lowercase_strings_and_ne_negates_them(): - query = {"isDefault": True, "isEntitySet": {"$ne": True}} - fields = {**OWNED_ENTITY_QUERY_FIELDS, "isEntitySet": "isEntitySet"} - - assert translate_query(query, fields) == {"isDefault": "true", "isEntitySet": "false"} - - -@pytest.mark.parametrize( - "query", - [ - {"formula": "Si"}, - {"$or": [{"name": "Si"}]}, - {"name": {"$regex": "Si", "$options": "i"}}, - {"name": {"$ne": "Si"}}, - {"name": {"$in": ["a"], "$ne": "b"}}, - ], -) -def test_rejects_what_the_endpoint_cannot_filter_on(query): - with pytest.raises(ValueError): - translate_query(query, OWNED_ENTITY_QUERY_FIELDS) - - def test_projection_limit_skip_and_sort(): projection = {"limit": 1, "skip": 2, "sort": {"precision.value": -1}, "fields": {"status": 1}} assert translate_projection(projection) == {"limit": 1, "skip": 2, "sort": "-precision.value"} assert translate_projection({"sort": {"name": 1}}) == {"sort": "name"} + assert translate_projection({"sort": "-name"}) == {"sort": "-name"} @pytest.mark.parametrize("projection", [{"hint": "x"}, {"sort": {"a": 1, "b": 1}}]) @@ -76,9 +33,25 @@ def test_rejects_unsupported_projection(projection): translate_projection(projection) +@pytest.mark.parametrize( + ("query", "expected"), + [ + ({"inSet._id": SET_ID}, {"setId": SET_ID}), + ({"owner._id": OWNER_ID, "inSet._id": SET_ID, "isEntitySet": {"$ne": True}}, {"setId": SET_ID}), + ({"isEntitySet": True}, {}), + ({"inSet._id": {"$in": [SET_ID]}}, {"globalSearch": "true"}), + ({"isEntitySet": {"$ne": True}}, {"globalSearch": "true"}), + ({"owner._id": OWNER_ID, "hash": "h"}, {"globalSearch": "true"}), + ({}, {"globalSearch": "true"}), + ], +) +def test_set_parameters(query, expected): + assert set_parameters(query) == expected + + class EndpointListQueryUnitTest(EndpointBaseUnitTest): """ - Class for testing the flat filters `list()` adds for endpoints that ignore the `query` blob. + Class for testing how `list()` sends a Mongo query to the endpoints that filter on `advancedSearches`. """ def sent_params(self, endpoint_class, query=None, projection=None): @@ -88,104 +61,63 @@ def sent_params(self, endpoint_class, query=None, projection=None): endpoint.list(query, projection) return mock_request.call_args[1]["params"] - def test_materials_send_flat_filters_next_to_the_blob(self): - query = {"hash": "abc", "owner._id": OWNER_ID} + def test_the_query_is_sent_as_advanced_searches_next_to_the_blob(self): + query = {"hash": "abc", "owner._id": OWNER_ID, "name": {"$regex": "Si", "$options": "i"}} params = self.sent_params(MaterialEndpoints, query, {"limit": 1}) + self.assertEqual(json.loads(params["advancedSearches"]), [query]) self.assertEqual(json.loads(params["query"]), query) self.assertEqual(json.loads(params["projection"]), {"limit": 1}) - self.assertEqual((params["hashes"], params["ownerId"], params["limit"]), ("abc", OWNER_ID, 1)) - - def test_materials_without_a_set_are_searched_everywhere(self): - self.assertEqual(self.sent_params(MaterialEndpoints, {"owner._id": OWNER_ID})["globalSearch"], "true") - self.assertEqual(self.sent_params(MaterialEndpoints)["globalSearch"], "true") - - def test_materials_filter_on_formula_and_properties_on_unit_and_precision(self): - direct = self.sent_params(MaterialEndpoints, {"ownerId": OWNER_ID, "formula": ["Si", "Ge"]}) - mongo_style = self.sent_params(MaterialEndpoints, {"owner._id": OWNER_ID, "formula": "Si"}) - properties = self.sent_params( - PropertiesEndpoints, {"source.info.unitId": "pw-nscf", "precision.value": 10, "jobId": "j"} - ) - - self.assertEqual(direct["formula"], ["Si", "Ge"]) - self.assertEqual(mongo_style["formula"], "Si") - self.assertEqual((properties["unitId"], properties["precisionValue"]), ("pw-nscf", 10)) - - def test_workflows_and_jobs_without_a_set_are_searched_everywhere(self): - self.assertEqual(self.sent_params(WorkflowEndpoints, {"hash": "h"})["globalSearch"], "true") - self.assertEqual(self.sent_params(JobEndpoints, {"status": "finished"})["globalSearch"], "true") - self.assertNotIn("globalSearch", self.sent_params(JobEndpoints, {"inSet._id": SET_ID})) - self.assertNotIn("globalSearch", self.sent_params(ProjectEndpoints, {"isDefault": True})) - - def test_jobs_requested_by_id_are_not_restricted_to_non_sets(self): - self.assertNotIn("globalSearch", self.sent_params(JobEndpoints, {"_id": {"$in": ["j1"]}})) - - def test_an_empty_in_matches_nothing_and_sends_no_request(self): - for endpoint_class in (MaterialEndpoints, JobEndpoints): - for query in ({"_id": {"$in": []}}, {"id": []}): - endpoint = endpoint_class(self.host, self.port, self.account_id, self.auth_token) - with mock.patch("requests.sessions.Session.request") as mock_request: - self.assertEqual(endpoint.list(query), []) - mock_request.assert_not_called() - - def test_materials_take_list_parameters_directly(self): - params = self.sent_params(MaterialEndpoints, {"name": "Si", "ownerId": OWNER_ID, "hashes": ["h1", "h2"]}) - - self.assertEqual((params["name"], params["ownerId"], params["hashes"]), ("Si", OWNER_ID, ["h1", "h2"])) - - def test_list_parameters_mean_what_the_endpoint_says(self): - top_level = self.sent_params(MaterialEndpoints, {"name": "Si", "ownerId": OWNER_ID}) - anywhere = self.sent_params(MaterialEndpoints, {"name": "Si", "ownerId": OWNER_ID, "globalSearch": True}) - mixed = self.sent_params(MaterialEndpoints, {"owner._id": OWNER_ID, "hashes": "h"}) - - self.assertNotIn("globalSearch", top_level) - self.assertEqual(anywhere["globalSearch"], "true") - self.assertNotIn("globalSearch", mixed) - - def test_the_unsupported_field_message_names_the_supported_ones(self): - endpoint = MaterialEndpoints(self.host, self.port, self.account_id, self.auth_token) - with self.assertRaisesRegex(ValueError, "ownerId.*owner._id"): - endpoint.list({"lattice.type": "FCC"}) - - def test_materials_in_a_set_or_sets_keep_the_set_semantics(self): - in_set = self.sent_params(MaterialEndpoints, {"inSet._id": SET_ID, "isEntitySet": {"$ne": True}}) + self.assertEqual(params["limit"], 1) + + def test_operators_and_empty_lists_are_left_to_the_server(self): + query = {"_id": {"$in": []}} + + self.assertEqual(json.loads(self.sent_params(JobEndpoints, query)["advancedSearches"]), [query]) + + def test_an_empty_query_sends_no_advanced_searches(self): + params = self.sent_params(ProjectEndpoints, None, {"limit": 5}) + + self.assertNotIn("advancedSearches", params) + self.assertEqual(params["limit"], 5) + + def test_every_advanced_searches_endpoint_sends_the_query(self): + for endpoint_class in ( + MaterialEndpoints, + WorkflowEndpoints, + ProjectEndpoints, + JobEndpoints, + PropertiesEndpoints, + BankMaterialEndpoints, + BankWorkflowEndpoints, + ): + params = self.sent_params(endpoint_class, {"name": "x"}) + self.assertEqual(json.loads(params["advancedSearches"]), [{"name": "x"}], endpoint_class.__name__) + + def test_entities_that_can_be_in_sets_are_searched_everywhere_unless_a_set_is_named(self): + for endpoint_class in (MaterialEndpoints, WorkflowEndpoints, JobEndpoints): + self.assertEqual(self.sent_params(endpoint_class, {"owner._id": OWNER_ID})["globalSearch"], "true") + self.assertEqual(self.sent_params(endpoint_class)["globalSearch"], "true") + in_set = self.sent_params(endpoint_class, {"inSet._id": SET_ID}) + self.assertEqual(in_set["setId"], SET_ID) + self.assertNotIn("globalSearch", in_set) sets = self.sent_params(MaterialEndpoints, {"owner._id": OWNER_ID, "isEntitySet": True}) - - self.assertEqual((in_set["setId"], in_set["isEntitySet"]), (SET_ID, "false")) - self.assertNotIn("globalSearch", in_set) - self.assertEqual(sets["isEntitySet"], "true") self.assertNotIn("globalSearch", sets) + self.assertNotIn("setId", sets) + + def test_entities_without_sets_get_no_global_search(self): + for endpoint_class in (ProjectEndpoints, PropertiesEndpoints, BankMaterialEndpoints, BankWorkflowEndpoints): + self.assertNotIn("globalSearch", self.sent_params(endpoint_class, {"name": "x"})) - def test_workflows_projects_jobs_properties_and_bank(self): - self.assertEqual(self.sent_params(WorkflowEndpoints, {"hash": "h"})["hash"], "h") - self.assertEqual(self.sent_params(ProjectEndpoints, {"isDefault": True})["isDefault"], "true") - self.assertEqual(self.sent_params(BankWorkflowEndpoints, {"systemName": "s"})["systemName"], "s") - jobs = self.sent_params(JobEndpoints, {"_material._id": {"$in": ["m1", "m2"]}, "status": "finished"}) - self.assertEqual((jobs["materialId"], jobs["status"]), (["m1", "m2"], "finished")) - properties = self.sent_params(PropertiesEndpoints, {"source.info.jobId": "j", "data.name": "total_energy"}) - self.assertEqual((properties["jobId"], properties["propertyName"]), ("j", "total_energy")) - unit_template_query = { - "exabyteId": {"$in": ["e1"]}, - "slug": "total_energy", - "group": "qe:dft", - "owner.slug": {"$in": ["me", "curators"]}, - } - template = self.sent_params( - PropertiesEndpoints, unit_template_query, {"sort": {"precision.value": -1}, "limit": 1} - ) - self.assertEqual( - (template["exabyteId"], template["group"], template["ownerSlug"], template["sort"], template["limit"]), - (["e1"], "qe:dft", ["me", "curators"], "-precision.value", 1), - ) - - def test_an_unsupported_field_raises_before_any_request(self): + def test_an_unsupported_option_raises_before_any_request(self): endpoint = MaterialEndpoints(self.host, self.port, self.account_id, self.auth_token) with mock.patch("requests.sessions.Session.request") as mock_request: with self.assertRaises(ValueError): - endpoint.list({"lattice.type": "FCC"}) + endpoint.list({"name": "x"}, {"hint": "y"}) mock_request.assert_not_called() def test_endpoints_that_still_read_the_blob_are_unchanged(self): - params = self.sent_params(ChargeEndpoints, {"jid": "1"}) + for endpoint_class in (ChargeEndpoints, MetaPropertiesEndpoints): + params = self.sent_params(endpoint_class, {"jid": "1"}) - self.assertEqual(set(params), {"query", "projection"}) + self.assertEqual(set(params), {"query", "projection"}, endpoint_class.__name__) diff --git a/tests/py/unit/test_materials.py b/tests/py/unit/test_materials.py index 1194bbe..4dcb849 100644 --- a/tests/py/unit/test_materials.py +++ b/tests/py/unit/test_materials.py @@ -31,7 +31,3 @@ def test_create(self, mock_request): @mock.patch("requests.sessions.Session.request") def test_delete(self, mock_request): self.create(mock_request) - - @mock.patch("requests.sessions.Session.request") - def test_show_default(self, mock_request): - self.show_default(mock_request) diff --git a/tests/py/unit/test_projects.py b/tests/py/unit/test_projects.py deleted file mode 100644 index a9b6ba9..0000000 --- a/tests/py/unit/test_projects.py +++ /dev/null @@ -1,29 +0,0 @@ -from unittest import mock - -from mat3ra.api_client.endpoints.projects import ProjectEndpoints -from tests.py.unit.entity import EntityEndpointsUnitTest - -ENDPOINT_NAME = "projects" - - -class EndpointProjectsUnitTest(EntityEndpointsUnitTest): - """ - Class for testing projects endpoint. - """ - - def __init__(self, *args, **kwargs): - super(EndpointProjectsUnitTest, self).__init__(*args, **kwargs) - self.endpoint_name = ENDPOINT_NAME - self.endpoints = ProjectEndpoints(self.host, self.port, self.account_id, self.auth_token) - - @mock.patch("requests.sessions.Session.request") - def test_list(self, mock_request): - self.list(mock_request) - - @mock.patch("requests.sessions.Session.request") - def test_get(self, mock_request): - self.get(mock_request) - - @mock.patch("requests.sessions.Session.request") - def test_show_default(self, mock_request): - self.show_default(mock_request) diff --git a/tests/py/unit/test_properties.py b/tests/py/unit/test_properties.py index 2426383..62f6ccb 100644 --- a/tests/py/unit/test_properties.py +++ b/tests/py/unit/test_properties.py @@ -84,21 +84,3 @@ def test_get_for_job_filtered_by_unit_id_and_name(self, mock_request): sent_query = json.loads(mock_request.call_args[1]["params"]["query"]) self.assertEqual(sent_query["source.info.unitId"], UNIT_ID_1) self.assertEqual(sent_query["data.name"], PROPERTY_NAME_1) - - @mock.patch("requests.sessions.Session.request") - def test_get_for_job_sends_flat_filters(self, mock_request): - mock_request.return_value = self.mock_response(MOCK_SINGLE_PROPERTY_RESPONSE) - self.endpoints.get_for_job(JOB_ID, PROPERTY_NAME_1, unit_id=UNIT_ID_1) - params = mock_request.call_args[1]["params"] - self.assertEqual( - (params["jobId"], params["propertyName"], params["unitId"]), (JOB_ID, PROPERTY_NAME_1, UNIT_ID_1) - ) - - @mock.patch("requests.sessions.Session.request") - def test_get_property_sends_flat_filters(self, mock_request): - mock_request.return_value = self.mock_response(MOCK_PROPERTIES_RESPONSE) - self.endpoints.get_property(JOB_ID, UNIT_ID_1, PROPERTY_NAME_1) - params = mock_request.call_args[1]["params"] - self.assertEqual( - (params["jobId"], params["propertyName"], params["unitId"]), (JOB_ID, PROPERTY_NAME_1, UNIT_ID_1) - ) diff --git a/tests/py/unit/test_workflows.py b/tests/py/unit/test_workflows.py index 49ba457..9586a85 100644 --- a/tests/py/unit/test_workflows.py +++ b/tests/py/unit/test_workflows.py @@ -31,7 +31,3 @@ def test_create(self, mock_request): @mock.patch("requests.sessions.Session.request") def test_delete(self, mock_request): self.delete(mock_request) - - @mock.patch("requests.sessions.Session.request") - def test_show_default(self, mock_request): - self.show_default(mock_request) From 701c54e55dab58935896a822a380132db7b90a81 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Wed, 7 Oct 2026 15:47:13 +0300 Subject: [PATCH 4/6] feat(SOF-8032): embed the workflow and default the project when creating jobs and job sets The server takes a full workflow document in a job and a project for a job set, while the notebooks give a workflow by its _id and a job set only a name and an owner. jobs.create (and create_by_ids) now fetches and embeds a workflow given by _id, and jobs.create_set uses the owner's default project when the config has no projectId, so the notebooks keep their original calls. Co-Authored-By: Claude Sonnet 5.5 --- src/py/mat3ra/api_client/endpoints/jobs.py | 57 +++++++++++++ tests/py/unit/test_jobs.py | 94 ++++++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/src/py/mat3ra/api_client/endpoints/jobs.py b/src/py/mat3ra/api_client/endpoints/jobs.py index 5844a72..b366a36 100644 --- a/src/py/mat3ra/api_client/endpoints/jobs.py +++ b/src/py/mat3ra/api_client/endpoints/jobs.py @@ -30,6 +30,63 @@ def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSI super(JobEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "jobs" + def create(self, config, owner_id=None): + """ + Creates a new job. + + The server takes the workflow as a full document (the job keeps a snapshot of it), so a workflow given only + by its `_id` is fetched and embedded first. + + Args: + config (dict): job config. + owner_id (str): owner ID. Job is created under user's default account if not specified. + + Returns: + dict: new job. + """ + workflow = config.get("workflow") + if isinstance(workflow, dict) and "_id" in workflow and "subworkflows" not in workflow: + workflow_document = self.request("GET", "/".join(("workflows", workflow["_id"])), headers=self.headers) + config = {**config, "workflow": workflow_document} + return super(JobEndpoints, self).create(config, owner_id) + + def create_set(self, config): + """ + Creates a new job set. + + A job set belongs to a project, so the default project of the owner (the account if the config has no + `owner`) is used when the config has no `projectId`. + + Args: + config (dict): job set config. + + Returns: + dict: new job set. + + Raises: + ValueError: if no project is given and the owner has no default one. + """ + if "projectId" not in config: + owner_id = config.get("owner", {}).get("_id") or self.headers["X-Account-Id"] + config = {**config, "projectId": self._get_default_project_id(owner_id)} + return super(JobEndpoints, self).create_set(config) + + def _get_default_project_id(self, owner_id): + """ + Returns the ID of the default project of the given owner. + + Args: + owner_id (str): owner ID. + + Returns: + str + """ + params = {"advancedSearches": json.dumps([{"isDefault": True, "owner._id": owner_id}])} + projects = self.request("GET", "projects", params=params, headers=self.headers) + if not projects: + raise ValueError(f"The owner {owner_id} has no default project, pass `projectId`.") + return projects[0]["_id"] + def submit(self, id_): """ Submits a given job. diff --git a/tests/py/unit/test_jobs.py b/tests/py/unit/test_jobs.py index 1e69c40..fb5ec80 100644 --- a/tests/py/unit/test_jobs.py +++ b/tests/py/unit/test_jobs.py @@ -1,9 +1,11 @@ +import json from unittest import mock from mat3ra.api_client.endpoints.jobs import JobEndpoints from tests.py.unit.entity import EntityEndpointsUnitTest ENDPOINT_NAME = "jobs" +WORKFLOW = {"_id": "workflowId", "name": "Total Energy", "subworkflows": [{"name": "scf", "units": []}]} class EndpointJobsUnitTest(EntityEndpointsUnitTest): @@ -31,3 +33,95 @@ def test_create(self, mock_request): @mock.patch("requests.sessions.Session.request") def test_delete(self, mock_request): self.create(mock_request) + + def respond(self, mock_request, *data): + responses = [json.dumps({"status": "success", "data": item}) for item in data] + mock_request.side_effect = [self.mock_response(response) for response in responses] + + @mock.patch("requests.sessions.Session.request") + def test_create_embeds_a_workflow_given_by_id(self, mock_request): + self.respond(mock_request, WORKFLOW, {"_id": "job"}) + config = {"name": "job", "workflow": {"_id": WORKFLOW["_id"]}} + + self.assertEqual(self.endpoints.create(config), {"_id": "job"}) + + fetch, create = mock_request.call_args_list + self.assertEqual(fetch[1]["method"], "get") + self.assertEqual(fetch[1]["url"], f"https://{self.host}:{self.port}/api/{self.version}/workflows/{WORKFLOW['_id']}") + self.assertEqual(create[1]["url"], f"{self.base_url}/create") + self.assertEqual(json.loads(create[1]["data"]), {"name": "job", "workflow": WORKFLOW}) + self.assertEqual(config["workflow"], {"_id": WORKFLOW["_id"]}) + + @mock.patch("requests.sessions.Session.request") + def test_create_leaves_a_full_workflow_alone(self, mock_request): + self.respond(mock_request, {"_id": "job"}) + + self.endpoints.create({"name": "job", "workflow": WORKFLOW}) + + self.assertEqual(mock_request.call_count, 1) + self.assertEqual(json.loads(mock_request.call_args[1]["data"])["workflow"], WORKFLOW) + + @mock.patch("requests.sessions.Session.request") + def test_create_without_a_workflow_fetches_nothing(self, mock_request): + self.respond(mock_request, {"_id": "job"}) + + self.endpoints.create({"name": "job"}, owner_id="owner") + + self.assertEqual(mock_request.call_count, 1) + self.assertEqual(json.loads(mock_request.call_args[1]["data"]), {"name": "job", "owner": {"_id": "owner"}}) + + @mock.patch("requests.sessions.Session.request") + def test_create_by_ids_embeds_the_workflow(self, mock_request): + self.respond(mock_request, WORKFLOW, {"_id": "job"}) + + jobs = self.endpoints.create_by_ids( + [{"_id": "material", "formula": "Si"}], WORKFLOW["_id"], "project", "prefix", owner_id="owner" + ) + + self.assertEqual(jobs, [{"_id": "job"}]) + self.assertEqual(json.loads(mock_request.call_args[1]["data"])["workflow"], WORKFLOW) + + @mock.patch("requests.sessions.Session.request") + def test_create_set_uses_the_default_project_of_the_owner(self, mock_request): + self.respond(mock_request, [{"_id": "project"}], {"_id": "set"}) + + self.assertEqual(self.endpoints.create_set({"name": "set", "owner": {"_id": "owner"}}), {"_id": "set"}) + + lookup, create = mock_request.call_args_list + self.assertTrue(lookup[1]["url"].endswith("/projects")) + self.assertEqual( + json.loads(lookup[1]["params"]["advancedSearches"]), [{"isDefault": True, "owner._id": "owner"}] + ) + self.assertEqual(create[1]["url"], f"{self.base_url}/create-set") + self.assertEqual( + json.loads(create[1]["data"]), {"name": "set", "owner": {"_id": "owner"}, "projectId": "project"} + ) + + @mock.patch("requests.sessions.Session.request") + def test_create_set_defaults_the_owner_to_the_account(self, mock_request): + self.respond(mock_request, [{"_id": "project"}], {"_id": "set"}) + + self.endpoints.create_set({"name": "set"}) + + lookup = mock_request.call_args_list[0] + self.assertEqual( + json.loads(lookup[1]["params"]["advancedSearches"]), [{"isDefault": True, "owner._id": self.account_id}] + ) + + @mock.patch("requests.sessions.Session.request") + def test_create_set_keeps_a_given_project(self, mock_request): + self.respond(mock_request, {"_id": "set"}) + + self.endpoints.create_set({"name": "set", "projectId": "mine"}) + + self.assertEqual(mock_request.call_count, 1) + self.assertEqual(json.loads(mock_request.call_args[1]["data"]), {"name": "set", "projectId": "mine"}) + + @mock.patch("requests.sessions.Session.request") + def test_create_set_without_any_default_project_raises(self, mock_request): + self.respond(mock_request, []) + + with self.assertRaises(ValueError): + self.endpoints.create_set({"name": "set"}) + + self.assertEqual(mock_request.call_count, 1) From bc836094efa031a451a6619873e46472fac6b6d5 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Wed, 7 Oct 2026 20:02:46 +0300 Subject: [PATCH 5/6] revert(SOF-8032): keep list() sending the query and projection params The web-app list endpoints accept the query and projection params again, so list() no longer translates them into advancedSearches and list parameters. Restores list() and the endpoint classes to main and removes utils/query.py with its tests. Keeps the job fixes: jobs.create embeds a workflow given by _id and jobs.create_set defaults the project, which are separate contract changes. The default project lookup now sends the query and projection params. Co-Authored-By: Claude Sonnet 5.5 --- .../api_client/endpoints/bank_materials.py | 2 - .../api_client/endpoints/bank_workflows.py | 2 - src/py/mat3ra/api_client/endpoints/entity.py | 34 +---- src/py/mat3ra/api_client/endpoints/jobs.py | 6 +- .../mat3ra/api_client/endpoints/materials.py | 3 - .../mat3ra/api_client/endpoints/projects.py | 2 - .../mat3ra/api_client/endpoints/properties.py | 2 - .../mat3ra/api_client/endpoints/workflows.py | 3 - src/py/mat3ra/api_client/utils/query.py | 68 ---------- tests/py/unit/test_jobs.py | 4 +- tests/py/unit/test_list_query.py | 123 ------------------ 11 files changed, 5 insertions(+), 244 deletions(-) delete mode 100644 src/py/mat3ra/api_client/utils/query.py delete mode 100644 tests/py/unit/test_list_query.py diff --git a/src/py/mat3ra/api_client/endpoints/bank_materials.py b/src/py/mat3ra/api_client/endpoints/bank_materials.py index 7eaba05..fb53de6 100644 --- a/src/py/mat3ra/api_client/endpoints/bank_materials.py +++ b/src/py/mat3ra/api_client/endpoints/bank_materials.py @@ -20,8 +20,6 @@ class BankMaterialEndpoints(BankEntityEndpoints): name (str): endpoint name. """ - advanced_searches = True - def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(BankMaterialEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "bank-materials" diff --git a/src/py/mat3ra/api_client/endpoints/bank_workflows.py b/src/py/mat3ra/api_client/endpoints/bank_workflows.py index 4ea0ebb..2267665 100644 --- a/src/py/mat3ra/api_client/endpoints/bank_workflows.py +++ b/src/py/mat3ra/api_client/endpoints/bank_workflows.py @@ -20,8 +20,6 @@ class BankWorkflowEndpoints(BankEntityEndpoints): name (str): endpoint name. """ - advanced_searches = True - def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(BankWorkflowEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "bank-workflows" diff --git a/src/py/mat3ra/api_client/endpoints/entity.py b/src/py/mat3ra/api_client/endpoints/entity.py index 9f7e0c8..be0ab1e 100644 --- a/src/py/mat3ra/api_client/endpoints/entity.py +++ b/src/py/mat3ra/api_client/endpoints/entity.py @@ -1,6 +1,5 @@ import json -from ..utils.query import set_parameters, translate_projection from . import BaseEndpoint from .enums import DEFAULT_API_VERSION, SECURE @@ -22,13 +21,8 @@ class EntityEndpoint(BaseEndpoint): Attributes: name (str): endpoint name. headers (dict): default HTTP headers. - advanced_searches (bool): whether the list endpoint filters on `advancedSearches` instead of the `query` blob. - searches_sets (bool): whether the entities can be in sets (list endpoints default to the top level). """ - advanced_searches = False - searches_sets = False - def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(EntityEndpoint, self).__init__(host, port, version, secure, **kwargs) self.name = None @@ -38,42 +32,16 @@ def list(self, query=None, projection=None): """ Returns a list of entities. - Endpoints that filter on `advancedSearches` get the query as such (the `query` blob is still sent, for servers - that read it). A query is answered "anywhere" unless it names a set, as it was before. - Args: query (dict): Mongo query. Defaults to {}. - projection (dict): options: limit, skip, sort. Defaults to {}. + projection (dict): Mongo projection. Defaults to {}. Returns: list[dict] - - Raises: - ValueError: for an option or sort the endpoint does not support. """ params = {"query": json.dumps(query or {}), "projection": json.dumps(projection or {})} - if self.advanced_searches: - params.update(self.build_advanced_search_parameters(query or {}, projection or {})) return self.request("GET", self.name, params=params, headers=self.headers) - def build_advanced_search_parameters(self, query, projection): - """ - Builds the parameters of an `advancedSearches` list. - - Args: - query (dict): Mongo query. - projection (dict): options: limit, skip, sort. - - Returns: - dict - """ - parameters = translate_projection(projection) - if query: - parameters["advancedSearches"] = json.dumps([query]) - if self.searches_sets: - parameters.update(set_parameters(query)) - return parameters - def get(self, id_): """ Returns a entity with given ID. diff --git a/src/py/mat3ra/api_client/endpoints/jobs.py b/src/py/mat3ra/api_client/endpoints/jobs.py index b366a36..a444e58 100644 --- a/src/py/mat3ra/api_client/endpoints/jobs.py +++ b/src/py/mat3ra/api_client/endpoints/jobs.py @@ -23,9 +23,6 @@ class JobEndpoints(EntitySetEndpointsMixin, EntityEndpoint): name (str): endpoint name. """ - advanced_searches = True - searches_sets = True - def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(JobEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "jobs" @@ -81,7 +78,8 @@ def _get_default_project_id(self, owner_id): Returns: str """ - params = {"advancedSearches": json.dumps([{"isDefault": True, "owner._id": owner_id}])} + query = {"isDefault": True, "owner._id": owner_id} + params = {"query": json.dumps(query), "projection": json.dumps({})} projects = self.request("GET", "projects", params=params, headers=self.headers) if not projects: raise ValueError(f"The owner {owner_id} has no default project, pass `projectId`.") diff --git a/src/py/mat3ra/api_client/endpoints/materials.py b/src/py/mat3ra/api_client/endpoints/materials.py index 9242d79..856e592 100644 --- a/src/py/mat3ra/api_client/endpoints/materials.py +++ b/src/py/mat3ra/api_client/endpoints/materials.py @@ -25,9 +25,6 @@ class MaterialEndpoints(EntitySetEndpointsMixin, DefaultableEntityEndpointsMixin name (str): endpoint name. """ - advanced_searches = True - searches_sets = True - def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(MaterialEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "materials" diff --git a/src/py/mat3ra/api_client/endpoints/projects.py b/src/py/mat3ra/api_client/endpoints/projects.py index 77a7a28..91adff4 100644 --- a/src/py/mat3ra/api_client/endpoints/projects.py +++ b/src/py/mat3ra/api_client/endpoints/projects.py @@ -24,8 +24,6 @@ class ProjectEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): headers (dict): default HTTP headers. """ - advanced_searches = True - def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(ProjectEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "projects" diff --git a/src/py/mat3ra/api_client/endpoints/properties.py b/src/py/mat3ra/api_client/endpoints/properties.py index 4b045eb..30b88c1 100644 --- a/src/py/mat3ra/api_client/endpoints/properties.py +++ b/src/py/mat3ra/api_client/endpoints/properties.py @@ -42,8 +42,6 @@ class PropertiesEndpoints(BasePropertiesEndpoints): headers (dict): default HTTP headers. """ - advanced_searches = True - def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(PropertiesEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "properties" diff --git a/src/py/mat3ra/api_client/endpoints/workflows.py b/src/py/mat3ra/api_client/endpoints/workflows.py index b24bc11..5e256e3 100644 --- a/src/py/mat3ra/api_client/endpoints/workflows.py +++ b/src/py/mat3ra/api_client/endpoints/workflows.py @@ -24,9 +24,6 @@ class WorkflowEndpoints(DefaultableEntityEndpointsMixin, EntityEndpoint): headers (dict): default HTTP headers. """ - advanced_searches = True - searches_sets = True - def __init__(self, host, port, account_id, auth_token, version=DEFAULT_API_VERSION, secure=SECURE, **kwargs): super(WorkflowEndpoints, self).__init__(host, port, account_id, auth_token, version, secure, **kwargs) self.name = "workflows" diff --git a/src/py/mat3ra/api_client/utils/query.py b/src/py/mat3ra/api_client/utils/query.py deleted file mode 100644 index dfcc408..0000000 --- a/src/py/mat3ra/api_client/utils/query.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -Helpers for the list endpoints that filter on `advancedSearches`. - -A migrated list endpoint validates its parameters against the flat keys of its use case and silently drops -anything else - including the `query=` blob `list()` sends. It does accept `advancedSearches`: raw Mongo -selectors, sent as a JSON string, applied next to the access scope. `list()` carries the whole query that way. -""" - -PROJECTION_OPTIONS = ("limit", "skip", "sort", "fields") - - -def _translate_sort(sort): - if isinstance(sort, str): - return sort - if not isinstance(sort, dict) or len(sort) != 1: - raise ValueError("Unsupported sort: a single field, as a string or {field: 1 | -1}, is supported.") - - ((field, direction),) = sort.items() - return f"-{field}" if direction in (-1, "desc") else field - - -def translate_projection(projection): - """ - Translates the Mongo-style options of a list call into the pagination parameters of the endpoint. - - `limit`, `skip` and a single-field `sort` are honored. `fields` is ignored: the endpoints return whole - documents, which can only be more than was asked for. - - Args: - projection (dict): options, e.g. {"limit": 1, "sort": {"precision.value": -1}}. - - Returns: - dict: flat parameters. - - Raises: - ValueError: for an unsupported option or sort. - """ - unsupported = [option for option in projection if option not in PROJECTION_OPTIONS] - if unsupported: - raise ValueError( - f'Unsupported projection option "{unsupported[0]}". Supported: {", ".join(PROJECTION_OPTIONS)}.' - ) - - parameters = {option: projection[option] for option in ("limit", "skip") if option in projection} - if "sort" in projection: - parameters["sort"] = _translate_sort(projection["sort"]) - return parameters - - -def set_parameters(query): - """ - The set parameters of a list that answers a Mongo query. - - The list endpoint defaults to top-level entities. A Mongo query means "anywhere" unless it names the set - (`setId`) or asks for the sets themselves, which sit at the top level. - - Args: - query (dict): Mongo query. - - Returns: - dict - """ - set_id = query.get("inSet._id") - if isinstance(set_id, str): - return {"setId": set_id} - if query.get("isEntitySet") is True: - return {} - return {"globalSearch": "true"} diff --git a/tests/py/unit/test_jobs.py b/tests/py/unit/test_jobs.py index fb5ec80..b51f56c 100644 --- a/tests/py/unit/test_jobs.py +++ b/tests/py/unit/test_jobs.py @@ -90,7 +90,7 @@ def test_create_set_uses_the_default_project_of_the_owner(self, mock_request): lookup, create = mock_request.call_args_list self.assertTrue(lookup[1]["url"].endswith("/projects")) self.assertEqual( - json.loads(lookup[1]["params"]["advancedSearches"]), [{"isDefault": True, "owner._id": "owner"}] + json.loads(lookup[1]["params"]["query"]), {"isDefault": True, "owner._id": "owner"} ) self.assertEqual(create[1]["url"], f"{self.base_url}/create-set") self.assertEqual( @@ -105,7 +105,7 @@ def test_create_set_defaults_the_owner_to_the_account(self, mock_request): lookup = mock_request.call_args_list[0] self.assertEqual( - json.loads(lookup[1]["params"]["advancedSearches"]), [{"isDefault": True, "owner._id": self.account_id}] + json.loads(lookup[1]["params"]["query"]), {"isDefault": True, "owner._id": self.account_id} ) @mock.patch("requests.sessions.Session.request") diff --git a/tests/py/unit/test_list_query.py b/tests/py/unit/test_list_query.py deleted file mode 100644 index 07697a7..0000000 --- a/tests/py/unit/test_list_query.py +++ /dev/null @@ -1,123 +0,0 @@ -import json -from unittest import mock - -import pytest -from mat3ra.api_client.endpoints.bank_materials import BankMaterialEndpoints -from mat3ra.api_client.endpoints.bank_workflows import BankWorkflowEndpoints -from mat3ra.api_client.endpoints.charges import ChargeEndpoints -from mat3ra.api_client.endpoints.jobs import JobEndpoints -from mat3ra.api_client.endpoints.materials import MaterialEndpoints -from mat3ra.api_client.endpoints.metaproperties import MetaPropertiesEndpoints -from mat3ra.api_client.endpoints.projects import ProjectEndpoints -from mat3ra.api_client.endpoints.properties import PropertiesEndpoints -from mat3ra.api_client.endpoints.workflows import WorkflowEndpoints -from mat3ra.api_client.utils.query import set_parameters, translate_projection -from tests.py.unit import EndpointBaseUnitTest -from tests.py.unit.entity import MOCK_SUCCESS_RESPONSE_LIST - -OWNER_ID = "ubxMkAyx37Rjn8qK9" -SET_ID = "setIdValue" - - -def test_projection_limit_skip_and_sort(): - projection = {"limit": 1, "skip": 2, "sort": {"precision.value": -1}, "fields": {"status": 1}} - - assert translate_projection(projection) == {"limit": 1, "skip": 2, "sort": "-precision.value"} - assert translate_projection({"sort": {"name": 1}}) == {"sort": "name"} - assert translate_projection({"sort": "-name"}) == {"sort": "-name"} - - -@pytest.mark.parametrize("projection", [{"hint": "x"}, {"sort": {"a": 1, "b": 1}}]) -def test_rejects_unsupported_projection(projection): - with pytest.raises(ValueError): - translate_projection(projection) - - -@pytest.mark.parametrize( - ("query", "expected"), - [ - ({"inSet._id": SET_ID}, {"setId": SET_ID}), - ({"owner._id": OWNER_ID, "inSet._id": SET_ID, "isEntitySet": {"$ne": True}}, {"setId": SET_ID}), - ({"isEntitySet": True}, {}), - ({"inSet._id": {"$in": [SET_ID]}}, {"globalSearch": "true"}), - ({"isEntitySet": {"$ne": True}}, {"globalSearch": "true"}), - ({"owner._id": OWNER_ID, "hash": "h"}, {"globalSearch": "true"}), - ({}, {"globalSearch": "true"}), - ], -) -def test_set_parameters(query, expected): - assert set_parameters(query) == expected - - -class EndpointListQueryUnitTest(EndpointBaseUnitTest): - """ - Class for testing how `list()` sends a Mongo query to the endpoints that filter on `advancedSearches`. - """ - - def sent_params(self, endpoint_class, query=None, projection=None): - endpoint = endpoint_class(self.host, self.port, self.account_id, self.auth_token) - with mock.patch("requests.sessions.Session.request") as mock_request: - mock_request.return_value = self.mock_response(MOCK_SUCCESS_RESPONSE_LIST) - endpoint.list(query, projection) - return mock_request.call_args[1]["params"] - - def test_the_query_is_sent_as_advanced_searches_next_to_the_blob(self): - query = {"hash": "abc", "owner._id": OWNER_ID, "name": {"$regex": "Si", "$options": "i"}} - params = self.sent_params(MaterialEndpoints, query, {"limit": 1}) - - self.assertEqual(json.loads(params["advancedSearches"]), [query]) - self.assertEqual(json.loads(params["query"]), query) - self.assertEqual(json.loads(params["projection"]), {"limit": 1}) - self.assertEqual(params["limit"], 1) - - def test_operators_and_empty_lists_are_left_to_the_server(self): - query = {"_id": {"$in": []}} - - self.assertEqual(json.loads(self.sent_params(JobEndpoints, query)["advancedSearches"]), [query]) - - def test_an_empty_query_sends_no_advanced_searches(self): - params = self.sent_params(ProjectEndpoints, None, {"limit": 5}) - - self.assertNotIn("advancedSearches", params) - self.assertEqual(params["limit"], 5) - - def test_every_advanced_searches_endpoint_sends_the_query(self): - for endpoint_class in ( - MaterialEndpoints, - WorkflowEndpoints, - ProjectEndpoints, - JobEndpoints, - PropertiesEndpoints, - BankMaterialEndpoints, - BankWorkflowEndpoints, - ): - params = self.sent_params(endpoint_class, {"name": "x"}) - self.assertEqual(json.loads(params["advancedSearches"]), [{"name": "x"}], endpoint_class.__name__) - - def test_entities_that_can_be_in_sets_are_searched_everywhere_unless_a_set_is_named(self): - for endpoint_class in (MaterialEndpoints, WorkflowEndpoints, JobEndpoints): - self.assertEqual(self.sent_params(endpoint_class, {"owner._id": OWNER_ID})["globalSearch"], "true") - self.assertEqual(self.sent_params(endpoint_class)["globalSearch"], "true") - in_set = self.sent_params(endpoint_class, {"inSet._id": SET_ID}) - self.assertEqual(in_set["setId"], SET_ID) - self.assertNotIn("globalSearch", in_set) - sets = self.sent_params(MaterialEndpoints, {"owner._id": OWNER_ID, "isEntitySet": True}) - self.assertNotIn("globalSearch", sets) - self.assertNotIn("setId", sets) - - def test_entities_without_sets_get_no_global_search(self): - for endpoint_class in (ProjectEndpoints, PropertiesEndpoints, BankMaterialEndpoints, BankWorkflowEndpoints): - self.assertNotIn("globalSearch", self.sent_params(endpoint_class, {"name": "x"})) - - def test_an_unsupported_option_raises_before_any_request(self): - endpoint = MaterialEndpoints(self.host, self.port, self.account_id, self.auth_token) - with mock.patch("requests.sessions.Session.request") as mock_request: - with self.assertRaises(ValueError): - endpoint.list({"name": "x"}, {"hint": "y"}) - mock_request.assert_not_called() - - def test_endpoints_that_still_read_the_blob_are_unchanged(self): - for endpoint_class in (ChargeEndpoints, MetaPropertiesEndpoints): - params = self.sent_params(endpoint_class, {"jid": "1"}) - - self.assertEqual(set(params), {"query", "projection"}, endpoint_class.__name__) From 54c9f90e735f087edfff72115a20364492e47726 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Wed, 7 Oct 2026 20:40:46 +0300 Subject: [PATCH 6/6] refactor(SOF-8032): leave the default project of job sets to the server The web-app now makes projectId optional for jobs/create-set and falls back to the account's default project, so the client no longer looks it up. jobs.create keeps embedding a workflow given by _id. Co-Authored-By: Claude Sonnet 5.5 --- src/py/mat3ra/api_client/endpoints/jobs.py | 38 ------------------ tests/py/unit/test_jobs.py | 45 ---------------------- 2 files changed, 83 deletions(-) diff --git a/src/py/mat3ra/api_client/endpoints/jobs.py b/src/py/mat3ra/api_client/endpoints/jobs.py index a444e58..c751858 100644 --- a/src/py/mat3ra/api_client/endpoints/jobs.py +++ b/src/py/mat3ra/api_client/endpoints/jobs.py @@ -47,44 +47,6 @@ def create(self, config, owner_id=None): config = {**config, "workflow": workflow_document} return super(JobEndpoints, self).create(config, owner_id) - def create_set(self, config): - """ - Creates a new job set. - - A job set belongs to a project, so the default project of the owner (the account if the config has no - `owner`) is used when the config has no `projectId`. - - Args: - config (dict): job set config. - - Returns: - dict: new job set. - - Raises: - ValueError: if no project is given and the owner has no default one. - """ - if "projectId" not in config: - owner_id = config.get("owner", {}).get("_id") or self.headers["X-Account-Id"] - config = {**config, "projectId": self._get_default_project_id(owner_id)} - return super(JobEndpoints, self).create_set(config) - - def _get_default_project_id(self, owner_id): - """ - Returns the ID of the default project of the given owner. - - Args: - owner_id (str): owner ID. - - Returns: - str - """ - query = {"isDefault": True, "owner._id": owner_id} - params = {"query": json.dumps(query), "projection": json.dumps({})} - projects = self.request("GET", "projects", params=params, headers=self.headers) - if not projects: - raise ValueError(f"The owner {owner_id} has no default project, pass `projectId`.") - return projects[0]["_id"] - def submit(self, id_): """ Submits a given job. diff --git a/tests/py/unit/test_jobs.py b/tests/py/unit/test_jobs.py index b51f56c..64907c0 100644 --- a/tests/py/unit/test_jobs.py +++ b/tests/py/unit/test_jobs.py @@ -80,48 +80,3 @@ def test_create_by_ids_embeds_the_workflow(self, mock_request): self.assertEqual(jobs, [{"_id": "job"}]) self.assertEqual(json.loads(mock_request.call_args[1]["data"])["workflow"], WORKFLOW) - - @mock.patch("requests.sessions.Session.request") - def test_create_set_uses_the_default_project_of_the_owner(self, mock_request): - self.respond(mock_request, [{"_id": "project"}], {"_id": "set"}) - - self.assertEqual(self.endpoints.create_set({"name": "set", "owner": {"_id": "owner"}}), {"_id": "set"}) - - lookup, create = mock_request.call_args_list - self.assertTrue(lookup[1]["url"].endswith("/projects")) - self.assertEqual( - json.loads(lookup[1]["params"]["query"]), {"isDefault": True, "owner._id": "owner"} - ) - self.assertEqual(create[1]["url"], f"{self.base_url}/create-set") - self.assertEqual( - json.loads(create[1]["data"]), {"name": "set", "owner": {"_id": "owner"}, "projectId": "project"} - ) - - @mock.patch("requests.sessions.Session.request") - def test_create_set_defaults_the_owner_to_the_account(self, mock_request): - self.respond(mock_request, [{"_id": "project"}], {"_id": "set"}) - - self.endpoints.create_set({"name": "set"}) - - lookup = mock_request.call_args_list[0] - self.assertEqual( - json.loads(lookup[1]["params"]["query"]), {"isDefault": True, "owner._id": self.account_id} - ) - - @mock.patch("requests.sessions.Session.request") - def test_create_set_keeps_a_given_project(self, mock_request): - self.respond(mock_request, {"_id": "set"}) - - self.endpoints.create_set({"name": "set", "projectId": "mine"}) - - self.assertEqual(mock_request.call_count, 1) - self.assertEqual(json.loads(mock_request.call_args[1]["data"]), {"name": "set", "projectId": "mine"}) - - @mock.patch("requests.sessions.Session.request") - def test_create_set_without_any_default_project_raises(self, mock_request): - self.respond(mock_request, []) - - with self.assertRaises(ValueError): - self.endpoints.create_set({"name": "set"}) - - self.assertEqual(mock_request.call_count, 1)