From 962be7eb326ac717fd872912bcd0adb8462646df Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Tue, 22 Sep 2026 00:13:31 +0300 Subject: [PATCH 01/10] fix(SOF-8032): stop sending raw query/projection blobs to migrated list endpoints exabyte-api-client's EntityEndpoint.list(query, projection) wraps filters as query=&projection= blob params. Several web-app REST list endpoints were migrated to use cases validated by a JSON schema with additionalProperties: false, which silently strips any undeclared param - so every one of these blob queries had become a silent no-op, returning unfiltered (or empty) results. Switches the shared material/property/job API helpers and the notebooks that called .list() directly to .request() with the flat params each use case actually declares. Co-Authored-By: Claude Sonnet 5 --- examples/job/create_and_submit_job.ipynb | 14 +++- examples/job/get-file-from-job.ipynb | 35 +++++++-- .../workflows/analyze_convex_hull.ipynb | 35 ++++++--- .../workflows/defect_formation_energy.ipynb | 40 +++++++--- .../workflows/dielectric_tensor.ipynb | 9 ++- .../workflows/equation_of_state.ipynb | 7 +- .../workflows/formation_energy.ipynb | 9 ++- .../workflows/homo_lumo_frequency.ipynb | 25 ++++-- other/materials_designer/workflows/neb.ipynb | 20 +++-- .../workflows/phonon_dos_dispersion.ipynb | 9 ++- .../workflows/relaxation.ipynb | 9 ++- .../workflows/surface_energy.ipynb | 32 +++++--- .../workflows/total_energy_convergence.ipynb | 7 +- .../workflows/valence_band_offset.ipynb | 7 +- .../notebooks_utils/core/entity/job/api.py | 6 +- .../core/entity/material/api.py | 78 +++++++++++++++---- .../core/entity/property/api.py | 61 +++++++++++---- .../ipython/entity/property/plot.py | 9 +-- 18 files changed, 308 insertions(+), 104 deletions(-) diff --git a/examples/job/create_and_submit_job.ipynb b/examples/job/create_and_submit_job.ipynb index 68b3745cf..8c4db0997 100644 --- a/examples/job/create_and_submit_job.ipynb +++ b/examples/job/create_and_submit_job.ipynb @@ -97,8 +97,15 @@ "outputs": [], "source": [ "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "default_material = client.materials.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0]\n", - "default_workflow = client.workflows.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0]\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its argument as a\n", + "# query= blob, which the materials/workflows list endpoints (migrated to validated use\n", + "# cases) silently drop since they only accept flat, declared keys.\n", + "default_material = client.materials.request(\n", + " \"GET\", client.materials.name, params={\"isDefault\": \"true\", \"ownerId\": OWNER_ID}, headers=client.materials.headers\n", + ")[0]\n", + "default_workflow = client.workflows.request(\n", + " \"GET\", client.workflows.name, params={\"isDefault\": \"true\", \"ownerId\": OWNER_ID}, headers=client.workflows.headers\n", + ")[0]\n", "\n", "material_id = default_material[\"_id\"]\n", "workflow_id = default_workflow[\"_id\"]" @@ -120,10 +127,11 @@ }, "outputs": [], "source": [ + "workflow = client.workflows.get(workflow_id)\n", "config = {\n", " \"owner\": {\"_id\": OWNER_ID},\n", " \"_material\": {\"_id\": material_id},\n", - " \"workflow\": {\"_id\": workflow_id},\n", + " \"workflow\": workflow,\n", " \"name\": JOB_NAME,\n", "}" ] diff --git a/examples/job/get-file-from-job.ipynb b/examples/job/get-file-from-job.ipynb index be2961815..42a394e17 100644 --- a/examples/job/get-file-from-job.ipynb +++ b/examples/job/get-file-from-job.ipynb @@ -93,7 +93,13 @@ "# Relevant functions from the API client\n", "# Get some account information\n", "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "project_metadata = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0]\n", + "# Using .request() with flat params instead of .list() here: .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "PROJECT_QUERY = {\"isDefault\": \"true\", \"ownerId\": OWNER_ID}\n", + "project_metadata = client.projects.request(\n", + " \"GET\", client.projects.name, params=PROJECT_QUERY, headers=client.projects.headers\n", + ")[0]\n", "project_id = project_metadata[\"_id\"]\n", "owner_id = project_metadata[\"owner\"][\"_id\"]\n", "\n", @@ -106,15 +112,30 @@ "\n", "# Get materials from bank and copy one to our account\n", "MATERIAL_QUERY = {\"formula\": \"Si\"}\n", - "bank_materials = client.bank_materials.list(MATERIAL_QUERY)\n", + "bank_materials = client.bank_materials.request(\n", + " \"GET\", client.bank_materials.name, params=MATERIAL_QUERY, headers=client.bank_materials.headers\n", + ")\n", "bank_material_id = bank_materials[0][\"_id\"]\n", "material_id = client.bank_materials.copy(bank_material_id, owner_id)[\"_id\"]\n", - "materials = client.materials.list({\"owner._id\": owner_id})\n", + "materials = client.materials.request(\n", + " \"GET\", client.materials.name, params={\"ownerId\": owner_id}, headers=client.materials.headers\n", + ")\n", "\n", - "# Create the job\n", - "job = client.jobs.create_by_ids(\n", - " materials=materials, workflow_id=workflow_id, project_id=project_id, owner_id=owner_id, prefix=\"Test_Job_Output\"\n", - ")[0]\n", + "# Create the job - JobsCreate's schema requires the full workflow document embedded (a job keeps\n", + "# an immutable snapshot of what it actually ran), not just a {\"_id\": ...} reference, so fetch it\n", + "# in full before building the job config (project/material references stay {\"_id\": ...}, which the\n", + "# schema does accept for those fields).\n", + "workflow = client.workflows.get(workflow_id)\n", + "material = materials[0]\n", + "job_config = client.jobs.build_config(\n", + " material_ids=[material[\"_id\"]],\n", + " workflow_id=workflow_id,\n", + " project_id=project_id,\n", + " owner_id=owner_id,\n", + " name=\" \".join((\"Test_Job_Output\", material[\"formula\"])),\n", + ")\n", + "job_config[\"workflow\"] = workflow\n", + "job = client.jobs.create(job_config)\n", "\n", "# Submit the job\n", "client.jobs.submit(job[\"_id\"])\n", diff --git a/other/materials_designer/workflows/analyze_convex_hull.ipynb b/other/materials_designer/workflows/analyze_convex_hull.ipynb index d50ebf299..4612869c8 100644 --- a/other/materials_designer/workflows/analyze_convex_hull.ipynb +++ b/other/materials_designer/workflows/analyze_convex_hull.ipynb @@ -168,12 +168,25 @@ "# Search materials by formula (optionally restricted to MATERIAL_SET)\n", "all_materials = []\n", "\n", + "# Using .request() with a flat ownerId param instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the materials list endpoint (migrated to a validated\n", + "# use case) silently drops - without it, accessibleByUser's broader scope let materials from\n", + "# outside this account leak into the result. MaterialsList has no server-side \"formula\" filter,\n", + "# so that part of the original per-formula query is applied client-side below instead (fetching\n", + "# once up front rather than once per formula, since the formula filter no longer narrows the\n", + "# request itself).\n", + "materials_by_owner = None\n", + "if set_materials is None:\n", + " materials_by_owner = client.materials.request(\n", + " \"GET\", client.materials.name, params={\"ownerId\": ACCOUNT_ID}, headers=client.materials.headers\n", + " )\n", + " materials_by_owner = [material for material in materials_by_owner if not material.get(\"isEntitySet\")]\n", + "\n", "for formula in FORMULAS:\n", " if set_materials is not None:\n", " matches = [material for material in set_materials if material.get(\"formula\") == formula]\n", " else:\n", - " matches = client.materials.list({\"formula\": formula, \"owner._id\": ACCOUNT_ID})\n", - " matches = [material for material in matches if not material.get(\"isEntitySet\")]\n", + " matches = [material for material in materials_by_owner if material.get(\"formula\") == formula]\n", " for material in matches:\n", " material[\"_search_formula\"] = formula # track which formula query found this\n", " all_materials.extend(matches)\n", @@ -237,18 +250,22 @@ "\n", "exabyte_ids = list(set(material[\"exabyteId\"] for material in all_materials if material.get(\"exabyteId\")))\n", "\n", - "# Query properties per exabyteId to avoid API pagination truncation\n", + "# Query properties per exabyteId to avoid API pagination truncation.\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its argument as a\n", + "# query= blob, which the properties list endpoint (migrated to a validated use case)\n", + "# silently drops since it only accepts flat, declared keys - exabyteId and slug (which mirrors\n", + "# data.name on every property document) both are declared, so use those directly. GROUP/PRECISION\n", + "# have no flat equivalent in the current schema, so they're left unfiltered here for now (as they\n", + "# already effectively were, since .list()'s blob dropped them too).\n", "properties_by_exabyte_id = {}\n", "for exabyte_id in exabyte_ids:\n", " query = {\n", " \"exabyteId\": exabyte_id,\n", - " \"data.name\": \"total_energy\",\n", + " \"slug\": \"total_energy\",\n", " }\n", - " if GROUP:\n", - " query[\"group\"] = {\"$regex\": GROUP}\n", - " if PRECISION is not None:\n", - " query[\"precision.value\"] = PRECISION\n", - " props = client.properties.list(query=query)\n", + " props = client.properties.request(\n", + " \"GET\", client.properties.name, params=query, headers=client.properties.headers\n", + " )\n", " for property_holder in props:\n", " eid = property_holder.get(\"exabyteId\")\n", " eid = eid if isinstance(eid, str) else (eid[0] if eid else None)\n", diff --git a/other/materials_designer/workflows/defect_formation_energy.ipynb b/other/materials_designer/workflows/defect_formation_energy.ipynb index 664d3e016..51b33face 100644 --- a/other/materials_designer/workflows/defect_formation_energy.ipynb +++ b/other/materials_designer/workflows/defect_formation_energy.ipynb @@ -241,7 +241,9 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] @@ -262,7 +264,6 @@ "metadata": {}, "outputs": [], "source": [ - "import re\n", "from mat3ra.made.material import Material\n", "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", @@ -273,12 +274,15 @@ " if loaded is not None:\n", " print(f\"✅ Loaded '{name}' from folder: {loaded.name}\")\n", " return loaded\n", - " matches = client.materials.list({\n", - " \"name\": {\"$regex\": re.escape(name), \"$options\": \"i\"},\n", - " \"owner._id\": ACCOUNT_ID,\n", - " })\n", + " # Using .request() with flat params instead of .list(): .list() always wraps its argument as\n", + " # a query= blob, which the materials list endpoint (migrated to a validated use case)\n", + " # silently drops since it only accepts flat, declared keys - it also has no $regex support, so\n", + " # this now matches by exact name instead of a case-insensitive partial match.\n", + " matches = client.materials.request(\n", + " \"GET\", client.materials.name, params={\"name\": name, \"ownerId\": ACCOUNT_ID}, headers=client.materials.headers\n", + " )\n", " if not matches:\n", - " raise ValueError(f\"No material containing '{name}' was found in '{FOLDER}' or on the platform.\")\n", + " raise ValueError(f\"No material named '{name}' was found in '{FOLDER}' or on the platform.\")\n", " material = Material.create(matches[0])\n", " print(f\"♻️ Loaded '{name}' from platform: {matches[0]['_id']}\")\n", " return material\n", @@ -353,9 +357,16 @@ "metadata": {}, "outputs": [], "source": [ - "elemental_materials_data = client.materials.list(\n", - " {\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}},\n", + "# .list()'s query= blob is silently dropped by the migrated materials list endpoint, and\n", + "# it has no flat \"tags\"/\"metadata.element\" filter to switch to either - fetch this account's\n", + "# materials once (scoped by the flat ownerId param) and filter for elemental references in Python.\n", + "account_materials = client.materials.request(\n", + " \"GET\", client.materials.name, params={\"ownerId\": ACCOUNT_ID}, headers=client.materials.headers\n", ")\n", + "elemental_materials_data = [\n", + " material for material in account_materials\n", + " if \"elemental\" in (material.get(\"tags\") or []) and material.get(\"metadata\", {}).get(\"element\") in elements\n", + "]\n", "element_materials_by_symbol = {\n", " element: [metadata for metadata in elemental_materials_data if\n", " metadata.get(\"metadata\", {}).get(\"element\") == element]\n", @@ -446,6 +457,7 @@ "from mat3ra.wode.context.providers import PointsGridDataProvider\n", "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", "from mat3ra.notebooks_utils.workflow import patch_workflow_qe_input\n", + "from mat3ra.notebooks_utils.core.entity.workflow.api import get_or_create_workflow\n", "\n", "defect_workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(\n", " WORKFLOW_SEARCH_TERM\n", @@ -468,7 +480,13 @@ "if CHARGE:\n", " patch_workflow_qe_input(defect_workflow, {\"system\": {\"tot_charge\": CHARGE}}, unit_names=[\"pw_scf\"])\n", "\n", - "visualize_workflow(defect_workflow)" + "visualize_workflow(defect_workflow)\n", + "\n", + "# JobsCreate's schema requires the full embedded workflow document, not a bare {\"_id\": ...}\n", + "# reference (a job keeps an immutable snapshot of what it actually ran) - saving here (rather than\n", + "# passing the local, never-saved defect_workflow object straight into job creation) matches the\n", + "# same pattern used for the material below.\n", + "saved_defect_workflow_response = get_or_create_workflow(client, defect_workflow, ACCOUNT_ID)" ] }, { @@ -542,7 +560,7 @@ "defect_job_response = create_job(\n", " api_client=client,\n", " materials=materials,\n", - " workflow=defect_workflow,\n", + " workflow=saved_defect_workflow_response,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=defect_job_name,\n", diff --git a/other/materials_designer/workflows/dielectric_tensor.ipynb b/other/materials_designer/workflows/dielectric_tensor.ipynb index 96639f975..f282c3cd7 100644 --- a/other/materials_designer/workflows/dielectric_tensor.ipynb +++ b/other/materials_designer/workflows/dielectric_tensor.ipynb @@ -230,7 +230,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] @@ -564,7 +569,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=workflow,\n", + " workflow=saved_workflow_response,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", diff --git a/other/materials_designer/workflows/equation_of_state.ipynb b/other/materials_designer/workflows/equation_of_state.ipynb index d03e74496..bf7602cdf 100644 --- a/other/materials_designer/workflows/equation_of_state.ipynb +++ b/other/materials_designer/workflows/equation_of_state.ipynb @@ -197,7 +197,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"Using project: {projects[0]['name']} ({project_id})\")" ] diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index d1125b41c..08cc6bacf 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -212,7 +212,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" ] @@ -548,7 +553,7 @@ "formation_job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=formation_workflow,\n", + " workflow=saved_formation_workflow_response,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=formation_job_name,\n", diff --git a/other/materials_designer/workflows/homo_lumo_frequency.ipynb b/other/materials_designer/workflows/homo_lumo_frequency.ipynb index be81e093e..9f43e7813 100644 --- a/other/materials_designer/workflows/homo_lumo_frequency.ipynb +++ b/other/materials_designer/workflows/homo_lumo_frequency.ipynb @@ -176,7 +176,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] @@ -406,7 +411,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=workflow,\n", + " workflow=saved_workflow_response,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", @@ -466,9 +471,10 @@ "metadata": {}, "outputs": [], "source": [ + "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", - "properties_data = client.properties.get_for_job(job_id)\n", + "properties_data = get_properties_for_job(client, job_id)\n", "visualize_properties(properties_data)\n" ] }, @@ -479,11 +485,12 @@ "metadata": {}, "outputs": [], "source": [ + "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", "if CALCULATE_HOMO_LUMO:\n", - " homo_data = client.properties.get_for_job(job_id, property_name=\"homo_energy\")\n", - " lumo_data = client.properties.get_for_job(job_id, property_name=\"lumo_energy\")\n", + " homo_data = get_properties_for_job(client, job_id, property_name=\"homo_energy\")\n", + " lumo_data = get_properties_for_job(client, job_id, property_name=\"lumo_energy\")\n", " visualize_properties(homo_data, title=\"HOMO Energy\")\n", " visualize_properties(lumo_data, title=\"LUMO Energy\")\n", " homo_lumo_gap = lumo_data[0][\"value\"] - homo_data[0][\"value\"]\n", @@ -497,10 +504,12 @@ "metadata": {}, "outputs": [], "source": [ + "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", + "\n", "if CALCULATE_FREQUENCY:\n", - " zero_point_energy_data = client.properties.get_for_job(job_id, property_name=\"zero_point_energy\")\n", - " thermal_correction_to_energy_data = client.properties.get_for_job(job_id, property_name=\"thermal_correction_to_energy\")\n", - " thermal_correction_to_enthalpy_data = client.properties.get_for_job(job_id, property_name=\"thermal_correction_to_enthalpy\")\n", + " zero_point_energy_data = get_properties_for_job(client, job_id, property_name=\"zero_point_energy\")\n", + " thermal_correction_to_energy_data = get_properties_for_job(client, job_id, property_name=\"thermal_correction_to_energy\")\n", + " thermal_correction_to_enthalpy_data = get_properties_for_job(client, job_id, property_name=\"thermal_correction_to_enthalpy\")\n", " visualize_properties(zero_point_energy_data, title=\"Zero Point Energy\")\n", " visualize_properties(thermal_correction_to_energy_data, title=\"Thermal Correction to Energy\")\n", " visualize_properties(thermal_correction_to_enthalpy_data, title=\"Thermal Correction to Enthalpy\")\n" diff --git a/other/materials_designer/workflows/neb.ipynb b/other/materials_designer/workflows/neb.ipynb index 11fc32bd6..f396378a5 100644 --- a/other/materials_designer/workflows/neb.ipynb +++ b/other/materials_designer/workflows/neb.ipynb @@ -224,7 +224,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] @@ -499,7 +504,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=saved_materials,\n", - " workflow=workflow,\n", + " workflow=saved_workflow_response,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", @@ -565,15 +570,16 @@ "metadata": {}, "outputs": [], "source": [ + "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", - "profile_data = client.properties.get_for_job(job_id, property_name=\"reaction_energy_profile\")\n", + "profile_data = get_properties_for_job(client, job_id, property_name=\"reaction_energy_profile\")\n", "if not profile_data:\n", " raise ValueError(f\"Job {job_id} finished but returned no reaction_energy_profile.\")\n", "visualize_properties(profile_data, title=\"Reaction Energy Profile\")\n", "\n", "energies = profile_data[0][\"yDataSeries\"][0]\n", - "print(f\"Peak reaction energy = {max(energies):.3f} eV\")" + "print(f\"Peak reaction energy = {max(energies):.3f} eV\")\n" ] }, { @@ -583,8 +589,10 @@ "metadata": {}, "outputs": [], "source": [ - "barrier_data = client.properties.get_for_job(job_id, property_name=\"reaction_energy_barrier\")\n", - "visualize_properties(barrier_data, title=\"Reaction Energy Barrier\")" + "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", + "\n", + "barrier_data = get_properties_for_job(client, job_id, property_name=\"reaction_energy_barrier\")\n", + "visualize_properties(barrier_data, title=\"Reaction Energy Barrier\")\n" ] } ], diff --git a/other/materials_designer/workflows/phonon_dos_dispersion.ipynb b/other/materials_designer/workflows/phonon_dos_dispersion.ipynb index 20472e800..d490fe228 100644 --- a/other/materials_designer/workflows/phonon_dos_dispersion.ipynb +++ b/other/materials_designer/workflows/phonon_dos_dispersion.ipynb @@ -229,7 +229,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] @@ -568,7 +573,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=workflow,\n", + " workflow=saved_workflow_response,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", diff --git a/other/materials_designer/workflows/relaxation.ipynb b/other/materials_designer/workflows/relaxation.ipynb index 07e426d76..6f53ad5f5 100644 --- a/other/materials_designer/workflows/relaxation.ipynb +++ b/other/materials_designer/workflows/relaxation.ipynb @@ -218,7 +218,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" ] @@ -512,7 +517,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=workflow,\n", + " workflow=saved_workflow_response,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", diff --git a/other/materials_designer/workflows/surface_energy.ipynb b/other/materials_designer/workflows/surface_energy.ipynb index 20cf21ebe..98c0db548 100644 --- a/other/materials_designer/workflows/surface_energy.ipynb +++ b/other/materials_designer/workflows/surface_energy.ipynb @@ -209,7 +209,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" ] @@ -230,7 +235,6 @@ "metadata": {}, "outputs": [], "source": [ - "import re\n", "from mat3ra.made.material import Material\n", "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", @@ -238,20 +242,23 @@ "slab = load_material_from_folder(FOLDER, SLAB_NAME)\n", "\n", "if slab is None:\n", - " slab_matches = client.materials.list({\n", - " \"name\": {\"$regex\": re.escape(SLAB_NAME), \"$options\": \"i\"},\n", - " \"owner._id\": ACCOUNT_ID,\n", - " })\n", + " # Using .request() with flat params instead of .list(): .list() always wraps its argument as\n", + " # a query= blob, which the materials list endpoint (migrated to a validated use case)\n", + " # silently drops since it only accepts flat, declared keys - it also has no $regex support, so\n", + " # this now matches by exact name instead of a case-insensitive partial match.\n", + " slab_matches = client.materials.request(\n", + " \"GET\", client.materials.name, params={\"name\": SLAB_NAME, \"ownerId\": ACCOUNT_ID}, headers=client.materials.headers\n", + " )\n", " if not slab_matches:\n", " raise ValueError(\n", - " f\"No slab containing '{SLAB_NAME}' was found in '{FOLDER}' or on the platform.\"\n", + " f\"No slab named '{SLAB_NAME}' was found in '{FOLDER}' or on the platform.\"\n", " )\n", " slab = Material.create(slab_matches[0])\n", " print(f\"♻️ Loaded slab from platform: {slab_matches[0]['_id']}\")\n", "else:\n", " print(f\"✅ Loaded slab from folder: {slab.name}\")\n", "\n", - "visualize(slab)" + "visualize(slab)\n" ] }, { @@ -517,11 +524,12 @@ "outputs": [], "source": [ "from mat3ra.prode import PropertyName\n", + "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", - "surface_energy_data = client.properties.get_for_job(surface_job_id, property_name=\"surface_energy\")\n", - "slab_total_energy_data = client.properties.get_for_job(\n", - " surface_job_id, property_name=PropertyName.scalar.total_energy.value\n", + "surface_energy_data = get_properties_for_job(client, surface_job_id, property_name=\"surface_energy\")\n", + "slab_total_energy_data = get_properties_for_job(\n", + " client, surface_job_id, property_name=PropertyName.scalar.total_energy.value\n", ")\n", "\n", "visualize_properties(surface_energy_data, title=\"Surface Energy\")\n", @@ -529,7 +537,7 @@ "print(f\"Slab: {saved_slab.name} ({saved_slab.id}), \"\n", " f\"total energy: {slab_total_energy_data[0]['value']} eV\")\n", "print(f\"Bulk: {bulk_material.name} ({bulk_material.id}), \"\n", - " f\"total energy: {bulk_total_energy_property['data']['value']} eV\")" + " f\"total energy: {bulk_total_energy_property['data']['value']} eV\")\n" ] } ], diff --git a/other/materials_designer/workflows/total_energy_convergence.ipynb b/other/materials_designer/workflows/total_energy_convergence.ipynb index 7beb3c53f..410602f14 100644 --- a/other/materials_designer/workflows/total_energy_convergence.ipynb +++ b/other/materials_designer/workflows/total_energy_convergence.ipynb @@ -181,7 +181,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "PROJECT_ID = projects[0][\"_id\"]\n", "print(f\"Using project: {projects[0]['name']} ({PROJECT_ID})\")\n" ] diff --git a/other/materials_designer/workflows/valence_band_offset.ipynb b/other/materials_designer/workflows/valence_band_offset.ipynb index 92f73b3a4..40cf7cb5e 100644 --- a/other/materials_designer/workflows/valence_band_offset.ipynb +++ b/other/materials_designer/workflows/valence_band_offset.ipynb @@ -234,7 +234,12 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "# Using .request() with flat params instead of .list(): .list() always wraps its\n", + "# argument as a query= blob, which the projects list endpoint (migrated to a\n", + "# validated use case) silently drops since it only accepts flat, declared keys.\n", + "projects = client.projects.request(\n", + " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", + ")\n", "project_id = projects[0][\"_id\"]\n", "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" ] diff --git a/src/py/mat3ra/notebooks_utils/core/entity/job/api.py b/src/py/mat3ra/notebooks_utils/core/entity/job/api.py index 4c72c0829..94fd43069 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/job/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/job/api.py @@ -35,7 +35,11 @@ def get_jobs_statuses_by_ids(endpoint: JobEndpoints, job_ids: List[str]) -> List Returns: list: list of job statuses """ - jobs = endpoint.list({"_id": {"$in": job_ids}}, {"fields": {"status": 1}}) + # .list()'s query= is silently dropped by the migrated JobsList use case, which + # only accepts flat, declared keys - unfiltered, it returns every job in the account, so the + # wait loop above never sees the tracked jobs reach a terminal status. `id` is such a flat key + # (accepts a single id or an array), so use .request() directly with it instead. + jobs = endpoint.request("GET", endpoint.name, params={"id": job_ids}, headers=endpoint.headers) return [job["status"] for job in jobs] diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py index 8d626c7c6..49b775a1b 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py @@ -27,7 +27,14 @@ def get_or_create_material(api_client: APIClient, material, owner_id: str) -> di Returns: dict: The material dict (existing or newly created). """ - existing = api_client.materials.list({"hash": material.hash, "owner._id": owner_id}) + # Using .request() with flat params instead of .list(): .list() always wraps its argument as + # a query= blob, which the materials list endpoint (migrated to a validated use case) + # silently drops since it only accepts flat, declared keys - "hashes" is the flat equivalent + # of a single "hash" (it maps onto the same $in-based DAO filter). + existing = api_client.materials.request( + "GET", api_client.materials.name, params={"hashes": material.hash, "ownerId": owner_id}, + headers=api_client.materials.headers, + ) if existing: print(f"♻️ Reusing already existing Material: {existing[0]['_id']}") return existing[0] @@ -56,7 +63,10 @@ def load_material(api_client: APIClient, folder: str, name: str, owner_id: str) loaded = load_material_from_folder(folder, name, verbose=False) if os.path.isdir(folder) else None if loaded is not None and loaded.name == name: return loaded - matches = api_client.materials.list({"name": name, "owner._id": owner_id}, {"limit": 1}) + matches = api_client.materials.request( + "GET", api_client.materials.name, params={"name": name, "ownerId": owner_id, "limit": 1}, + headers=api_client.materials.headers, + ) if not matches: raise ValueError(f"No material named '{name}' in '{folder}' or in the account") return Material.create(matches[0]) @@ -83,9 +93,19 @@ def find_relaxed_material(api_client: APIClient, material, owner_id: str) -> Opt Returns: Material, optional: The relaxed structure, or None if none exists. """ - ids = [m["_id"] for m in api_client.materials.list({"hash": material.hash, "owner._id": owner_id})] - query = {"_material._id": {"$in": ids}, "owner._id": owner_id, "status": "finished"} - for job in api_client.jobs.list(query): + matching_materials = api_client.materials.request( + "GET", api_client.materials.name, params={"hashes": material.hash, "ownerId": owner_id}, + headers=api_client.materials.headers, + ) + ids = {m["_id"] for m in matching_materials} + # JobsList has no flat "materialId"/"_material._id" filter, so fetch this account's finished + # jobs (flat ownerId + status params) and narrow to these material ids in Python. + finished_jobs = api_client.jobs.request( + "GET", api_client.jobs.name, params={"ownerId": owner_id, "status": "finished"}, + headers=api_client.jobs.headers, + ) + matching_jobs = [job for job in finished_jobs if job.get("_material", {}).get("_id") in ids] + for job in matching_jobs: properties = api_client.properties.get_for_job(job["_id"], PropertyName.non_scalar.final_structure.value) if not properties: continue @@ -133,7 +153,28 @@ def get_bulk_material_by_crystal(api_client: APIClient, bulk_crystal: Material, def _require_material_for_owner(api_client: APIClient, query: dict, owner_id: str) -> Material: - matches = api_client.materials.list({**query, "owner._id": owner_id}) + # Using .request() with flat params instead of .list(): .list() always wraps its argument as + # a query= blob, which the materials list endpoint (migrated to a validated use case) + # silently drops. "_id" and "hash" have flat equivalents ("id"/"hashes"); "scaledHash" (the + # last fallback in resolve_bulk_query_from_crystal) does not, so that case fetches this + # account's materials and filters in Python instead. + if "_id" in query: + matches = api_client.materials.request( + "GET", api_client.materials.name, params={"id": query["_id"], "ownerId": owner_id}, + headers=api_client.materials.headers, + ) + elif "hash" in query: + matches = api_client.materials.request( + "GET", api_client.materials.name, params={"hashes": query["hash"], "ownerId": owner_id}, + headers=api_client.materials.headers, + ) + else: + scaled_hash = query["scaledHash"] + account_materials = api_client.materials.request( + "GET", api_client.materials.name, params={"ownerId": owner_id}, + headers=api_client.materials.headers, + ) + matches = [m for m in account_materials if m.get("scaledHash") == scaled_hash] material_response = next(iter(matches), None) if material_response is None: raise ValueError( @@ -176,13 +217,17 @@ def find_material_set( Raises: ValueError: If no set matches, or if `require_ordered` and the match is unordered. """ - material_sets = api_client.materials.list( - { - "owner._id": owner_id, - "isEntitySet": True, - "name": {"$regex": re.escape(material_set_name), "$options": "i"}, - } + # Using .request() with flat params instead of .list(): .list() always wraps its argument as + # a query= blob, which the materials list endpoint (migrated to a validated use case) + # silently drops. "isEntitySet" is a flat key, but there's no flat regex/substring match on + # "name", so that part is applied in Python instead (entity sets per account are few, so + # fetching them all and filtering here is cheap). + account_entity_sets = api_client.materials.request( + "GET", api_client.materials.name, params={"ownerId": owner_id, "isEntitySet": "true"}, + headers=api_client.materials.headers, ) + name_pattern = re.compile(re.escape(material_set_name), re.IGNORECASE) + material_sets = [s for s in account_entity_sets if name_pattern.search(s.get("name", ""))] if not material_sets: raise ValueError(f"No material set matching '{material_set_name}'") material_set = material_sets[0] @@ -205,8 +250,13 @@ def list_materials_in_set(api_client: APIClient, owner_id: str, material_set: Di that already have one do not re-query for it. """ material_set_id = material_set["_id"] - matches = api_client.materials.list( - {"owner._id": owner_id, "inSet._id": material_set_id, "isEntitySet": {"$ne": True}} + # Using .request() with flat params instead of .list(): "setId"/"isEntitySet" are both flat, + # declared keys ("isEntitySet": "false" is the exact-match equivalent of "$ne": True for a + # boolean field), unlike the dot-notation "inSet._id" .list() sent before. + matches = api_client.materials.request( + "GET", api_client.materials.name, + params={"ownerId": owner_id, "setId": material_set_id, "isEntitySet": "false"}, + headers=api_client.materials.headers, ) members = [material for material in matches if not material.get("isEntitySet")] return sorted(members, key=lambda material: _index_in_set(material, material_set_id)) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/property/api.py b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py index 050c16f4f..d0cb47134 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/property/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py @@ -11,19 +11,42 @@ } +def _get_properties_for_job( + client: APIClient, job_id: str, property_name: Optional[str] = None, unit_id: Optional[str] = None +) -> List[dict]: + """ + Replacement for the broken `client.properties.get_for_job()`: that method builds its filter + as `{"source.info.jobId": ..., "data.name": ...}` via `.list()`, which wraps it as a + query= blob - the properties list endpoint (migrated to a validated use case) silently + drops it, since it only accepts flat, declared keys. "jobId"/"slug" are the flat equivalents + of "source.info.jobId"/"data.name"; there's no flat equivalent for "source.info.unitId", so + that part (rare - only used for disambiguating fermiEnergy by flowchart unit) is filtered in + Python. Returns each match's "data" sub-dict, matching get_for_job()'s own return shape. + """ + params = {"jobId": job_id} + if property_name: + params["slug"] = property_name + holders = client.properties.request( + "GET", client.properties.name, params=params, headers=client.properties.headers + ) + if unit_id: + holders = [h for h in holders if h.get("source", {}).get("info", {}).get("unitId") == unit_id] + return [holder["data"] for holder in holders] + + def get_properties_for_job(client: APIClient, job_id: str, property_name: Optional[str] = None) -> List[dict]: """ Fetch properties for a job, automatically enriching band_structure/DOS results with fermiEnergy. Use instead of client.properties.get_for_job when passing results to visualize_properties. """ job = client.jobs.get(job_id) - properties = client.properties.get_for_job(job_id, property_name) + properties = _get_properties_for_job(client, job_id, property_name) if property_name not in FERMI_ENERGY_PROPERTIES: return properties flowchart_id = get_fermi_energy_flowchart_id(job) fermi_energy = None if flowchart_id: - fe_props = client.properties.get_for_job(job_id, PropertyName.scalar.fermi_energy.value, flowchart_id) + fe_props = _get_properties_for_job(client, job_id, PropertyName.scalar.fermi_energy.value, flowchart_id) if fe_props: fermi_energy = fe_props[0].get("value") return [{**prop, "fermiEnergy": fermi_energy} for prop in properties] @@ -44,13 +67,15 @@ def get_property_holder_for_job( Returns: dict: Full property holder document. """ - query = { - "source.info.jobId": job_id, - "data.name": property_name, - } + # Using .request() with flat params instead of .list(): see _get_properties_for_job above for + # why, and note this function (unlike get_properties_for_job) needs the full property holder, + # not just its "data" sub-dict. + params = {"jobId": job_id, "slug": property_name} + holders = client.properties.request( + "GET", client.properties.name, params=params, headers=client.properties.headers + ) if unit_id: - query["source.info.unitId"] = unit_id - holders = client.properties.list(query=query) + holders = [h for h in holders if h.get("source", {}).get("info", {}).get("unitId") == unit_id] if not holders: raise ValueError(f"Property '{property_name}' not found for job '{job_id}'") return holders[0] @@ -94,18 +119,24 @@ def find_total_energy_for_material(client: APIClient, material_id: str, source: exabyte_id = material.get("exabyteId") if not exabyte_id: return None - query = {"exabyteId": exabyte_id, "slug": "total_energy"} + # Using .request() with flat params instead of .list(): see _get_properties_for_job above for + # why. "owner.slug"/"owner._id" map onto the flat "ownerSlug"/"ownerId" keys; sorting/limiting + # by best precision has no flat equivalent (the endpoint's "sort" param expects a nested + # object, which a simple flat query param can't carry), so that's done in Python instead - + # there are only ever a few differently-precise total_energy properties per material. + params = {"exabyteId": exabyte_id, "slug": "total_energy"} if source == "curators": - query["owner.slug"] = "curators" + params["ownerSlug"] = "curators" elif source == "my_account": - query["owner._id"] = client.my_account.id + params["ownerId"] = client.my_account.id elif source != "public": raise ValueError(f"Invalid source: {source!r}. Expected 'public', 'curators', or 'my_account'.") - properties = client.properties.list( - query=query, - projection={"sort": {"precision.value": -1}, "limit": 1}, + properties = client.properties.request( + "GET", client.properties.name, params=params, headers=client.properties.headers ) - return properties[0] if properties else None + if not properties: + return None + return max(properties, key=lambda prop: prop.get("precision", {}).get("value", 0)) def get_property_by_subworkflow_and_unit_indicies( diff --git a/src/py/mat3ra/notebooks_utils/ipython/entity/property/plot.py b/src/py/mat3ra/notebooks_utils/ipython/entity/property/plot.py index 2fddefed8..926247380 100644 --- a/src/py/mat3ra/notebooks_utils/ipython/entity/property/plot.py +++ b/src/py/mat3ra/notebooks_utils/ipython/entity/property/plot.py @@ -23,7 +23,6 @@ FONT_SIZE_LABEL_STABLE = 16 FONT_SIZE_LABEL_UNSTABLE = 14 FONT_SIZE_LEGEND = 14 -TEXT_SHADOW = "2px 2px 4px rgba(0,0,0,0.8), " "-2px -2px 4px rgba(0,0,0,0.8), " "0px 0px 8px rgba(0,0,0,0.9)" # Markers MARKER_SIZE = 20 @@ -60,14 +59,10 @@ def plot_convex_hull(phase_diagram: PhaseDiagram, show_unstable: float = 0.2) -> trace.marker.size = MARKER_SIZE if trace.name == "Stable": trace.textposition = "top center" - trace.textfont = dict( - size=FONT_SIZE_LABEL_STABLE, color=TEXT_COLOR_PRIMARY, family=FONT_FAMILY, shadow=TEXT_SHADOW - ) + trace.textfont = dict(size=FONT_SIZE_LABEL_STABLE, color=TEXT_COLOR_PRIMARY, family=FONT_FAMILY) else: trace.textposition = "bottom center" - trace.textfont = dict( - size=FONT_SIZE_LABEL_UNSTABLE, color=TEXT_COLOR_UNSTABLE, family=FONT_FAMILY, shadow=TEXT_SHADOW - ) + trace.textfont = dict(size=FONT_SIZE_LABEL_UNSTABLE, color=TEXT_COLOR_UNSTABLE, family=FONT_FAMILY) axis_style = dict( title=dict(font=dict(size=FONT_SIZE_TITLE, color=TEXT_COLOR_PRIMARY)), From 3bfeebfa3d7f162bdf768d15a97e0d77defb08d4 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Thu, 24 Sep 2026 14:44:12 +0300 Subject: [PATCH 02/10] fix(SOF-8032): replace broken client.list()/get_for_job() calls and add missing pandas dep Notebooks used client.projects.list()/client.jobs.create_set()/client.properties.get_for_job(), which build their filters as raw dot-notation query blobs or the old {name, owner} shape. Endpoints migrated to validated use cases silently drop undeclared keys (AJV removeAdditional) or reject the request outright, so these calls returned unfiltered/wrong results (e.g. always the account's first project or property) or a 422. Added get_default_project (mirroring the already-fixed get_or_create_material/get_or_create_workflow pattern) and applied it across the affected notebooks; fixed jobs.create_set's param shape and get_or_create_workflow's own .list() call in equation_of_state.ipynb and workflow/api.py; swapped equation_of_state.ipynb's direct client.properties.get_for_job() (broken in the vendored api-client itself) for the already-fixed get_properties_for_job() helper. Also added the pandas dependency the mat3ra-standata package needs, missing from specific_examples/api_examples' package lists. Co-Authored-By: Claude Sonnet 5 --- config.yml | 2 ++ .../workflows/band_gap.ipynb | 8 +++-- .../workflows/band_structure.ipynb | 8 +++-- .../workflows/band_structure_hse.ipynb | 8 +++-- .../workflows/band_structure_magn.ipynb | 8 +++-- .../workflows/convergence.ipynb | 8 +++-- .../workflows/custom_python_calculation.ipynb | 8 +++-- .../workflows/custom_shell_calculation.ipynb | 8 +++-- .../workflows/equation_of_state.ipynb | 7 +++-- .../workflows/homo_lumo_frequency.ipynb | 26 +++++++++-------- .../workflows/interfacial_energy.ipynb | 8 +++-- .../workflows/total_energy.ipynb | 8 +++-- .../total_energy_post_processing.ipynb | 8 +++-- .../workflows/zero_point_energy.ipynb | 8 +++-- .../core/entity/project/__init__.py | 0 .../core/entity/project/api.py | 29 +++++++++++++++++++ .../core/entity/workflow/api.py | 9 +++++- 17 files changed, 112 insertions(+), 49 deletions(-) create mode 100644 src/py/mat3ra/notebooks_utils/core/entity/project/__init__.py create mode 100644 src/py/mat3ra/notebooks_utils/core/entity/project/api.py diff --git a/config.yml b/config.yml index 5a5c5477d..c3e961b35 100644 --- a/config.yml +++ b/config.yml @@ -51,6 +51,7 @@ notebooks: - emfs:/drive/packages/watchdog-2.3.1-py3-none-any.whl - name: specific_examples packages_pyodide: + - pandas - mat3ra-standata # - mat3ra-notebooks-utils[utils_standata] - name: analyze_convex_hull @@ -75,6 +76,7 @@ notebooks: packages_pyodide: - annotated_types>=0.6.0 - jinja2 + - pandas - emfs:/drive/packages/pydantic_core-2.18.2-py3-none-any.whl - emfs:/drive/packages/pydantic-2.7.1-py3-none-any.whl - mat3ra-standata diff --git a/other/materials_designer/workflows/band_gap.ipynb b/other/materials_designer/workflows/band_gap.ipynb index 7bb2f819a..d672ab48f 100644 --- a/other/materials_designer/workflows/band_gap.ipynb +++ b/other/materials_designer/workflows/band_gap.ipynb @@ -218,9 +218,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/band_structure.ipynb b/other/materials_designer/workflows/band_structure.ipynb index 76790c41c..3a261d050 100644 --- a/other/materials_designer/workflows/band_structure.ipynb +++ b/other/materials_designer/workflows/band_structure.ipynb @@ -225,9 +225,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/band_structure_hse.ipynb b/other/materials_designer/workflows/band_structure_hse.ipynb index 62fa647ba..d9e8aff69 100644 --- a/other/materials_designer/workflows/band_structure_hse.ipynb +++ b/other/materials_designer/workflows/band_structure_hse.ipynb @@ -219,9 +219,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/band_structure_magn.ipynb b/other/materials_designer/workflows/band_structure_magn.ipynb index f8240c26f..ba15b7042 100644 --- a/other/materials_designer/workflows/band_structure_magn.ipynb +++ b/other/materials_designer/workflows/band_structure_magn.ipynb @@ -252,9 +252,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/convergence.ipynb b/other/materials_designer/workflows/convergence.ipynb index 7ecc4d298..8e17788df 100644 --- a/other/materials_designer/workflows/convergence.ipynb +++ b/other/materials_designer/workflows/convergence.ipynb @@ -185,9 +185,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "PROJECT_ID = projects[0][\"_id\"]\n", - "print(f\"Using project: {projects[0]['name']} ({PROJECT_ID})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "PROJECT_ID = project[\"_id\"]\n", + "print(f\"Using project: {project['name']} ({PROJECT_ID})\")" ] }, { diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 5f12e976b..ea736b954 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -247,9 +247,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/custom_shell_calculation.ipynb b/other/materials_designer/workflows/custom_shell_calculation.ipynb index 25a44f821..8079b92eb 100644 --- a/other/materials_designer/workflows/custom_shell_calculation.ipynb +++ b/other/materials_designer/workflows/custom_shell_calculation.ipynb @@ -338,9 +338,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/equation_of_state.ipynb b/other/materials_designer/workflows/equation_of_state.ipynb index bf7602cdf..0ba08702e 100644 --- a/other/materials_designer/workflows/equation_of_state.ipynb +++ b/other/materials_designer/workflows/equation_of_state.ipynb @@ -534,7 +534,7 @@ "outputs": [], "source": [ "jobs_set_name = f\"{JOBS_SET_PREFIX} {base_label} {timestamp}\"\n", - "jobs_set = client.jobs.create_set({\"name\": jobs_set_name, \"owner\": {\"_id\": ACCOUNT_ID}})\n", + "jobs_set = client.jobs.create_set({\"name\": jobs_set_name, \"projectId\": project_id})\n", "print(f\"Created jobs set: {jobs_set['name']} ({jobs_set['_id']})\")" ] }, @@ -641,12 +641,13 @@ "outputs": [], "source": [ "from mat3ra.prode import PropertyName\n", + "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "\n", "results_records = []\n", "for record in job_records:\n", " job = client.jobs.get(record[\"job_id\"])\n", - " properties = client.properties.get_for_job(\n", - " record[\"job_id\"], property_name=PropertyName.scalar.total_energy.value\n", + " properties = get_properties_for_job(\n", + " client, record[\"job_id\"], property_name=PropertyName.scalar.total_energy.value\n", " )\n", " total_energy = properties[0].get(\"value\") if properties else None\n", " results_records.append(\n", diff --git a/other/materials_designer/workflows/homo_lumo_frequency.ipynb b/other/materials_designer/workflows/homo_lumo_frequency.ipynb index 9f43e7813..b8b8b9013 100644 --- a/other/materials_designer/workflows/homo_lumo_frequency.ipynb +++ b/other/materials_designer/workflows/homo_lumo_frequency.ipynb @@ -158,7 +158,7 @@ " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", "\n", "ACCOUNT_ID = selected_account.id\n", - "print(f\"✅ Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" + "print(f\"\u2705 Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" ] }, { @@ -183,7 +183,7 @@ " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", ")\n", "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "print(f\"\u2705 Using project: {projects[0]['name']} ({project_id})\")" ] }, { @@ -202,7 +202,6 @@ "metadata": {}, "outputs": [], "source": [ - "import re\n", "from mat3ra.made.material import Material\n", "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", @@ -211,16 +210,19 @@ "def load_material(name):\n", " loaded = load_material_from_folder(FOLDER, name)\n", " if loaded is not None:\n", - " print(f\"✅ Loaded '{name}' from folder: {loaded.name}\")\n", + " print(f\"\u2705 Loaded '{name}' from folder: {loaded.name}\")\n", " return loaded\n", - " matches = client.materials.list({\n", - " \"name\": {\"$regex\": re.escape(name), \"$options\": \"i\"},\n", - " \"owner._id\": ACCOUNT_ID,\n", - " })\n", + " # Using .request() with flat params instead of .list(): .list() always wraps its\n", + " # argument as a query= blob, which the materials list endpoint (migrated to a\n", + " # validated use case) silently drops since it only accepts flat, declared keys.\n", + " matches = client.materials.request(\n", + " \"GET\", client.materials.name, params={\"name\": name, \"ownerId\": ACCOUNT_ID},\n", + " headers=client.materials.headers,\n", + " )\n", " if not matches:\n", - " raise ValueError(f\"No material containing '{name}' was found in '{FOLDER}' or on the platform.\")\n", + " raise ValueError(f\"No material named '{name}' was found in '{FOLDER}' or on the platform.\")\n", " material = Material.create(matches[0])\n", - " print(f\"♻️ Loaded '{name}' from platform: {matches[0]['_id']}\")\n", + " print(f\"\u267b\ufe0f Loaded '{name}' from platform: {matches[0]['_id']}\")\n", " return material\n", "\n", "\n", @@ -420,7 +422,7 @@ "\n", "job = dict_to_namespace_recursive(job_response)\n", "job_id = job._id\n", - "print(\"✅ Job created successfully!\")\n", + "print(\"\u2705 Job created successfully!\")\n", "print(f\"Job ID: {job_id}\")\n", "display_JSON(job_response)" ] @@ -441,7 +443,7 @@ "outputs": [], "source": [ "client.jobs.submit(job_id)\n", - "print(f\"✅ Job {job_id} submitted successfully!\")" + "print(f\"\u2705 Job {job_id} submitted successfully!\")" ] }, { diff --git a/other/materials_designer/workflows/interfacial_energy.ipynb b/other/materials_designer/workflows/interfacial_energy.ipynb index dd82f217d..c06905bf2 100644 --- a/other/materials_designer/workflows/interfacial_energy.ipynb +++ b/other/materials_designer/workflows/interfacial_energy.ipynb @@ -219,9 +219,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" ] }, { diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 6b2e81711..3f2b2b880 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -195,9 +195,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/total_energy_post_processing.ipynb b/other/materials_designer/workflows/total_energy_post_processing.ipynb index 0ab88dca9..1654568fd 100644 --- a/other/materials_designer/workflows/total_energy_post_processing.ipynb +++ b/other/materials_designer/workflows/total_energy_post_processing.ipynb @@ -210,9 +210,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/zero_point_energy.ipynb b/other/materials_designer/workflows/zero_point_energy.ipynb index 55029e0d8..e4eddecd4 100644 --- a/other/materials_designer/workflows/zero_point_energy.ipynb +++ b/other/materials_designer/workflows/zero_point_energy.ipynb @@ -230,9 +230,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/src/py/mat3ra/notebooks_utils/core/entity/project/__init__.py b/src/py/mat3ra/notebooks_utils/core/entity/project/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/core/entity/project/api.py b/src/py/mat3ra/notebooks_utils/core/entity/project/api.py new file mode 100644 index 000000000..8174a99e1 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/project/api.py @@ -0,0 +1,29 @@ +from mat3ra.api_client import APIClient + + +def get_default_project(api_client: APIClient, owner_id: str) -> dict: + """ + Returns the given account's default project. + + Uses .request() with flat params instead of .list(): .list() always wraps its argument as + a query= blob, which the projects list endpoint (migrated to a validated use case) + silently drops since it only accepts flat, declared keys - unfiltered, it returns every + project visible to the session, so the first result is not necessarily this account's own. + + Args: + api_client (APIClient): API client instance carrying the authorization context. + owner_id (str): Account ID whose default project should be returned. + + Returns: + dict: The account's default project document. + + Raises: + ValueError: If the account has no default project. + """ + projects = api_client.projects.request( + "GET", api_client.projects.name, params={"isDefault": "true", "ownerId": owner_id}, + headers=api_client.projects.headers, + ) + if not projects: + raise ValueError(f"Account {owner_id} has no default project.") + return projects[0] diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py index 91a590cfc..b554f5802 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -14,7 +14,14 @@ def get_or_create_workflow(api_client: APIClient, workflow: Workflow, owner_id: Returns: dict: The workflow dict (existing or newly created). """ - existing = api_client.workflows.list({"hash": workflow.hash, "owner._id": owner_id}) + # Using .request() with flat params instead of .list(): .list() always wraps its argument as + # a query= blob, which the workflows list endpoint (migrated to a validated use case) + # silently drops since it only accepts flat, declared keys - "hash" and "ownerId" are those + # flat equivalents. + existing = api_client.workflows.request( + "GET", api_client.workflows.name, params={"hash": workflow.hash, "ownerId": owner_id}, + headers=api_client.workflows.headers, + ) if existing: print(f"♻️ Reusing already existing Workflow: {existing[0]['_id']}") return existing[0] From 1692a3f8087658229974a5f14c94ae19cd2eac51 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Thu, 24 Sep 2026 17:07:10 +0300 Subject: [PATCH 03/10] fix(SOF-8032): build jobs from the locally-built workflow, not the server round-trip create_job(workflow=...) was passing saved_workflow_response (the server's saved copy) instead of the notebook's own locally-built workflow object. The server round-trip drops context providers' defaults (e.g. the gamma-point kgrid), so job creation's unconditional renderAsync() recomputes them from the material instead - for neb.ipynb this silently swapped a gamma-point NEB calculation for a material-derived k-grid, converging to a qualitatively different (and wrong-signed) reaction profile. create_job() already strips _id from the workflow it's given, so passing the local object works exactly like main did before saved_workflow_response was substituted in as an incidental part of an unrelated query-blob fix. Also fixes stray \uXXXX-escaped unicode (from an earlier json.dump without ensure_ascii=False) in homo_lumo_frequency.ipynb. Co-Authored-By: Claude Sonnet 5 --- .../workflows/defect_formation_energy.ipynb | 2 +- .../workflows/dielectric_tensor.ipynb | 2 +- .../workflows/homo_lumo_frequency.ipynb | 14 +++++++------- other/materials_designer/workflows/neb.ipynb | 2 +- .../workflows/phonon_dos_dispersion.ipynb | 2 +- .../materials_designer/workflows/relaxation.ipynb | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/other/materials_designer/workflows/defect_formation_energy.ipynb b/other/materials_designer/workflows/defect_formation_energy.ipynb index 51b33face..7b80ee974 100644 --- a/other/materials_designer/workflows/defect_formation_energy.ipynb +++ b/other/materials_designer/workflows/defect_formation_energy.ipynb @@ -560,7 +560,7 @@ "defect_job_response = create_job(\n", " api_client=client,\n", " materials=materials,\n", - " workflow=saved_defect_workflow_response,\n", + " workflow=defect_workflow,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=defect_job_name,\n", diff --git a/other/materials_designer/workflows/dielectric_tensor.ipynb b/other/materials_designer/workflows/dielectric_tensor.ipynb index f282c3cd7..f1dedb2ee 100644 --- a/other/materials_designer/workflows/dielectric_tensor.ipynb +++ b/other/materials_designer/workflows/dielectric_tensor.ipynb @@ -569,7 +569,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=saved_workflow_response,\n", + " workflow=workflow,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", diff --git a/other/materials_designer/workflows/homo_lumo_frequency.ipynb b/other/materials_designer/workflows/homo_lumo_frequency.ipynb index b8b8b9013..fb9414085 100644 --- a/other/materials_designer/workflows/homo_lumo_frequency.ipynb +++ b/other/materials_designer/workflows/homo_lumo_frequency.ipynb @@ -158,7 +158,7 @@ " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", "\n", "ACCOUNT_ID = selected_account.id\n", - "print(f\"\u2705 Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" + "print(f\"✅ Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" ] }, { @@ -183,7 +183,7 @@ " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", ")\n", "project_id = projects[0][\"_id\"]\n", - "print(f\"\u2705 Using project: {projects[0]['name']} ({project_id})\")" + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { @@ -210,7 +210,7 @@ "def load_material(name):\n", " loaded = load_material_from_folder(FOLDER, name)\n", " if loaded is not None:\n", - " print(f\"\u2705 Loaded '{name}' from folder: {loaded.name}\")\n", + " print(f\"✅ Loaded '{name}' from folder: {loaded.name}\")\n", " return loaded\n", " # Using .request() with flat params instead of .list(): .list() always wraps its\n", " # argument as a query= blob, which the materials list endpoint (migrated to a\n", @@ -222,7 +222,7 @@ " if not matches:\n", " raise ValueError(f\"No material named '{name}' was found in '{FOLDER}' or on the platform.\")\n", " material = Material.create(matches[0])\n", - " print(f\"\u267b\ufe0f Loaded '{name}' from platform: {matches[0]['_id']}\")\n", + " print(f\"♻️ Loaded '{name}' from platform: {matches[0]['_id']}\")\n", " return material\n", "\n", "\n", @@ -413,7 +413,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=saved_workflow_response,\n", + " workflow=workflow,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", @@ -422,7 +422,7 @@ "\n", "job = dict_to_namespace_recursive(job_response)\n", "job_id = job._id\n", - "print(\"\u2705 Job created successfully!\")\n", + "print(\"✅ Job created successfully!\")\n", "print(f\"Job ID: {job_id}\")\n", "display_JSON(job_response)" ] @@ -443,7 +443,7 @@ "outputs": [], "source": [ "client.jobs.submit(job_id)\n", - "print(f\"\u2705 Job {job_id} submitted successfully!\")" + "print(f\"✅ Job {job_id} submitted successfully!\")" ] }, { diff --git a/other/materials_designer/workflows/neb.ipynb b/other/materials_designer/workflows/neb.ipynb index f396378a5..73b651d96 100644 --- a/other/materials_designer/workflows/neb.ipynb +++ b/other/materials_designer/workflows/neb.ipynb @@ -504,7 +504,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=saved_materials,\n", - " workflow=saved_workflow_response,\n", + " workflow=workflow,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", diff --git a/other/materials_designer/workflows/phonon_dos_dispersion.ipynb b/other/materials_designer/workflows/phonon_dos_dispersion.ipynb index d490fe228..c7267dfb6 100644 --- a/other/materials_designer/workflows/phonon_dos_dispersion.ipynb +++ b/other/materials_designer/workflows/phonon_dos_dispersion.ipynb @@ -573,7 +573,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=saved_workflow_response,\n", + " workflow=workflow,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", diff --git a/other/materials_designer/workflows/relaxation.ipynb b/other/materials_designer/workflows/relaxation.ipynb index 6f53ad5f5..09a65d756 100644 --- a/other/materials_designer/workflows/relaxation.ipynb +++ b/other/materials_designer/workflows/relaxation.ipynb @@ -517,7 +517,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=saved_workflow_response,\n", + " workflow=workflow,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", From e0d1ae6f9e5cb4148792f6bd751ee4a0a4e9dd31 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Fri, 25 Sep 2026 14:01:34 +0300 Subject: [PATCH 04/10] chore(SOF-8032): apply black formatting to the query-blob fix's call sites The 3 fix commits on this branch broke the repo's black formatting (long .request(...) calls not wrapped) - CI's run-linter failed, which also skipped run-py-tests and publish-py-package downstream. No behavior change. Co-Authored-By: Claude Sonnet 5 --- .../core/entity/material/api.py | 35 ++++++++++++++----- .../core/entity/project/api.py | 4 ++- .../core/entity/property/api.py | 8 ++--- .../core/entity/workflow/api.py | 4 ++- 4 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py index 49b775a1b..f7bf0a0ad 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py @@ -32,7 +32,9 @@ def get_or_create_material(api_client: APIClient, material, owner_id: str) -> di # silently drops since it only accepts flat, declared keys - "hashes" is the flat equivalent # of a single "hash" (it maps onto the same $in-based DAO filter). existing = api_client.materials.request( - "GET", api_client.materials.name, params={"hashes": material.hash, "ownerId": owner_id}, + "GET", + api_client.materials.name, + params={"hashes": material.hash, "ownerId": owner_id}, headers=api_client.materials.headers, ) if existing: @@ -64,7 +66,9 @@ def load_material(api_client: APIClient, folder: str, name: str, owner_id: str) if loaded is not None and loaded.name == name: return loaded matches = api_client.materials.request( - "GET", api_client.materials.name, params={"name": name, "ownerId": owner_id, "limit": 1}, + "GET", + api_client.materials.name, + params={"name": name, "ownerId": owner_id, "limit": 1}, headers=api_client.materials.headers, ) if not matches: @@ -94,14 +98,18 @@ def find_relaxed_material(api_client: APIClient, material, owner_id: str) -> Opt Material, optional: The relaxed structure, or None if none exists. """ matching_materials = api_client.materials.request( - "GET", api_client.materials.name, params={"hashes": material.hash, "ownerId": owner_id}, + "GET", + api_client.materials.name, + params={"hashes": material.hash, "ownerId": owner_id}, headers=api_client.materials.headers, ) ids = {m["_id"] for m in matching_materials} # JobsList has no flat "materialId"/"_material._id" filter, so fetch this account's finished # jobs (flat ownerId + status params) and narrow to these material ids in Python. finished_jobs = api_client.jobs.request( - "GET", api_client.jobs.name, params={"ownerId": owner_id, "status": "finished"}, + "GET", + api_client.jobs.name, + params={"ownerId": owner_id, "status": "finished"}, headers=api_client.jobs.headers, ) matching_jobs = [job for job in finished_jobs if job.get("_material", {}).get("_id") in ids] @@ -160,18 +168,24 @@ def _require_material_for_owner(api_client: APIClient, query: dict, owner_id: st # account's materials and filters in Python instead. if "_id" in query: matches = api_client.materials.request( - "GET", api_client.materials.name, params={"id": query["_id"], "ownerId": owner_id}, + "GET", + api_client.materials.name, + params={"id": query["_id"], "ownerId": owner_id}, headers=api_client.materials.headers, ) elif "hash" in query: matches = api_client.materials.request( - "GET", api_client.materials.name, params={"hashes": query["hash"], "ownerId": owner_id}, + "GET", + api_client.materials.name, + params={"hashes": query["hash"], "ownerId": owner_id}, headers=api_client.materials.headers, ) else: scaled_hash = query["scaledHash"] account_materials = api_client.materials.request( - "GET", api_client.materials.name, params={"ownerId": owner_id}, + "GET", + api_client.materials.name, + params={"ownerId": owner_id}, headers=api_client.materials.headers, ) matches = [m for m in account_materials if m.get("scaledHash") == scaled_hash] @@ -223,7 +237,9 @@ def find_material_set( # "name", so that part is applied in Python instead (entity sets per account are few, so # fetching them all and filtering here is cheap). account_entity_sets = api_client.materials.request( - "GET", api_client.materials.name, params={"ownerId": owner_id, "isEntitySet": "true"}, + "GET", + api_client.materials.name, + params={"ownerId": owner_id, "isEntitySet": "true"}, headers=api_client.materials.headers, ) name_pattern = re.compile(re.escape(material_set_name), re.IGNORECASE) @@ -254,7 +270,8 @@ def list_materials_in_set(api_client: APIClient, owner_id: str, material_set: Di # declared keys ("isEntitySet": "false" is the exact-match equivalent of "$ne": True for a # boolean field), unlike the dot-notation "inSet._id" .list() sent before. matches = api_client.materials.request( - "GET", api_client.materials.name, + "GET", + api_client.materials.name, params={"ownerId": owner_id, "setId": material_set_id, "isEntitySet": "false"}, headers=api_client.materials.headers, ) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/project/api.py b/src/py/mat3ra/notebooks_utils/core/entity/project/api.py index 8174a99e1..8f61ea0c6 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/project/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/project/api.py @@ -21,7 +21,9 @@ def get_default_project(api_client: APIClient, owner_id: str) -> dict: ValueError: If the account has no default project. """ projects = api_client.projects.request( - "GET", api_client.projects.name, params={"isDefault": "true", "ownerId": owner_id}, + "GET", + api_client.projects.name, + params={"isDefault": "true", "ownerId": owner_id}, headers=api_client.projects.headers, ) if not projects: diff --git a/src/py/mat3ra/notebooks_utils/core/entity/property/api.py b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py index d0cb47134..0b64f8dcb 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/property/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py @@ -26,9 +26,7 @@ def _get_properties_for_job( params = {"jobId": job_id} if property_name: params["slug"] = property_name - holders = client.properties.request( - "GET", client.properties.name, params=params, headers=client.properties.headers - ) + holders = client.properties.request("GET", client.properties.name, params=params, headers=client.properties.headers) if unit_id: holders = [h for h in holders if h.get("source", {}).get("info", {}).get("unitId") == unit_id] return [holder["data"] for holder in holders] @@ -71,9 +69,7 @@ def get_property_holder_for_job( # why, and note this function (unlike get_properties_for_job) needs the full property holder, # not just its "data" sub-dict. params = {"jobId": job_id, "slug": property_name} - holders = client.properties.request( - "GET", client.properties.name, params=params, headers=client.properties.headers - ) + holders = client.properties.request("GET", client.properties.name, params=params, headers=client.properties.headers) if unit_id: holders = [h for h in holders if h.get("source", {}).get("info", {}).get("unitId") == unit_id] if not holders: diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py index b554f5802..72ab276da 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -19,7 +19,9 @@ def get_or_create_workflow(api_client: APIClient, workflow: Workflow, owner_id: # silently drops since it only accepts flat, declared keys - "hash" and "ownerId" are those # flat equivalents. existing = api_client.workflows.request( - "GET", api_client.workflows.name, params={"hash": workflow.hash, "ownerId": owner_id}, + "GET", + api_client.workflows.name, + params={"hash": workflow.hash, "ownerId": owner_id}, headers=api_client.workflows.headers, ) if existing: From 3d772c999454357033271385bb37c181543de2b2 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Fri, 25 Sep 2026 14:15:19 +0300 Subject: [PATCH 05/10] fix(SOF-8032): update unit tests for the .list()->.request() migration The 3 fix commits on this branch moved several material/property functions from client.materials.list()/client.properties.list() (raw query=/projection= blobs, silently dropped by the migrated list endpoints) to client.materials.request()/client.properties.request() with flat params - but left these two test files mocking/asserting the old .list() call shape, so every test touching a migrated function failed. - Retarget all mocks/assertions from `.list(...)` to `.request("GET", .name, params=..., headers=...)`, translating each query's dot-notation/operator shape to the flat params the production code now actually sends (e.g. "owner._id" -> "ownerId", "hash" -> "hashes", "$regex" name matching -> no server-side equivalent, filtered in Python). - find_relaxed_material's job-matching narrowing ("_material._id": {"$in": [...]}) has no flat equivalent either and is now done client-side, so FINISHED_JOB (and the two job fixtures in its "skips a final structure with the same hash" test) needed a "_material" field to match against - they had none, so every finished-job branch was silently unreachable. - _get_properties_for_job now unwraps holder["data"] (matching get_for_job()'s old return shape), so test_get_final_structure_for_job's fixture needed the same {"data": {...}} wrapping its mock is standing in for. Verified locally: 72 passed (2 pre-existing, unrelated asyncio-marker failures in test_jupyterlite_interrupts.py excluded - confirmed present on origin/main too, needs pytest-asyncio which isn't a project dependency). Co-Authored-By: Claude Sonnet 5 --- .../py/unit/core/entity/test_material_api.py | 151 ++++++++++++------ ...erty_api_find_total_energy_for_material.py | 44 ++--- 2 files changed, 127 insertions(+), 68 deletions(-) diff --git a/tests/py/unit/core/entity/test_material_api.py b/tests/py/unit/core/entity/test_material_api.py index 44e7df45d..ce5df1589 100644 --- a/tests/py/unit/core/entity/test_material_api.py +++ b/tests/py/unit/core/entity/test_material_api.py @@ -57,7 +57,7 @@ def _client_with_list_responses(responses: List[List[Dict[str, Any]]]) -> MagicMock: client = MagicMock() - client.materials.list.side_effect = responses + client.materials.request.side_effect = responses return client @@ -67,12 +67,11 @@ def test_find_material_set_returns_first_match(): material_set = find_material_set(client, OWNER_ID, MATERIAL_SET_NAME) assert material_set["_id"] == MATERIAL_SET_ID - client.materials.list.assert_called_once_with( - { - "owner._id": OWNER_ID, - "isEntitySet": True, - "name": {"$regex": re.escape(MATERIAL_SET_NAME), "$options": "i"}, - } + client.materials.request.assert_called_once_with( + "GET", + client.materials.name, + params={"ownerId": OWNER_ID, "isEntitySet": "true"}, + headers=client.materials.headers, ) assert "+" in MATERIAL_SET_NAME assert re.escape(MATERIAL_SET_NAME) != MATERIAL_SET_NAME @@ -97,7 +96,7 @@ def test_list_materials_by_set_rejects_unordered_when_order_required(): with pytest.raises(ValueError, match="is 'unordered', not 'ordered'"): list_materials_by_set(client, OWNER_ID, MATERIAL_SET_NAME, require_ordered=True) - assert client.materials.list.call_count == 1 + assert client.materials.request.call_count == 1 def test_list_materials_in_set_does_not_re_resolve_the_set(): @@ -106,8 +105,11 @@ def test_list_materials_in_set_does_not_re_resolve_the_set(): materials = list_materials_in_set(client, OWNER_ID, ENTITY_SET) assert [material["_id"] for material in materials] == EXPECTED_ORDERED_IDS - client.materials.list.assert_called_once_with( - {"owner._id": OWNER_ID, "inSet._id": MATERIAL_SET_ID, "isEntitySet": {"$ne": True}} + client.materials.request.assert_called_once_with( + "GET", + client.materials.name, + params={"ownerId": OWNER_ID, "setId": MATERIAL_SET_ID, "isEntitySet": "false"}, + headers=client.materials.headers, ) @@ -124,10 +126,10 @@ def test_list_materials_by_set_orders_by_inset_index(members, expected_ids): materials = list_materials_by_set(client, OWNER_ID, MATERIAL_SET_NAME) assert [material["_id"] for material in materials] == expected_ids - assert client.materials.list.call_args_list[1].args[0] == { - "owner._id": OWNER_ID, - "inSet._id": MATERIAL_SET_ID, - "isEntitySet": {"$ne": True}, + assert client.materials.request.call_args_list[1].kwargs["params"] == { + "ownerId": OWNER_ID, + "setId": MATERIAL_SET_ID, + "isEntitySet": "false", } @@ -215,7 +217,7 @@ def test_get_or_create_materials_set_ordered_requires_two_materials(): [MATERIAL_INITIAL], is_ordered=True, ) - client.materials.list.assert_not_called() + client.materials.request.assert_not_called() def test_get_or_create_materials_set_requires_one_material(): @@ -229,14 +231,19 @@ def test_get_or_create_materials_set_requires_one_material(): [], is_ordered=False, ) - client.materials.list.assert_not_called() + client.materials.request.assert_not_called() DEFECTIVE_HASH = "hash-defective" RELAXED_HASH = "hash-relaxed" DEFECTIVE_MATERIAL = SimpleNamespace(hash=DEFECTIVE_HASH) SAVED_DEFECTIVE: Dict[str, Any] = {"_id": "m-defective", "name": "B-vacancy h-BN", "hash": DEFECTIVE_HASH} -FINISHED_JOB: Dict[str, Any] = {"_id": "job-1", "name": "Fixed-cell Relaxation", "status": "finished"} +FINISHED_JOB: Dict[str, Any] = { + "_id": "job-1", + "name": "Fixed-cell Relaxation", + "status": "finished", + "_material": {"_id": "m-defective"}, +} RELAXED_MATERIAL_DOC: Dict[str, Any] = { **Materials.get_by_name_first_match("Silicon"), "name": "B-vacancy h-BN relaxed", @@ -252,8 +259,8 @@ def test_get_or_create_materials_set_requires_one_material(): def test_find_relaxed_material_returns_final_structure_from_the_job(): client = MagicMock() - client.materials.list.return_value = [SAVED_DEFECTIVE] - client.jobs.list.return_value = [FINISHED_JOB] + client.materials.request.return_value = [SAVED_DEFECTIVE] + client.jobs.request.return_value = [FINISHED_JOB] client.properties.get_for_job.return_value = [{"materialId": "m-relaxed"}] client.materials.get.return_value = RELAXED_MATERIAL_DOC @@ -261,9 +268,17 @@ def test_find_relaxed_material_returns_final_structure_from_the_job(): assert relaxed is not None assert relaxed.name == "B-vacancy h-BN relaxed" - client.materials.list.assert_called_once_with({"hash": DEFECTIVE_HASH, "owner._id": OWNER_ID}) - client.jobs.list.assert_called_once_with( - {"_material._id": {"$in": [SAVED_DEFECTIVE["_id"]]}, "owner._id": OWNER_ID, "status": "finished"} + client.materials.request.assert_called_once_with( + "GET", + client.materials.name, + params={"hashes": DEFECTIVE_HASH, "ownerId": OWNER_ID}, + headers=client.materials.headers, + ) + client.jobs.request.assert_called_once_with( + "GET", + client.jobs.name, + params={"ownerId": OWNER_ID, "status": "finished"}, + headers=client.jobs.headers, ) client.properties.get_for_job.assert_called_once_with(FINISHED_JOB["_id"], "final_structure") client.materials.get.assert_called_once_with("m-relaxed") @@ -273,8 +288,8 @@ def test_find_relaxed_material_uses_the_last_final_structure_entry(): # A relaxation job's final_structure property can carry more than one entry: the initial # structure (same hash as the input) first, the relaxed one last -- PLAN #27. client = MagicMock() - client.materials.list.return_value = [SAVED_DEFECTIVE] - client.jobs.list.return_value = [FINISHED_JOB] + client.materials.request.return_value = [SAVED_DEFECTIVE] + client.jobs.request.return_value = [FINISHED_JOB] client.properties.get_for_job.return_value = [{"materialId": "m-initial"}, {"materialId": "m-relaxed"}] client.materials.get.side_effect = lambda material_id: { "m-initial": SCF_MATERIAL_DOC, @@ -290,17 +305,24 @@ def test_find_relaxed_material_uses_the_last_final_structure_entry(): def test_find_relaxed_material_returns_none_when_material_is_not_on_the_platform(): client = MagicMock() - client.materials.list.return_value = [] - client.jobs.list.return_value = [] + client.materials.request.return_value = [] + client.jobs.request.return_value = [] assert find_relaxed_material(client, DEFECTIVE_MATERIAL, OWNER_ID) is None - assert client.jobs.list.call_args.args[0]["_material._id"] == {"$in": []} + # JobsList has no flat material-id filter, so this account's finished jobs are always fetched + # unfiltered by material and narrowed to matching material ids in Python (see find_relaxed_material). + client.jobs.request.assert_called_once_with( + "GET", + client.jobs.name, + params={"ownerId": OWNER_ID, "status": "finished"}, + headers=client.jobs.headers, + ) def test_find_relaxed_material_returns_none_when_no_job_exists(): client = MagicMock() - client.materials.list.return_value = [SAVED_DEFECTIVE] - client.jobs.list.return_value = [] + client.materials.request.return_value = [SAVED_DEFECTIVE] + client.jobs.request.return_value = [] assert find_relaxed_material(client, DEFECTIVE_MATERIAL, OWNER_ID) is None client.properties.get_for_job.assert_not_called() @@ -308,8 +330,8 @@ def test_find_relaxed_material_returns_none_when_no_job_exists(): def test_find_relaxed_material_returns_none_when_job_has_no_final_structure(): client = MagicMock() - client.materials.list.return_value = [SAVED_DEFECTIVE] - client.jobs.list.return_value = [FINISHED_JOB] + client.materials.request.return_value = [SAVED_DEFECTIVE] + client.jobs.request.return_value = [FINISHED_JOB] client.properties.get_for_job.return_value = [] assert find_relaxed_material(client, DEFECTIVE_MATERIAL, OWNER_ID) is None @@ -319,8 +341,8 @@ def test_find_relaxed_material_returns_none_when_job_has_no_final_structure(): def test_find_relaxed_material_checks_every_same_hash_material(): other_material: Dict[str, Any] = {"_id": "m-other", "name": "B-vacancy h-BN", "hash": DEFECTIVE_HASH} client = MagicMock() - client.materials.list.return_value = [other_material, SAVED_DEFECTIVE] - client.jobs.list.return_value = [FINISHED_JOB] + client.materials.request.return_value = [other_material, SAVED_DEFECTIVE] + client.jobs.request.return_value = [FINISHED_JOB] client.properties.get_for_job.return_value = [{"materialId": "m-relaxed"}] client.materials.get.return_value = RELAXED_MATERIAL_DOC @@ -328,17 +350,31 @@ def test_find_relaxed_material_checks_every_same_hash_material(): assert relaxed is not None assert relaxed.name == "B-vacancy h-BN relaxed" - client.jobs.list.assert_called_once_with( - {"_material._id": {"$in": ["m-other", "m-defective"]}, "owner._id": OWNER_ID, "status": "finished"} + # As above: the material-id narrowing happens in Python, not in the jobs query itself. + client.jobs.request.assert_called_once_with( + "GET", + client.jobs.name, + params={"ownerId": OWNER_ID, "status": "finished"}, + headers=client.jobs.headers, ) def test_find_relaxed_material_skips_a_final_structure_with_the_same_hash(): - scf_job: Dict[str, Any] = {"_id": "job-scf", "name": "Total Energy", "status": "finished"} - relax_job: Dict[str, Any] = {"_id": "job-relax", "name": "Fixed-cell Relaxation", "status": "finished"} + scf_job: Dict[str, Any] = { + "_id": "job-scf", + "name": "Total Energy", + "status": "finished", + "_material": {"_id": "m-defective"}, + } + relax_job: Dict[str, Any] = { + "_id": "job-relax", + "name": "Fixed-cell Relaxation", + "status": "finished", + "_material": {"_id": "m-defective"}, + } client = MagicMock() - client.materials.list.return_value = [SAVED_DEFECTIVE] - client.jobs.list.return_value = [scf_job, relax_job] + client.materials.request.return_value = [SAVED_DEFECTIVE] + client.jobs.request.return_value = [scf_job, relax_job] client.properties.get_for_job.side_effect = [[{"materialId": "m-scf"}], [{"materialId": "m-relaxed"}]] client.materials.get.side_effect = [SCF_MATERIAL_DOC, RELAXED_MATERIAL_DOC] @@ -359,23 +395,28 @@ def test_load_material_finds_an_exact_match_in_the_folder(tmp_path): material = load_material(client, str(tmp_path), "Silicon", OWNER_ID) assert material.name == "Silicon" - client.materials.list.assert_not_called() + client.materials.request.assert_not_called() def test_load_material_falls_back_to_the_account(tmp_path): (tmp_path / "silicon.json").write_text(json.dumps(Materials.get_by_name_first_match("Silicon"))) client = MagicMock() - client.materials.list.return_value = [SILICON_NAMED] + client.materials.request.return_value = [SILICON_NAMED] material = load_material(client, str(tmp_path), "Silicon", OWNER_ID) assert material.name == "Silicon" - client.materials.list.assert_called_once_with({"name": "Silicon", "owner._id": OWNER_ID}, {"limit": 1}) + client.materials.request.assert_called_once_with( + "GET", + client.materials.name, + params={"name": "Silicon", "ownerId": OWNER_ID, "limit": 1}, + headers=client.materials.headers, + ) def test_load_material_raises_when_neither_has_it(tmp_path): client = MagicMock() - client.materials.list.return_value = [] + client.materials.request.return_value = [] with pytest.raises(ValueError, match="Germanium"): load_material(client, str(tmp_path), "Germanium", OWNER_ID) @@ -385,12 +426,17 @@ def test_load_material_falls_through_a_folder_near_miss(tmp_path): (tmp_path / "silicon.json").write_text(json.dumps(SILICON_NAMED)) (tmp_path / "silicon relaxed.json").write_text(json.dumps({**SILICON_NAMED, "name": "Silicon relaxed"})) client = MagicMock() - client.materials.list.return_value = [SILICON_NAMED] + client.materials.request.return_value = [SILICON_NAMED] material = load_material(client, str(tmp_path), "Silicon", OWNER_ID) assert material.name == "Silicon" - client.materials.list.assert_called_once_with({"name": "Silicon", "owner._id": OWNER_ID}, {"limit": 1}) + client.materials.request.assert_called_once_with( + "GET", + client.materials.name, + params={"name": "Silicon", "ownerId": OWNER_ID, "limit": 1}, + headers=client.materials.headers, + ) JOB_ID = "job-1" @@ -398,15 +444,15 @@ def test_load_material_falls_through_a_folder_near_miss(tmp_path): @pytest.mark.parametrize( - ("properties", "error"), + ("holders", "error"), [ - ([{"materialId": FINAL_STRUCTURE_MATERIAL_ID}], None), + ([{"data": {"materialId": FINAL_STRUCTURE_MATERIAL_ID}}], None), ([], "reported no 'final_structure'"), ], ) -def test_get_final_structure_for_job(properties, error): +def test_get_final_structure_for_job(holders, error): client = MagicMock() - client.properties.get_for_job.return_value = properties + client.properties.request.return_value = holders client.materials.get.return_value = Materials.get_by_name_first_match("Silicon") if error: with pytest.raises(RuntimeError, match=error): @@ -414,5 +460,10 @@ def test_get_final_structure_for_job(properties, error): return material = get_final_structure_for_job(client, JOB_ID) assert material.basis.elements.values == ["Si", "Si"] - client.properties.get_for_job.assert_called_once_with(JOB_ID, "final_structure") + client.properties.request.assert_called_once_with( + "GET", + client.properties.name, + params={"jobId": JOB_ID, "slug": "final_structure"}, + headers=client.properties.headers, + ) client.materials.get.assert_called_once_with(FINAL_STRUCTURE_MATERIAL_ID) diff --git a/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py b/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py index a7596ca70..787dbf862 100644 --- a/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py +++ b/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py @@ -15,7 +15,7 @@ def _client(): client = MagicMock() client.materials.get.return_value = MATERIAL - client.properties.list.return_value = [TOTAL_ENERGY_PROPERTY] + client.properties.request.return_value = [TOTAL_ENERGY_PROPERTY] client.my_account.id = OWNER_ACCOUNT_ID # entity_cache is often None right after auth -- this must not be relied on. client.my_account.entity_cache = None @@ -28,13 +28,15 @@ def test_find_total_energy_for_material_defaults_to_my_account_scope(): result = find_total_energy_for_material(client, MATERIAL_ID) client.materials.get.assert_called_once_with(MATERIAL_ID) - client.properties.list.assert_called_once_with( - query={ + client.properties.request.assert_called_once_with( + "GET", + client.properties.name, + params={ "exabyteId": EXABYTE_ID, "slug": "total_energy", - "owner._id": OWNER_ACCOUNT_ID, + "ownerId": OWNER_ACCOUNT_ID, }, - projection={"sort": {"precision.value": -1}, "limit": 1}, + headers=client.properties.headers, ) client.jobs.list.assert_not_called() assert result == TOTAL_ENERGY_PROPERTY @@ -45,13 +47,15 @@ def test_find_total_energy_for_material_my_account_scope(): find_total_energy_for_material(client, MATERIAL_ID, source="my_account") - client.properties.list.assert_called_once_with( - query={ + client.properties.request.assert_called_once_with( + "GET", + client.properties.name, + params={ "exabyteId": EXABYTE_ID, "slug": "total_energy", - "owner._id": OWNER_ACCOUNT_ID, + "ownerId": OWNER_ACCOUNT_ID, }, - projection={"sort": {"precision.value": -1}, "limit": 1}, + headers=client.properties.headers, ) @@ -60,13 +64,15 @@ def test_find_total_energy_for_material_curators_scope(): find_total_energy_for_material(client, MATERIAL_ID, source="curators") - client.properties.list.assert_called_once_with( - query={ + client.properties.request.assert_called_once_with( + "GET", + client.properties.name, + params={ "exabyteId": EXABYTE_ID, "slug": "total_energy", - "owner.slug": "curators", + "ownerSlug": "curators", }, - projection={"sort": {"precision.value": -1}, "limit": 1}, + headers=client.properties.headers, ) @@ -75,12 +81,14 @@ def test_find_total_energy_for_material_public_scope_has_no_owner_filter(): find_total_energy_for_material(client, MATERIAL_ID, source="public") - client.properties.list.assert_called_once_with( - query={ + client.properties.request.assert_called_once_with( + "GET", + client.properties.name, + params={ "exabyteId": EXABYTE_ID, "slug": "total_energy", }, - projection={"sort": {"precision.value": -1}, "limit": 1}, + headers=client.properties.headers, ) @@ -93,7 +101,7 @@ def test_find_total_energy_for_material_rejects_invalid_source(): def test_find_total_energy_for_material_returns_none_when_no_property_found(): client = _client() - client.properties.list.return_value = [] + client.properties.request.return_value = [] result = find_total_energy_for_material(client, MATERIAL_ID) @@ -106,5 +114,5 @@ def test_find_total_energy_for_material_returns_none_when_material_has_no_exabyt result = find_total_energy_for_material(client, MATERIAL_ID) - client.properties.list.assert_not_called() + client.properties.request.assert_not_called() assert result is None From 086f7b55528d21090d82e833d3e551bd8630046c Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Fri, 25 Sep 2026 15:02:06 +0300 Subject: [PATCH 06/10] fix(SOF-8032): use JobsList's flat materialId filter in find_relaxed_material JobsList already declares and implements a flat "materialId" param (JobDAO#filterByMaterialId, $in-matching _material._id/_materials._id server-side) - the earlier fix's comment claiming no such filter exists was wrong. Narrowing by material id server-side means we no longer have to fetch every one of the account's finished jobs and filter in Python, and skip the jobs call entirely when no materials match by hash. Co-Authored-By: Claude Sonnet 5 --- .../core/entity/material/api.py | 13 +++--- .../py/unit/core/entity/test_material_api.py | 40 +++++-------------- 2 files changed, 18 insertions(+), 35 deletions(-) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py index f7bf0a0ad..230bf72cd 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py @@ -103,16 +103,17 @@ def find_relaxed_material(api_client: APIClient, material, owner_id: str) -> Opt params={"hashes": material.hash, "ownerId": owner_id}, headers=api_client.materials.headers, ) - ids = {m["_id"] for m in matching_materials} - # JobsList has no flat "materialId"/"_material._id" filter, so fetch this account's finished - # jobs (flat ownerId + status params) and narrow to these material ids in Python. - finished_jobs = api_client.jobs.request( + ids = [m["_id"] for m in matching_materials] + if not ids: + return None + # JobsList's flat "materialId" param ($in-matches _material._id/_materials._id server-side - + # see JobDAO#filterByMaterialId), so no need to fetch every finished job and narrow in Python. + matching_jobs = api_client.jobs.request( "GET", api_client.jobs.name, - params={"ownerId": owner_id, "status": "finished"}, + params={"materialId": ids, "ownerId": owner_id, "status": "finished"}, headers=api_client.jobs.headers, ) - matching_jobs = [job for job in finished_jobs if job.get("_material", {}).get("_id") in ids] for job in matching_jobs: properties = api_client.properties.get_for_job(job["_id"], PropertyName.non_scalar.final_structure.value) if not properties: diff --git a/tests/py/unit/core/entity/test_material_api.py b/tests/py/unit/core/entity/test_material_api.py index ce5df1589..ae91286a4 100644 --- a/tests/py/unit/core/entity/test_material_api.py +++ b/tests/py/unit/core/entity/test_material_api.py @@ -238,12 +238,7 @@ def test_get_or_create_materials_set_requires_one_material(): RELAXED_HASH = "hash-relaxed" DEFECTIVE_MATERIAL = SimpleNamespace(hash=DEFECTIVE_HASH) SAVED_DEFECTIVE: Dict[str, Any] = {"_id": "m-defective", "name": "B-vacancy h-BN", "hash": DEFECTIVE_HASH} -FINISHED_JOB: Dict[str, Any] = { - "_id": "job-1", - "name": "Fixed-cell Relaxation", - "status": "finished", - "_material": {"_id": "m-defective"}, -} +FINISHED_JOB: Dict[str, Any] = {"_id": "job-1", "name": "Fixed-cell Relaxation", "status": "finished"} RELAXED_MATERIAL_DOC: Dict[str, Any] = { **Materials.get_by_name_first_match("Silicon"), "name": "B-vacancy h-BN relaxed", @@ -277,7 +272,7 @@ def test_find_relaxed_material_returns_final_structure_from_the_job(): client.jobs.request.assert_called_once_with( "GET", client.jobs.name, - params={"ownerId": OWNER_ID, "status": "finished"}, + params={"materialId": [SAVED_DEFECTIVE["_id"]], "ownerId": OWNER_ID, "status": "finished"}, headers=client.jobs.headers, ) client.properties.get_for_job.assert_called_once_with(FINISHED_JOB["_id"], "final_structure") @@ -309,14 +304,8 @@ def test_find_relaxed_material_returns_none_when_material_is_not_on_the_platform client.jobs.request.return_value = [] assert find_relaxed_material(client, DEFECTIVE_MATERIAL, OWNER_ID) is None - # JobsList has no flat material-id filter, so this account's finished jobs are always fetched - # unfiltered by material and narrowed to matching material ids in Python (see find_relaxed_material). - client.jobs.request.assert_called_once_with( - "GET", - client.jobs.name, - params={"ownerId": OWNER_ID, "status": "finished"}, - headers=client.jobs.headers, - ) + # No matching materials -> nothing to filter jobs by, so the jobs endpoint is never queried. + client.jobs.request.assert_not_called() def test_find_relaxed_material_returns_none_when_no_job_exists(): @@ -350,28 +339,21 @@ def test_find_relaxed_material_checks_every_same_hash_material(): assert relaxed is not None assert relaxed.name == "B-vacancy h-BN relaxed" - # As above: the material-id narrowing happens in Python, not in the jobs query itself. client.jobs.request.assert_called_once_with( "GET", client.jobs.name, - params={"ownerId": OWNER_ID, "status": "finished"}, + params={ + "materialId": [other_material["_id"], SAVED_DEFECTIVE["_id"]], + "ownerId": OWNER_ID, + "status": "finished", + }, headers=client.jobs.headers, ) def test_find_relaxed_material_skips_a_final_structure_with_the_same_hash(): - scf_job: Dict[str, Any] = { - "_id": "job-scf", - "name": "Total Energy", - "status": "finished", - "_material": {"_id": "m-defective"}, - } - relax_job: Dict[str, Any] = { - "_id": "job-relax", - "name": "Fixed-cell Relaxation", - "status": "finished", - "_material": {"_id": "m-defective"}, - } + scf_job: Dict[str, Any] = {"_id": "job-scf", "name": "Total Energy", "status": "finished"} + relax_job: Dict[str, Any] = {"_id": "job-relax", "name": "Fixed-cell Relaxation", "status": "finished"} client = MagicMock() client.materials.request.return_value = [SAVED_DEFECTIVE] client.jobs.request.return_value = [scf_job, relax_job] From 3ed8a2ff90e4370e40eca604ab5ea036030cac18 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Tue, 6 Oct 2026 18:17:30 +0300 Subject: [PATCH 07/10] fix(SOF-8032): look up default material, workflow and project with show_default Replaces the `.request("GET", ..., {"isDefault": "true", "ownerId": ...})` workaround with the API client's `show_default(account_id)`, so notebooks no longer need to know about the flat-param list queries. - `get_default_material` / `get_default_workflow` live in their own small `default.py` modules next to `get_default_project`. They are kept apart from `api.py` because notebooks that install only the `api` package group cannot import `mat3ra.made` / `mat3ra.wode`; a test guards that they import only the API client. - `create_and_submit_job` uses the two new helpers and no longer re-fetches the workflow it already has. - `get-file-from-job` and the materials_designer workflow notebooks that inlined the projects lookup now use `get_default_project`. Needs a `mat3ra-api-client` release that includes `show_default` (mat3ra/api-client feature/SOF-8032). Co-Authored-By: Claude Sonnet 5.5 --- examples/job/create_and_submit_job.ipynb | 20 +++----- examples/job/get-file-from-job.ipynb | 9 +--- .../workflows/defect_formation_energy.ipynb | 10 ++-- .../workflows/dielectric_tensor.ipynb | 13 ++--- .../workflows/equation_of_state.ipynb | 13 ++--- .../workflows/formation_energy.ipynb | 13 ++--- .../workflows/homo_lumo_frequency.ipynb | 13 ++--- other/materials_designer/workflows/neb.ipynb | 13 ++--- .../workflows/phonon_dos_dispersion.ipynb | 13 ++--- .../workflows/relaxation.ipynb | 13 ++--- .../workflows/surface_energy.ipynb | 13 ++--- .../workflows/total_energy_convergence.ipynb | 13 ++--- .../workflows/valence_band_offset.ipynb | 13 ++--- .../core/entity/material/default.py | 22 +++++++++ .../core/entity/project/api.py | 18 +------ .../core/entity/workflow/default.py | 22 +++++++++ .../core/entity/test_default_entity_api.py | 47 +++++++++++++++++++ 17 files changed, 155 insertions(+), 123 deletions(-) create mode 100644 src/py/mat3ra/notebooks_utils/core/entity/material/default.py create mode 100644 src/py/mat3ra/notebooks_utils/core/entity/workflow/default.py create mode 100644 tests/py/unit/core/entity/test_default_entity_api.py diff --git a/examples/job/create_and_submit_job.ipynb b/examples/job/create_and_submit_job.ipynb index 8c4db0997..36abcbc7a 100644 --- a/examples/job/create_and_submit_job.ipynb +++ b/examples/job/create_and_submit_job.ipynb @@ -97,18 +97,11 @@ "outputs": [], "source": [ "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "# Using .request() with flat params instead of .list(): .list() always wraps its argument as a\n", - "# query= blob, which the materials/workflows list endpoints (migrated to validated use\n", - "# cases) silently drop since they only accept flat, declared keys.\n", - "default_material = client.materials.request(\n", - " \"GET\", client.materials.name, params={\"isDefault\": \"true\", \"ownerId\": OWNER_ID}, headers=client.materials.headers\n", - ")[0]\n", - "default_workflow = client.workflows.request(\n", - " \"GET\", client.workflows.name, params={\"isDefault\": \"true\", \"ownerId\": OWNER_ID}, headers=client.workflows.headers\n", - ")[0]\n", + "from mat3ra.notebooks_utils.core.entity.material.default import get_default_material\n", + "from mat3ra.notebooks_utils.core.entity.workflow.default import get_default_workflow\n", "\n", - "material_id = default_material[\"_id\"]\n", - "workflow_id = default_workflow[\"_id\"]" + "default_material = get_default_material(client, OWNER_ID)\n", + "default_workflow = get_default_workflow(client, OWNER_ID)" ] }, { @@ -127,11 +120,10 @@ }, "outputs": [], "source": [ - "workflow = client.workflows.get(workflow_id)\n", "config = {\n", " \"owner\": {\"_id\": OWNER_ID},\n", - " \"_material\": {\"_id\": material_id},\n", - " \"workflow\": workflow,\n", + " \"_material\": {\"_id\": default_material[\"_id\"]},\n", + " \"workflow\": default_workflow,\n", " \"name\": JOB_NAME,\n", "}" ] diff --git a/examples/job/get-file-from-job.ipynb b/examples/job/get-file-from-job.ipynb index 42a394e17..57f92503e 100644 --- a/examples/job/get-file-from-job.ipynb +++ b/examples/job/get-file-from-job.ipynb @@ -88,18 +88,13 @@ "execution_count": null, "source": [ "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", "from mat3ra.notebooks_utils.ipython.ui import display_JSON\n", "\n", "# Relevant functions from the API client\n", "# Get some account information\n", "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "# Using .request() with flat params instead of .list() here: .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "PROJECT_QUERY = {\"isDefault\": \"true\", \"ownerId\": OWNER_ID}\n", - "project_metadata = client.projects.request(\n", - " \"GET\", client.projects.name, params=PROJECT_QUERY, headers=client.projects.headers\n", - ")[0]\n", + "project_metadata = get_default_project(client, OWNER_ID)\n", "project_id = project_metadata[\"_id\"]\n", "owner_id = project_metadata[\"owner\"][\"_id\"]\n", "\n", diff --git a/other/materials_designer/workflows/defect_formation_energy.ipynb b/other/materials_designer/workflows/defect_formation_energy.ipynb index 7b80ee974..8999c3742 100644 --- a/other/materials_designer/workflows/defect_formation_energy.ipynb +++ b/other/materials_designer/workflows/defect_formation_energy.ipynb @@ -241,11 +241,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/dielectric_tensor.ipynb b/other/materials_designer/workflows/dielectric_tensor.ipynb index f1dedb2ee..beb841be2 100644 --- a/other/materials_designer/workflows/dielectric_tensor.ipynb +++ b/other/materials_designer/workflows/dielectric_tensor.ipynb @@ -230,14 +230,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/equation_of_state.ipynb b/other/materials_designer/workflows/equation_of_state.ipynb index 0ba08702e..72971d031 100644 --- a/other/materials_designer/workflows/equation_of_state.ipynb +++ b/other/materials_designer/workflows/equation_of_state.ipynb @@ -197,14 +197,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index 08cc6bacf..0efcc605b 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -212,14 +212,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" ] }, { diff --git a/other/materials_designer/workflows/homo_lumo_frequency.ipynb b/other/materials_designer/workflows/homo_lumo_frequency.ipynb index fb9414085..bcba29d5a 100644 --- a/other/materials_designer/workflows/homo_lumo_frequency.ipynb +++ b/other/materials_designer/workflows/homo_lumo_frequency.ipynb @@ -176,14 +176,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/neb.ipynb b/other/materials_designer/workflows/neb.ipynb index 73b651d96..0ac2102b1 100644 --- a/other/materials_designer/workflows/neb.ipynb +++ b/other/materials_designer/workflows/neb.ipynb @@ -224,14 +224,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/phonon_dos_dispersion.ipynb b/other/materials_designer/workflows/phonon_dos_dispersion.ipynb index c7267dfb6..f19bb9f15 100644 --- a/other/materials_designer/workflows/phonon_dos_dispersion.ipynb +++ b/other/materials_designer/workflows/phonon_dos_dispersion.ipynb @@ -229,14 +229,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/relaxation.ipynb b/other/materials_designer/workflows/relaxation.ipynb index 09a65d756..8f5b84345 100644 --- a/other/materials_designer/workflows/relaxation.ipynb +++ b/other/materials_designer/workflows/relaxation.ipynb @@ -218,14 +218,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/surface_energy.ipynb b/other/materials_designer/workflows/surface_energy.ipynb index 98c0db548..1205905d7 100644 --- a/other/materials_designer/workflows/surface_energy.ipynb +++ b/other/materials_designer/workflows/surface_energy.ipynb @@ -209,14 +209,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" ] }, { diff --git a/other/materials_designer/workflows/total_energy_convergence.ipynb b/other/materials_designer/workflows/total_energy_convergence.ipynb index 410602f14..e4178421f 100644 --- a/other/materials_designer/workflows/total_energy_convergence.ipynb +++ b/other/materials_designer/workflows/total_energy_convergence.ipynb @@ -181,14 +181,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "PROJECT_ID = projects[0][\"_id\"]\n", - "print(f\"Using project: {projects[0]['name']} ({PROJECT_ID})\")\n" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "PROJECT_ID = project[\"_id\"]\n", + "print(f\"Using project: {project['name']} ({PROJECT_ID})\")\n" ] }, { diff --git a/other/materials_designer/workflows/valence_band_offset.ipynb b/other/materials_designer/workflows/valence_band_offset.ipynb index 40cf7cb5e..a74a9ab6e 100644 --- a/other/materials_designer/workflows/valence_band_offset.ipynb +++ b/other/materials_designer/workflows/valence_band_offset.ipynb @@ -234,14 +234,11 @@ "metadata": {}, "outputs": [], "source": [ - "# Using .request() with flat params instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the projects list endpoint (migrated to a\n", - "# validated use case) silently drops since it only accepts flat, declared keys.\n", - "projects = client.projects.request(\n", - " \"GET\", client.projects.name, params={\"isDefault\": \"true\", \"ownerId\": ACCOUNT_ID}, headers=client.projects.headers\n", - ")\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" ] }, { diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/default.py b/src/py/mat3ra/notebooks_utils/core/entity/material/default.py new file mode 100644 index 000000000..3bbb205ab --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/default.py @@ -0,0 +1,22 @@ +""" +Default-material lookup. + +Kept apart from api.py on purpose: notebooks that install only the `api` package group cannot import +mat3ra.made (which api.py needs), so this module may depend on mat3ra.api_client alone. +""" + +from mat3ra.api_client import APIClient + + +def get_default_material(api_client: APIClient, owner_id: str) -> dict: + """ + Returns the given account's default material. + + Args: + api_client (APIClient): API client instance carrying the authorization context. + owner_id (str): Account ID whose default material should be returned. + + Returns: + dict: The account's default material document. + """ + return api_client.materials.show_default(owner_id) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/project/api.py b/src/py/mat3ra/notebooks_utils/core/entity/project/api.py index 8f61ea0c6..c92ba55d2 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/project/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/project/api.py @@ -5,27 +5,11 @@ def get_default_project(api_client: APIClient, owner_id: str) -> dict: """ Returns the given account's default project. - Uses .request() with flat params instead of .list(): .list() always wraps its argument as - a query= blob, which the projects list endpoint (migrated to a validated use case) - silently drops since it only accepts flat, declared keys - unfiltered, it returns every - project visible to the session, so the first result is not necessarily this account's own. - Args: api_client (APIClient): API client instance carrying the authorization context. owner_id (str): Account ID whose default project should be returned. Returns: dict: The account's default project document. - - Raises: - ValueError: If the account has no default project. """ - projects = api_client.projects.request( - "GET", - api_client.projects.name, - params={"isDefault": "true", "ownerId": owner_id}, - headers=api_client.projects.headers, - ) - if not projects: - raise ValueError(f"Account {owner_id} has no default project.") - return projects[0] + return api_client.projects.show_default(owner_id) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/default.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/default.py new file mode 100644 index 000000000..27da76637 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/default.py @@ -0,0 +1,22 @@ +""" +Default-workflow lookup. + +Kept apart from api.py on purpose: notebooks that install only the `api` package group cannot import +mat3ra.wode (which api.py needs), so this module may depend on mat3ra.api_client alone. +""" + +from mat3ra.api_client import APIClient + + +def get_default_workflow(api_client: APIClient, owner_id: str) -> dict: + """ + Returns the given account's default workflow. + + Args: + api_client (APIClient): API client instance carrying the authorization context. + owner_id (str): Account ID whose default workflow should be returned. + + Returns: + dict: The account's default workflow document. + """ + return api_client.workflows.show_default(owner_id) diff --git a/tests/py/unit/core/entity/test_default_entity_api.py b/tests/py/unit/core/entity/test_default_entity_api.py new file mode 100644 index 000000000..22352a800 --- /dev/null +++ b/tests/py/unit/core/entity/test_default_entity_api.py @@ -0,0 +1,47 @@ +import ast +from unittest.mock import MagicMock + +import pytest +from mat3ra.notebooks_utils.core.entity.material import default as material_default +from mat3ra.notebooks_utils.core.entity.material.default import get_default_material +from mat3ra.notebooks_utils.core.entity.project import api as project_api +from mat3ra.notebooks_utils.core.entity.project.api import get_default_project +from mat3ra.notebooks_utils.core.entity.workflow import default as workflow_default +from mat3ra.notebooks_utils.core.entity.workflow.default import get_default_workflow + +ACCOUNT_ID = "account-1" +DEFAULT_ENTITY = {"_id": "default-1", "isDefault": True} +# The `api` package group of config.yml installs only these, so the default lookups may import nothing else. +ALLOWED_IMPORTED_MODULES = {"mat3ra.api_client"} + + +@pytest.mark.parametrize( + ("get_default", "endpoint_name"), + [ + (get_default_material, "materials"), + (get_default_workflow, "workflows"), + (get_default_project, "projects"), + ], +) +def test_returns_the_default_entity_of_the_given_account(get_default, endpoint_name): + client = MagicMock() + getattr(client, endpoint_name).show_default.return_value = DEFAULT_ENTITY + + assert get_default(client, ACCOUNT_ID) == DEFAULT_ENTITY + getattr(client, endpoint_name).show_default.assert_called_once_with(ACCOUNT_ID) + + +@pytest.mark.parametrize("module", [material_default, workflow_default, project_api]) +def test_default_lookups_import_only_the_api_client(module): + """Notebooks installing just the `api` package group must be able to import these.""" + with open(module.__file__) as module_file: + tree = ast.parse(module_file.read()) + + imported_modules = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported_modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported_modules.add(node.module) + + assert imported_modules <= ALLOWED_IMPORTED_MODULES From 70454d902783594c2aa6d2689b18b8406687966e Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Tue, 6 Oct 2026 19:42:56 +0300 Subject: [PATCH 08/10] fix(SOF-8032): query with list() and the list parameters instead of .request() workarounds The list endpoints filter on the flat parameters of their use cases and drop the Mongo query `list()` used to send. Instead of calling `.request()` with flat params, notebooks and helpers use `list()` with those parameters directly, e.g. `client.materials.list({"name": name, "ownerId": ACCOUNT_ID})`. - Helpers (`material`, `workflow`, `job`, `property` api): lookups by hash, name, id, set and job use `ownerId`, `hashes`, `setId`, `materialId`, `status`, ... Lookups that must find entities inside sets pass `globalSearch`. An empty id list returns early in `find_relaxed_material` (an empty filter matches all). `get_properties_for_job` is back to `get_for_job`, which filters on the unit. - Notebooks: the project, material and workflow default lookups use `get_default_*`; elemental references use `tags` + `metadataElement`; `get_materials_by_formula` filters on `formula`; `get_workflows` searches with `quickSearch`; `analyze_convex_hull` uses `formula`, `groupPrefix` and `precisionValue`. Name lookups match the exact name. - Their "Set Parameters" cells describe the list filters instead of a Mongo selector. - `formation_energy` keeps the Mongo-style query of its standata template, which the client translates. - Tests follow the new call shapes. Needs the web-app list filters (`formula`, `unitId`, `precisionValue`) deployed and a `mat3ra-api-client` release with `list()` parameters and `show_default` (mat3ra/api-client feature/SOF-8032). Co-Authored-By: Claude Sonnet 5.5 --- examples/job/get-file-from-job.ipynb | 14 +- .../ml-train-model-predict-properties.ipynb | 13 +- ...n-simulations-and-extract-properties.ipynb | 8 +- .../material/get_materials_by_formula.ipynb | 8 +- ...bilayer_twisted_molybdenum_disulfide.ipynb | 14 +- ...bilayer_twisted_molybdenum_disulfide.ipynb | 14 +- examples/workflow/get_workflows.ipynb | 10 +- ...int_substitution_graphene_SIMULATION.ipynb | 8 +- ...int_vacancy_boron_nitride_SIMULATION.ipynb | 12 +- ...ices_molybdenum_disulfide_SIMULATION.ipynb | 8 +- ..._position_graphene_nickel_SIMULATION.ipynb | 8 +- .../workflows/analyze_convex_hull.ipynb | 32 +---- .../workflows/defect_formation_energy.ipynb | 20 +-- .../workflows/equation_of_state.ipynb | 5 +- .../workflows/formation_energy.ipynb | 2 +- .../workflows/homo_lumo_frequency.ipynb | 24 +--- .../workflows/interfacial_energy.ipynb | 9 +- other/materials_designer/workflows/neb.ipynb | 11 +- .../workflows/surface_energy.ipynb | 18 +-- .../notebooks_utils/core/entity/job/api.py | 19 ++- .../core/entity/material/analysis.py | 6 +- .../core/entity/material/api.py | 87 ++---------- .../core/entity/property/api.py | 54 ++------ .../core/entity/workflow/api.py | 11 +- tests/py/unit/core/entity/test_job_api.py | 15 ++- .../core/entity/test_material_analysis.py | 10 +- .../py/unit/core/entity/test_material_api.py | 126 +++++++----------- ...erty_api_find_total_energy_for_material.py | 38 +++--- 28 files changed, 211 insertions(+), 393 deletions(-) diff --git a/examples/job/get-file-from-job.ipynb b/examples/job/get-file-from-job.ipynb index 57f92503e..a5112d67d 100644 --- a/examples/job/get-file-from-job.ipynb +++ b/examples/job/get-file-from-job.ipynb @@ -99,22 +99,16 @@ "owner_id = project_metadata[\"owner\"][\"_id\"]\n", "\n", "# Get a workflow for the job from the bank, and copy it to our account\n", - "WORKFLOW_QUERY = {\"systemName\": \"espresso-total-energy\", \"limit\": 1}\n", - "bank_workflow_id = client.bank_workflows.request(\n", - " \"GET\", client.bank_workflows.name, params=WORKFLOW_QUERY, headers=client.bank_workflows.headers\n", - ")[0][\"_id\"]\n", + "WORKFLOW_QUERY = {\"systemName\": \"espresso-total-energy\"}\n", + "bank_workflow_id = client.bank_workflows.list(WORKFLOW_QUERY, {\"limit\": 1})[0][\"_id\"]\n", "workflow_id = client.bank_workflows.copy(bank_workflow_id, owner_id)[\"_id\"]\n", "\n", "# Get materials from bank and copy one to our account\n", "MATERIAL_QUERY = {\"formula\": \"Si\"}\n", - "bank_materials = client.bank_materials.request(\n", - " \"GET\", client.bank_materials.name, params=MATERIAL_QUERY, headers=client.bank_materials.headers\n", - ")\n", + "bank_materials = client.bank_materials.list(MATERIAL_QUERY)\n", "bank_material_id = bank_materials[0][\"_id\"]\n", "material_id = client.bank_materials.copy(bank_material_id, owner_id)[\"_id\"]\n", - "materials = client.materials.request(\n", - " \"GET\", client.materials.name, params={\"ownerId\": owner_id}, headers=client.materials.headers\n", - ")\n", + "materials = client.materials.list({\"ownerId\": owner_id})\n", "\n", "# Create the job - JobsCreate's schema requires the full workflow document embedded (a job keeps\n", "# an immutable snapshot of what it actually ran), not just a {\"_id\": ...} reference, so fetch it\n", diff --git a/examples/job/ml-train-model-predict-properties.ipynb b/examples/job/ml-train-model-predict-properties.ipynb index 2d5e56bfc..4453462c0 100644 --- a/examples/job/ml-train-model-predict-properties.ipynb +++ b/examples/job/ml-train-model-predict-properties.ipynb @@ -213,9 +213,12 @@ }, "outputs": [], "source": [ + "from mat3ra.notebooks_utils.core.entity.material.default import get_default_material\n", + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "owner_id = client.materials.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"owner\"][\"_id\"]\n", - "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]" + "owner_id = get_default_material(client, OWNER_ID)[\"owner\"][\"_id\"]\n", + "project_id = get_default_project(client, OWNER_ID)[\"_id\"]" ] }, { @@ -274,9 +277,9 @@ "train_material_id_ge = client.materials.create(train_material_json_ge, OWNER_ID)[\"_id\"]\n", "target_material_id_sige = client.materials.create(target_material_json_sige, OWNER_ID)[\"_id\"]\n", "\n", - "train_material_si = client.materials.list({\"_id\": train_material_id_si})[0]\n", - "train_material_ge = client.materials.list({\"_id\": train_material_id_ge})[0]\n", - "target_material_sige = client.materials.list({\"_id\": target_material_id_sige})[0]\n", + "train_material_si = client.materials.list({\"id\": train_material_id_si})[0]\n", + "train_material_ge = client.materials.list({\"id\": train_material_id_ge})[0]\n", + "target_material_sige = client.materials.list({\"id\": target_material_id_sige})[0]\n", "\n", "train_materials = [train_material_ge, train_material_si]\n", "target_materials = [target_material_sige]" diff --git a/examples/job/run-simulations-and-extract-properties.ipynb b/examples/job/run-simulations-and-extract-properties.ipynb index bb8a5db74..65917a0f2 100644 --- a/examples/job/run-simulations-and-extract-properties.ipynb +++ b/examples/job/run-simulations-and-extract-properties.ipynb @@ -252,7 +252,9 @@ "outputs": [], "source": [ "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project_id = get_default_project(client, OWNER_ID)[\"_id\"]" ] }, { @@ -328,8 +330,8 @@ "material_id_si = client.materials.create(material_json_si, OWNER_ID)[\"_id\"]\n", "material_id_ge = client.materials.create(material_json_ge, OWNER_ID)[\"_id\"]\n", "\n", - "material_si = client.materials.list({\"_id\": material_id_si})[0]\n", - "material_sige = client.materials.list({\"_id\": material_id_ge})[0]\n", + "material_si = client.materials.list({\"id\": material_id_si})[0]\n", + "material_sige = client.materials.list({\"id\": material_id_ge})[0]\n", "\n", "materials = [material_sige, material_si]" ] diff --git a/examples/material/get_materials_by_formula.ipynb b/examples/material/get_materials_by_formula.ipynb index 68ef5dde2..d1d0d2436 100644 --- a/examples/material/get_materials_by_formula.ipynb +++ b/examples/material/get_materials_by_formula.ipynb @@ -77,7 +77,11 @@ "metadata": { "id": "mbtRgwoBCzPE" }, - "source": "## Set Parameters\n\n- **QUERY**: A query describing the documents to find. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information." + "source": [ + "## Set Parameters\n", + "\n", + "- **QUERY**: The filters of the materials list: the owner account (`ownerId`) and the chemical formula (`formula`). Other filters include `name`, `hashes`, `tags` and `quickSearch`." + ] }, { "cell_type": "code", @@ -88,7 +92,7 @@ "outputs": [], "source": [ "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "QUERY = {\"formula\": \"Si\", \"owner._id\": OWNER_ID}" + "QUERY = {\"ownerId\": OWNER_ID, \"formula\": \"Si\"}" ] }, { diff --git a/examples/reproducing_publications/band_gaps_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb b/examples/reproducing_publications/band_gaps_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb index 7f7a26811..09927a9cb 100644 --- a/examples/reproducing_publications/band_gaps_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb +++ b/examples/reproducing_publications/band_gaps_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb @@ -336,9 +336,9 @@ "source": [ "### 7.1. Set Parameters\n", "\n", - "- **QUERY**: A query describing the documents to find. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information.\n", + "- **QUERY**: The filters of the workflows list: the workflow `id` and the owner account (`ownerId`).\n", "\n", - "- **limit**: Maximum number of results to return. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information." + "- **limit**: Maximum number of results to return." ] }, { @@ -415,7 +415,7 @@ "metadata": {}, "outputs": [], "source": [ - "QUERY = {\"_id\": band_gap_workflow_id, \"owner._id\": OWNER_ID}\n", + "QUERY = {\"id\": band_gap_workflow_id, \"ownerId\": OWNER_ID}\n", "workflow = client.workflows.list(QUERY, OPTIONS)[0]\n", "\n", "modifier_0 = {\n", @@ -561,9 +561,11 @@ "source": [ "from datetime import datetime\n", "\n", + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", "\n", - "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]\n", + "project_id = get_default_project(client, OWNER_ID)[\"_id\"]\n", "\n", "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\") # for name to be found easily\n", "\n", @@ -607,7 +609,9 @@ "metadata": {}, "outputs": [], "source": [ - "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]\n", + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project_id = get_default_project(client, OWNER_ID)[\"_id\"]\n", "\n", "JOBS_SET_NAME = \"MoS2 Twisted Interfaces Band Gaps\"\n", "\n", diff --git a/examples/reproducing_publications/band_structure_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb b/examples/reproducing_publications/band_structure_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb index 270ab08e8..06103ce06 100644 --- a/examples/reproducing_publications/band_structure_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb +++ b/examples/reproducing_publications/band_structure_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb @@ -334,9 +334,9 @@ "source": [ "### 7.1. Set Parameters\n", "\n", - "- **QUERY**: A query describing the documents to find. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information.\n", + "- **QUERY**: The filters of the workflows list: the workflow `id` and the owner account (`ownerId`).\n", "\n", - "- **limit**: Maximum number of results to return. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information." + "- **limit**: Maximum number of results to return." ], "id": "9378dd5bdba80208" }, @@ -409,7 +409,7 @@ "cell_type": "code", "metadata": {}, "source": [ - "QUERY = {\"_id\": band_structure_workflow_id, \"owner._id\": OWNER_ID}\n", + "QUERY = {\"id\": band_structure_workflow_id, \"ownerId\": OWNER_ID}\n", "workflow = client.workflows.list(QUERY, OPTIONS)[0]\n", "\n", "modifier_0 = {\n", @@ -559,9 +559,11 @@ "source": [ "from datetime import datetime\n", "\n", + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", "\n", - "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]\n", + "project_id = get_default_project(client, OWNER_ID)[\"_id\"]\n", "\n", "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\") # for name to be found easily\n", "\n", @@ -605,7 +607,9 @@ "cell_type": "code", "metadata": {}, "source": [ - "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]\n", + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project_id = get_default_project(client, OWNER_ID)[\"_id\"]\n", "\n", "JOBS_SET_NAME = None\n", "\n", diff --git a/examples/workflow/get_workflows.ipynb b/examples/workflow/get_workflows.ipynb index b55906c2c..59a576077 100644 --- a/examples/workflow/get_workflows.ipynb +++ b/examples/workflow/get_workflows.ipynb @@ -77,7 +77,13 @@ "metadata": { "id": "h8JubXPxIQAY" }, - "source": "## Set Parameters\n\n- **QUERY**: A query describing the documents to find. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information. \n\n- **limit**: Maximum number of results to return. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information." + "source": [ + "## Set Parameters\n", + "\n", + "- **QUERY**: The filters of the workflows list: the owner account (`ownerId`) and a text search over the workflow name, slug and system name (`quickSearch`, a case-insensitive pattern).\n", + "\n", + "- **limit**: Maximum number of results to return." + ] }, { "cell_type": "code", @@ -88,7 +94,7 @@ "outputs": [], "source": [ "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "QUERY = {\"name\": {\"$regex\": \"Total Energy\"}, \"owner._id\": OWNER_ID}\n", + "QUERY = {\"ownerId\": OWNER_ID, \"quickSearch\": \"Total Energy\"}\n", "\n", "OPTIONS = {\"limit\": 2}" ] diff --git a/other/materials_designer/specific_examples/defect_point_substitution_graphene_SIMULATION.ipynb b/other/materials_designer/specific_examples/defect_point_substitution_graphene_SIMULATION.ipynb index 4902ba679..bbeb2d4bf 100644 --- a/other/materials_designer/specific_examples/defect_point_substitution_graphene_SIMULATION.ipynb +++ b/other/materials_designer/specific_examples/defect_point_substitution_graphene_SIMULATION.ipynb @@ -246,9 +246,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/specific_examples/defect_point_vacancy_boron_nitride_SIMULATION.ipynb b/other/materials_designer/specific_examples/defect_point_vacancy_boron_nitride_SIMULATION.ipynb index 72cc22ced..b3492b8b2 100644 --- a/other/materials_designer/specific_examples/defect_point_vacancy_boron_nitride_SIMULATION.ipynb +++ b/other/materials_designer/specific_examples/defect_point_vacancy_boron_nitride_SIMULATION.ipynb @@ -246,9 +246,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" ] }, { @@ -301,7 +303,9 @@ "\n", "elements = sorted(set(pristine.basis.elements.values) | set(defective.basis.elements.values))\n", "\n", - "elemental_materials_data = client.materials.list({\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}})\n", + "elemental_materials_data = client.materials.list(\n", + " {\"tags\": \"elemental\", \"metadataElement\": elements, \"globalSearch\": True, \"limit\": 500}\n", + ")\n", "elemental_materials = {}\n", "for element in elements:\n", " matches = [m for m in elemental_materials_data if m.get(\"metadata\", {}).get(\"element\") == element]\n", diff --git a/other/materials_designer/specific_examples/interface_bilayer_twisted_commensurate_lattices_molybdenum_disulfide_SIMULATION.ipynb b/other/materials_designer/specific_examples/interface_bilayer_twisted_commensurate_lattices_molybdenum_disulfide_SIMULATION.ipynb index 66d77a21d..76f469978 100644 --- a/other/materials_designer/specific_examples/interface_bilayer_twisted_commensurate_lattices_molybdenum_disulfide_SIMULATION.ipynb +++ b/other/materials_designer/specific_examples/interface_bilayer_twisted_commensurate_lattices_molybdenum_disulfide_SIMULATION.ipynb @@ -237,9 +237,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"✅ Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/specific_examples/optimization_interface_film_xy_position_graphene_nickel_SIMULATION.ipynb b/other/materials_designer/specific_examples/optimization_interface_film_xy_position_graphene_nickel_SIMULATION.ipynb index 3fb5e1926..19ec152bf 100644 --- a/other/materials_designer/specific_examples/optimization_interface_film_xy_position_graphene_nickel_SIMULATION.ipynb +++ b/other/materials_designer/specific_examples/optimization_interface_film_xy_position_graphene_nickel_SIMULATION.ipynb @@ -613,9 +613,11 @@ "metadata": {}, "outputs": [], "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"Using project: {projects[0]['name']} ({project_id})\")" + "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", + "\n", + "project = get_default_project(client, ACCOUNT_ID)\n", + "project_id = project[\"_id\"]\n", + "print(f\"Using project: {project['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/analyze_convex_hull.ipynb b/other/materials_designer/workflows/analyze_convex_hull.ipynb index 4612869c8..b1edcf655 100644 --- a/other/materials_designer/workflows/analyze_convex_hull.ipynb +++ b/other/materials_designer/workflows/analyze_convex_hull.ipynb @@ -168,25 +168,11 @@ "# Search materials by formula (optionally restricted to MATERIAL_SET)\n", "all_materials = []\n", "\n", - "# Using .request() with a flat ownerId param instead of .list(): .list() always wraps its\n", - "# argument as a query= blob, which the materials list endpoint (migrated to a validated\n", - "# use case) silently drops - without it, accessibleByUser's broader scope let materials from\n", - "# outside this account leak into the result. MaterialsList has no server-side \"formula\" filter,\n", - "# so that part of the original per-formula query is applied client-side below instead (fetching\n", - "# once up front rather than once per formula, since the formula filter no longer narrows the\n", - "# request itself).\n", - "materials_by_owner = None\n", - "if set_materials is None:\n", - " materials_by_owner = client.materials.request(\n", - " \"GET\", client.materials.name, params={\"ownerId\": ACCOUNT_ID}, headers=client.materials.headers\n", - " )\n", - " materials_by_owner = [material for material in materials_by_owner if not material.get(\"isEntitySet\")]\n", - "\n", "for formula in FORMULAS:\n", " if set_materials is not None:\n", " matches = [material for material in set_materials if material.get(\"formula\") == formula]\n", " else:\n", - " matches = [material for material in materials_by_owner if material.get(\"formula\") == formula]\n", + " matches = client.materials.list({\"ownerId\": ACCOUNT_ID, \"formula\": formula, \"globalSearch\": True})\n", " for material in matches:\n", " material[\"_search_formula\"] = formula # track which formula query found this\n", " all_materials.extend(matches)\n", @@ -251,21 +237,17 @@ "exabyte_ids = list(set(material[\"exabyteId\"] for material in all_materials if material.get(\"exabyteId\")))\n", "\n", "# Query properties per exabyteId to avoid API pagination truncation.\n", - "# Using .request() with flat params instead of .list(): .list() always wraps its argument as a\n", - "# query= blob, which the properties list endpoint (migrated to a validated use case)\n", - "# silently drops since it only accepts flat, declared keys - exabyteId and slug (which mirrors\n", - "# data.name on every property document) both are declared, so use those directly. GROUP/PRECISION\n", - "# have no flat equivalent in the current schema, so they're left unfiltered here for now (as they\n", - "# already effectively were, since .list()'s blob dropped them too).\n", "properties_by_exabyte_id = {}\n", "for exabyte_id in exabyte_ids:\n", " query = {\n", " \"exabyteId\": exabyte_id,\n", - " \"slug\": \"total_energy\",\n", + " \"propertyName\": \"total_energy\",\n", " }\n", - " props = client.properties.request(\n", - " \"GET\", client.properties.name, params=query, headers=client.properties.headers\n", - " )\n", + " if GROUP:\n", + " query[\"groupPrefix\"] = GROUP\n", + " if PRECISION is not None:\n", + " query[\"precisionValue\"] = PRECISION\n", + " props = client.properties.list(query)\n", " for property_holder in props:\n", " eid = property_holder.get(\"exabyteId\")\n", " eid = eid if isinstance(eid, str) else (eid[0] if eid else None)\n", diff --git a/other/materials_designer/workflows/defect_formation_energy.ipynb b/other/materials_designer/workflows/defect_formation_energy.ipynb index 8999c3742..3e12d9383 100644 --- a/other/materials_designer/workflows/defect_formation_energy.ipynb +++ b/other/materials_designer/workflows/defect_formation_energy.ipynb @@ -274,13 +274,8 @@ " if loaded is not None:\n", " print(f\"✅ Loaded '{name}' from folder: {loaded.name}\")\n", " return loaded\n", - " # Using .request() with flat params instead of .list(): .list() always wraps its argument as\n", - " # a query= blob, which the materials list endpoint (migrated to a validated use case)\n", - " # silently drops since it only accepts flat, declared keys - it also has no $regex support, so\n", - " # this now matches by exact name instead of a case-insensitive partial match.\n", - " matches = client.materials.request(\n", - " \"GET\", client.materials.name, params={\"name\": name, \"ownerId\": ACCOUNT_ID}, headers=client.materials.headers\n", - " )\n", + " # a name condition matches the exact name, not a pattern\n", + " matches = client.materials.list({\"name\": name, \"ownerId\": ACCOUNT_ID})\n", " if not matches:\n", " raise ValueError(f\"No material named '{name}' was found in '{FOLDER}' or on the platform.\")\n", " material = Material.create(matches[0])\n", @@ -357,16 +352,9 @@ "metadata": {}, "outputs": [], "source": [ - "# .list()'s query= blob is silently dropped by the migrated materials list endpoint, and\n", - "# it has no flat \"tags\"/\"metadata.element\" filter to switch to either - fetch this account's\n", - "# materials once (scoped by the flat ownerId param) and filter for elemental references in Python.\n", - "account_materials = client.materials.request(\n", - " \"GET\", client.materials.name, params={\"ownerId\": ACCOUNT_ID}, headers=client.materials.headers\n", + "elemental_materials_data = client.materials.list(\n", + " {\"tags\": \"elemental\", \"metadataElement\": elements, \"globalSearch\": True, \"limit\": 500}\n", ")\n", - "elemental_materials_data = [\n", - " material for material in account_materials\n", - " if \"elemental\" in (material.get(\"tags\") or []) and material.get(\"metadata\", {}).get(\"element\") in elements\n", - "]\n", "element_materials_by_symbol = {\n", " element: [metadata for metadata in elemental_materials_data if\n", " metadata.get(\"metadata\", {}).get(\"element\") == element]\n", diff --git a/other/materials_designer/workflows/equation_of_state.ipynb b/other/materials_designer/workflows/equation_of_state.ipynb index 72971d031..6e8ac9cae 100644 --- a/other/materials_designer/workflows/equation_of_state.ipynb +++ b/other/materials_designer/workflows/equation_of_state.ipynb @@ -638,13 +638,12 @@ "outputs": [], "source": [ "from mat3ra.prode import PropertyName\n", - "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "\n", "results_records = []\n", "for record in job_records:\n", " job = client.jobs.get(record[\"job_id\"])\n", - " properties = get_properties_for_job(\n", - " client, record[\"job_id\"], property_name=PropertyName.scalar.total_energy.value\n", + " properties = client.properties.get_for_job(\n", + " record[\"job_id\"], property_name=PropertyName.scalar.total_energy.value\n", " )\n", " total_energy = properties[0].get(\"value\") if properties else None\n", " results_records.append(\n", diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index 0efcc605b..7b6947e17 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -282,7 +282,7 @@ "outputs": [], "source": [ "elemental_materials_data = client.materials.list(\n", - " {\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}},\n", + " {\"tags\": \"elemental\", \"metadataElement\": elements, \"globalSearch\": True, \"limit\": 500}\n", ")\n", "\n", "element_materials_by_symbol = {\n", diff --git a/other/materials_designer/workflows/homo_lumo_frequency.ipynb b/other/materials_designer/workflows/homo_lumo_frequency.ipynb index bcba29d5a..d980932e4 100644 --- a/other/materials_designer/workflows/homo_lumo_frequency.ipynb +++ b/other/materials_designer/workflows/homo_lumo_frequency.ipynb @@ -209,13 +209,7 @@ " if loaded is not None:\n", " print(f\"✅ Loaded '{name}' from folder: {loaded.name}\")\n", " return loaded\n", - " # Using .request() with flat params instead of .list(): .list() always wraps its\n", - " # argument as a query= blob, which the materials list endpoint (migrated to a\n", - " # validated use case) silently drops since it only accepts flat, declared keys.\n", - " matches = client.materials.request(\n", - " \"GET\", client.materials.name, params={\"name\": name, \"ownerId\": ACCOUNT_ID},\n", - " headers=client.materials.headers,\n", - " )\n", + " matches = client.materials.list({\"name\": name, \"ownerId\": ACCOUNT_ID})\n", " if not matches:\n", " raise ValueError(f\"No material named '{name}' was found in '{FOLDER}' or on the platform.\")\n", " material = Material.create(matches[0])\n", @@ -470,10 +464,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", - "properties_data = get_properties_for_job(client, job_id)\n", + "properties_data = client.properties.get_for_job(job_id)\n", "visualize_properties(properties_data)\n" ] }, @@ -484,12 +477,11 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", "if CALCULATE_HOMO_LUMO:\n", - " homo_data = get_properties_for_job(client, job_id, property_name=\"homo_energy\")\n", - " lumo_data = get_properties_for_job(client, job_id, property_name=\"lumo_energy\")\n", + " homo_data = client.properties.get_for_job(job_id, property_name=\"homo_energy\")\n", + " lumo_data = client.properties.get_for_job(job_id, property_name=\"lumo_energy\")\n", " visualize_properties(homo_data, title=\"HOMO Energy\")\n", " visualize_properties(lumo_data, title=\"LUMO Energy\")\n", " homo_lumo_gap = lumo_data[0][\"value\"] - homo_data[0][\"value\"]\n", @@ -503,12 +495,10 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", - "\n", "if CALCULATE_FREQUENCY:\n", - " zero_point_energy_data = get_properties_for_job(client, job_id, property_name=\"zero_point_energy\")\n", - " thermal_correction_to_energy_data = get_properties_for_job(client, job_id, property_name=\"thermal_correction_to_energy\")\n", - " thermal_correction_to_enthalpy_data = get_properties_for_job(client, job_id, property_name=\"thermal_correction_to_enthalpy\")\n", + " zero_point_energy_data = client.properties.get_for_job(job_id, property_name=\"zero_point_energy\")\n", + " thermal_correction_to_energy_data = client.properties.get_for_job(job_id, property_name=\"thermal_correction_to_energy\")\n", + " thermal_correction_to_enthalpy_data = client.properties.get_for_job(job_id, property_name=\"thermal_correction_to_enthalpy\")\n", " visualize_properties(zero_point_energy_data, title=\"Zero Point Energy\")\n", " visualize_properties(thermal_correction_to_energy_data, title=\"Thermal Correction to Energy\")\n", " visualize_properties(thermal_correction_to_enthalpy_data, title=\"Thermal Correction to Enthalpy\")\n" diff --git a/other/materials_designer/workflows/interfacial_energy.ipynb b/other/materials_designer/workflows/interfacial_energy.ipynb index c06905bf2..c45620517 100644 --- a/other/materials_designer/workflows/interfacial_energy.ipynb +++ b/other/materials_designer/workflows/interfacial_energy.ipynb @@ -242,7 +242,6 @@ "metadata": {}, "outputs": [], "source": [ - "import re\n", "from mat3ra.made.material import Material\n", "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", @@ -250,13 +249,11 @@ "interface = load_material_from_folder(FOLDER, INTERFACE_NAME)\n", "\n", "if interface is None:\n", - " interface_matches = client.materials.list({\n", - " \"name\": {\"$regex\": re.escape(INTERFACE_NAME), \"$options\": \"i\"},\n", - " \"owner._id\": ACCOUNT_ID,\n", - " })\n", + " # a name condition matches the exact name, not a pattern\n", + " interface_matches = client.materials.list({\"name\": INTERFACE_NAME, \"ownerId\": ACCOUNT_ID})\n", " if not interface_matches:\n", " raise ValueError(\n", - " f\"No interface containing '{INTERFACE_NAME}' was found in '{FOLDER}' or on the platform.\"\n", + " f\"No interface named '{INTERFACE_NAME}' was found in '{FOLDER}' or on the platform.\"\n", " )\n", " interface = Material.create(interface_matches[0])\n", " print(f\"♻️ Loaded interface from platform: {interface_matches[0]['_id']}\")\n", diff --git a/other/materials_designer/workflows/neb.ipynb b/other/materials_designer/workflows/neb.ipynb index 0ac2102b1..7eb73389c 100644 --- a/other/materials_designer/workflows/neb.ipynb +++ b/other/materials_designer/workflows/neb.ipynb @@ -567,16 +567,15 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", - "profile_data = get_properties_for_job(client, job_id, property_name=\"reaction_energy_profile\")\n", + "profile_data = client.properties.get_for_job(job_id, property_name=\"reaction_energy_profile\")\n", "if not profile_data:\n", " raise ValueError(f\"Job {job_id} finished but returned no reaction_energy_profile.\")\n", "visualize_properties(profile_data, title=\"Reaction Energy Profile\")\n", "\n", "energies = profile_data[0][\"yDataSeries\"][0]\n", - "print(f\"Peak reaction energy = {max(energies):.3f} eV\")\n" + "print(f\"Peak reaction energy = {max(energies):.3f} eV\")" ] }, { @@ -586,10 +585,8 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", - "\n", - "barrier_data = get_properties_for_job(client, job_id, property_name=\"reaction_energy_barrier\")\n", - "visualize_properties(barrier_data, title=\"Reaction Energy Barrier\")\n" + "barrier_data = client.properties.get_for_job(job_id, property_name=\"reaction_energy_barrier\")\n", + "visualize_properties(barrier_data, title=\"Reaction Energy Barrier\")" ] } ], diff --git a/other/materials_designer/workflows/surface_energy.ipynb b/other/materials_designer/workflows/surface_energy.ipynb index 1205905d7..6a734dbae 100644 --- a/other/materials_designer/workflows/surface_energy.ipynb +++ b/other/materials_designer/workflows/surface_energy.ipynb @@ -239,13 +239,8 @@ "slab = load_material_from_folder(FOLDER, SLAB_NAME)\n", "\n", "if slab is None:\n", - " # Using .request() with flat params instead of .list(): .list() always wraps its argument as\n", - " # a query= blob, which the materials list endpoint (migrated to a validated use case)\n", - " # silently drops since it only accepts flat, declared keys - it also has no $regex support, so\n", - " # this now matches by exact name instead of a case-insensitive partial match.\n", - " slab_matches = client.materials.request(\n", - " \"GET\", client.materials.name, params={\"name\": SLAB_NAME, \"ownerId\": ACCOUNT_ID}, headers=client.materials.headers\n", - " )\n", + " # a name condition matches the exact name, not a pattern\n", + " slab_matches = client.materials.list({\"name\": SLAB_NAME, \"ownerId\": ACCOUNT_ID})\n", " if not slab_matches:\n", " raise ValueError(\n", " f\"No slab named '{SLAB_NAME}' was found in '{FOLDER}' or on the platform.\"\n", @@ -521,12 +516,11 @@ "outputs": [], "source": [ "from mat3ra.prode import PropertyName\n", - "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", - "surface_energy_data = get_properties_for_job(client, surface_job_id, property_name=\"surface_energy\")\n", - "slab_total_energy_data = get_properties_for_job(\n", - " client, surface_job_id, property_name=PropertyName.scalar.total_energy.value\n", + "surface_energy_data = client.properties.get_for_job(surface_job_id, property_name=\"surface_energy\")\n", + "slab_total_energy_data = client.properties.get_for_job(\n", + " surface_job_id, property_name=PropertyName.scalar.total_energy.value\n", ")\n", "\n", "visualize_properties(surface_energy_data, title=\"Surface Energy\")\n", @@ -534,7 +528,7 @@ "print(f\"Slab: {saved_slab.name} ({saved_slab.id}), \"\n", " f\"total energy: {slab_total_energy_data[0]['value']} eV\")\n", "print(f\"Bulk: {bulk_material.name} ({bulk_material.id}), \"\n", - " f\"total energy: {bulk_total_energy_property['data']['value']} eV\")\n" + " f\"total energy: {bulk_total_energy_property['data']['value']} eV\")" ] } ], diff --git a/src/py/mat3ra/notebooks_utils/core/entity/job/api.py b/src/py/mat3ra/notebooks_utils/core/entity/job/api.py index 94fd43069..b221bbcb1 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/job/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/job/api.py @@ -35,11 +35,7 @@ def get_jobs_statuses_by_ids(endpoint: JobEndpoints, job_ids: List[str]) -> List Returns: list: list of job statuses """ - # .list()'s query= is silently dropped by the migrated JobsList use case, which - # only accepts flat, declared keys - unfiltered, it returns every job in the account, so the - # wait loop above never sees the tracked jobs reach a terminal status. `id` is such a flat key - # (accepts a single id or an array), so use .request() directly with it instead. - jobs = endpoint.request("GET", endpoint.name, params={"id": job_ids}, headers=endpoint.headers) + jobs = endpoint.list({"id": job_ids}) return [job["status"] for job in jobs] @@ -142,12 +138,13 @@ def find_job_for_material( """ existing = api_client.jobs.list( { - "_material._id": material_id, - "owner._id": owner_id, - "workflow.name": workflow_name, - "status": {"$in": list(statuses)}, - }, - {"limit": 1}, + "materialId": material_id, + "ownerId": owner_id, + "workflowName": workflow_name, + "status": list(statuses), + "globalSearch": True, + "limit": 1, + } ) return existing[0] if existing else None diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py b/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py index bd329339a..2805e5c71 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py @@ -120,10 +120,10 @@ def get_slab_bulk_crystal(slab_material: Material) -> dict: def resolve_bulk_query_from_crystal(bulk_crystal: dict) -> dict: """Builds a materials.list query that resolves a bulk crystal to a platform material.""" - for key in ("_id", "hash", "scaledHash"): + for key, parameter in (("_id", "id"), ("hash", "hashes"), ("scaledHash", "scaledHashes")): if bulk_crystal.get(key) is not None: - return {key: bulk_crystal[key]} + return {parameter: bulk_crystal[key]} try: - return {"hash": Material.create(bulk_crystal).hash} + return {"hashes": Material.create(bulk_crystal).hash} except Exception as exc: raise ValueError("Could not resolve a bulk query from crystal metadata.") from exc diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py index 230bf72cd..833e928c8 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py @@ -27,16 +27,7 @@ def get_or_create_material(api_client: APIClient, material, owner_id: str) -> di Returns: dict: The material dict (existing or newly created). """ - # Using .request() with flat params instead of .list(): .list() always wraps its argument as - # a query= blob, which the materials list endpoint (migrated to a validated use case) - # silently drops since it only accepts flat, declared keys - "hashes" is the flat equivalent - # of a single "hash" (it maps onto the same $in-based DAO filter). - existing = api_client.materials.request( - "GET", - api_client.materials.name, - params={"hashes": material.hash, "ownerId": owner_id}, - headers=api_client.materials.headers, - ) + existing = api_client.materials.list({"hashes": material.hash, "ownerId": owner_id, "globalSearch": True}) if existing: print(f"♻️ Reusing already existing Material: {existing[0]['_id']}") return existing[0] @@ -65,12 +56,7 @@ def load_material(api_client: APIClient, folder: str, name: str, owner_id: str) loaded = load_material_from_folder(folder, name, verbose=False) if os.path.isdir(folder) else None if loaded is not None and loaded.name == name: return loaded - matches = api_client.materials.request( - "GET", - api_client.materials.name, - params={"name": name, "ownerId": owner_id, "limit": 1}, - headers=api_client.materials.headers, - ) + matches = api_client.materials.list({"name": name, "ownerId": owner_id, "globalSearch": True, "limit": 1}) if not matches: raise ValueError(f"No material named '{name}' in '{folder}' or in the account") return Material.create(matches[0]) @@ -97,24 +83,12 @@ def find_relaxed_material(api_client: APIClient, material, owner_id: str) -> Opt Returns: Material, optional: The relaxed structure, or None if none exists. """ - matching_materials = api_client.materials.request( - "GET", - api_client.materials.name, - params={"hashes": material.hash, "ownerId": owner_id}, - headers=api_client.materials.headers, - ) + matching_materials = api_client.materials.list({"hashes": material.hash, "ownerId": owner_id, "globalSearch": True}) ids = [m["_id"] for m in matching_materials] if not ids: return None - # JobsList's flat "materialId" param ($in-matches _material._id/_materials._id server-side - - # see JobDAO#filterByMaterialId), so no need to fetch every finished job and narrow in Python. - matching_jobs = api_client.jobs.request( - "GET", - api_client.jobs.name, - params={"materialId": ids, "ownerId": owner_id, "status": "finished"}, - headers=api_client.jobs.headers, - ) - for job in matching_jobs: + query = {"materialId": ids, "ownerId": owner_id, "status": "finished", "globalSearch": True} + for job in api_client.jobs.list(query): properties = api_client.properties.get_for_job(job["_id"], PropertyName.non_scalar.final_structure.value) if not properties: continue @@ -162,34 +136,7 @@ def get_bulk_material_by_crystal(api_client: APIClient, bulk_crystal: Material, def _require_material_for_owner(api_client: APIClient, query: dict, owner_id: str) -> Material: - # Using .request() with flat params instead of .list(): .list() always wraps its argument as - # a query= blob, which the materials list endpoint (migrated to a validated use case) - # silently drops. "_id" and "hash" have flat equivalents ("id"/"hashes"); "scaledHash" (the - # last fallback in resolve_bulk_query_from_crystal) does not, so that case fetches this - # account's materials and filters in Python instead. - if "_id" in query: - matches = api_client.materials.request( - "GET", - api_client.materials.name, - params={"id": query["_id"], "ownerId": owner_id}, - headers=api_client.materials.headers, - ) - elif "hash" in query: - matches = api_client.materials.request( - "GET", - api_client.materials.name, - params={"hashes": query["hash"], "ownerId": owner_id}, - headers=api_client.materials.headers, - ) - else: - scaled_hash = query["scaledHash"] - account_materials = api_client.materials.request( - "GET", - api_client.materials.name, - params={"ownerId": owner_id}, - headers=api_client.materials.headers, - ) - matches = [m for m in account_materials if m.get("scaledHash") == scaled_hash] + matches = api_client.materials.list({**query, "ownerId": owner_id, "globalSearch": True}) material_response = next(iter(matches), None) if material_response is None: raise ValueError( @@ -232,17 +179,7 @@ def find_material_set( Raises: ValueError: If no set matches, or if `require_ordered` and the match is unordered. """ - # Using .request() with flat params instead of .list(): .list() always wraps its argument as - # a query= blob, which the materials list endpoint (migrated to a validated use case) - # silently drops. "isEntitySet" is a flat key, but there's no flat regex/substring match on - # "name", so that part is applied in Python instead (entity sets per account are few, so - # fetching them all and filtering here is cheap). - account_entity_sets = api_client.materials.request( - "GET", - api_client.materials.name, - params={"ownerId": owner_id, "isEntitySet": "true"}, - headers=api_client.materials.headers, - ) + account_entity_sets = api_client.materials.list({"ownerId": owner_id, "isEntitySet": True}) name_pattern = re.compile(re.escape(material_set_name), re.IGNORECASE) material_sets = [s for s in account_entity_sets if name_pattern.search(s.get("name", ""))] if not material_sets: @@ -267,15 +204,7 @@ def list_materials_in_set(api_client: APIClient, owner_id: str, material_set: Di that already have one do not re-query for it. """ material_set_id = material_set["_id"] - # Using .request() with flat params instead of .list(): "setId"/"isEntitySet" are both flat, - # declared keys ("isEntitySet": "false" is the exact-match equivalent of "$ne": True for a - # boolean field), unlike the dot-notation "inSet._id" .list() sent before. - matches = api_client.materials.request( - "GET", - api_client.materials.name, - params={"ownerId": owner_id, "setId": material_set_id, "isEntitySet": "false"}, - headers=api_client.materials.headers, - ) + matches = api_client.materials.list({"ownerId": owner_id, "setId": material_set_id, "isEntitySet": False}) members = [material for material in matches if not material.get("isEntitySet")] return sorted(members, key=lambda material: _index_in_set(material, material_set_id)) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/property/api.py b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py index 0b64f8dcb..82bb31ad7 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/property/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py @@ -11,40 +11,19 @@ } -def _get_properties_for_job( - client: APIClient, job_id: str, property_name: Optional[str] = None, unit_id: Optional[str] = None -) -> List[dict]: - """ - Replacement for the broken `client.properties.get_for_job()`: that method builds its filter - as `{"source.info.jobId": ..., "data.name": ...}` via `.list()`, which wraps it as a - query= blob - the properties list endpoint (migrated to a validated use case) silently - drops it, since it only accepts flat, declared keys. "jobId"/"slug" are the flat equivalents - of "source.info.jobId"/"data.name"; there's no flat equivalent for "source.info.unitId", so - that part (rare - only used for disambiguating fermiEnergy by flowchart unit) is filtered in - Python. Returns each match's "data" sub-dict, matching get_for_job()'s own return shape. - """ - params = {"jobId": job_id} - if property_name: - params["slug"] = property_name - holders = client.properties.request("GET", client.properties.name, params=params, headers=client.properties.headers) - if unit_id: - holders = [h for h in holders if h.get("source", {}).get("info", {}).get("unitId") == unit_id] - return [holder["data"] for holder in holders] - - def get_properties_for_job(client: APIClient, job_id: str, property_name: Optional[str] = None) -> List[dict]: """ Fetch properties for a job, automatically enriching band_structure/DOS results with fermiEnergy. Use instead of client.properties.get_for_job when passing results to visualize_properties. """ job = client.jobs.get(job_id) - properties = _get_properties_for_job(client, job_id, property_name) + properties = client.properties.get_for_job(job_id, property_name) if property_name not in FERMI_ENERGY_PROPERTIES: return properties flowchart_id = get_fermi_energy_flowchart_id(job) fermi_energy = None if flowchart_id: - fe_props = _get_properties_for_job(client, job_id, PropertyName.scalar.fermi_energy.value, flowchart_id) + fe_props = client.properties.get_for_job(job_id, PropertyName.scalar.fermi_energy.value, flowchart_id) if fe_props: fermi_energy = fe_props[0].get("value") return [{**prop, "fermiEnergy": fermi_energy} for prop in properties] @@ -65,13 +44,10 @@ def get_property_holder_for_job( Returns: dict: Full property holder document. """ - # Using .request() with flat params instead of .list(): see _get_properties_for_job above for - # why, and note this function (unlike get_properties_for_job) needs the full property holder, - # not just its "data" sub-dict. - params = {"jobId": job_id, "slug": property_name} - holders = client.properties.request("GET", client.properties.name, params=params, headers=client.properties.headers) + query = {"jobId": job_id, "propertyName": property_name} if unit_id: - holders = [h for h in holders if h.get("source", {}).get("info", {}).get("unitId") == unit_id] + query["unitId"] = unit_id + holders = client.properties.list(query) if not holders: raise ValueError(f"Property '{property_name}' not found for job '{job_id}'") return holders[0] @@ -115,24 +91,18 @@ def find_total_energy_for_material(client: APIClient, material_id: str, source: exabyte_id = material.get("exabyteId") if not exabyte_id: return None - # Using .request() with flat params instead of .list(): see _get_properties_for_job above for - # why. "owner.slug"/"owner._id" map onto the flat "ownerSlug"/"ownerId" keys; sorting/limiting - # by best precision has no flat equivalent (the endpoint's "sort" param expects a nested - # object, which a simple flat query param can't carry), so that's done in Python instead - - # there are only ever a few differently-precise total_energy properties per material. - params = {"exabyteId": exabyte_id, "slug": "total_energy"} + query = {"exabyteId": exabyte_id, "slug": "total_energy"} if source == "curators": - params["ownerSlug"] = "curators" + query["ownerSlug"] = "curators" elif source == "my_account": - params["ownerId"] = client.my_account.id + query["ownerId"] = client.my_account.id elif source != "public": raise ValueError(f"Invalid source: {source!r}. Expected 'public', 'curators', or 'my_account'.") - properties = client.properties.request( - "GET", client.properties.name, params=params, headers=client.properties.headers + properties = client.properties.list( + query=query, + projection={"sort": "-precision.value", "limit": 1}, ) - if not properties: - return None - return max(properties, key=lambda prop: prop.get("precision", {}).get("value", 0)) + return properties[0] if properties else None def get_property_by_subworkflow_and_unit_indicies( diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py index 72ab276da..069c12495 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -14,16 +14,7 @@ def get_or_create_workflow(api_client: APIClient, workflow: Workflow, owner_id: Returns: dict: The workflow dict (existing or newly created). """ - # Using .request() with flat params instead of .list(): .list() always wraps its argument as - # a query= blob, which the workflows list endpoint (migrated to a validated use case) - # silently drops since it only accepts flat, declared keys - "hash" and "ownerId" are those - # flat equivalents. - existing = api_client.workflows.request( - "GET", - api_client.workflows.name, - params={"hash": workflow.hash, "ownerId": owner_id}, - headers=api_client.workflows.headers, - ) + existing = api_client.workflows.list({"hash": workflow.hash, "ownerId": owner_id, "globalSearch": True}) if existing: print(f"♻️ Reusing already existing Workflow: {existing[0]['_id']}") return existing[0] diff --git a/tests/py/unit/core/entity/test_job_api.py b/tests/py/unit/core/entity/test_job_api.py index a0942a2dc..6cd1ab057 100644 --- a/tests/py/unit/core/entity/test_job_api.py +++ b/tests/py/unit/core/entity/test_job_api.py @@ -87,12 +87,13 @@ def test_find_job_for_material_returns_the_job_when_found(statuses): assert job == EXISTING_JOB client.jobs.list.assert_called_once_with( { - "_material._id": MATERIAL_INITIAL["_id"], - "owner._id": OWNER_ID, - "workflow.name": RELAX_WORKFLOW_NAME, - "status": {"$in": list(statuses)}, - }, - {"limit": 1}, + "materialId": MATERIAL_INITIAL["_id"], + "ownerId": OWNER_ID, + "workflowName": RELAX_WORKFLOW_NAME, + "status": list(statuses), + "globalSearch": True, + "limit": 1, + } ) @@ -111,4 +112,4 @@ def test_find_job_for_material_defaults_to_finished_only(): find_job_for_material(client, MATERIAL_INITIAL["_id"], RELAX_WORKFLOW_NAME, OWNER_ID) - assert client.jobs.list.call_args.args[0]["status"] == {"$in": ["finished"]} + assert client.jobs.list.call_args.args[0]["status"] == ["finished"] diff --git a/tests/py/unit/core/entity/test_material_analysis.py b/tests/py/unit/core/entity/test_material_analysis.py index ac99995f8..3acecb607 100644 --- a/tests/py/unit/core/entity/test_material_analysis.py +++ b/tests/py/unit/core/entity/test_material_analysis.py @@ -15,10 +15,10 @@ [ ( {"scaledHash": "scaled-hash-value", "hash": "hash-value", "_id": "material-id"}, - {"_id": "material-id"}, + {"id": "material-id"}, ), - ({"scaledHash": "scaled-hash-value", "hash": "hash-value"}, {"hash": "hash-value"}), - ({"scaledHash": "scaled-hash-value"}, {"scaledHash": "scaled-hash-value"}), + ({"scaledHash": "scaled-hash-value", "hash": "hash-value"}, {"hashes": "hash-value"}), + ({"scaledHash": "scaled-hash-value"}, {"scaledHashes": "scaled-hash-value"}), ], ) def test_resolve_bulk_query_prefers_id_then_hash_then_scaled_hash(extra_keys, expected): @@ -27,8 +27,8 @@ def test_resolve_bulk_query_prefers_id_then_hash_then_scaled_hash(extra_keys, ex def test_resolve_bulk_query_computes_hash_when_none_present(): query = resolve_bulk_query_from_crystal(SILICON) - assert set(query) == {"hash"} - assert query["hash"] + assert set(query) == {"hashes"} + assert query["hashes"] def test_slab_bulk_crystal_is_the_material_the_slab_was_built_from(): diff --git a/tests/py/unit/core/entity/test_material_api.py b/tests/py/unit/core/entity/test_material_api.py index ae91286a4..16f18d0b4 100644 --- a/tests/py/unit/core/entity/test_material_api.py +++ b/tests/py/unit/core/entity/test_material_api.py @@ -57,22 +57,18 @@ def _client_with_list_responses(responses: List[List[Dict[str, Any]]]) -> MagicMock: client = MagicMock() - client.materials.request.side_effect = responses + client.materials.list.side_effect = responses return client -def test_find_material_set_returns_first_match(): - client = _client_with_list_responses([[ENTITY_SET]]) +def test_find_material_set_matches_the_name_literally_and_case_insensitively(): + decoy_set = {**ENTITY_SET, "_id": "set-decoy", "name": "H22H"} + client = _client_with_list_responses([[decoy_set, ENTITY_SET]]) - material_set = find_material_set(client, OWNER_ID, MATERIAL_SET_NAME) + material_set = find_material_set(client, OWNER_ID, MATERIAL_SET_NAME.lower()) assert material_set["_id"] == MATERIAL_SET_ID - client.materials.request.assert_called_once_with( - "GET", - client.materials.name, - params={"ownerId": OWNER_ID, "isEntitySet": "true"}, - headers=client.materials.headers, - ) + client.materials.list.assert_called_once_with({"ownerId": OWNER_ID, "isEntitySet": True}) assert "+" in MATERIAL_SET_NAME assert re.escape(MATERIAL_SET_NAME) != MATERIAL_SET_NAME @@ -96,7 +92,7 @@ def test_list_materials_by_set_rejects_unordered_when_order_required(): with pytest.raises(ValueError, match="is 'unordered', not 'ordered'"): list_materials_by_set(client, OWNER_ID, MATERIAL_SET_NAME, require_ordered=True) - assert client.materials.request.call_count == 1 + assert client.materials.list.call_count == 1 def test_list_materials_in_set_does_not_re_resolve_the_set(): @@ -105,12 +101,7 @@ def test_list_materials_in_set_does_not_re_resolve_the_set(): materials = list_materials_in_set(client, OWNER_ID, ENTITY_SET) assert [material["_id"] for material in materials] == EXPECTED_ORDERED_IDS - client.materials.request.assert_called_once_with( - "GET", - client.materials.name, - params={"ownerId": OWNER_ID, "setId": MATERIAL_SET_ID, "isEntitySet": "false"}, - headers=client.materials.headers, - ) + client.materials.list.assert_called_once_with({"ownerId": OWNER_ID, "setId": MATERIAL_SET_ID, "isEntitySet": False}) @pytest.mark.parametrize( @@ -126,10 +117,10 @@ def test_list_materials_by_set_orders_by_inset_index(members, expected_ids): materials = list_materials_by_set(client, OWNER_ID, MATERIAL_SET_NAME) assert [material["_id"] for material in materials] == expected_ids - assert client.materials.request.call_args_list[1].kwargs["params"] == { + assert client.materials.list.call_args_list[1].args[0] == { "ownerId": OWNER_ID, "setId": MATERIAL_SET_ID, - "isEntitySet": "false", + "isEntitySet": False, } @@ -217,7 +208,7 @@ def test_get_or_create_materials_set_ordered_requires_two_materials(): [MATERIAL_INITIAL], is_ordered=True, ) - client.materials.request.assert_not_called() + client.materials.list.assert_not_called() def test_get_or_create_materials_set_requires_one_material(): @@ -231,7 +222,7 @@ def test_get_or_create_materials_set_requires_one_material(): [], is_ordered=False, ) - client.materials.request.assert_not_called() + client.materials.list.assert_not_called() DEFECTIVE_HASH = "hash-defective" @@ -254,8 +245,8 @@ def test_get_or_create_materials_set_requires_one_material(): def test_find_relaxed_material_returns_final_structure_from_the_job(): client = MagicMock() - client.materials.request.return_value = [SAVED_DEFECTIVE] - client.jobs.request.return_value = [FINISHED_JOB] + client.materials.list.return_value = [SAVED_DEFECTIVE] + client.jobs.list.return_value = [FINISHED_JOB] client.properties.get_for_job.return_value = [{"materialId": "m-relaxed"}] client.materials.get.return_value = RELAXED_MATERIAL_DOC @@ -263,17 +254,9 @@ def test_find_relaxed_material_returns_final_structure_from_the_job(): assert relaxed is not None assert relaxed.name == "B-vacancy h-BN relaxed" - client.materials.request.assert_called_once_with( - "GET", - client.materials.name, - params={"hashes": DEFECTIVE_HASH, "ownerId": OWNER_ID}, - headers=client.materials.headers, - ) - client.jobs.request.assert_called_once_with( - "GET", - client.jobs.name, - params={"materialId": [SAVED_DEFECTIVE["_id"]], "ownerId": OWNER_ID, "status": "finished"}, - headers=client.jobs.headers, + client.materials.list.assert_called_once_with({"hashes": DEFECTIVE_HASH, "ownerId": OWNER_ID, "globalSearch": True}) + client.jobs.list.assert_called_once_with( + {"materialId": [SAVED_DEFECTIVE["_id"]], "ownerId": OWNER_ID, "status": "finished", "globalSearch": True} ) client.properties.get_for_job.assert_called_once_with(FINISHED_JOB["_id"], "final_structure") client.materials.get.assert_called_once_with("m-relaxed") @@ -283,8 +266,8 @@ def test_find_relaxed_material_uses_the_last_final_structure_entry(): # A relaxation job's final_structure property can carry more than one entry: the initial # structure (same hash as the input) first, the relaxed one last -- PLAN #27. client = MagicMock() - client.materials.request.return_value = [SAVED_DEFECTIVE] - client.jobs.request.return_value = [FINISHED_JOB] + client.materials.list.return_value = [SAVED_DEFECTIVE] + client.jobs.list.return_value = [FINISHED_JOB] client.properties.get_for_job.return_value = [{"materialId": "m-initial"}, {"materialId": "m-relaxed"}] client.materials.get.side_effect = lambda material_id: { "m-initial": SCF_MATERIAL_DOC, @@ -300,18 +283,17 @@ def test_find_relaxed_material_uses_the_last_final_structure_entry(): def test_find_relaxed_material_returns_none_when_material_is_not_on_the_platform(): client = MagicMock() - client.materials.request.return_value = [] - client.jobs.request.return_value = [] + client.materials.list.return_value = [] assert find_relaxed_material(client, DEFECTIVE_MATERIAL, OWNER_ID) is None - # No matching materials -> nothing to filter jobs by, so the jobs endpoint is never queried. - client.jobs.request.assert_not_called() + # an empty $in drops the job filter and would match every finished job + client.jobs.list.assert_not_called() def test_find_relaxed_material_returns_none_when_no_job_exists(): client = MagicMock() - client.materials.request.return_value = [SAVED_DEFECTIVE] - client.jobs.request.return_value = [] + client.materials.list.return_value = [SAVED_DEFECTIVE] + client.jobs.list.return_value = [] assert find_relaxed_material(client, DEFECTIVE_MATERIAL, OWNER_ID) is None client.properties.get_for_job.assert_not_called() @@ -319,8 +301,8 @@ def test_find_relaxed_material_returns_none_when_no_job_exists(): def test_find_relaxed_material_returns_none_when_job_has_no_final_structure(): client = MagicMock() - client.materials.request.return_value = [SAVED_DEFECTIVE] - client.jobs.request.return_value = [FINISHED_JOB] + client.materials.list.return_value = [SAVED_DEFECTIVE] + client.jobs.list.return_value = [FINISHED_JOB] client.properties.get_for_job.return_value = [] assert find_relaxed_material(client, DEFECTIVE_MATERIAL, OWNER_ID) is None @@ -330,8 +312,8 @@ def test_find_relaxed_material_returns_none_when_job_has_no_final_structure(): def test_find_relaxed_material_checks_every_same_hash_material(): other_material: Dict[str, Any] = {"_id": "m-other", "name": "B-vacancy h-BN", "hash": DEFECTIVE_HASH} client = MagicMock() - client.materials.request.return_value = [other_material, SAVED_DEFECTIVE] - client.jobs.request.return_value = [FINISHED_JOB] + client.materials.list.return_value = [other_material, SAVED_DEFECTIVE] + client.jobs.list.return_value = [FINISHED_JOB] client.properties.get_for_job.return_value = [{"materialId": "m-relaxed"}] client.materials.get.return_value = RELAXED_MATERIAL_DOC @@ -339,15 +321,8 @@ def test_find_relaxed_material_checks_every_same_hash_material(): assert relaxed is not None assert relaxed.name == "B-vacancy h-BN relaxed" - client.jobs.request.assert_called_once_with( - "GET", - client.jobs.name, - params={ - "materialId": [other_material["_id"], SAVED_DEFECTIVE["_id"]], - "ownerId": OWNER_ID, - "status": "finished", - }, - headers=client.jobs.headers, + client.jobs.list.assert_called_once_with( + {"materialId": ["m-other", "m-defective"], "ownerId": OWNER_ID, "status": "finished", "globalSearch": True} ) @@ -355,8 +330,8 @@ def test_find_relaxed_material_skips_a_final_structure_with_the_same_hash(): scf_job: Dict[str, Any] = {"_id": "job-scf", "name": "Total Energy", "status": "finished"} relax_job: Dict[str, Any] = {"_id": "job-relax", "name": "Fixed-cell Relaxation", "status": "finished"} client = MagicMock() - client.materials.request.return_value = [SAVED_DEFECTIVE] - client.jobs.request.return_value = [scf_job, relax_job] + client.materials.list.return_value = [SAVED_DEFECTIVE] + client.jobs.list.return_value = [scf_job, relax_job] client.properties.get_for_job.side_effect = [[{"materialId": "m-scf"}], [{"materialId": "m-relaxed"}]] client.materials.get.side_effect = [SCF_MATERIAL_DOC, RELAXED_MATERIAL_DOC] @@ -377,28 +352,25 @@ def test_load_material_finds_an_exact_match_in_the_folder(tmp_path): material = load_material(client, str(tmp_path), "Silicon", OWNER_ID) assert material.name == "Silicon" - client.materials.request.assert_not_called() + client.materials.list.assert_not_called() def test_load_material_falls_back_to_the_account(tmp_path): (tmp_path / "silicon.json").write_text(json.dumps(Materials.get_by_name_first_match("Silicon"))) client = MagicMock() - client.materials.request.return_value = [SILICON_NAMED] + client.materials.list.return_value = [SILICON_NAMED] material = load_material(client, str(tmp_path), "Silicon", OWNER_ID) assert material.name == "Silicon" - client.materials.request.assert_called_once_with( - "GET", - client.materials.name, - params={"name": "Silicon", "ownerId": OWNER_ID, "limit": 1}, - headers=client.materials.headers, + client.materials.list.assert_called_once_with( + {"name": "Silicon", "ownerId": OWNER_ID, "globalSearch": True, "limit": 1} ) def test_load_material_raises_when_neither_has_it(tmp_path): client = MagicMock() - client.materials.request.return_value = [] + client.materials.list.return_value = [] with pytest.raises(ValueError, match="Germanium"): load_material(client, str(tmp_path), "Germanium", OWNER_ID) @@ -408,16 +380,13 @@ def test_load_material_falls_through_a_folder_near_miss(tmp_path): (tmp_path / "silicon.json").write_text(json.dumps(SILICON_NAMED)) (tmp_path / "silicon relaxed.json").write_text(json.dumps({**SILICON_NAMED, "name": "Silicon relaxed"})) client = MagicMock() - client.materials.request.return_value = [SILICON_NAMED] + client.materials.list.return_value = [SILICON_NAMED] material = load_material(client, str(tmp_path), "Silicon", OWNER_ID) assert material.name == "Silicon" - client.materials.request.assert_called_once_with( - "GET", - client.materials.name, - params={"name": "Silicon", "ownerId": OWNER_ID, "limit": 1}, - headers=client.materials.headers, + client.materials.list.assert_called_once_with( + {"name": "Silicon", "ownerId": OWNER_ID, "globalSearch": True, "limit": 1} ) @@ -426,15 +395,15 @@ def test_load_material_falls_through_a_folder_near_miss(tmp_path): @pytest.mark.parametrize( - ("holders", "error"), + ("properties", "error"), [ - ([{"data": {"materialId": FINAL_STRUCTURE_MATERIAL_ID}}], None), + ([{"materialId": FINAL_STRUCTURE_MATERIAL_ID}], None), ([], "reported no 'final_structure'"), ], ) -def test_get_final_structure_for_job(holders, error): +def test_get_final_structure_for_job(properties, error): client = MagicMock() - client.properties.request.return_value = holders + client.properties.get_for_job.return_value = properties client.materials.get.return_value = Materials.get_by_name_first_match("Silicon") if error: with pytest.raises(RuntimeError, match=error): @@ -442,10 +411,5 @@ def test_get_final_structure_for_job(holders, error): return material = get_final_structure_for_job(client, JOB_ID) assert material.basis.elements.values == ["Si", "Si"] - client.properties.request.assert_called_once_with( - "GET", - client.properties.name, - params={"jobId": JOB_ID, "slug": "final_structure"}, - headers=client.properties.headers, - ) + client.properties.get_for_job.assert_called_once_with(JOB_ID, "final_structure") client.materials.get.assert_called_once_with(FINAL_STRUCTURE_MATERIAL_ID) diff --git a/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py b/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py index 787dbf862..3a36cd78f 100644 --- a/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py +++ b/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py @@ -15,7 +15,7 @@ def _client(): client = MagicMock() client.materials.get.return_value = MATERIAL - client.properties.request.return_value = [TOTAL_ENERGY_PROPERTY] + client.properties.list.return_value = [TOTAL_ENERGY_PROPERTY] client.my_account.id = OWNER_ACCOUNT_ID # entity_cache is often None right after auth -- this must not be relied on. client.my_account.entity_cache = None @@ -28,15 +28,13 @@ def test_find_total_energy_for_material_defaults_to_my_account_scope(): result = find_total_energy_for_material(client, MATERIAL_ID) client.materials.get.assert_called_once_with(MATERIAL_ID) - client.properties.request.assert_called_once_with( - "GET", - client.properties.name, - params={ + client.properties.list.assert_called_once_with( + query={ "exabyteId": EXABYTE_ID, "slug": "total_energy", "ownerId": OWNER_ACCOUNT_ID, }, - headers=client.properties.headers, + projection={"sort": "-precision.value", "limit": 1}, ) client.jobs.list.assert_not_called() assert result == TOTAL_ENERGY_PROPERTY @@ -47,15 +45,13 @@ def test_find_total_energy_for_material_my_account_scope(): find_total_energy_for_material(client, MATERIAL_ID, source="my_account") - client.properties.request.assert_called_once_with( - "GET", - client.properties.name, - params={ + client.properties.list.assert_called_once_with( + query={ "exabyteId": EXABYTE_ID, "slug": "total_energy", "ownerId": OWNER_ACCOUNT_ID, }, - headers=client.properties.headers, + projection={"sort": "-precision.value", "limit": 1}, ) @@ -64,15 +60,13 @@ def test_find_total_energy_for_material_curators_scope(): find_total_energy_for_material(client, MATERIAL_ID, source="curators") - client.properties.request.assert_called_once_with( - "GET", - client.properties.name, - params={ + client.properties.list.assert_called_once_with( + query={ "exabyteId": EXABYTE_ID, "slug": "total_energy", "ownerSlug": "curators", }, - headers=client.properties.headers, + projection={"sort": "-precision.value", "limit": 1}, ) @@ -81,14 +75,12 @@ def test_find_total_energy_for_material_public_scope_has_no_owner_filter(): find_total_energy_for_material(client, MATERIAL_ID, source="public") - client.properties.request.assert_called_once_with( - "GET", - client.properties.name, - params={ + client.properties.list.assert_called_once_with( + query={ "exabyteId": EXABYTE_ID, "slug": "total_energy", }, - headers=client.properties.headers, + projection={"sort": "-precision.value", "limit": 1}, ) @@ -101,7 +93,7 @@ def test_find_total_energy_for_material_rejects_invalid_source(): def test_find_total_energy_for_material_returns_none_when_no_property_found(): client = _client() - client.properties.request.return_value = [] + client.properties.list.return_value = [] result = find_total_energy_for_material(client, MATERIAL_ID) @@ -114,5 +106,5 @@ def test_find_total_energy_for_material_returns_none_when_material_has_no_exabyt result = find_total_energy_for_material(client, MATERIAL_ID) - client.properties.request.assert_not_called() + client.properties.list.assert_not_called() assert result is None From 235f3d36f83d05d29a516edf007896310e21977d Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Wed, 7 Oct 2026 01:25:43 +0300 Subject: [PATCH 09/10] revert(SOF-8032): keep the original list() queries in the notebooks and helpers The api-client list() now sends Mongo-style queries as advancedSearches, so the notebooks keep their original form: no show_default lookups, no flat list parameters, no get_default_* helpers. What stays from the PR: pandas in config.yml, the TEXT_SHADOW removal in plot.py, and the notebook fixes that do not depend on list() (embedding the full workflow when creating a job, the saved-workflow response variables, and the project id in the equation_of_state set). Co-Authored-By: Claude Sonnet 5.5 --- examples/job/create_and_submit_job.ipynb | 13 ++--- examples/job/get-file-from-job.ipynb | 11 +++-- .../ml-train-model-predict-properties.ipynb | 13 ++--- ...n-simulations-and-extract-properties.ipynb | 8 ++-- .../material/get_materials_by_formula.ipynb | 8 +--- ...bilayer_twisted_molybdenum_disulfide.ipynb | 14 ++---- ...bilayer_twisted_molybdenum_disulfide.ipynb | 14 ++---- examples/workflow/get_workflows.ipynb | 10 +--- ...int_substitution_graphene_SIMULATION.ipynb | 8 ++-- ...int_vacancy_boron_nitride_SIMULATION.ipynb | 12 ++--- ...ices_molybdenum_disulfide_SIMULATION.ipynb | 8 ++-- ..._position_graphene_nickel_SIMULATION.ipynb | 8 ++-- .../workflows/analyze_convex_hull.ipynb | 13 ++--- .../workflows/band_gap.ipynb | 8 ++-- .../workflows/band_structure.ipynb | 8 ++-- .../workflows/band_structure_hse.ipynb | 8 ++-- .../workflows/band_structure_magn.ipynb | 8 ++-- .../workflows/convergence.ipynb | 8 ++-- .../workflows/custom_python_calculation.ipynb | 8 ++-- .../workflows/custom_shell_calculation.ipynb | 8 ++-- .../workflows/defect_formation_energy.ipynb | 19 ++++---- .../workflows/dielectric_tensor.ipynb | 8 ++-- .../workflows/equation_of_state.ipynb | 8 ++-- .../workflows/formation_energy.ipynb | 10 ++-- .../workflows/homo_lumo_frequency.ipynb | 16 ++++--- .../workflows/interfacial_energy.ipynb | 17 +++---- other/materials_designer/workflows/neb.ipynb | 8 ++-- .../workflows/phonon_dos_dispersion.ipynb | 8 ++-- .../workflows/relaxation.ipynb | 8 ++-- .../workflows/surface_energy.ipynb | 19 ++++---- .../workflows/total_energy.ipynb | 8 ++-- .../workflows/total_energy_convergence.ipynb | 8 ++-- .../total_energy_post_processing.ipynb | 8 ++-- .../workflows/valence_band_offset.ipynb | 8 ++-- .../workflows/zero_point_energy.ipynb | 8 ++-- .../notebooks_utils/core/entity/job/api.py | 15 +++--- .../core/entity/material/analysis.py | 6 +-- .../core/entity/material/api.py | 27 ++++++----- .../core/entity/material/default.py | 22 --------- .../core/entity/project/__init__.py | 0 .../core/entity/project/api.py | 15 ------ .../core/entity/property/api.py | 15 +++--- .../core/entity/workflow/api.py | 2 +- .../core/entity/workflow/default.py | 22 --------- .../core/entity/test_default_entity_api.py | 47 ------------------- tests/py/unit/core/entity/test_job_api.py | 15 +++--- .../core/entity/test_material_analysis.py | 10 ++-- .../py/unit/core/entity/test_material_api.py | 43 +++++++++-------- ...erty_api_find_total_energy_for_material.py | 14 +++--- 49 files changed, 225 insertions(+), 385 deletions(-) delete mode 100644 src/py/mat3ra/notebooks_utils/core/entity/material/default.py delete mode 100644 src/py/mat3ra/notebooks_utils/core/entity/project/__init__.py delete mode 100644 src/py/mat3ra/notebooks_utils/core/entity/project/api.py delete mode 100644 src/py/mat3ra/notebooks_utils/core/entity/workflow/default.py delete mode 100644 tests/py/unit/core/entity/test_default_entity_api.py diff --git a/examples/job/create_and_submit_job.ipynb b/examples/job/create_and_submit_job.ipynb index 36abcbc7a..205d16eca 100644 --- a/examples/job/create_and_submit_job.ipynb +++ b/examples/job/create_and_submit_job.ipynb @@ -97,11 +97,11 @@ "outputs": [], "source": [ "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "from mat3ra.notebooks_utils.core.entity.material.default import get_default_material\n", - "from mat3ra.notebooks_utils.core.entity.workflow.default import get_default_workflow\n", + "default_material = client.materials.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0]\n", + "default_workflow = client.workflows.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0]\n", "\n", - "default_material = get_default_material(client, OWNER_ID)\n", - "default_workflow = get_default_workflow(client, OWNER_ID)" + "material_id = default_material[\"_id\"]\n", + "workflow_id = default_workflow[\"_id\"]" ] }, { @@ -120,10 +120,11 @@ }, "outputs": [], "source": [ + "workflow = client.workflows.get(workflow_id)\n", "config = {\n", " \"owner\": {\"_id\": OWNER_ID},\n", - " \"_material\": {\"_id\": default_material[\"_id\"]},\n", - " \"workflow\": default_workflow,\n", + " \"_material\": {\"_id\": material_id},\n", + " \"workflow\": workflow,\n", " \"name\": JOB_NAME,\n", "}" ] diff --git a/examples/job/get-file-from-job.ipynb b/examples/job/get-file-from-job.ipynb index a5112d67d..43a401d79 100644 --- a/examples/job/get-file-from-job.ipynb +++ b/examples/job/get-file-from-job.ipynb @@ -88,19 +88,20 @@ "execution_count": null, "source": [ "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", "from mat3ra.notebooks_utils.ipython.ui import display_JSON\n", "\n", "# Relevant functions from the API client\n", "# Get some account information\n", "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "project_metadata = get_default_project(client, OWNER_ID)\n", + "project_metadata = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0]\n", "project_id = project_metadata[\"_id\"]\n", "owner_id = project_metadata[\"owner\"][\"_id\"]\n", "\n", "# Get a workflow for the job from the bank, and copy it to our account\n", - "WORKFLOW_QUERY = {\"systemName\": \"espresso-total-energy\"}\n", - "bank_workflow_id = client.bank_workflows.list(WORKFLOW_QUERY, {\"limit\": 1})[0][\"_id\"]\n", + "WORKFLOW_QUERY = {\"systemName\": \"espresso-total-energy\", \"limit\": 1}\n", + "bank_workflow_id = client.bank_workflows.request(\n", + " \"GET\", client.bank_workflows.name, params=WORKFLOW_QUERY, headers=client.bank_workflows.headers\n", + ")[0][\"_id\"]\n", "workflow_id = client.bank_workflows.copy(bank_workflow_id, owner_id)[\"_id\"]\n", "\n", "# Get materials from bank and copy one to our account\n", @@ -108,7 +109,7 @@ "bank_materials = client.bank_materials.list(MATERIAL_QUERY)\n", "bank_material_id = bank_materials[0][\"_id\"]\n", "material_id = client.bank_materials.copy(bank_material_id, owner_id)[\"_id\"]\n", - "materials = client.materials.list({\"ownerId\": owner_id})\n", + "materials = client.materials.list({\"owner._id\": owner_id})\n", "\n", "# Create the job - JobsCreate's schema requires the full workflow document embedded (a job keeps\n", "# an immutable snapshot of what it actually ran), not just a {\"_id\": ...} reference, so fetch it\n", diff --git a/examples/job/ml-train-model-predict-properties.ipynb b/examples/job/ml-train-model-predict-properties.ipynb index 4453462c0..2d5e56bfc 100644 --- a/examples/job/ml-train-model-predict-properties.ipynb +++ b/examples/job/ml-train-model-predict-properties.ipynb @@ -213,12 +213,9 @@ }, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.material.default import get_default_material\n", - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "owner_id = get_default_material(client, OWNER_ID)[\"owner\"][\"_id\"]\n", - "project_id = get_default_project(client, OWNER_ID)[\"_id\"]" + "owner_id = client.materials.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"owner\"][\"_id\"]\n", + "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]" ] }, { @@ -277,9 +274,9 @@ "train_material_id_ge = client.materials.create(train_material_json_ge, OWNER_ID)[\"_id\"]\n", "target_material_id_sige = client.materials.create(target_material_json_sige, OWNER_ID)[\"_id\"]\n", "\n", - "train_material_si = client.materials.list({\"id\": train_material_id_si})[0]\n", - "train_material_ge = client.materials.list({\"id\": train_material_id_ge})[0]\n", - "target_material_sige = client.materials.list({\"id\": target_material_id_sige})[0]\n", + "train_material_si = client.materials.list({\"_id\": train_material_id_si})[0]\n", + "train_material_ge = client.materials.list({\"_id\": train_material_id_ge})[0]\n", + "target_material_sige = client.materials.list({\"_id\": target_material_id_sige})[0]\n", "\n", "train_materials = [train_material_ge, train_material_si]\n", "target_materials = [target_material_sige]" diff --git a/examples/job/run-simulations-and-extract-properties.ipynb b/examples/job/run-simulations-and-extract-properties.ipynb index 65917a0f2..bb8a5db74 100644 --- a/examples/job/run-simulations-and-extract-properties.ipynb +++ b/examples/job/run-simulations-and-extract-properties.ipynb @@ -252,9 +252,7 @@ "outputs": [], "source": [ "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project_id = get_default_project(client, OWNER_ID)[\"_id\"]" + "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]" ] }, { @@ -330,8 +328,8 @@ "material_id_si = client.materials.create(material_json_si, OWNER_ID)[\"_id\"]\n", "material_id_ge = client.materials.create(material_json_ge, OWNER_ID)[\"_id\"]\n", "\n", - "material_si = client.materials.list({\"id\": material_id_si})[0]\n", - "material_sige = client.materials.list({\"id\": material_id_ge})[0]\n", + "material_si = client.materials.list({\"_id\": material_id_si})[0]\n", + "material_sige = client.materials.list({\"_id\": material_id_ge})[0]\n", "\n", "materials = [material_sige, material_si]" ] diff --git a/examples/material/get_materials_by_formula.ipynb b/examples/material/get_materials_by_formula.ipynb index d1d0d2436..68ef5dde2 100644 --- a/examples/material/get_materials_by_formula.ipynb +++ b/examples/material/get_materials_by_formula.ipynb @@ -77,11 +77,7 @@ "metadata": { "id": "mbtRgwoBCzPE" }, - "source": [ - "## Set Parameters\n", - "\n", - "- **QUERY**: The filters of the materials list: the owner account (`ownerId`) and the chemical formula (`formula`). Other filters include `name`, `hashes`, `tags` and `quickSearch`." - ] + "source": "## Set Parameters\n\n- **QUERY**: A query describing the documents to find. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information." }, { "cell_type": "code", @@ -92,7 +88,7 @@ "outputs": [], "source": [ "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "QUERY = {\"ownerId\": OWNER_ID, \"formula\": \"Si\"}" + "QUERY = {\"formula\": \"Si\", \"owner._id\": OWNER_ID}" ] }, { diff --git a/examples/reproducing_publications/band_gaps_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb b/examples/reproducing_publications/band_gaps_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb index 09927a9cb..7f7a26811 100644 --- a/examples/reproducing_publications/band_gaps_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb +++ b/examples/reproducing_publications/band_gaps_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb @@ -336,9 +336,9 @@ "source": [ "### 7.1. Set Parameters\n", "\n", - "- **QUERY**: The filters of the workflows list: the workflow `id` and the owner account (`ownerId`).\n", + "- **QUERY**: A query describing the documents to find. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information.\n", "\n", - "- **limit**: Maximum number of results to return." + "- **limit**: Maximum number of results to return. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information." ] }, { @@ -415,7 +415,7 @@ "metadata": {}, "outputs": [], "source": [ - "QUERY = {\"id\": band_gap_workflow_id, \"ownerId\": OWNER_ID}\n", + "QUERY = {\"_id\": band_gap_workflow_id, \"owner._id\": OWNER_ID}\n", "workflow = client.workflows.list(QUERY, OPTIONS)[0]\n", "\n", "modifier_0 = {\n", @@ -561,11 +561,9 @@ "source": [ "from datetime import datetime\n", "\n", - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", "\n", - "project_id = get_default_project(client, OWNER_ID)[\"_id\"]\n", + "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]\n", "\n", "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\") # for name to be found easily\n", "\n", @@ -609,9 +607,7 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project_id = get_default_project(client, OWNER_ID)[\"_id\"]\n", + "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]\n", "\n", "JOBS_SET_NAME = \"MoS2 Twisted Interfaces Band Gaps\"\n", "\n", diff --git a/examples/reproducing_publications/band_structure_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb b/examples/reproducing_publications/band_structure_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb index 06103ce06..270ab08e8 100644 --- a/examples/reproducing_publications/band_structure_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb +++ b/examples/reproducing_publications/band_structure_for_interface_bilayer_twisted_molybdenum_disulfide.ipynb @@ -334,9 +334,9 @@ "source": [ "### 7.1. Set Parameters\n", "\n", - "- **QUERY**: The filters of the workflows list: the workflow `id` and the owner account (`ownerId`).\n", + "- **QUERY**: A query describing the documents to find. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information.\n", "\n", - "- **limit**: Maximum number of results to return." + "- **limit**: Maximum number of results to return. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information." ], "id": "9378dd5bdba80208" }, @@ -409,7 +409,7 @@ "cell_type": "code", "metadata": {}, "source": [ - "QUERY = {\"id\": band_structure_workflow_id, \"ownerId\": OWNER_ID}\n", + "QUERY = {\"_id\": band_structure_workflow_id, \"owner._id\": OWNER_ID}\n", "workflow = client.workflows.list(QUERY, OPTIONS)[0]\n", "\n", "modifier_0 = {\n", @@ -559,11 +559,9 @@ "source": [ "from datetime import datetime\n", "\n", - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", "\n", - "project_id = get_default_project(client, OWNER_ID)[\"_id\"]\n", + "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]\n", "\n", "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\") # for name to be found easily\n", "\n", @@ -607,9 +605,7 @@ "cell_type": "code", "metadata": {}, "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project_id = get_default_project(client, OWNER_ID)[\"_id\"]\n", + "project_id = client.projects.list({\"isDefault\": True, \"owner._id\": OWNER_ID})[0][\"_id\"]\n", "\n", "JOBS_SET_NAME = None\n", "\n", diff --git a/examples/workflow/get_workflows.ipynb b/examples/workflow/get_workflows.ipynb index 59a576077..b55906c2c 100644 --- a/examples/workflow/get_workflows.ipynb +++ b/examples/workflow/get_workflows.ipynb @@ -77,13 +77,7 @@ "metadata": { "id": "h8JubXPxIQAY" }, - "source": [ - "## Set Parameters\n", - "\n", - "- **QUERY**: The filters of the workflows list: the owner account (`ownerId`) and a text search over the workflow name, slug and system name (`quickSearch`, a case-insensitive pattern).\n", - "\n", - "- **limit**: Maximum number of results to return." - ] + "source": "## Set Parameters\n\n- **QUERY**: A query describing the documents to find. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information. \n\n- **limit**: Maximum number of results to return. See [Meteor collection](https://docs.meteor.com/api/collections.html#Mongo-Collection-find) for more information." }, { "cell_type": "code", @@ -94,7 +88,7 @@ "outputs": [], "source": [ "OWNER_ID = os.getenv(\"ORGANIZATION_ID\") or selected_account.id\n", - "QUERY = {\"ownerId\": OWNER_ID, \"quickSearch\": \"Total Energy\"}\n", + "QUERY = {\"name\": {\"$regex\": \"Total Energy\"}, \"owner._id\": OWNER_ID}\n", "\n", "OPTIONS = {\"limit\": 2}" ] diff --git a/other/materials_designer/specific_examples/defect_point_substitution_graphene_SIMULATION.ipynb b/other/materials_designer/specific_examples/defect_point_substitution_graphene_SIMULATION.ipynb index bbeb2d4bf..4902ba679 100644 --- a/other/materials_designer/specific_examples/defect_point_substitution_graphene_SIMULATION.ipynb +++ b/other/materials_designer/specific_examples/defect_point_substitution_graphene_SIMULATION.ipynb @@ -246,11 +246,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/specific_examples/defect_point_vacancy_boron_nitride_SIMULATION.ipynb b/other/materials_designer/specific_examples/defect_point_vacancy_boron_nitride_SIMULATION.ipynb index b3492b8b2..72cc22ced 100644 --- a/other/materials_designer/specific_examples/defect_point_vacancy_boron_nitride_SIMULATION.ipynb +++ b/other/materials_designer/specific_examples/defect_point_vacancy_boron_nitride_SIMULATION.ipynb @@ -246,11 +246,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" ] }, { @@ -303,9 +301,7 @@ "\n", "elements = sorted(set(pristine.basis.elements.values) | set(defective.basis.elements.values))\n", "\n", - "elemental_materials_data = client.materials.list(\n", - " {\"tags\": \"elemental\", \"metadataElement\": elements, \"globalSearch\": True, \"limit\": 500}\n", - ")\n", + "elemental_materials_data = client.materials.list({\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}})\n", "elemental_materials = {}\n", "for element in elements:\n", " matches = [m for m in elemental_materials_data if m.get(\"metadata\", {}).get(\"element\") == element]\n", diff --git a/other/materials_designer/specific_examples/interface_bilayer_twisted_commensurate_lattices_molybdenum_disulfide_SIMULATION.ipynb b/other/materials_designer/specific_examples/interface_bilayer_twisted_commensurate_lattices_molybdenum_disulfide_SIMULATION.ipynb index 76f469978..66d77a21d 100644 --- a/other/materials_designer/specific_examples/interface_bilayer_twisted_commensurate_lattices_molybdenum_disulfide_SIMULATION.ipynb +++ b/other/materials_designer/specific_examples/interface_bilayer_twisted_commensurate_lattices_molybdenum_disulfide_SIMULATION.ipynb @@ -237,11 +237,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/specific_examples/optimization_interface_film_xy_position_graphene_nickel_SIMULATION.ipynb b/other/materials_designer/specific_examples/optimization_interface_film_xy_position_graphene_nickel_SIMULATION.ipynb index 19ec152bf..3fb5e1926 100644 --- a/other/materials_designer/specific_examples/optimization_interface_film_xy_position_graphene_nickel_SIMULATION.ipynb +++ b/other/materials_designer/specific_examples/optimization_interface_film_xy_position_graphene_nickel_SIMULATION.ipynb @@ -613,11 +613,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/analyze_convex_hull.ipynb b/other/materials_designer/workflows/analyze_convex_hull.ipynb index b1edcf655..d50ebf299 100644 --- a/other/materials_designer/workflows/analyze_convex_hull.ipynb +++ b/other/materials_designer/workflows/analyze_convex_hull.ipynb @@ -172,7 +172,8 @@ " if set_materials is not None:\n", " matches = [material for material in set_materials if material.get(\"formula\") == formula]\n", " else:\n", - " matches = client.materials.list({\"ownerId\": ACCOUNT_ID, \"formula\": formula, \"globalSearch\": True})\n", + " matches = client.materials.list({\"formula\": formula, \"owner._id\": ACCOUNT_ID})\n", + " matches = [material for material in matches if not material.get(\"isEntitySet\")]\n", " for material in matches:\n", " material[\"_search_formula\"] = formula # track which formula query found this\n", " all_materials.extend(matches)\n", @@ -236,18 +237,18 @@ "\n", "exabyte_ids = list(set(material[\"exabyteId\"] for material in all_materials if material.get(\"exabyteId\")))\n", "\n", - "# Query properties per exabyteId to avoid API pagination truncation.\n", + "# Query properties per exabyteId to avoid API pagination truncation\n", "properties_by_exabyte_id = {}\n", "for exabyte_id in exabyte_ids:\n", " query = {\n", " \"exabyteId\": exabyte_id,\n", - " \"propertyName\": \"total_energy\",\n", + " \"data.name\": \"total_energy\",\n", " }\n", " if GROUP:\n", - " query[\"groupPrefix\"] = GROUP\n", + " query[\"group\"] = {\"$regex\": GROUP}\n", " if PRECISION is not None:\n", - " query[\"precisionValue\"] = PRECISION\n", - " props = client.properties.list(query)\n", + " query[\"precision.value\"] = PRECISION\n", + " props = client.properties.list(query=query)\n", " for property_holder in props:\n", " eid = property_holder.get(\"exabyteId\")\n", " eid = eid if isinstance(eid, str) else (eid[0] if eid else None)\n", diff --git a/other/materials_designer/workflows/band_gap.ipynb b/other/materials_designer/workflows/band_gap.ipynb index d672ab48f..7bb2f819a 100644 --- a/other/materials_designer/workflows/band_gap.ipynb +++ b/other/materials_designer/workflows/band_gap.ipynb @@ -218,11 +218,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/band_structure.ipynb b/other/materials_designer/workflows/band_structure.ipynb index 3a261d050..76790c41c 100644 --- a/other/materials_designer/workflows/band_structure.ipynb +++ b/other/materials_designer/workflows/band_structure.ipynb @@ -225,11 +225,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/band_structure_hse.ipynb b/other/materials_designer/workflows/band_structure_hse.ipynb index d9e8aff69..62fa647ba 100644 --- a/other/materials_designer/workflows/band_structure_hse.ipynb +++ b/other/materials_designer/workflows/band_structure_hse.ipynb @@ -219,11 +219,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/band_structure_magn.ipynb b/other/materials_designer/workflows/band_structure_magn.ipynb index ba15b7042..f8240c26f 100644 --- a/other/materials_designer/workflows/band_structure_magn.ipynb +++ b/other/materials_designer/workflows/band_structure_magn.ipynb @@ -252,11 +252,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/convergence.ipynb b/other/materials_designer/workflows/convergence.ipynb index 8e17788df..7ecc4d298 100644 --- a/other/materials_designer/workflows/convergence.ipynb +++ b/other/materials_designer/workflows/convergence.ipynb @@ -185,11 +185,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "PROJECT_ID = project[\"_id\"]\n", - "print(f\"Using project: {project['name']} ({PROJECT_ID})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "PROJECT_ID = projects[0][\"_id\"]\n", + "print(f\"Using project: {projects[0]['name']} ({PROJECT_ID})\")" ] }, { diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index ea736b954..5f12e976b 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -247,11 +247,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/custom_shell_calculation.ipynb b/other/materials_designer/workflows/custom_shell_calculation.ipynb index 8079b92eb..25a44f821 100644 --- a/other/materials_designer/workflows/custom_shell_calculation.ipynb +++ b/other/materials_designer/workflows/custom_shell_calculation.ipynb @@ -338,11 +338,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/defect_formation_energy.ipynb b/other/materials_designer/workflows/defect_formation_energy.ipynb index 3e12d9383..fba19957e 100644 --- a/other/materials_designer/workflows/defect_formation_energy.ipynb +++ b/other/materials_designer/workflows/defect_formation_energy.ipynb @@ -241,11 +241,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { @@ -264,6 +262,7 @@ "metadata": {}, "outputs": [], "source": [ + "import re\n", "from mat3ra.made.material import Material\n", "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", @@ -274,10 +273,12 @@ " if loaded is not None:\n", " print(f\"✅ Loaded '{name}' from folder: {loaded.name}\")\n", " return loaded\n", - " # a name condition matches the exact name, not a pattern\n", - " matches = client.materials.list({\"name\": name, \"ownerId\": ACCOUNT_ID})\n", + " matches = client.materials.list({\n", + " \"name\": {\"$regex\": re.escape(name), \"$options\": \"i\"},\n", + " \"owner._id\": ACCOUNT_ID,\n", + " })\n", " if not matches:\n", - " raise ValueError(f\"No material named '{name}' was found in '{FOLDER}' or on the platform.\")\n", + " raise ValueError(f\"No material containing '{name}' was found in '{FOLDER}' or on the platform.\")\n", " material = Material.create(matches[0])\n", " print(f\"♻️ Loaded '{name}' from platform: {matches[0]['_id']}\")\n", " return material\n", @@ -353,7 +354,7 @@ "outputs": [], "source": [ "elemental_materials_data = client.materials.list(\n", - " {\"tags\": \"elemental\", \"metadataElement\": elements, \"globalSearch\": True, \"limit\": 500}\n", + " {\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}},\n", ")\n", "element_materials_by_symbol = {\n", " element: [metadata for metadata in elemental_materials_data if\n", diff --git a/other/materials_designer/workflows/dielectric_tensor.ipynb b/other/materials_designer/workflows/dielectric_tensor.ipynb index beb841be2..96639f975 100644 --- a/other/materials_designer/workflows/dielectric_tensor.ipynb +++ b/other/materials_designer/workflows/dielectric_tensor.ipynb @@ -230,11 +230,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/equation_of_state.ipynb b/other/materials_designer/workflows/equation_of_state.ipynb index 6e8ac9cae..40642fd06 100644 --- a/other/materials_designer/workflows/equation_of_state.ipynb +++ b/other/materials_designer/workflows/equation_of_state.ipynb @@ -197,11 +197,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index 7b6947e17..56704eec8 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -212,11 +212,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" ] }, { @@ -282,7 +280,7 @@ "outputs": [], "source": [ "elemental_materials_data = client.materials.list(\n", - " {\"tags\": \"elemental\", \"metadataElement\": elements, \"globalSearch\": True, \"limit\": 500}\n", + " {\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}},\n", ")\n", "\n", "element_materials_by_symbol = {\n", diff --git a/other/materials_designer/workflows/homo_lumo_frequency.ipynb b/other/materials_designer/workflows/homo_lumo_frequency.ipynb index d980932e4..be81e093e 100644 --- a/other/materials_designer/workflows/homo_lumo_frequency.ipynb +++ b/other/materials_designer/workflows/homo_lumo_frequency.ipynb @@ -176,11 +176,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { @@ -199,6 +197,7 @@ "metadata": {}, "outputs": [], "source": [ + "import re\n", "from mat3ra.made.material import Material\n", "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", @@ -209,9 +208,12 @@ " if loaded is not None:\n", " print(f\"✅ Loaded '{name}' from folder: {loaded.name}\")\n", " return loaded\n", - " matches = client.materials.list({\"name\": name, \"ownerId\": ACCOUNT_ID})\n", + " matches = client.materials.list({\n", + " \"name\": {\"$regex\": re.escape(name), \"$options\": \"i\"},\n", + " \"owner._id\": ACCOUNT_ID,\n", + " })\n", " if not matches:\n", - " raise ValueError(f\"No material named '{name}' was found in '{FOLDER}' or on the platform.\")\n", + " raise ValueError(f\"No material containing '{name}' was found in '{FOLDER}' or on the platform.\")\n", " material = Material.create(matches[0])\n", " print(f\"♻️ Loaded '{name}' from platform: {matches[0]['_id']}\")\n", " return material\n", diff --git a/other/materials_designer/workflows/interfacial_energy.ipynb b/other/materials_designer/workflows/interfacial_energy.ipynb index c45620517..dd82f217d 100644 --- a/other/materials_designer/workflows/interfacial_energy.ipynb +++ b/other/materials_designer/workflows/interfacial_energy.ipynb @@ -219,11 +219,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" ] }, { @@ -242,6 +240,7 @@ "metadata": {}, "outputs": [], "source": [ + "import re\n", "from mat3ra.made.material import Material\n", "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", @@ -249,11 +248,13 @@ "interface = load_material_from_folder(FOLDER, INTERFACE_NAME)\n", "\n", "if interface is None:\n", - " # a name condition matches the exact name, not a pattern\n", - " interface_matches = client.materials.list({\"name\": INTERFACE_NAME, \"ownerId\": ACCOUNT_ID})\n", + " interface_matches = client.materials.list({\n", + " \"name\": {\"$regex\": re.escape(INTERFACE_NAME), \"$options\": \"i\"},\n", + " \"owner._id\": ACCOUNT_ID,\n", + " })\n", " if not interface_matches:\n", " raise ValueError(\n", - " f\"No interface named '{INTERFACE_NAME}' was found in '{FOLDER}' or on the platform.\"\n", + " f\"No interface containing '{INTERFACE_NAME}' was found in '{FOLDER}' or on the platform.\"\n", " )\n", " interface = Material.create(interface_matches[0])\n", " print(f\"♻️ Loaded interface from platform: {interface_matches[0]['_id']}\")\n", diff --git a/other/materials_designer/workflows/neb.ipynb b/other/materials_designer/workflows/neb.ipynb index 7eb73389c..11fc32bd6 100644 --- a/other/materials_designer/workflows/neb.ipynb +++ b/other/materials_designer/workflows/neb.ipynb @@ -224,11 +224,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/phonon_dos_dispersion.ipynb b/other/materials_designer/workflows/phonon_dos_dispersion.ipynb index f19bb9f15..20472e800 100644 --- a/other/materials_designer/workflows/phonon_dos_dispersion.ipynb +++ b/other/materials_designer/workflows/phonon_dos_dispersion.ipynb @@ -229,11 +229,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/relaxation.ipynb b/other/materials_designer/workflows/relaxation.ipynb index 8f5b84345..07e426d76 100644 --- a/other/materials_designer/workflows/relaxation.ipynb +++ b/other/materials_designer/workflows/relaxation.ipynb @@ -218,11 +218,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"\\u2705 Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/surface_energy.ipynb b/other/materials_designer/workflows/surface_energy.ipynb index 6a734dbae..20cf21ebe 100644 --- a/other/materials_designer/workflows/surface_energy.ipynb +++ b/other/materials_designer/workflows/surface_energy.ipynb @@ -209,11 +209,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" ] }, { @@ -232,6 +230,7 @@ "metadata": {}, "outputs": [], "source": [ + "import re\n", "from mat3ra.made.material import Material\n", "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", @@ -239,18 +238,20 @@ "slab = load_material_from_folder(FOLDER, SLAB_NAME)\n", "\n", "if slab is None:\n", - " # a name condition matches the exact name, not a pattern\n", - " slab_matches = client.materials.list({\"name\": SLAB_NAME, \"ownerId\": ACCOUNT_ID})\n", + " slab_matches = client.materials.list({\n", + " \"name\": {\"$regex\": re.escape(SLAB_NAME), \"$options\": \"i\"},\n", + " \"owner._id\": ACCOUNT_ID,\n", + " })\n", " if not slab_matches:\n", " raise ValueError(\n", - " f\"No slab named '{SLAB_NAME}' was found in '{FOLDER}' or on the platform.\"\n", + " f\"No slab containing '{SLAB_NAME}' was found in '{FOLDER}' or on the platform.\"\n", " )\n", " slab = Material.create(slab_matches[0])\n", " print(f\"♻️ Loaded slab from platform: {slab_matches[0]['_id']}\")\n", "else:\n", " print(f\"✅ Loaded slab from folder: {slab.name}\")\n", "\n", - "visualize(slab)\n" + "visualize(slab)" ] }, { diff --git a/other/materials_designer/workflows/total_energy.ipynb b/other/materials_designer/workflows/total_energy.ipynb index 3f2b2b880..6b2e81711 100644 --- a/other/materials_designer/workflows/total_energy.ipynb +++ b/other/materials_designer/workflows/total_energy.ipynb @@ -195,11 +195,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/total_energy_convergence.ipynb b/other/materials_designer/workflows/total_energy_convergence.ipynb index e4178421f..7beb3c53f 100644 --- a/other/materials_designer/workflows/total_energy_convergence.ipynb +++ b/other/materials_designer/workflows/total_energy_convergence.ipynb @@ -181,11 +181,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "PROJECT_ID = project[\"_id\"]\n", - "print(f\"Using project: {project['name']} ({PROJECT_ID})\")\n" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "PROJECT_ID = projects[0][\"_id\"]\n", + "print(f\"Using project: {projects[0]['name']} ({PROJECT_ID})\")\n" ] }, { diff --git a/other/materials_designer/workflows/total_energy_post_processing.ipynb b/other/materials_designer/workflows/total_energy_post_processing.ipynb index 1654568fd..0ab88dca9 100644 --- a/other/materials_designer/workflows/total_energy_post_processing.ipynb +++ b/other/materials_designer/workflows/total_energy_post_processing.ipynb @@ -210,11 +210,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/other/materials_designer/workflows/valence_band_offset.ipynb b/other/materials_designer/workflows/valence_band_offset.ipynb index a74a9ab6e..92f73b3a4 100644 --- a/other/materials_designer/workflows/valence_band_offset.ipynb +++ b/other/materials_designer/workflows/valence_band_offset.ipynb @@ -234,11 +234,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")\n" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" ] }, { diff --git a/other/materials_designer/workflows/zero_point_energy.ipynb b/other/materials_designer/workflows/zero_point_energy.ipynb index e4eddecd4..55029e0d8 100644 --- a/other/materials_designer/workflows/zero_point_energy.ipynb +++ b/other/materials_designer/workflows/zero_point_energy.ipynb @@ -230,11 +230,9 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.project.api import get_default_project\n", - "\n", - "project = get_default_project(client, ACCOUNT_ID)\n", - "project_id = project[\"_id\"]\n", - "print(f\"✅ Using project: {project['name']} ({project_id})\")" + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" ] }, { diff --git a/src/py/mat3ra/notebooks_utils/core/entity/job/api.py b/src/py/mat3ra/notebooks_utils/core/entity/job/api.py index b221bbcb1..4c72c0829 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/job/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/job/api.py @@ -35,7 +35,7 @@ def get_jobs_statuses_by_ids(endpoint: JobEndpoints, job_ids: List[str]) -> List Returns: list: list of job statuses """ - jobs = endpoint.list({"id": job_ids}) + jobs = endpoint.list({"_id": {"$in": job_ids}}, {"fields": {"status": 1}}) return [job["status"] for job in jobs] @@ -138,13 +138,12 @@ def find_job_for_material( """ existing = api_client.jobs.list( { - "materialId": material_id, - "ownerId": owner_id, - "workflowName": workflow_name, - "status": list(statuses), - "globalSearch": True, - "limit": 1, - } + "_material._id": material_id, + "owner._id": owner_id, + "workflow.name": workflow_name, + "status": {"$in": list(statuses)}, + }, + {"limit": 1}, ) return existing[0] if existing else None diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py b/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py index 2805e5c71..bd329339a 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py @@ -120,10 +120,10 @@ def get_slab_bulk_crystal(slab_material: Material) -> dict: def resolve_bulk_query_from_crystal(bulk_crystal: dict) -> dict: """Builds a materials.list query that resolves a bulk crystal to a platform material.""" - for key, parameter in (("_id", "id"), ("hash", "hashes"), ("scaledHash", "scaledHashes")): + for key in ("_id", "hash", "scaledHash"): if bulk_crystal.get(key) is not None: - return {parameter: bulk_crystal[key]} + return {key: bulk_crystal[key]} try: - return {"hashes": Material.create(bulk_crystal).hash} + return {"hash": Material.create(bulk_crystal).hash} except Exception as exc: raise ValueError("Could not resolve a bulk query from crystal metadata.") from exc diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py index 833e928c8..8d626c7c6 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py @@ -27,7 +27,7 @@ def get_or_create_material(api_client: APIClient, material, owner_id: str) -> di Returns: dict: The material dict (existing or newly created). """ - existing = api_client.materials.list({"hashes": material.hash, "ownerId": owner_id, "globalSearch": True}) + existing = api_client.materials.list({"hash": material.hash, "owner._id": owner_id}) if existing: print(f"♻️ Reusing already existing Material: {existing[0]['_id']}") return existing[0] @@ -56,7 +56,7 @@ def load_material(api_client: APIClient, folder: str, name: str, owner_id: str) loaded = load_material_from_folder(folder, name, verbose=False) if os.path.isdir(folder) else None if loaded is not None and loaded.name == name: return loaded - matches = api_client.materials.list({"name": name, "ownerId": owner_id, "globalSearch": True, "limit": 1}) + matches = api_client.materials.list({"name": name, "owner._id": owner_id}, {"limit": 1}) if not matches: raise ValueError(f"No material named '{name}' in '{folder}' or in the account") return Material.create(matches[0]) @@ -83,11 +83,8 @@ def find_relaxed_material(api_client: APIClient, material, owner_id: str) -> Opt Returns: Material, optional: The relaxed structure, or None if none exists. """ - matching_materials = api_client.materials.list({"hashes": material.hash, "ownerId": owner_id, "globalSearch": True}) - ids = [m["_id"] for m in matching_materials] - if not ids: - return None - query = {"materialId": ids, "ownerId": owner_id, "status": "finished", "globalSearch": True} + ids = [m["_id"] for m in api_client.materials.list({"hash": material.hash, "owner._id": owner_id})] + query = {"_material._id": {"$in": ids}, "owner._id": owner_id, "status": "finished"} for job in api_client.jobs.list(query): properties = api_client.properties.get_for_job(job["_id"], PropertyName.non_scalar.final_structure.value) if not properties: @@ -136,7 +133,7 @@ def get_bulk_material_by_crystal(api_client: APIClient, bulk_crystal: Material, def _require_material_for_owner(api_client: APIClient, query: dict, owner_id: str) -> Material: - matches = api_client.materials.list({**query, "ownerId": owner_id, "globalSearch": True}) + matches = api_client.materials.list({**query, "owner._id": owner_id}) material_response = next(iter(matches), None) if material_response is None: raise ValueError( @@ -179,9 +176,13 @@ def find_material_set( Raises: ValueError: If no set matches, or if `require_ordered` and the match is unordered. """ - account_entity_sets = api_client.materials.list({"ownerId": owner_id, "isEntitySet": True}) - name_pattern = re.compile(re.escape(material_set_name), re.IGNORECASE) - material_sets = [s for s in account_entity_sets if name_pattern.search(s.get("name", ""))] + material_sets = api_client.materials.list( + { + "owner._id": owner_id, + "isEntitySet": True, + "name": {"$regex": re.escape(material_set_name), "$options": "i"}, + } + ) if not material_sets: raise ValueError(f"No material set matching '{material_set_name}'") material_set = material_sets[0] @@ -204,7 +205,9 @@ def list_materials_in_set(api_client: APIClient, owner_id: str, material_set: Di that already have one do not re-query for it. """ material_set_id = material_set["_id"] - matches = api_client.materials.list({"ownerId": owner_id, "setId": material_set_id, "isEntitySet": False}) + matches = api_client.materials.list( + {"owner._id": owner_id, "inSet._id": material_set_id, "isEntitySet": {"$ne": True}} + ) members = [material for material in matches if not material.get("isEntitySet")] return sorted(members, key=lambda material: _index_in_set(material, material_set_id)) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/default.py b/src/py/mat3ra/notebooks_utils/core/entity/material/default.py deleted file mode 100644 index 3bbb205ab..000000000 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/default.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Default-material lookup. - -Kept apart from api.py on purpose: notebooks that install only the `api` package group cannot import -mat3ra.made (which api.py needs), so this module may depend on mat3ra.api_client alone. -""" - -from mat3ra.api_client import APIClient - - -def get_default_material(api_client: APIClient, owner_id: str) -> dict: - """ - Returns the given account's default material. - - Args: - api_client (APIClient): API client instance carrying the authorization context. - owner_id (str): Account ID whose default material should be returned. - - Returns: - dict: The account's default material document. - """ - return api_client.materials.show_default(owner_id) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/project/__init__.py b/src/py/mat3ra/notebooks_utils/core/entity/project/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/py/mat3ra/notebooks_utils/core/entity/project/api.py b/src/py/mat3ra/notebooks_utils/core/entity/project/api.py deleted file mode 100644 index c92ba55d2..000000000 --- a/src/py/mat3ra/notebooks_utils/core/entity/project/api.py +++ /dev/null @@ -1,15 +0,0 @@ -from mat3ra.api_client import APIClient - - -def get_default_project(api_client: APIClient, owner_id: str) -> dict: - """ - Returns the given account's default project. - - Args: - api_client (APIClient): API client instance carrying the authorization context. - owner_id (str): Account ID whose default project should be returned. - - Returns: - dict: The account's default project document. - """ - return api_client.projects.show_default(owner_id) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/property/api.py b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py index 82bb31ad7..050c16f4f 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/property/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py @@ -44,10 +44,13 @@ def get_property_holder_for_job( Returns: dict: Full property holder document. """ - query = {"jobId": job_id, "propertyName": property_name} + query = { + "source.info.jobId": job_id, + "data.name": property_name, + } if unit_id: - query["unitId"] = unit_id - holders = client.properties.list(query) + query["source.info.unitId"] = unit_id + holders = client.properties.list(query=query) if not holders: raise ValueError(f"Property '{property_name}' not found for job '{job_id}'") return holders[0] @@ -93,14 +96,14 @@ def find_total_energy_for_material(client: APIClient, material_id: str, source: return None query = {"exabyteId": exabyte_id, "slug": "total_energy"} if source == "curators": - query["ownerSlug"] = "curators" + query["owner.slug"] = "curators" elif source == "my_account": - query["ownerId"] = client.my_account.id + query["owner._id"] = client.my_account.id elif source != "public": raise ValueError(f"Invalid source: {source!r}. Expected 'public', 'curators', or 'my_account'.") properties = client.properties.list( query=query, - projection={"sort": "-precision.value", "limit": 1}, + projection={"sort": {"precision.value": -1}, "limit": 1}, ) return properties[0] if properties else None diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py index 069c12495..91a590cfc 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -14,7 +14,7 @@ def get_or_create_workflow(api_client: APIClient, workflow: Workflow, owner_id: Returns: dict: The workflow dict (existing or newly created). """ - existing = api_client.workflows.list({"hash": workflow.hash, "ownerId": owner_id, "globalSearch": True}) + existing = api_client.workflows.list({"hash": workflow.hash, "owner._id": owner_id}) if existing: print(f"♻️ Reusing already existing Workflow: {existing[0]['_id']}") return existing[0] diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/default.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/default.py deleted file mode 100644 index 27da76637..000000000 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/default.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Default-workflow lookup. - -Kept apart from api.py on purpose: notebooks that install only the `api` package group cannot import -mat3ra.wode (which api.py needs), so this module may depend on mat3ra.api_client alone. -""" - -from mat3ra.api_client import APIClient - - -def get_default_workflow(api_client: APIClient, owner_id: str) -> dict: - """ - Returns the given account's default workflow. - - Args: - api_client (APIClient): API client instance carrying the authorization context. - owner_id (str): Account ID whose default workflow should be returned. - - Returns: - dict: The account's default workflow document. - """ - return api_client.workflows.show_default(owner_id) diff --git a/tests/py/unit/core/entity/test_default_entity_api.py b/tests/py/unit/core/entity/test_default_entity_api.py deleted file mode 100644 index 22352a800..000000000 --- a/tests/py/unit/core/entity/test_default_entity_api.py +++ /dev/null @@ -1,47 +0,0 @@ -import ast -from unittest.mock import MagicMock - -import pytest -from mat3ra.notebooks_utils.core.entity.material import default as material_default -from mat3ra.notebooks_utils.core.entity.material.default import get_default_material -from mat3ra.notebooks_utils.core.entity.project import api as project_api -from mat3ra.notebooks_utils.core.entity.project.api import get_default_project -from mat3ra.notebooks_utils.core.entity.workflow import default as workflow_default -from mat3ra.notebooks_utils.core.entity.workflow.default import get_default_workflow - -ACCOUNT_ID = "account-1" -DEFAULT_ENTITY = {"_id": "default-1", "isDefault": True} -# The `api` package group of config.yml installs only these, so the default lookups may import nothing else. -ALLOWED_IMPORTED_MODULES = {"mat3ra.api_client"} - - -@pytest.mark.parametrize( - ("get_default", "endpoint_name"), - [ - (get_default_material, "materials"), - (get_default_workflow, "workflows"), - (get_default_project, "projects"), - ], -) -def test_returns_the_default_entity_of_the_given_account(get_default, endpoint_name): - client = MagicMock() - getattr(client, endpoint_name).show_default.return_value = DEFAULT_ENTITY - - assert get_default(client, ACCOUNT_ID) == DEFAULT_ENTITY - getattr(client, endpoint_name).show_default.assert_called_once_with(ACCOUNT_ID) - - -@pytest.mark.parametrize("module", [material_default, workflow_default, project_api]) -def test_default_lookups_import_only_the_api_client(module): - """Notebooks installing just the `api` package group must be able to import these.""" - with open(module.__file__) as module_file: - tree = ast.parse(module_file.read()) - - imported_modules = set() - for node in ast.walk(tree): - if isinstance(node, ast.Import): - imported_modules.update(alias.name for alias in node.names) - elif isinstance(node, ast.ImportFrom): - imported_modules.add(node.module) - - assert imported_modules <= ALLOWED_IMPORTED_MODULES diff --git a/tests/py/unit/core/entity/test_job_api.py b/tests/py/unit/core/entity/test_job_api.py index 6cd1ab057..a0942a2dc 100644 --- a/tests/py/unit/core/entity/test_job_api.py +++ b/tests/py/unit/core/entity/test_job_api.py @@ -87,13 +87,12 @@ def test_find_job_for_material_returns_the_job_when_found(statuses): assert job == EXISTING_JOB client.jobs.list.assert_called_once_with( { - "materialId": MATERIAL_INITIAL["_id"], - "ownerId": OWNER_ID, - "workflowName": RELAX_WORKFLOW_NAME, - "status": list(statuses), - "globalSearch": True, - "limit": 1, - } + "_material._id": MATERIAL_INITIAL["_id"], + "owner._id": OWNER_ID, + "workflow.name": RELAX_WORKFLOW_NAME, + "status": {"$in": list(statuses)}, + }, + {"limit": 1}, ) @@ -112,4 +111,4 @@ def test_find_job_for_material_defaults_to_finished_only(): find_job_for_material(client, MATERIAL_INITIAL["_id"], RELAX_WORKFLOW_NAME, OWNER_ID) - assert client.jobs.list.call_args.args[0]["status"] == ["finished"] + assert client.jobs.list.call_args.args[0]["status"] == {"$in": ["finished"]} diff --git a/tests/py/unit/core/entity/test_material_analysis.py b/tests/py/unit/core/entity/test_material_analysis.py index 3acecb607..ac99995f8 100644 --- a/tests/py/unit/core/entity/test_material_analysis.py +++ b/tests/py/unit/core/entity/test_material_analysis.py @@ -15,10 +15,10 @@ [ ( {"scaledHash": "scaled-hash-value", "hash": "hash-value", "_id": "material-id"}, - {"id": "material-id"}, + {"_id": "material-id"}, ), - ({"scaledHash": "scaled-hash-value", "hash": "hash-value"}, {"hashes": "hash-value"}), - ({"scaledHash": "scaled-hash-value"}, {"scaledHashes": "scaled-hash-value"}), + ({"scaledHash": "scaled-hash-value", "hash": "hash-value"}, {"hash": "hash-value"}), + ({"scaledHash": "scaled-hash-value"}, {"scaledHash": "scaled-hash-value"}), ], ) def test_resolve_bulk_query_prefers_id_then_hash_then_scaled_hash(extra_keys, expected): @@ -27,8 +27,8 @@ def test_resolve_bulk_query_prefers_id_then_hash_then_scaled_hash(extra_keys, ex def test_resolve_bulk_query_computes_hash_when_none_present(): query = resolve_bulk_query_from_crystal(SILICON) - assert set(query) == {"hashes"} - assert query["hashes"] + assert set(query) == {"hash"} + assert query["hash"] def test_slab_bulk_crystal_is_the_material_the_slab_was_built_from(): diff --git a/tests/py/unit/core/entity/test_material_api.py b/tests/py/unit/core/entity/test_material_api.py index 16f18d0b4..44e7df45d 100644 --- a/tests/py/unit/core/entity/test_material_api.py +++ b/tests/py/unit/core/entity/test_material_api.py @@ -61,14 +61,19 @@ def _client_with_list_responses(responses: List[List[Dict[str, Any]]]) -> MagicM return client -def test_find_material_set_matches_the_name_literally_and_case_insensitively(): - decoy_set = {**ENTITY_SET, "_id": "set-decoy", "name": "H22H"} - client = _client_with_list_responses([[decoy_set, ENTITY_SET]]) +def test_find_material_set_returns_first_match(): + client = _client_with_list_responses([[ENTITY_SET]]) - material_set = find_material_set(client, OWNER_ID, MATERIAL_SET_NAME.lower()) + material_set = find_material_set(client, OWNER_ID, MATERIAL_SET_NAME) assert material_set["_id"] == MATERIAL_SET_ID - client.materials.list.assert_called_once_with({"ownerId": OWNER_ID, "isEntitySet": True}) + client.materials.list.assert_called_once_with( + { + "owner._id": OWNER_ID, + "isEntitySet": True, + "name": {"$regex": re.escape(MATERIAL_SET_NAME), "$options": "i"}, + } + ) assert "+" in MATERIAL_SET_NAME assert re.escape(MATERIAL_SET_NAME) != MATERIAL_SET_NAME @@ -101,7 +106,9 @@ def test_list_materials_in_set_does_not_re_resolve_the_set(): materials = list_materials_in_set(client, OWNER_ID, ENTITY_SET) assert [material["_id"] for material in materials] == EXPECTED_ORDERED_IDS - client.materials.list.assert_called_once_with({"ownerId": OWNER_ID, "setId": MATERIAL_SET_ID, "isEntitySet": False}) + client.materials.list.assert_called_once_with( + {"owner._id": OWNER_ID, "inSet._id": MATERIAL_SET_ID, "isEntitySet": {"$ne": True}} + ) @pytest.mark.parametrize( @@ -118,9 +125,9 @@ def test_list_materials_by_set_orders_by_inset_index(members, expected_ids): assert [material["_id"] for material in materials] == expected_ids assert client.materials.list.call_args_list[1].args[0] == { - "ownerId": OWNER_ID, - "setId": MATERIAL_SET_ID, - "isEntitySet": False, + "owner._id": OWNER_ID, + "inSet._id": MATERIAL_SET_ID, + "isEntitySet": {"$ne": True}, } @@ -254,9 +261,9 @@ def test_find_relaxed_material_returns_final_structure_from_the_job(): assert relaxed is not None assert relaxed.name == "B-vacancy h-BN relaxed" - client.materials.list.assert_called_once_with({"hashes": DEFECTIVE_HASH, "ownerId": OWNER_ID, "globalSearch": True}) + client.materials.list.assert_called_once_with({"hash": DEFECTIVE_HASH, "owner._id": OWNER_ID}) client.jobs.list.assert_called_once_with( - {"materialId": [SAVED_DEFECTIVE["_id"]], "ownerId": OWNER_ID, "status": "finished", "globalSearch": True} + {"_material._id": {"$in": [SAVED_DEFECTIVE["_id"]]}, "owner._id": OWNER_ID, "status": "finished"} ) client.properties.get_for_job.assert_called_once_with(FINISHED_JOB["_id"], "final_structure") client.materials.get.assert_called_once_with("m-relaxed") @@ -284,10 +291,10 @@ def test_find_relaxed_material_uses_the_last_final_structure_entry(): def test_find_relaxed_material_returns_none_when_material_is_not_on_the_platform(): client = MagicMock() client.materials.list.return_value = [] + client.jobs.list.return_value = [] assert find_relaxed_material(client, DEFECTIVE_MATERIAL, OWNER_ID) is None - # an empty $in drops the job filter and would match every finished job - client.jobs.list.assert_not_called() + assert client.jobs.list.call_args.args[0]["_material._id"] == {"$in": []} def test_find_relaxed_material_returns_none_when_no_job_exists(): @@ -322,7 +329,7 @@ def test_find_relaxed_material_checks_every_same_hash_material(): assert relaxed is not None assert relaxed.name == "B-vacancy h-BN relaxed" client.jobs.list.assert_called_once_with( - {"materialId": ["m-other", "m-defective"], "ownerId": OWNER_ID, "status": "finished", "globalSearch": True} + {"_material._id": {"$in": ["m-other", "m-defective"]}, "owner._id": OWNER_ID, "status": "finished"} ) @@ -363,9 +370,7 @@ def test_load_material_falls_back_to_the_account(tmp_path): material = load_material(client, str(tmp_path), "Silicon", OWNER_ID) assert material.name == "Silicon" - client.materials.list.assert_called_once_with( - {"name": "Silicon", "ownerId": OWNER_ID, "globalSearch": True, "limit": 1} - ) + client.materials.list.assert_called_once_with({"name": "Silicon", "owner._id": OWNER_ID}, {"limit": 1}) def test_load_material_raises_when_neither_has_it(tmp_path): @@ -385,9 +390,7 @@ def test_load_material_falls_through_a_folder_near_miss(tmp_path): material = load_material(client, str(tmp_path), "Silicon", OWNER_ID) assert material.name == "Silicon" - client.materials.list.assert_called_once_with( - {"name": "Silicon", "ownerId": OWNER_ID, "globalSearch": True, "limit": 1} - ) + client.materials.list.assert_called_once_with({"name": "Silicon", "owner._id": OWNER_ID}, {"limit": 1}) JOB_ID = "job-1" diff --git a/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py b/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py index 3a36cd78f..a7596ca70 100644 --- a/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py +++ b/tests/py/unit/core/entity/test_property_api_find_total_energy_for_material.py @@ -32,9 +32,9 @@ def test_find_total_energy_for_material_defaults_to_my_account_scope(): query={ "exabyteId": EXABYTE_ID, "slug": "total_energy", - "ownerId": OWNER_ACCOUNT_ID, + "owner._id": OWNER_ACCOUNT_ID, }, - projection={"sort": "-precision.value", "limit": 1}, + projection={"sort": {"precision.value": -1}, "limit": 1}, ) client.jobs.list.assert_not_called() assert result == TOTAL_ENERGY_PROPERTY @@ -49,9 +49,9 @@ def test_find_total_energy_for_material_my_account_scope(): query={ "exabyteId": EXABYTE_ID, "slug": "total_energy", - "ownerId": OWNER_ACCOUNT_ID, + "owner._id": OWNER_ACCOUNT_ID, }, - projection={"sort": "-precision.value", "limit": 1}, + projection={"sort": {"precision.value": -1}, "limit": 1}, ) @@ -64,9 +64,9 @@ def test_find_total_energy_for_material_curators_scope(): query={ "exabyteId": EXABYTE_ID, "slug": "total_energy", - "ownerSlug": "curators", + "owner.slug": "curators", }, - projection={"sort": "-precision.value", "limit": 1}, + projection={"sort": {"precision.value": -1}, "limit": 1}, ) @@ -80,7 +80,7 @@ def test_find_total_energy_for_material_public_scope_has_no_owner_filter(): "exabyteId": EXABYTE_ID, "slug": "total_energy", }, - projection={"sort": "-precision.value", "limit": 1}, + projection={"sort": {"precision.value": -1}, "limit": 1}, ) From 49062467df204979f3dd05ccc37822e82e41d761 Mon Sep 17 00:00:00 2001 From: Kostiantyn Dvornik Date: Wed, 7 Oct 2026 01:44:29 +0300 Subject: [PATCH 10/10] fix(SOF-8032): build the formation energy job from the local workflow create_job was given the server's saved copy of the workflow, which drops the context providers' defaults, so job creation recomputes them from the material. Same fix as 1692a3f8 for the other notebooks; the notebook is back to main's version. Co-Authored-By: Claude Sonnet 5.5 --- other/materials_designer/workflows/formation_energy.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index 56704eec8..d1125b41c 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -548,7 +548,7 @@ "formation_job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=saved_formation_workflow_response,\n", + " workflow=formation_workflow,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=formation_job_name,\n",