diff --git a/src/prerna/auth/utils/AbstractSecurityUtils.java b/src/prerna/auth/utils/AbstractSecurityUtils.java index b5a050b8c9d..d3692be877c 100644 --- a/src/prerna/auth/utils/AbstractSecurityUtils.java +++ b/src/prerna/auth/utils/AbstractSecurityUtils.java @@ -152,6 +152,12 @@ public abstract class AbstractSecurityUtils { static boolean adminOnlyGuardrailSetPublic = false; static boolean adminOnlyGuardrailSetDiscoverable = false; + static boolean adminOnlyVenvAdd = false; + static boolean adminOnlyVenvDelete = false; + static boolean adminOnlyVenvAddAccess = false; + static boolean adminOnlyVenvSetPublic = false; + static boolean adminOnlyVenvSetDiscoverable = false; + static boolean adminOnlyInsightSetPublic = false; static boolean adminOnlyInsightAddAccess = false; @@ -273,6 +279,12 @@ public static void loadSecurityDatabase() throws Exception { adminOnlyGuardrailSetPublic = Utility.getApplicationAdminOnlyGuardrailSetPublic(); adminOnlyGuardrailSetDiscoverable = Utility.getApplicationAdminOnlyGuardrailSetDiscoverable(); + adminOnlyVenvAdd = Utility.getApplicationAdminOnlyVenvAdd(); + adminOnlyVenvDelete = Utility.getApplicationAdminOnlyVenvDelete(); + adminOnlyVenvAddAccess = Utility.getApplicationAdminOnlyVenvAddAccess(); + adminOnlyVenvSetPublic = Utility.getApplicationAdminOnlyVenvSetPublic(); + adminOnlyVenvSetDiscoverable = Utility.getApplicationAdminOnlyVenvSetDiscoverable(); + adminOnlyInsightSetPublic = Utility.getApplicationAdminOnlyInsightSetPublic(); adminOnlyInsightAddAccess = Utility.getApplicationAdminOnlyInsightAddAccess(); @@ -551,6 +563,26 @@ public static boolean adminOnlyGuardrailSetDiscoverable() { return adminOnlyGuardrailSetDiscoverable; } + public static boolean adminOnlyVenvAdd() { + return adminOnlyVenvAdd; + } + + public static boolean adminOnlyVenvDelete() { + return adminOnlyVenvDelete; + } + + public static boolean adminOnlyVenvAddAccess() { + return adminOnlyVenvAddAccess; + } + + public static boolean adminOnlyVenvSetPublic() { + return adminOnlyVenvSetPublic; + } + + public static boolean adminOnlyVenvSetDiscoverable() { + return adminOnlyVenvSetDiscoverable; + } + public static boolean adminOnlyInsightSetPublic() { return adminOnlyInsightSetPublic; } @@ -580,6 +612,8 @@ public static boolean adminOnlyEngineAdd(IEngine.CATALOG_TYPE type) { return adminOnlyFunctionAdd; } else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) { return adminOnlyGuardrailAdd; + } else if (IEngine.CATALOG_TYPE.VENV == type) { + return adminOnlyVenvAdd; } throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type); @@ -602,6 +636,8 @@ public static boolean adminOnlyEngineDelete(IEngine.CATALOG_TYPE type) { return adminOnlyFunctionDelete; } else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) { return adminOnlyGuardrailDelete; + } else if (IEngine.CATALOG_TYPE.VENV == type) { + return adminOnlyVenvDelete; } throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type); @@ -624,6 +660,8 @@ public static boolean adminOnlyEngineAddAccess(IEngine.CATALOG_TYPE type) { return adminOnlyFunctionAddAccess; } else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) { return adminOnlyGuardrailAddAccess; + } else if (IEngine.CATALOG_TYPE.VENV == type) { + return adminOnlyVenvAddAccess; } throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type); @@ -646,6 +684,8 @@ public static boolean adminOnlyEngineSetPublic(IEngine.CATALOG_TYPE type) { return adminOnlyFunctionSetPublic; } else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) { return adminOnlyGuardrailSetPublic; + } else if (IEngine.CATALOG_TYPE.VENV == type) { + return adminOnlyVenvSetPublic; } throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type); @@ -668,6 +708,8 @@ public static boolean adminOnlyEngineSetDiscoverable(IEngine.CATALOG_TYPE type) return adminOnlyFunctionSetDiscoverable; } else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) { return adminOnlyGuardrailSetDiscoverable; + } else if (IEngine.CATALOG_TYPE.VENV == type) { + return adminOnlyVenvSetDiscoverable; } throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type); diff --git a/src/prerna/auth/utils/SecurityProjectUtils.java b/src/prerna/auth/utils/SecurityProjectUtils.java index d16fb2ff463..4e21890e70b 100644 --- a/src/prerna/auth/utils/SecurityProjectUtils.java +++ b/src/prerna/auth/utils/SecurityProjectUtils.java @@ -1161,15 +1161,16 @@ public static boolean projectIsTemplate(String projectId) { } /** - * Determine whether a user may clone a project. The user must be able to view - * the project and the owner must have explicitly enabled it as a template. + * Determine whether a user may clone a project. The project's owners can always + * clone it; anyone else must be able to view the project, and its owner must + * have enabled it as a template. * * @param user current user * @param projectId project identifier * @return whether the user may clone the project */ public static boolean userCanCloneProject(User user, String projectId) { - return userCanViewProject(user, projectId) && projectIsTemplate(projectId); + return userIsOwner(user, projectId) || (userCanViewProject(user, projectId) && projectIsTemplate(projectId)); } /** diff --git a/src/prerna/cluster/util/ClusterSyncMethod.java b/src/prerna/cluster/util/ClusterSyncMethod.java index 58aa50144cd..7fd65c7bf3f 100644 --- a/src/prerna/cluster/util/ClusterSyncMethod.java +++ b/src/prerna/cluster/util/ClusterSyncMethod.java @@ -30,74 +30,155 @@ import java.util.HashMap; import java.util.Map; +import prerna.util.EngineSyncUtility; +import prerna.util.ProjectSyncUtility; + /** - * Type-safe catalog of the {@link ClusterUtil} "pull" actions that one cluster - * node can ask its peers to run in response to a change published through an - * {@link prerna.cluster.sync.IClusterSynchronizer}. + * Type-safe catalog of the actions that one cluster node can ask its peers to + * run in response to a change published through an + * {@link prerna.cluster.sync.IClusterSynchronizer}. Most are + * {@link ClusterUtil} "pull" actions that refresh a local copy from cloud + * storage; the rest act on what a peer has open, such as closing an engine or + * clearing a project's python imports. *

* Each constant maps a stable {@code wireName} (the token published across * nodes by the active synchronizer) to a direct, compiled call of the - * corresponding {@link ClusterUtil} method. Because {@link #invoke(Object[])} - * calls the real method rather than resolving it reflectively by name, renaming - * or changing the signature of a target method becomes a compile-time error - * here instead of a runtime failure on a peer node. + * corresponding method. Because {@link #invoke(Object[])} calls the real method + * rather than resolving it reflectively by name, renaming or changing the + * signature of a target method becomes a compile-time error here instead of a + * runtime failure on a peer node. *

* The {@code wireName} values intentionally match the historical method-name * strings so the payloads stay compatible with nodes running older builds. */ public enum ClusterSyncMethod { + /** + * Pulls an engine's folder and smss file from cloud storage so a peer that has + * the engine replaces its local copy with the one just pushed, such as after an + * smss edit. A peer that has the engine open closes it before replacing its + * files and opens it again from them, so the running engine and its python + * process pick up the change. Params: the engine id. + */ PULL_ENGINE("pullEngine") { @Override public void invoke(Object[] params) { - ClusterUtil.pullEngine((String) params[0]); + String engineId = (String) params[0]; + ClusterUtil.pullEngine(engineId, null, EngineSyncUtility.isEngineOpen(engineId)); } }, + /** + * Pulls one folder of an engine from cloud storage, such as a folder of files + * edited in the engine's assets, without pulling the rest of the engine. + * Params: the engine id, the folder's absolute path on the node, and its path + * in cloud storage relative to the engine. + */ PULL_ENGINE_FOLDER("pullEngineFolder") { @Override public void invoke(Object[] params) { ClusterUtil.pullEngineFolder((String) params[0], (String) params[1], (String) params[2]); } }, + /** + * Pulls a database's OWL file from cloud storage after its metadata changed, so + * a peer reads the current tables, columns and relationships. Params: the + * database id. + */ PULL_OWL("pullOwl") { @Override public void invoke(Object[] params) { ClusterUtil.pullOwl((String) params[0]); } }, + /** + * Pulls a project's folder and smss file from cloud storage so a peer that has + * the project replaces its local copy with the one just pushed, such as after + * an smss edit. A peer that has the project open closes it before replacing its + * files and opens it again from them. Params: the project id. + */ PULL_PROJECT("pullProject") { @Override public void invoke(Object[] params) { - ClusterUtil.pullProject((String) params[0]); + String projectId = (String) params[0]; + ClusterUtil.pullProject(projectId, ProjectSyncUtility.isProjectOpen(projectId)); } }, + /** + * Pulls one folder of a project from cloud storage, such as a folder of app + * files that was saved, without pulling the rest of the project. Params: the + * project id, the folder's absolute path on the node, and its path in cloud + * storage relative to the project. + */ PULL_PROJECT_FOLDER("pullProjectFolder") { @Override public void invoke(Object[] params) { ClusterUtil.pullProjectFolder((String) params[0], (String) params[1], (String) params[2]); } }, + /** + * Pulls one insight's folder of a project from cloud storage after the insight + * was saved. Params: the project id and the insight id. + */ PULL_INSIGHT("pullInsight") { @Override public void invoke(Object[] params) { ClusterUtil.pullInsight((String) params[0], (String) params[1]); } }, + /** + * Pulls a project's insights database from cloud storage after an insight was + * added, changed or removed, so a peer lists the current insights. Params: the + * project id. + */ PULL_INSIGHTS_DB("pullInsightsDB") { @Override public void invoke(Object[] params) { ClusterUtil.pullInsightsDB((String) params[0]); } }, + /** + * Pulls a user's asset project from cloud storage after the user's own files + * changed, on a peer that already has a copy of them. Params: the id of the + * user's asset project. + */ PULL_USER_ASSET("pullUserAsset") { @Override public void invoke(Object[] params) { ClusterUtil.pullUserAsset((String) params[0]); } + }, + /** + * Closes the engine where a peer has it open, so the peer opens it again from + * its current files on next use. This is how a restart of an engine reaches + * every node. Params: the engine id. + */ + CLOSE_ENGINE("closeEngine") { + @Override + public void invoke(Object[] params) { + EngineSyncUtility.closeLoadedEngine((String) params[0]); + } + }, + /** + * Clears the project modules cached in the project python process where a peer + * has it running, so the next import reads the current .py files. Params: the + * project id. + */ + RELOAD_PROJECT("reloadProject") { + @Override + public void invoke(Object[] params) { + ProjectSyncUtility.clearLoadedProjectPythonImports((String) params[0]); + } }; + /** + * The token published across nodes to name this action. It never changes, so + * nodes on older and newer builds read each other's events. + */ private final String wireName; + /** + * Every action keyed by its wire token, for reading a published event back. + */ private static final Map BY_WIRE_NAME = new HashMap<>(); static { for (ClusterSyncMethod method : values()) { @@ -105,11 +186,19 @@ public void invoke(Object[] params) { } } + /** + * Creates the action published under the given token. + * + * @param wireName the token published across nodes to name this action + */ ClusterSyncMethod(String wireName) { this.wireName = wireName; } /** + * Returns the token published across nodes to name this action, which + * {@link #fromWireName(String)} reads back on the receiving node. + * * @return the stable token published across nodes to identify this action */ public String getWireName() { @@ -117,7 +206,7 @@ public String getWireName() { } /** - * Executes the {@link ClusterUtil} action represented by this constant. + * Executes the action represented by this constant. * * @param params deserialized arguments, in the same order they were published */ diff --git a/src/prerna/cluster/util/ClusterUtil.java b/src/prerna/cluster/util/ClusterUtil.java index 6c208c226a7..5223b2dbdb0 100644 --- a/src/prerna/cluster/util/ClusterUtil.java +++ b/src/prerna/cluster/util/ClusterUtil.java @@ -127,6 +127,14 @@ public class ClusterUtil { * "storageAccount"; public static final String REDIS_TIMESTAMP = "timestamp"; */ + /** + * Determines whether this node runs the scheduled jobs. Outside a cluster every + * node does. In a cluster the SCHEDULER_EXECUTOR setting decides, read from + * RDF_Map and then the environment, and without it the ZooKeeper leader + * election does. + * + * @return true if this node runs the scheduled jobs + */ public static boolean isSchedulerExecutor() { if (ClusterUtil.IS_CLUSTER) { classLogger.info("Checking if pod is leader"); @@ -174,9 +182,11 @@ public static boolean isSchedulerExecutor() { } /** - * - * @return - * @throws Exception + * Returns the client for the cluster's central cloud storage, which holds the + * engines, projects, insights, rooms, user assets and images every node pulls. + * + * @return the central cloud storage client + * @throws Exception if the client cannot be created from the storage settings */ public static CentralCloudStorage getCentralStorageClient() throws Exception { return CentralCloudStorage.getInstance(); @@ -194,8 +204,14 @@ public static IClusterSynchronizer getClusterSynchronizer() throws Exception { } /** - * - * @param databaseId + * Downloads an engine's folder and smss file from cloud storage to this node, + * reading its type from the security database. An engine open on this node + * keeps running on its old files; see + * {@link #pullEngine(String, IEngine.CATALOG_TYPE, boolean)} to close and + * reopen it. Does nothing outside a cluster. + * + * @param engineId the engine to pull + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullEngine(String engineId) { if (ClusterUtil.IS_CLUSTER) { @@ -212,8 +228,13 @@ public static void pullEngine(String engineId) { } /** - * - * @param databaseId + * Downloads an engine's folder and smss file from cloud storage to this node. + * An engine open on this node keeps running on its old files. Does nothing + * outside a cluster. + * + * @param engineId the engine to pull + * @param engineType the engine's type + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullEngine(String engineId, IEngine.CATALOG_TYPE engineType) { if (ClusterUtil.IS_CLUSTER) { @@ -231,9 +252,16 @@ public static void pullEngine(String engineId, IEngine.CATALOG_TYPE engineType) } /** - * - * @param engineId - * @param engineAlreadyLoaded + * Downloads an engine's folder and smss file from cloud storage to this node. + * When the engine is open here, it is closed first, under its engine lock, and + * opened again from the new files after, so the running engine picks them up. + * Does nothing outside a cluster. + * + * @param engineId the engine to pull + * @param engineType the engine's type, or null to read it from the + * open engine or the security database + * @param engineAlreadyLoaded whether the engine is open on this node + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullEngine(String engineId, IEngine.CATALOG_TYPE engineType, boolean engineAlreadyLoaded) { if (ClusterUtil.IS_CLUSTER) { @@ -251,8 +279,15 @@ public static void pullEngine(String engineId, IEngine.CATALOG_TYPE engineType, } /** - * - * @param engineId + * Uploads an engine's folder and smss file to cloud storage, then tells the + * other nodes in the cluster to pull the engine + * ({@link ClusterSyncMethod#PULL_ENGINE}). The upload runs only in a cluster, + * and the notice only when a cluster synchronizer is set up. + * + * @param engineId the engine to push + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushEngine(String engineId) { if (ClusterUtil.IS_CLUSTER) { @@ -281,8 +316,16 @@ public static void pushEngine(String engineId) { } /** - * - * @param engineId + * Uploads an engine's smss file to cloud storage after it changed, reading its + * type from the security database, then tells the other nodes to pull the + * engine ({@link ClusterSyncMethod#PULL_ENGINE}), so a node that has it open + * closes it and opens it again with the new smss. The upload runs only in a + * cluster, and the notice only when a cluster synchronizer is set up. + * + * @param engineId the engine whose smss changed + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushEngineSmss(String engineId) { if (ClusterUtil.IS_CLUSTER) { @@ -310,8 +353,17 @@ public static void pushEngineSmss(String engineId) { } /** - * - * @param engineId + * Uploads an engine's smss file to cloud storage after it changed, then tells + * the other nodes to pull the engine ({@link ClusterSyncMethod#PULL_ENGINE}), + * so a node that has it open closes it and opens it again with the new smss. + * The upload runs only in a cluster, and the notice only when a cluster + * synchronizer is set up. + * + * @param engineId the engine whose smss changed + * @param engineType the engine's type + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushEngineSmss(String engineId, IEngine.CATALOG_TYPE engineType) { if (ClusterUtil.IS_CLUSTER) { @@ -341,8 +393,33 @@ public static void pushEngineSmss(String engineId, IEngine.CATALOG_TYPE engineTy } /** - * - * @param engineId + * Asks the other nodes in the cluster to close the engine where they have it + * open, so each one opens it again from its current files on next use. Runs + * through the active cluster synchronizer, ZooKeeper or Redis, and does nothing + * when no synchronizer is configured. + * + * @param engineId the engine to close on the other nodes + */ + public static void publishEngineClose(String engineId) { + if (ClusterSynchronizerFactory.IS_CLUSTER_SYNC_SETUP) { + try { + getClusterSynchronizer().publishEngineChange(engineId, ClusterSyncMethod.CLOSE_ENGINE, engineId); + } catch (Exception e) { + classLogger.error("Failed to publish the close of engine '{}' to the cluster", engineId, e); + SemossPixelException err = new SemossPixelException( + "Failed to publish the close of engine '" + engineId + "' to the other nodes in the cluster"); + err.setContinueThreadOfExecution(true); + throw err; + } + } + } + + /** + * Deletes an engine's folder and smss file from cloud storage, reading its type + * from the security database. Does nothing outside a cluster. + * + * @param engineId the engine to delete + * @throws SemossPixelException if the delete fails, which stops the pixel */ public static void deleteEngine(String engineId) { if (ClusterUtil.IS_CLUSTER) { @@ -359,9 +436,12 @@ public static void deleteEngine(String engineId) { } /** - * - * @param engineId - * @param engineType + * Deletes an engine's folder and smss file from cloud storage. Does nothing + * outside a cluster. + * + * @param engineId the engine to delete + * @param engineType the engine's type + * @throws SemossPixelException if the delete fails, which stops the pixel */ public static void deleteEngine(String engineId, IEngine.CATALOG_TYPE engineType) { if (ClusterUtil.IS_CLUSTER) { @@ -379,10 +459,14 @@ public static void deleteEngine(String engineId, IEngine.CATALOG_TYPE engineType } /** - * - * @param engineId - * @param engineType - * @param filePath + * Uploads one file of an engine to cloud storage, at the same path relative to + * the engine folder that it has on this node. Does nothing outside a cluster. + * + * @param engineId the engine the file belongs to + * @param engineType the engine's type + * @param filePath the file's absolute path, inside the engine folder on this + * node + * @throws SemossPixelException if the upload fails, which stops the pixel */ public static void copyLocalFileToEngineCloudFolder(String engineId, IEngine.CATALOG_TYPE engineType, String filePath) { @@ -399,10 +483,15 @@ public static void copyLocalFileToEngineCloudFolder(String engineId, IEngine.CAT } /** - * - * @param engineId - * @param engineType - * @param filePath + * Downloads one file of an engine from cloud storage to the given path on this + * node, taking it from the same path relative to the engine folder. Does + * nothing outside a cluster. + * + * @param engineId the engine the file belongs to + * @param engineType the engine's type + * @param filePath the absolute path to write the file to, inside the engine + * folder on this node + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void copyEngineCloudFileToLocalFile(String engineId, IEngine.CATALOG_TYPE engineType, String filePath) { @@ -419,10 +508,15 @@ public static void copyEngineCloudFileToLocalFile(String engineId, IEngine.CATAL } /** - * - * @param engineId - * @param engineType - * @param filePath + * Deletes one file of an engine from cloud storage, the one at the same path + * relative to the engine folder as the given local path. Does nothing outside a + * cluster. + * + * @param engineId the engine the file belongs to + * @param engineType the engine's type + * @param filePath the file's absolute path inside the engine folder on this + * node + * @throws SemossPixelException if the delete fails, which stops the pixel */ public static void deleteEngineCloudFile(String engineId, IEngine.CATALOG_TYPE engineType, String filePath) { if (ClusterUtil.IS_CLUSTER) { @@ -438,8 +532,12 @@ public static void deleteEngineCloudFile(String engineId, IEngine.CATALOG_TYPE e } /** - * - * @param engineType + * Downloads the image folder of one catalog type, such as databases, models or + * projects, from cloud storage, so this node has every uploaded image of that + * type. Does nothing outside a cluster. + * + * @param engineType the catalog type whose images to pull + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullEngineAndProjectImageFolder(IEngine.CATALOG_TYPE engineType) { if (ClusterUtil.IS_CLUSTER) { @@ -457,9 +555,13 @@ public static void pullEngineAndProjectImageFolder(IEngine.CATALOG_TYPE engineTy } /** - * - * @param engineType - * @param fileName + * Uploads one engine or project image from the local image folder of its + * catalog type to cloud storage. Does nothing outside a cluster. + * + * @param engineType the catalog type the image belongs to + * @param fileName the image's file name in that folder, named by the engine + * or project id + * @throws SemossPixelException if the upload fails, which stops the pixel */ public static void pushEngineAndProjectImage(IEngine.CATALOG_TYPE engineType, String fileName) { if (ClusterUtil.IS_CLUSTER) { @@ -477,9 +579,12 @@ public static void pushEngineAndProjectImage(IEngine.CATALOG_TYPE engineType, St } /** - * - * @param engineType - * @param fileName + * Deletes one engine or project image from cloud storage. Does nothing outside + * a cluster. + * + * @param engineType the catalog type the image belongs to + * @param fileName the image's file name + * @throws SemossPixelException if the delete fails, which stops the pixel */ public static void deleteEngineAndProjectImage(IEngine.CATALOG_TYPE engineType, String fileName) { if (ClusterUtil.IS_CLUSTER) { @@ -497,9 +602,13 @@ public static void deleteEngineAndProjectImage(IEngine.CATALOG_TYPE engineType, } /** - * - * @param engineType - * @param fileName + * Deletes every image of an engine or project from cloud storage: the files in + * the local image folder of its catalog type whose names start with its id. + * Does nothing outside a cluster. + * + * @param engineType the catalog type the engine or project belongs to + * @param engineId the engine or project id + * @throws SemossPixelException if the delete fails, which stops the pixel */ public static void deleteEngineAndProjectImageById(IEngine.CATALOG_TYPE engineType, String engineId) { if (ClusterUtil.IS_CLUSTER) { @@ -517,8 +626,13 @@ public static void deleteEngineAndProjectImageById(IEngine.CATALOG_TYPE engineTy } /** - * - * @param projectId + * Downloads a project's folder and smss file from cloud storage to this node. A + * project open on this node keeps running on its old files; see + * {@link #pullProject(String, boolean)} to close and reopen it. Does nothing + * outside a cluster. + * + * @param projectId the project to pull + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullProject(String projectId) { if (ClusterUtil.IS_CLUSTER) { @@ -535,9 +649,13 @@ public static void pullProject(String projectId) { } /** - * - * @param projectId - * @param projectAlreadyLoaded + * Downloads a project's folder and smss file from cloud storage to this node. + * When the project is open here, it is closed first, under its project lock, + * and opened again from the new files after. Does nothing outside a cluster. + * + * @param projectId the project to pull + * @param projectAlreadyLoaded whether the project is open on this node + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullProject(String projectId, boolean projectAlreadyLoaded) { if (ClusterUtil.IS_CLUSTER) { @@ -555,8 +673,11 @@ public static void pullProject(String projectId, boolean projectAlreadyLoaded) { } /** - * - * @param projectId + * Deletes a project's folder and smss file from cloud storage. Does nothing + * outside a cluster. + * + * @param projectId the project to delete + * @throws SemossPixelException if the delete fails, which stops the pixel */ public static void deleteProject(String projectId) { if (ClusterUtil.IS_CLUSTER) { @@ -573,8 +694,11 @@ public static void deleteProject(String projectId) { } /** - * - * @param projectId + * Downloads a project's insights database from cloud storage, so this node + * lists the project's current insights. Does nothing outside a cluster. + * + * @param projectId the project whose insights database to pull + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullInsightsDB(String projectId) { if (ClusterUtil.IS_CLUSTER) { @@ -592,8 +716,15 @@ public static void pullInsightsDB(String projectId) { } /** - * - * @param projectId + * Uploads a project's insights database to cloud storage after an insight was + * added, changed or removed, then tells the other nodes to pull it + * ({@link ClusterSyncMethod#PULL_INSIGHTS_DB}). The upload runs only in a + * cluster, and the notice only when a cluster synchronizer is set up. + * + * @param projectId the project whose insights database to push + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushInsightDB(String projectId) { if (ClusterUtil.IS_CLUSTER) { @@ -622,16 +753,25 @@ public static void pushInsightDB(String projectId) { } /** - * - * @param databaseId + * Downloads a database's OWL file from cloud storage, through a write OWL + * engine taken from the database. See {@link #pullOwl(String, WriteOWLEngine)}. + * + * @param databaseId the database whose OWL to pull + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullOwl(String databaseId) { pullOwl(databaseId, null); } /** - * - * @param databaseId + * Downloads a database's OWL file and its positions file from cloud storage. + * The OWL is closed for the download and reloaded from the new file after. Does + * nothing outside a cluster. + * + * @param databaseId the database whose OWL to pull + * @param owlEngine the write OWL engine to close and reload, or null to take + * one from the database + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullOwl(String databaseId, WriteOWLEngine owlEngine) { if (ClusterUtil.IS_CLUSTER) { @@ -649,17 +789,30 @@ public static void pullOwl(String databaseId, WriteOWLEngine owlEngine) { } /** - * - * @param databaseId + * Uploads a database's OWL file to cloud storage, through a write OWL engine + * taken from the database. See {@link #pushOwl(String, WriteOWLEngine)}. + * + * @param databaseId the database whose OWL to push + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushOwl(String databaseId) { pushOwl(databaseId, null); } /** - * - * @param databaseId - * @param owlEngine + * Uploads a database's OWL file and its positions file to cloud storage after + * the metadata changed, then tells the other nodes to pull the OWL + * ({@link ClusterSyncMethod#PULL_OWL}). The upload runs only in a cluster, and + * the notice only when a cluster synchronizer is set up. + * + * @param databaseId the database whose OWL to push + * @param owlEngine the write OWL engine holding the OWL, or null to take one + * from the database + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushOwl(String databaseId, WriteOWLEngine owlEngine) { if (ClusterUtil.IS_CLUSTER) { @@ -688,8 +841,16 @@ public static void pushOwl(String databaseId, WriteOWLEngine owlEngine) { } /** - * - * @param projectId + * Uploads a project's folder and smss file to cloud storage, then tells the + * other nodes to pull the project ({@link ClusterSyncMethod#PULL_PROJECT}), so + * a node that has it open closes it and opens it again from the new files. The + * upload runs only in a cluster, and the notice only when a cluster + * synchronizer is set up. + * + * @param projectId the project to push + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushProject(String projectId) { if (ClusterUtil.IS_CLUSTER) { @@ -719,8 +880,33 @@ public static void pushProject(String projectId) { } /** + * Asks the other nodes in the cluster to clear the project modules cached in + * the project python process where they have it running, so the next import + * reads the current .py files. Runs through the active cluster synchronizer, + * ZooKeeper or Redis, and does nothing when no synchronizer is configured. + * + * @param projectId the project to reload on the other nodes + */ + public static void publishProjectReload(String projectId) { + if (ClusterSynchronizerFactory.IS_CLUSTER_SYNC_SETUP) { + try { + getClusterSynchronizer().publishProjectChange(projectId, ClusterSyncMethod.RELOAD_PROJECT, projectId); + } catch (Exception e) { + classLogger.error("Failed to publish the reload of project '{}' to the cluster", projectId, e); + SemossPixelException err = new SemossPixelException("Failed to publish the reload of project '" + + projectId + "' to the other nodes in the cluster"); + err.setContinueThreadOfExecution(true); + throw err; + } + } + } + + /** + * Pushes a project's smss file to cloud storage after it changed, then tells + * the other nodes in the cluster to pull the project, so a node that has it + * open closes it and opens it again with the new smss. * - * @param projectId + * @param projectId the project whose smss changed */ public static void pushProjectSmss(String projectId) { if (ClusterUtil.IS_CLUSTER) { @@ -735,12 +921,30 @@ public static void pushProjectSmss(String projectId) { throw err; } } + + if (ClusterSynchronizerFactory.IS_CLUSTER_SYNC_SETUP) { + try { + getClusterSynchronizer().publishProjectChange(projectId, ClusterSyncMethod.PULL_PROJECT, projectId); + } catch (Exception e) { + classLogger.error("Failed to publish project '{}' smss change to the cluster", projectId, e); + SemossPixelException err = new SemossPixelException( + "Failed to publish project '" + projectId + "' to sync with the cluster"); + err.setContinueThreadOfExecution(true); + throw err; + } + } } /** - * - * @param project - * @param absolutePath + * Uploads one folder of a project to cloud storage, then tells the other nodes + * to pull it. See {@link #pushProjectFolder(IProject, String, String)}. + * + * @param project the project the folder belongs to + * @param absolutePath the folder's absolute path, inside the project folder on + * this node + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushProjectFolder(IProject project, String absolutePath) { if (ClusterUtil.IS_CLUSTER) { @@ -749,10 +953,19 @@ public static void pushProjectFolder(IProject project, String absolutePath) { } /** - * - * @param project - * @param absolutePath - * @param relativePath + * Uploads one folder of a project to cloud storage, at the same path relative + * to the project folder that it has on this node, then tells the other nodes to + * pull it ({@link ClusterSyncMethod#PULL_PROJECT_FOLDER}). Does nothing outside + * a cluster. + * + * @param project the project the folder belongs to + * @param absolutePath the folder's absolute path, or a folder above it, inside + * the project folder on this node + * @param relativePath the folder's path below absolutePath, or null or blank + * when absolutePath is the folder + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushProjectFolder(IProject project, String absolutePath, String relativePath) { if (ClusterUtil.IS_CLUSTER) { @@ -773,10 +986,19 @@ public static void pushProjectFolder(IProject project, String absolutePath, Stri } /** - * - * @param projectId - * @param absolutePath - * @param remoteRelativePath + * Uploads one folder of a project to cloud storage at the given path, then + * tells the other nodes to pull it + * ({@link ClusterSyncMethod#PULL_PROJECT_FOLDER}) with the same paths. The + * upload runs only in a cluster, and the notice only when a cluster + * synchronizer is set up. + * + * @param projectId the project the folder belongs to + * @param absolutePath the folder's absolute path on this node + * @param remoteRelativePath the folder's path in cloud storage, relative to the + * project + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushProjectFolder(String projectId, String absolutePath, String remoteRelativePath) { if (ClusterUtil.IS_CLUSTER) { @@ -804,9 +1026,13 @@ public static void pushProjectFolder(String projectId, String absolutePath, Stri } /** - * - * @param project - * @param absolutePath + * Downloads one folder of a project from cloud storage. See + * {@link #pullProjectFolder(IProject, String, String)}. + * + * @param project the project the folder belongs to + * @param absolutePath the folder's absolute path, inside the project folder on + * this node + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullProjectFolder(IProject project, String absolutePath) { if (ClusterUtil.IS_CLUSTER) { @@ -815,10 +1041,16 @@ public static void pullProjectFolder(IProject project, String absolutePath) { } /** - * - * @param project - * @param absolutePath - * @param relativePath + * Downloads one folder of a project from cloud storage, from the same path + * relative to the project folder that it has on this node. Does nothing outside + * a cluster. + * + * @param project the project the folder belongs to + * @param absolutePath the folder's absolute path, or a folder above it, inside + * the project folder on this node + * @param relativePath the folder's path below absolutePath, or null or blank + * when absolutePath is the folder + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullProjectFolder(IProject project, String absolutePath, String relativePath) { if (ClusterUtil.IS_CLUSTER) { @@ -839,10 +1071,14 @@ public static void pullProjectFolder(IProject project, String absolutePath, Stri } /** - * - * @param projectId - * @param absolutePath - * @param remoteRelativePath + * Downloads one folder of a project from the given path in cloud storage to the + * given path on this node. Does nothing outside a cluster. + * + * @param projectId the project the folder belongs to + * @param absolutePath the folder's absolute path on this node + * @param remoteRelativePath the folder's path in cloud storage, relative to the + * project + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullProjectFolder(String projectId, String absolutePath, String remoteRelativePath) { if (ClusterUtil.IS_CLUSTER) { @@ -859,9 +1095,15 @@ public static void pullProjectFolder(String projectId, String absolutePath, Stri } /** - * - * @param project - * @param absolutePath + * Uploads one folder of an engine to cloud storage, then tells the other nodes + * to pull it. See {@link #pushEngineFolder(IEngine, String, String)}. + * + * @param engine the engine the folder belongs to + * @param absolutePath the folder's absolute path, inside the engine folder on + * this node + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushEngineFolder(IEngine engine, String absolutePath) { if (ClusterUtil.IS_CLUSTER) { @@ -870,10 +1112,20 @@ public static void pushEngineFolder(IEngine engine, String absolutePath) { } /** - * - * @param project - * @param absolutePath - * @param relativePath + * Uploads one folder of an engine to cloud storage, at the same path relative + * to the engine folder that it has on this node, then tells the other nodes to + * pull it ({@link ClusterSyncMethod#PULL_ENGINE_FOLDER}). The other nodes + * replace the files without restarting the engine. Does nothing outside a + * cluster. + * + * @param engine the engine the folder belongs to + * @param absolutePath the folder's absolute path, or a folder above it, inside + * the engine folder on this node + * @param relativePath the folder's path below absolutePath, or null or blank + * when absolutePath is the folder + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushEngineFolder(IEngine engine, String absolutePath, String relativePath) { if (ClusterUtil.IS_CLUSTER) { @@ -894,10 +1146,19 @@ public static void pushEngineFolder(IEngine engine, String absolutePath, String } /** - * - * @param projectId - * @param absolutePath - * @param remoteRelativePath + * Uploads one folder of an engine to cloud storage at the given path, then + * tells the other nodes to pull it + * ({@link ClusterSyncMethod#PULL_ENGINE_FOLDER}) with the same paths. The + * upload runs only in a cluster, and the notice only when a cluster + * synchronizer is set up. + * + * @param engineId the engine the folder belongs to + * @param absolutePath the folder's absolute path on this node + * @param remoteRelativePath the folder's path in cloud storage, relative to the + * engine + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushEngineFolder(String engineId, String absolutePath, String remoteRelativePath) { if (ClusterUtil.IS_CLUSTER) { @@ -924,9 +1185,13 @@ public static void pushEngineFolder(String engineId, String absolutePath, String } /** - * - * @param project - * @param absolutePath + * Downloads one folder of an engine from cloud storage. See + * {@link #pullEngineFolder(IEngine, String, String)}. + * + * @param engine the engine the folder belongs to + * @param absolutePath the folder's absolute path, inside the engine folder on + * this node + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullEngineFolder(IEngine engine, String absolutePath) { if (ClusterUtil.IS_CLUSTER) { @@ -935,10 +1200,16 @@ public static void pullEngineFolder(IEngine engine, String absolutePath) { } /** - * - * @param engine - * @param absolutePath - * @param relativePath + * Downloads one folder of an engine from cloud storage, from the same path + * relative to the engine folder that it has on this node. Does nothing outside + * a cluster. + * + * @param engine the engine the folder belongs to + * @param absolutePath the folder's absolute path, or a folder above it, inside + * the engine folder on this node + * @param relativePath the folder's path below absolutePath, or null or blank + * when absolutePath is the folder + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullEngineFolder(IEngine engine, String absolutePath, String relativePath) { if (ClusterUtil.IS_CLUSTER) { @@ -959,10 +1230,14 @@ public static void pullEngineFolder(IEngine engine, String absolutePath, String } /** - * - * @param engineId - * @param absolutePath - * @param remoteRelativePath + * Downloads one folder of an engine from the given path in cloud storage to the + * given path on this node. Does nothing outside a cluster. + * + * @param engineId the engine the folder belongs to + * @param absolutePath the folder's absolute path on this node + * @param remoteRelativePath the folder's path in cloud storage, relative to the + * engine + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullEngineFolder(String engineId, String absolutePath, String remoteRelativePath) { if (ClusterUtil.IS_CLUSTER) { @@ -978,9 +1253,16 @@ public static void pullEngineFolder(String engineId, String absolutePath, String } /** - * - * @param projectId - * @param rdbmsId + * Uploads one insight's folder of a project to cloud storage after the insight + * was saved, then tells the other nodes to pull it + * ({@link ClusterSyncMethod#PULL_INSIGHT}). The upload runs only in a cluster, + * and the notice only when a cluster synchronizer is set up. + * + * @param projectId the project the insight belongs to + * @param rdbmsId the insight to push + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushInsight(String projectId, String rdbmsId) { if (ClusterUtil.IS_CLUSTER) { @@ -1010,9 +1292,12 @@ public static void pushInsight(String projectId, String rdbmsId) { } /** - * - * @param projectId - * @param rdbmsId + * Downloads one insight's folder of a project from cloud storage. Does nothing + * outside a cluster. + * + * @param projectId the project the insight belongs to + * @param rdbmsId the insight to pull + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullInsight(String projectId, String rdbmsId) { if (ClusterUtil.IS_CLUSTER) { @@ -1029,6 +1314,18 @@ public static void pullInsight(String projectId, String rdbmsId) { } } + /** + * Uploads an insight's image to cloud storage, deleting the image it replaces + * there first. Does nothing outside a cluster. + * + * @param projectId the project the insight belongs to + * @param insightId the insight the image belongs to + * @param oldImageName the file name of the image it replaces, or null when + * there is none + * @param imageFileName the file name of the new image, in the insight folder on + * this node + * @throws SemossPixelException if the upload fails, which stops the pixel + */ public static void pushInsightImage(String projectId, String insightId, String oldImageName, String imageFileName) { if (ClusterUtil.IS_CLUSTER) { try { @@ -1043,6 +1340,13 @@ public static void pushInsightImage(String projectId, String insightId, String o } } + /** + * Uploads a chat room's folder to cloud storage and waits for the upload. Does + * nothing outside a cluster. + * + * @param roomId the room to push + * @throws SemossPixelException if the upload fails, which stops the pixel + */ public static void pushRoom(String roomId) { if (ClusterUtil.IS_CLUSTER) { classLogger.info("ClusterUtil.pushRoom - pushing room '{}' to cloud storage", roomId); @@ -1057,6 +1361,13 @@ public static void pushRoom(String roomId) { } } + /** + * Uploads a chat room's folder to cloud storage on a virtual thread, without + * waiting for it. Nothing waits for the upload, so a failure is only logged on + * that thread. Does nothing outside a cluster. + * + * @param roomId the room to push + */ public static void pushRoomAsync(String roomId) { if (ClusterUtil.IS_CLUSTER) { Thread.ofVirtual().start(() -> { @@ -1073,6 +1384,13 @@ public static void pushRoomAsync(String roomId) { } } + /** + * Downloads a chat room's folder from cloud storage. Does nothing outside a + * cluster. + * + * @param roomId the room to pull + * @throws SemossPixelException if the download fails, which stops the pixel + */ public static void pullRoom(String roomId) { if (ClusterUtil.IS_CLUSTER) { try { @@ -1093,8 +1411,15 @@ public static void pullRoom(String roomId) { */ /** - * - * @param project + * Uploads a user's asset project, which holds the user's own files, to cloud + * storage, then tells the other nodes to pull it + * ({@link ClusterSyncMethod#PULL_USER_ASSET}). The upload runs only in a + * cluster, and the notice only when a cluster synchronizer is set up. + * + * @param projectId the user's asset project + * @throws SemossPixelException if the upload fails, which stops the pixel, or + * if the other nodes cannot be told, which lets + * the pixel continue */ public static void pushUserAsset(String projectId) { if (ClusterUtil.IS_CLUSTER) { @@ -1122,8 +1447,11 @@ public static void pushUserAsset(String projectId) { } /** - * - * @param project + * Downloads a user's asset project from cloud storage, as one that is not open + * on this node. See {@link #pullUserAsset(String, boolean)}. + * + * @param projectId the user's asset project + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullUserAsset(String projectId) { if (ClusterUtil.IS_CLUSTER) { @@ -1139,9 +1467,13 @@ public static void pullUserAsset(String projectId) { } /** - * - * @param project - * @param alreadyLoaded + * Downloads a user's asset project from cloud storage. When it is open on this + * node, it is closed first and opened again from the new files after. Does + * nothing outside a cluster. + * + * @param projectId the user's asset project + * @param alreadyLoaded whether the asset project is open on this node + * @throws SemossPixelException if the download fails, which stops the pixel */ public static void pullUserAsset(String projectId, boolean alreadyLoaded) { if (ClusterUtil.IS_CLUSTER) { @@ -1157,18 +1489,34 @@ public static void pullUserAsset(String projectId, boolean alreadyLoaded) { } } + /** + * Returns an engine or project's image, with the default theme for the stock + * image. See + * {@link #getEngineAndProjectImage(String, IEngine.CATALOG_TYPE, String)}. + * + * @param engineId the engine or project id + * @param engineType the catalog type the engine or project belongs to + * @return the uploaded image, or the shared stock image when there is none + * @throws Exception if the image folder cannot be pulled from cloud storage + */ + public static File getEngineAndProjectImage(String engineId, IEngine.CATALOG_TYPE engineType) throws Exception { + return getEngineAndProjectImage(engineId, engineType, null); + } + /** * Returns an existing resource image, refreshing the cloud image folder at most * once per 30 seconds when an image is missing. A missing custom image uses the * shared stock file without creating or uploading a per-resource copy. Locally * available uploads take precedence immediately, even during the refresh * window. + * + * @param engineId the engine or project id + * @param engineType the catalog type the engine or project belongs to + * @param theme the theme of the stock image, used only when the resource + * has no uploaded image; null for the default + * @return the uploaded image, or the shared stock image when there is none + * @throws Exception if the image folder cannot be pulled from cloud storage */ - public static File getEngineAndProjectImage(String engineId, IEngine.CATALOG_TYPE engineType) throws Exception { - return getEngineAndProjectImage(engineId, engineType, null); - } - - /** Uses the request's theme only when the resource has no uploaded image. */ public static File getEngineAndProjectImage(String engineId, IEngine.CATALOG_TYPE engineType, String theme) throws Exception { File localEngineImageFolder = new File(EngineUtility.getLocalEngineImageDirectory(engineType)); @@ -1190,14 +1538,24 @@ public static File getEngineAndProjectImage(String engineId, IEngine.CATALOG_TYP return DefaultImageGeneratorUtil.getStockImageForPath(imageFilePath, theme); } + /** + * Finds an engine or project's uploaded image in an image folder. + * + * @param folder the image folder of the catalog type + * @param engineId the engine or project id + * @return the first file whose name contains the id, or null when there is none + */ private static File findEngineAndProjectImage(File folder, String engineId) { File[] images = folder.listFiles((dir, name) -> name.contains(engineId)); return images != null && images.length > 0 ? images[0] : null; } /** - * - * @param folderPath + * Adds a hidden placeholder file to every empty folder under the given folder, + * at any depth, so the empty folders survive an upload to cloud storage, which + * keeps files rather than folders. + * + * @param folderPath the folder to check */ public static void validateFolder(String folderPath) { List subdirs = getSubdirs(folderPath); @@ -1209,9 +1567,11 @@ public static void validateFolder(String folderPath) { } /** - * - * @param path - * @return + * Lists every folder under a folder, at any depth. + * + * @param path the folder to list + * @return the folders under it, each folder before the ones inside it + * @throws IllegalArgumentException if the path is not a folder */ public static List getSubdirs(String path) { File file = new File(path); @@ -1235,8 +1595,10 @@ public boolean accept(File f) { } /** - * - * @param folder + * Creates the hidden placeholder file ({@link Constants#HIDDEN_FILE_EXTENSION}) + * in a folder. A failure to create it is logged. + * + * @param folder the folder to add it to */ public static void addHiddenFileToDir(File folder) { File hiddenFile = new File(folder.getAbsolutePath() + DIR_SEPARATOR + Constants.HIDDEN_FILE_EXTENSION); diff --git a/src/prerna/cluster/util/clients/CentralCloudStorage.java b/src/prerna/cluster/util/clients/CentralCloudStorage.java index e6fc2ca0cdf..617cf5a6457 100644 --- a/src/prerna/cluster/util/clients/CentralCloudStorage.java +++ b/src/prerna/cluster/util/clients/CentralCloudStorage.java @@ -1617,7 +1617,7 @@ public void pullProjectFolder(String projectId, String localAbsoluteFilePath, St */ @Override - public void pushInsight(String projectId, String insightId) throws IOException, InterruptedException { + public void pushInsight(String projectId, String rdbmsId) throws IOException, InterruptedException { IProject project = Utility.getProject(projectId); if (project == null) { throw new IllegalArgumentException("Project not found..."); @@ -1626,13 +1626,13 @@ public void pushInsight(String projectId, String insightId) throws IOException, // only need to pull the insight folder - 99% the project is always already // loaded to get to this point String localInsightFolderPath = Utility.normalizePath( - AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId) + "/" + insightId); + AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId) + "/" + rdbmsId); File localInsightF = new File(localInsightFolderPath); if (!localInsightF.exists() || !localInsightF.exists()) { localInsightF.mkdirs(); } String storageInsightFolder = PROJECT_CONTAINER_PREFIX + projectId + "/" + Constants.APP_ROOT_FOLDER + "/" - + Constants.VERSION_FOLDER + "/" + insightId; + + Constants.VERSION_FOLDER + "/" + rdbmsId; classLogger.info("Pushing insight from local={} to remote={}", Utility.cleanLogString(localInsightFolderPath), Utility.cleanLogString(storageInsightFolder)); @@ -1642,7 +1642,7 @@ public void pushInsight(String projectId, String insightId) throws IOException, } @Override - public void pullInsight(String projectId, String insightId) throws IOException, InterruptedException { + public void pullInsight(String projectId, String rdbmsId) throws IOException, InterruptedException { IProject project = Utility.getProject(projectId); if (project == null) { throw new IllegalArgumentException("Project not found..."); @@ -1651,13 +1651,13 @@ public void pullInsight(String projectId, String insightId) throws IOException, // only need to pull the insight folder - 99% the project is always already // loaded to get to this point String localInsightFolderPath = Utility.normalizePath( - AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId) + "/" + insightId); + AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId) + "/" + rdbmsId); File localInsightF = new File(localInsightFolderPath); if (!localInsightF.exists() || !localInsightF.exists()) { localInsightF.mkdirs(); } String storageInsightFolder = PROJECT_CONTAINER_PREFIX + projectId + "/" + Constants.APP_ROOT_FOLDER + "/" - + Constants.VERSION_FOLDER + "/" + insightId; + + Constants.VERSION_FOLDER + "/" + rdbmsId; classLogger.info("Pulling insight from remote={} to target={}", Utility.cleanLogString(storageInsightFolder), Utility.cleanLogString(localInsightFolderPath)); diff --git a/src/prerna/cluster/util/clients/ICloudClient.java b/src/prerna/cluster/util/clients/ICloudClient.java index 909d8b19d29..92929e0ad13 100644 --- a/src/prerna/cluster/util/clients/ICloudClient.java +++ b/src/prerna/cluster/util/clients/ICloudClient.java @@ -38,161 +38,183 @@ public interface ICloudClient { - /////////////////////////////////////////////////////////////////////////////////// - - /* - * Moving away from specific types and going to engine interface - * - */ - /** - * - * @param engineId - * @throws IOException - * @throws InterruptedException + * Uploads an engine's folder and its smss file to cloud storage. An engine that + * holds file locks, such as an H2 database, is closed for the upload and opened + * again after. + * + * @param engineId the engine to upload + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushEngine(String engineId) throws IOException, InterruptedException; /** - * - * @param engineId - * @throws IOException - * @throws InterruptedException + * Downloads an engine's folder and its smss file from cloud storage and + * catalogs the engine on this node, for an engine that is not open here. The + * engine's type is read from the security database. + * + * @param engineId the engine to download + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullEngine(String engineId) throws IOException, InterruptedException; /** - * - * @param engineId - * @param engineType - * @throws IOException - * @throws InterruptedException + * Downloads an engine's folder and its smss file from cloud storage and + * catalogs the engine on this node, for an engine that is not open here. + * + * @param engineId the engine to download + * @param engineType the engine's type, which places its folders + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullEngine(String engineId, IEngine.CATALOG_TYPE engineType) throws IOException, InterruptedException; /** - * - * @param engineId - * @param engineType - * @param engineAlreadyLoaded - * @throws IOException - * @throws InterruptedException + * Downloads an engine's folder and its smss file from cloud storage. An engine + * open on this node is closed for the download and opened again after, so it + * reads the new files; one that is not open is cataloged on this node. Nothing + * is downloaded for an engine the security database does not have. + * + * @param engineId the engine to download + * @param engineType the engine's type, or null to read it from the + * security database + * @param engineAlreadyLoaded true if the engine is open on this node + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullEngine(String engineId, IEngine.CATALOG_TYPE engineType, boolean engineAlreadyLoaded) throws IOException, InterruptedException; /** - * Push only the smss file for a engine - * - * @param engineId - * @throws Exception + * Uploads only an engine's smss file to cloud storage. The engine's type is + * read from the security database. + * + * @param engineId the engine whose smss file to upload + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushEngineSmss(String engineId) throws IOException, InterruptedException; /** - * Push only the smss file for a engine - * - * @param engineId - * @param engineType - * @throws IOException - * @throws InterruptedException + * Uploads only an engine's smss file to cloud storage. + * + * @param engineId the engine whose smss file to upload + * @param engineType the engine's type, which places its folders + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushEngineSmss(String engineId, IEngine.CATALOG_TYPE engineType) throws IOException, InterruptedException; /** - * Pull only the smss file for a engine - * - * @param engineId - * @throws Exception + * Downloads only an engine's smss file from cloud storage. The engine's type is + * read from the security database. + * + * @param engineId the engine whose smss file to download + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullEngineSmss(String engineId) throws IOException, InterruptedException; /** - * Pull only the smss file for a engine - * - * @param engineId - * @param engineType - * @throws IOException - * @throws InterruptedException + * Downloads only an engine's smss file from cloud storage. + * + * @param engineId the engine whose smss file to download + * @param engineType the engine's type, which places its folders + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullEngineSmss(String engineId, IEngine.CATALOG_TYPE engineType) throws IOException, InterruptedException; /** - * - * @param engineId - * @param localAbsoluteFilePath - * @param storageRelativePath - * @throws IOException - * @throws InterruptedException + * Uploads one folder of an engine, making the folder at a path in the engine's + * cloud folder match the local folder. + * + * @param engineId the engine + * @param localAbsoluteFilePath the absolute path of the local folder to upload + * @param storageRelativePath the folder's path in the engine's cloud folder, + * or null for the engine's cloud folder itself + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushEngineFolder(String engineId, String localAbsoluteFilePath, String storageRelativePath) throws IOException, InterruptedException; /** - * - * @param engineId - * @param localAbsoluteFilePath - * @param storageRelativePath - * @throws IOException - * @throws InterruptedException + * Downloads one folder of an engine, making the local folder match the folder + * at a path in the engine's cloud folder. + * + * @param engineId the engine + * @param localAbsoluteFilePath the absolute path of the local folder to + * download into + * @param storageRelativePath the folder's path in the engine's cloud folder + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullEngineFolder(String engineId, String localAbsoluteFilePath, String storageRelativePath) throws IOException, InterruptedException; /** - * Delete the engine from cloud storage - * - * @param engineId - * @throws IOException - * @throws InterruptedException + * Deletes an engine's folder, its smss file and its images from cloud storage. + * The engine's type is read from the security database. + * + * @param engineId the engine to delete + * @throws IOException if the delete fails + * @throws InterruptedException if the delete is interrupted */ void deleteEngine(String engineId) throws IOException, InterruptedException; /** - * Delete the engine from cloud storage - * - * @param engineId - * @param engineType - * @throws IOException - * @throws InterruptedException + * Deletes an engine's folder, its smss file and its images from cloud storage. + * + * @param engineId the engine to delete + * @param engineType the engine's type, which places its folders + * @throws IOException if the delete fails + * @throws InterruptedException if the delete is interrupted */ void deleteEngine(String engineId, IEngine.CATALOG_TYPE engineType) throws IOException, InterruptedException; /** - * Copy the engine local file to the corresponding storage location - * - * @param engineId - * @param engineType - * @param localFilePath - * @throws IOException - * @throws InterruptedException + * Uploads one file or folder of an engine to the place in the engine's cloud + * folder that matches its place in the engine's local folder. + * + * @param engineId the engine + * @param engineType the engine's type, which places its folders + * @param localFilePath the absolute path of a file or folder in the engine's + * local folder + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void copyLocalFileToEngineCloudFolder(String engineId, CATALOG_TYPE engineType, String localFilePath) throws IOException, InterruptedException; /** - * Copy engine file path to local file path, based on where the corresponding - * storage location should be - * - * @param engineId - * @param engineType - * @param storageRelativePath - * @param localFilePath - * @throws IOException - * @throws InterruptedException + * Downloads one file or folder of an engine to a local path, from the place in + * the engine's cloud folder that matches the path's place in the engine's local + * folder. + * + * @param engineId the engine + * @param engineType the engine's type, which places its folders + * @param localFilePath the absolute path in the engine's local folder to + * download to + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void copyEngineCloudFileToLocalFile(String engineId, CATALOG_TYPE engineType, String localFilePath) throws IOException, InterruptedException; /** - * Delete where the corresponding local file path would be from the - * corresponding storage location - * - * @param engineId - * @param engineType - * @param localFilePath - * @throws IOException - * @throws InterruptedException + * Deletes one file or folder of an engine from cloud storage: the one at the + * place in the engine's cloud folder that matches a local path's place in the + * engine's local folder. + * + * @param engineId the engine + * @param engineType the engine's type, which places its folders + * @param localFilePath the absolute path in the engine's local folder whose + * cloud copy to delete + * @throws IOException if the delete fails + * @throws InterruptedException if the delete is interrupted */ void deleteEngineCloudFile(String engineId, CATALOG_TYPE engineType, String localFilePath) throws IOException, InterruptedException; @@ -204,37 +226,45 @@ void deleteEngineCloudFile(String engineId, CATALOG_TYPE engineType, String loca */ /** - * - * @param engineType - * @throws IOException - * @throws InterruptedException + * Downloads every image of one catalog type, such as databases or projects, + * from cloud storage into the local image folder for that type. + * + * @param engineType the catalog type whose images to download + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullEngineAndProjectImageFolder(CATALOG_TYPE engineType) throws IOException, InterruptedException; /** - * - * @param engineType - * @param fileName - * @throws IOException - * @throws InterruptedException + * Uploads one image from the local image folder of a catalog type to cloud + * storage. + * + * @param engineType the catalog type the image belongs to + * @param fileName the image's file name in the local image folder + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushEngineAndProjectImage(CATALOG_TYPE engineType, String fileName) throws IOException, InterruptedException; /** - * - * @param engineType - * @param fileName - * @throws IOException - * @throws InterruptedException + * Deletes one image of a catalog type from cloud storage. + * + * @param engineType the catalog type the image belongs to + * @param fileName the image's file name + * @throws IOException if the delete fails + * @throws InterruptedException if the delete is interrupted */ void deleteEngineAndProjectImage(CATALOG_TYPE engineType, String fileName) throws IOException, InterruptedException; /** - * - * @param engineType - * @param engineId - * @throws IOException - * @throws InterruptedException + * Deletes the images of one engine or project, the files in the local image + * folder whose names start with its id, from cloud storage and from the local + * folder. + * + * @param engineType the catalog type of the engine or project + * @param engineId the id of the engine or project whose images to delete + * @throws IOException if the delete fails + * @throws InterruptedException if the delete is interrupted */ void deleteEngineAndProjectImageById(CATALOG_TYPE engineType, String engineId) throws IOException, InterruptedException; @@ -246,54 +276,71 @@ void deleteEngineAndProjectImageById(CATALOG_TYPE engineType, String engineId) */ /** - * - * @param databaseId - * @param dbType - * @throws IOException - * @throws InterruptedException + * Uploads the database file of an H2 or SQLite database to cloud storage. The + * database is closed for the upload and opened again after. + * + * @param databaseId the database + * @param dbType the database's type, H2 or SQLite + * @throws IllegalArgumentException if the database is not H2 or SQLite, or is + * not found + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushLocalDatabaseFile(String databaseId, RdbmsTypeEnum dbType) throws IOException, InterruptedException; /** - * - * @param databaseId - * @param rdbmsType - * @throws IOException - * @throws InterruptedException + * Downloads the database file of an H2 or SQLite database from cloud storage. + * The database is closed for the download and opened again after. + * + * @param databaseId the database + * @param rdbmsType the database's type, H2 or SQLite + * @throws IllegalArgumentException if the database is not H2 or SQLite, or is + * not found + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullLocalDatabaseFile(String databaseId, RdbmsTypeEnum rdbmsType) throws IOException, InterruptedException; /** - * - * @param databaseId - * @throws IOException - * @throws InterruptedException + * Uploads a database's OWL file, and its positions file when it has one, to + * cloud storage. + * + * @param databaseId the database + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushOwl(String databaseId) throws IOException, InterruptedException; /** - * - * @param databaseId - * @param owlEngine - * @throws IOException - * @throws InterruptedException + * Uploads a database's OWL file, and its positions file when it has one, to + * cloud storage. The upload reads the files on disk. + * + * @param databaseId the database + * @param owlEngine the write OWL engine the caller holds, if any + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushOwl(String databaseId, WriteOWLEngine owlEngine) throws IOException, InterruptedException; /** - * - * @param databaseId - * @throws IOException - * @throws InterruptedException + * Downloads a database's OWL file and its positions file from cloud storage, + * then reloads the OWL. + * + * @param databaseId the database + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullOwl(String databaseId) throws IOException, InterruptedException; /** - * - * @param databaseId - * @param owlEngine - * @throws IOException - * @throws InterruptedException + * Downloads a database's OWL file and its positions file from cloud storage. + * The write OWL engine is closed for the download and reloads the OWL after. + * + * @param databaseId the database + * @param owlEngine the write OWL engine the caller holds, or null to open one + * for the download and close it after + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullOwl(String databaseId, WriteOWLEngine owlEngine) throws IOException, InterruptedException; @@ -304,85 +351,111 @@ void deleteEngineAndProjectImageById(CATALOG_TYPE engineType, String engineId) */ /** - * - * @param projectId - * @throws IOException - * @throws InterruptedException + * Uploads a project's folder and its smss file to cloud storage. The project is + * closed for the upload and opened again after. + * + * @param projectId the project to upload + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushProject(String projectId) throws IOException, InterruptedException; /** - * - * @param projectId - * @throws IOException - * @throws InterruptedException + * Downloads a project's folder and its smss file from cloud storage and + * catalogs the project on this node, for a project that is not open here. + * + * @param projectId the project to download + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullProject(String projectId) throws IOException, InterruptedException; /** - * - * @param projectId - * @param projectAlreadyLoaded - * @throws IOException - * @throws InterruptedException + * Downloads a project's folder and its smss file from cloud storage. A project + * open on this node is closed for the download and opened again after; one that + * is not open is cataloged on this node. Nothing is downloaded for a project + * the security database does not have. + * + * @param projectId the project to download + * @param projectAlreadyLoaded true if the project is open on this node + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullProject(String projectId, boolean projectAlreadyLoaded) throws IOException, InterruptedException; /** - * Push only the smss file for a project - * - * @param projectId - * @throws Exception + * Uploads only a project's smss file to cloud storage. + * + * @param projectId the project whose smss file to upload + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushProjectSmss(String projectId) throws IOException, InterruptedException; /** - * Pull only the smss file for a project - * - * @param projectId - * @throws Exception + * Downloads only a project's smss file from cloud storage. + * + * @param projectId the project whose smss file to download + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullProjectSmss(String projectId) throws IOException, InterruptedException; /** - * - * @param projectId - * @throws Exception + * Deletes a project's folder, its smss file and its images from cloud storage. + * + * @param projectId the project to delete + * @throws IOException if the delete fails + * @throws InterruptedException if the delete is interrupted */ void deleteProject(String projectId) throws IOException, InterruptedException; /** - * - * @param projectId - * @throws Exception + * Downloads a project's insights database file from cloud storage. The insights + * database is closed for the download and opened again after, whether or not + * the download works. + * + * @param projectId the project + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullInsightsDB(String projectId) throws IOException, InterruptedException; /** - * - * @param projectId - * @throws Exception + * Uploads a project's insights database file to cloud storage. The insights + * database is closed for the upload and opened again after, whether or not the + * upload works. + * + * @param projectId the project + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushInsightDB(String projectId) throws IOException, InterruptedException; /** - * - * @param projectId - * @param localAbsoluteFilePath - * @param storageRelativePath - * @throws IOException - * @throws InterruptedException + * Uploads one folder of a project, making the folder at a path in the project's + * cloud folder match the local folder. + * + * @param projectId the project + * @param localAbsoluteFilePath the absolute path of the local folder to upload + * @param storageRelativePath the folder's path in the project's cloud folder, + * or null for the project's cloud folder itself + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushProjectFolder(String projectId, String localAbsoluteFilePath, String storageRelativePath) throws IOException, InterruptedException; /** - * - * @param projectId - * @param localAbsoluteFilePath - * @param storageRelativePath - * @throws IOException - * @throws InterruptedException + * Downloads one folder of a project, making the local folder match the folder + * at a path in the project's cloud folder. + * + * @param projectId the project + * @param localAbsoluteFilePath the absolute path of the local folder to + * download into + * @param storageRelativePath the folder's path in the project's cloud folder + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullProjectFolder(String projectId, String localAbsoluteFilePath, String storageRelativePath) throws IOException, InterruptedException; @@ -394,28 +467,38 @@ void pullProjectFolder(String projectId, String localAbsoluteFilePath, String st */ /** - * - * @param projectId - * @param insightId - * @throws Exception + * Uploads one insight's folder of a project to cloud storage. + * + * @param projectId the project the insight belongs to + * @param rdbmsId the id of the insight + * @throws IOException if the upload fails + * @throws InterruptedException if the upload is interrupted */ - void pushInsight(String projectId, String insightId) throws IOException, InterruptedException; + void pushInsight(String projectId, String rdbmsId) throws IOException, InterruptedException; /** - * - * @param projectId - * @param insightId - * @throws Exception + * Downloads one insight's folder of a project from cloud storage. + * + * @param projectId the project the insight belongs to + * @param rdbmsId the id of the insight + * @throws IOException if the download fails + * @throws InterruptedException if the download is interrupted */ - void pullInsight(String projectId, String insightId) throws IOException, InterruptedException; + void pullInsight(String projectId, String rdbmsId) throws IOException, InterruptedException; /** - * - * @param projectId - * @param insightId - * @param oldImageFileName - * @param newImageFileName - * @throws Exception + * Replaces an insight's image in cloud storage: deletes the old image file by + * name, since the new one can have another extension, then uploads the new one + * from the insight's local folder. + * + * @param projectId the project the insight belongs to + * @param insightId the id of the insight + * @param oldImageFileName the file name of the image to delete, or null when + * there is none + * @param newImageFileName the file name of the image to upload, or null to + * upload none + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushInsightImage(String projectId, String insightId, String oldImageFileName, String newImageFileName) throws IOException, InterruptedException; @@ -427,27 +510,33 @@ void pushInsightImage(String projectId, String insightId, String oldImageFileNam */ /** - * - * @param projectId - * @param projectAlreadyLoaded - * @throws IOException - * @throws InterruptedException + * Downloads a user's asset project folder and its smss file from cloud storage. + * An asset project open on this node is closed for the download and opened + * again after. + * + * @param projectId the id of the user's asset project + * @param projectAlreadyLoaded true if the asset project is open on this node + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pullUserAsset(String projectId, boolean projectAlreadyLoaded) throws IOException, InterruptedException; /** - * - * @param projectId - * @throws IOException - * @throws InterruptedException + * Uploads a user's asset project folder and its smss file to cloud storage. The + * project is closed for the upload and opened again after. + * + * @param projectId the id of the user's asset project + * @throws IOException if the transfer fails + * @throws InterruptedException if the transfer is interrupted */ void pushUserAsset(String projectId) throws IOException, InterruptedException; /** - * - * @return - * @throws IOException - * @throws InterruptedException + * Lists what each engine and project bucket in cloud storage holds. + * + * @return the entries of each bucket, keyed by the bucket's name + * @throws IOException if the listing fails + * @throws InterruptedException if the listing is interrupted */ Map> listAllContainersByBucket() throws IOException, InterruptedException; diff --git a/src/prerna/ds/py/PyUtils.java b/src/prerna/ds/py/PyUtils.java index 95da6cb82e0..50e07c141c3 100644 --- a/src/prerna/ds/py/PyUtils.java +++ b/src/prerna/ds/py/PyUtils.java @@ -54,6 +54,14 @@ public final class PyUtils { public static final String PY_COMMAND_SEPARATOR = ";"; + /** + * Python call that drops the cached imports of the project modules under the + * asset paths of the current execution, plus the MCP driver aliases in the + * insight globals, so the next import reads the current .py files. Defined by + * {@code smss_clear_app_imports} in gaas_tcp_server_thread_local.py. + */ + public static final String CLEAR_APP_IMPORTS_SCRIPT = "smss_clear_app_imports()"; + private static Boolean pyEnabled = null; /** diff --git a/src/prerna/project/api/IProject.java b/src/prerna/project/api/IProject.java index d58ddd39f4b..d8eca61e895 100644 --- a/src/prerna/project/api/IProject.java +++ b/src/prerna/project/api/IProject.java @@ -347,6 +347,19 @@ enum PROJECT_TYPE { @IgnoreEngineLogging PyTranslator getProjectPyTranslator(); + /** + * Clears the project modules cached in the project's dedicated python process + * by running {@code smss_clear_app_imports()} with the project asset paths, so + * the next import reads the current .py files. The process keeps running, and a + * process that is not running is not started. + * + * @return the result of {@code smss_clear_app_imports()}, a map of the cleared + * module keys and MCP aliases, or null when the project python process + * is not running + */ + @IgnoreEngineLogging + Object clearPythonImports(); + /** * * @return diff --git a/src/prerna/project/impl/Project.java b/src/prerna/project/impl/Project.java index f995dc2cfc4..a46a8dd6ae2 100644 --- a/src/prerna/project/impl/Project.java +++ b/src/prerna/project/impl/Project.java @@ -74,6 +74,7 @@ import prerna.cluster.util.ClusterUtil; import prerna.date.SemossDate; import prerna.ds.py.PyTranslator; +import prerna.ds.py.PyUtils; import prerna.engine.api.IEngine; import prerna.engine.api.IHeadersDataRow; import prerna.engine.api.IMCP; @@ -421,31 +422,16 @@ public void close() { } } - // TODO: do we want to close the py process everything time we close? - // we close when we push insights (cause of the insights database) or update - // smss - -// try { -// if(tcpClient != null) { -// // this should destroy the process as well -// tcpClient.stopPyServe(this.tcpServerDirectory); -// } -// } catch(Exception e) { -// classLogger.error(Constants.STACKTRACE, e); -// } -// // but just in case above doesn't destroy it -// try { -// if(tcpServerProcess != null) { -// tcpServerProcess.destroy(); -// } -// } catch(Exception e) { -// classLogger.error(Constants.STACKTRACE, e); -// try { -// tcpServerProcess.destroy(); -// } catch(Exception e1) { -// classLogger.error(Constants.STACKTRACE, e1); -// } -// } + // shut down the project's python process when it is running; a project opened + // again is a new instance that starts its own + if (this.cpw != null) { + try { + this.cpw.shutdown(false); + } catch (Exception e) { + classLogger.error("Failed to shut down the python process of project '{}'", + SmssUtilities.getUniqueName(this.projectName, this.projectId), e); + } + } } @Override @@ -1320,6 +1306,17 @@ public PyTranslator getProjectPyTranslator() { return this.pyTranslator; } + @Override + public Object clearPythonImports() { + if (this.cpw == null || this.cpw.getSocketClient() == null || !this.cpw.getSocketClient().isConnected()) { + return null; + } + // the translator's insight carries this project as its context, so the + // script runs with the project assets as its asset paths + this.pyTranslator.setSocketClient(this.cpw.getSocketClient()); + return this.pyTranslator.runScript(PyUtils.CLEAR_APP_IMPORTS_SCRIPT); + } + /** * */ diff --git a/src/prerna/reactor/agent/skill/CloneSkillReactor.java b/src/prerna/reactor/agent/skill/CloneSkillReactor.java index e0440b82c1f..b3cab77b72b 100644 --- a/src/prerna/reactor/agent/skill/CloneSkillReactor.java +++ b/src/prerna/reactor/agent/skill/CloneSkillReactor.java @@ -117,7 +117,7 @@ public NounMetadata execute() { "Skill " + sourceSkillId + " does not exist or user does not have permission to view it"); } if (!SecurityProjectUtils.userCanCloneProject(user, sourceSkillId)) { - throw new IllegalArgumentException("This skill is not enabled as a template and cannot be cloned."); + throw new IllegalArgumentException("This skill is not a template, so only its owners can clone it."); } if (!SkillProjects.isSkillProject(sourceSkillId)) { diff --git a/src/prerna/reactor/engine/CloneEngineReactor.java b/src/prerna/reactor/engine/CloneEngineReactor.java new file mode 100644 index 00000000000..5b943bc3acd --- /dev/null +++ b/src/prerna/reactor/engine/CloneEngineReactor.java @@ -0,0 +1,166 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.engine; + +import java.util.Map; + +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.AbstractSecurityUtils; +import prerna.auth.utils.SecurityAdminUtils; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.engine.api.IEngine; +import prerna.engine.api.StorageTypeEnum; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.execptions.SemossPixelException; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.usertracking.AuditEventType; +import prerna.usertracking.AuditTargetType; +import prerna.usertracking.UserAuditTrailUtils; +import prerna.util.EngineCloneUtility; +import prerna.util.UploadUtilities; +import prerna.util.Utility; + +/** + * Clones an engine of any type into a new private engine owned by the user, + * with a copy of every asset the source keeps on the server. See + * {@link EngineCloneUtility} for what is copied. The user must own the source + * engine, or be an admin, and be allowed to add an engine of its type. + */ +public class CloneEngineReactor extends AbstractReactor { + + private static final String CLASS_NAME = CloneEngineReactor.class.getName(); + + /** Appended to the source display name when no name is given for the clone. */ + private static final String DEFAULT_NAME_SUFFIX = " Copy"; + /** Audit detail key holding the id of the engine that was cloned. */ + private static final String CLONED_FROM_DETAIL = "clonedFrom"; + + public CloneEngineReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.ENGINE.getKey(), ReactorKeysEnum.NAME.getKey() }; + this.keyRequired = new int[] { 1, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + Logger logger = getLogger(CLASS_NAME); + + User user = this.insight.getUser(); + if (user == null) { + NounMetadata noun = new NounMetadata("User must be signed into an account in order to clone an engine", + PixelDataType.CONST_STRING, PixelOperationType.ERROR, PixelOperationType.LOGGIN_REQUIRED_ERROR); + SemossPixelException err = new SemossPixelException(noun); + err.setContinueThreadOfExecution(false); + throw err; + } + if (AbstractSecurityUtils.anonymousUsersEnabled() && user.isAnonymous()) { + throwAnonymousUserError(); + } + if (AbstractSecurityUtils.adminSetPublisher() && !SecurityQueryUtils.userIsPublisher(user)) { + throwUserNotPublisherError(); + } + + String engineId = this.keyValue.get(ReactorKeysEnum.ENGINE.getKey()); + boolean isAdmin = SecurityAdminUtils.userIsAdmin(user); + if (!isAdmin && !SecurityEngineUtils.userIsOwner(user, engineId)) { + throw new IllegalArgumentException("Engine " + engineId + + " does not exist or user does not have permissions to clone the engine. User must be the owner to perform this function."); + } + + // the clone is a new engine, so the rules for adding one apply + Object[] typeAndSubtype = SecurityEngineUtils.getEngineTypeAndSubtype(engineId); + IEngine.CATALOG_TYPE engineType = (IEngine.CATALOG_TYPE) typeAndSubtype[0]; + if (AbstractSecurityUtils.adminOnlyEngineAdd(engineType) && !isAdmin) { + throwFunctionalityOnlyExposedForAdminsError(); + } + if (IEngine.CATALOG_TYPE.STORAGE == engineType && !isAdmin + && StorageTypeEnum.LOCAL_FILE_SYSTEM.getStorageName().equalsIgnoreCase((String) typeAndSubtype[1])) { + throw new IllegalArgumentException("Only an admin can create a local file system storage engine"); + } + + String cloneName = this.keyValue.get(ReactorKeysEnum.NAME.getKey()); + if (cloneName == null || cloneName.trim().isEmpty()) { + IEngine source = Utility.getEngine(engineId); + if (source == null) { + throw new IllegalArgumentException("Engine " + engineId + " does not exist or could not be loaded"); + } + String sourceName = source.getDisplayName(); + if (sourceName == null || sourceName.trim().isEmpty()) { + sourceName = source.getEngineName(); + } + // a display name can hold the folder separators a clone name cannot + cloneName = sourceName.replace('/', '-').replace('\\', '-') + DEFAULT_NAME_SUFFIX; + } + cloneName = cloneName.trim(); + // the name is part of the engine folder name + if (cloneName.contains("/") || cloneName.contains("\\")) { + throw new IllegalArgumentException("The name of the clone cannot contain / or \\"); + } + + logger.info("Cloning engine {} as {}", Utility.cleanLogString(engineId), Utility.cleanLogString(cloneName)); + String cloneId = EngineCloneUtility.cloneEngine(user, engineId, cloneName, logger); + logger.info("Cloned engine {} into {}", Utility.cleanLogString(engineId), cloneId); + + UserAuditTrailUtils.recordEngineLifecycle(user, AuditEventType.ENGINE_CREATE, + AuditTargetType.ofEngine(engineType), cloneId, cloneName, Map.of(CLONED_FROM_DETAIL, engineId)); + + Map retMap = UploadUtilities.getEngineReturnData(user, cloneId); + NounMetadata noun = new NounMetadata(retMap, PixelDataType.UPLOAD_RETURN_MAP, + PixelOperationType.MARKET_PLACE_ADDITION); + noun.addAdditionalReturn(NounMetadata.getSuccessNounMessage("Successfully cloned the engine as " + cloneName)); + return noun; + } + + @Override + public String getReactorDescription() { + return """ + Clone an engine into a new private engine owned by the user. The clone gets a new id and a copy \ + of everything the engine keeps on the server: its folder (database file, OWL, vector indexes, \ + python files, virtual environment, version history), its smss file and secrets, its metadata, \ + and its image. Connection settings are copied as they are, so a clone of an engine backed by an \ + external system uses the same external data. The user must own the engine."""; + } + + @Override + protected String getDescriptionForKey(String key) { + if (key.equals(ReactorKeysEnum.ENGINE.getKey())) { + return "The id of the engine to clone"; + } else if (key.equals(ReactorKeysEnum.NAME.getKey())) { + return "The name of the clone. Defaults to the engine name followed by \"" + DEFAULT_NAME_SUFFIX.trim() + + "\""; + } + return super.getDescriptionForKey(key); + } + +} diff --git a/src/prerna/reactor/project/CreateAppFromTemplateReactor.java b/src/prerna/reactor/project/CreateAppFromTemplateReactor.java index 5f5beecfda9..02fc970487d 100644 --- a/src/prerna/reactor/project/CreateAppFromTemplateReactor.java +++ b/src/prerna/reactor/project/CreateAppFromTemplateReactor.java @@ -107,7 +107,7 @@ public NounMetadata execute() { // Use the template to populate the parameters needed to create the new project IProject.PROJECT_TYPE projectEnumType = templateProject.getProjectType(); if (!SecurityProjectUtils.userCanCloneProject(this.insight.getUser(), projectTemplateId)) { - throw new IllegalArgumentException("This project is not enabled as a template and cannot be cloned."); + throw new IllegalArgumentException("This project is not a template, so only its owners can clone it."); } // Create new project diff --git a/src/prerna/reactor/project/ReloadProjectReactor.java b/src/prerna/reactor/project/ReloadProjectReactor.java new file mode 100644 index 00000000000..c40240d7074 --- /dev/null +++ b/src/prerna/reactor/project/ReloadProjectReactor.java @@ -0,0 +1,194 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.project; + +import java.util.Collection; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.cluster.util.ClusterUtil; +import prerna.ds.py.PyUtils; +import prerna.engine.api.IEngine; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.execptions.SemossPixelException; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.tcp.client.SocketClient; +import prerna.util.EngineUtility; +import prerna.util.ProjectSyncUtility; +import prerna.util.Utility; + +/** + * Reloads the python files of a project without restarting any python process. + * Runs {@code smss_clear_app_imports()} with the project asset paths in the + * project's shared python process and in the calling user's python process, + * wherever each one is running, so the next import of a project module reads + * the current .py files. In a cluster the reload is published to the other + * nodes, which clear the imports in the project python process they have + * running. The user must be able to edit the project. + */ +public class ReloadProjectReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(ReloadProjectReactor.class); + + private static final String MODULES_KEY = "modules"; + + public ReloadProjectReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = resolveContextEngineId(this.keyValue.get(this.keysToGet[0])); + + User user = this.insight.getUser(); + if (!SecurityProjectUtils.userCanEditProject(user, projectId)) { + throw new IllegalArgumentException( + "Project does not exist or user does not have access to edit the project"); + } + + NounMetadata noun = new NounMetadata(true, PixelDataType.BOOLEAN); + if (!PyUtils.pyEnabled()) { + noun.addAdditionalReturn( + NounMetadata.getWarningNounMessage("Python is not enabled, there are no python imports to reload")); + return noun; + } + + // each process is cleared on its own, so a failure in one still clears the + // other and tells the rest of the cluster + boolean failed = false; + Object projectCleared = null; + try { + // the project's shared python process on this node + projectCleared = ProjectSyncUtility.clearLoadedProjectPythonImports(projectId); + } catch (Exception e) { + classLogger.error("Failed to clear the python imports of project {} in its python process", + Utility.cleanLogString(projectId), e); + failed = true; + noun.addAdditionalReturn(NounMetadata.getWarningNounMessage( + "Could not reload the python files in the project python process: " + e.getMessage())); + } + Object userCleared = null; + try { + // the user's own python process, where the app python runs when the project + // does not use a dedicated process + userCleared = clearUserPythonImports(user, projectId); + } catch (Exception e) { + classLogger.error("Failed to clear the python imports of project {} in the python process of the user", + Utility.cleanLogString(projectId), e); + failed = true; + noun.addAdditionalReturn(NounMetadata.getWarningNounMessage( + "Could not reload the python files in your python process: " + e.getMessage())); + } + + try { + ClusterUtil.publishProjectReload(projectId); + } catch (SemossPixelException e) { + noun.addAdditionalReturn(NounMetadata.getWarningNounMessage(e.getMessage())); + } + + if (projectCleared != null || userCleared != null) { + int moduleCount = countClearedModules(projectCleared) + countClearedModules(userCleared); + noun.addAdditionalReturn(NounMetadata.getSuccessNounMessage("Reloaded the python files. Cleared " + + moduleCount + " cached " + (moduleCount == 1 ? "module" : "modules") + ".")); + } else if (!failed) { + noun.addAdditionalReturn(NounMetadata.getSuccessNounMessage( + "No python process has these python files loaded on this server. The current files load on their next run.")); + } + return noun; + } + + /** + * Clears the project modules cached in the calling user's python process, + * passing the project asset paths explicitly so the modules imported from the + * project are the ones cleared. A process that is not running is not started. + * + * @param user the calling user + * @param projectId the project whose modules to clear + * @return the result of {@code smss_clear_app_imports()}, or null when the user + * has no python process running + */ + private Object clearUserPythonImports(User user, String projectId) { + if (user == null) { + return null; + } + SocketClient userClient = user.getPythonSocketClient(false); + if (userClient == null || !userClient.isConnected()) { + return null; + } + String projectName = SecurityProjectUtils.getProjectAliasForId(projectId); + String assetsDir = EngineUtility + .getSpecificEngineAssetsFolder(IEngine.CATALOG_TYPE.PROJECT, projectId, projectName).replace('\\', '/'); + Object cleared = this.insight.getPyTranslator().runScriptWithExplicitAssetPaths(this.insight, + PyUtils.CLEAR_APP_IMPORTS_SCRIPT, assetsDir, new String[] { assetsDir + "/py" }); + classLogger.info("Cleared python imports for project {} in the python process of the user: {}", + Utility.cleanLogString(projectId), cleared); + return cleared; + } + + /** + * Counts the module keys a {@code smss_clear_app_imports()} result lists as + * removed from {@code sys.modules}. + * + * @param cleared the result of {@code smss_clear_app_imports()}, may be null + * @return the number of modules cleared, or 0 when the result has none + */ + private static int countClearedModules(Object cleared) { + if (cleared instanceof Map) { + Object modules = ((Map) cleared).get(MODULES_KEY); + if (modules instanceof Collection) { + return ((Collection) modules).size(); + } + } + return 0; + } + + @Override + public String getReactorDescription() { + return """ + Reload the python files of a project without restarting its python process. Clears the \ + project modules cached in the project python process and in the python process of the user, \ + so the next import reads the current .py files. In a cluster every node also clears the \ + imports in the project python process it has running. The user must be able to edit the project."""; + } + + @Override + protected String getDescriptionForKey(String key) { + if (key.equals(ReactorKeysEnum.PROJECT.getKey())) { + return "The id of the project to reload. Defaults to the project of the current app context"; + } + return super.getDescriptionForKey(key); + } + +} diff --git a/src/prerna/reactor/utils/CloseEngineReactor.java b/src/prerna/reactor/utils/CloseEngineReactor.java index 4bdcafec624..6d08258f2f5 100644 --- a/src/prerna/reactor/utils/CloseEngineReactor.java +++ b/src/prerna/reactor/utils/CloseEngineReactor.java @@ -27,7 +27,6 @@ *******************************************************************************/ package prerna.reactor.utils; -import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -37,18 +36,24 @@ import prerna.auth.User; import prerna.auth.utils.SecurityAdminUtils; import prerna.auth.utils.SecurityEngineUtils; -import prerna.auth.utils.SecurityQueryUtils; -import prerna.engine.api.IEngine; +import prerna.cluster.util.ClusterUtil; import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.GenRowStruct; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.execptions.SemossPixelException; import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.util.Constants; -import prerna.util.UploadUtilities; +import prerna.util.EngineSyncUtility; import prerna.util.Utility; +/** + * Closes engines so the next use opens them again from their smss file and + * files. This is how an engine is restarted: closing shuts down the python + * process the engine owns, so the reopened engine picks up changed files. In a + * cluster the close is published to the other nodes, which close the engine + * where they have it open. The user must own every engine, or be an admin. + */ public class CloseEngineReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(CloseEngineReactor.class); @@ -60,42 +65,39 @@ public CloseEngineReactor() { @Override public NounMetadata execute() { - List engineIds = getEngineIds(); - User user = this.insight.getUser(); boolean isAdmin = SecurityAdminUtils.userIsAdmin(user); - if (!isAdmin) { - for (String engineId : engineIds) { - // we may have the alias - engineId = SecurityQueryUtils.testUserEngineIdForAlias(this.insight.getUser(), engineId); - boolean isOwner = SecurityEngineUtils.userIsOwner(user, engineId); - if (!isOwner) { - throw new IllegalArgumentException("Engine " + engineId - + " does not exist or user does not have permissions to delete the engine. User must be the owner to perform this function."); - } + + // check every engine before closing any + List engineIds = getEngineIds(); + for (String engineId : engineIds) { + if (!isAdmin && !SecurityEngineUtils.userIsOwner(user, engineId)) { + throw new IllegalArgumentException("Engine " + engineId + + " does not exist or user does not have permissions to close the engine. User must be the owner to perform this function."); } } - // once all are good, we can close + NounMetadata noun = new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.ENGINE_INFO); for (String engineId : engineIds) { - classLogger.info("Attempting to close engine: " + engineId); - // we may have the alias - engineId = SecurityQueryUtils.testUserEngineIdForAlias(this.insight.getUser(), engineId); - IEngine engine = Utility.getEngine(engineId); + classLogger.info("Closing engine {} for a restart", Utility.cleanLogString(engineId)); + boolean closed = EngineSyncUtility.closeLoadedEngine(engineId); try { - classLogger.info("Shutting down engine: " + engineId); - engine.close(); - UploadUtilities.removeEngineExcludingSMSSFromDIHelper(engineId); - classLogger.info("Shut down engine: " + engineId); - } catch (IOException e) { - classLogger.error(Constants.STACKTRACE, e); + ClusterUtil.publishEngineClose(engineId); + } catch (SemossPixelException e) { + // keep closing the remaining engines, and tell the user which nodes + // still have the old instance open + noun.addAdditionalReturn(NounMetadata.getWarningNounMessage(e.getMessage())); } + String message = closed ? "Closed engine " + engineId + ". It restarts on its next use." + : "Engine " + engineId + " was not running. It starts on its next use."; + noun.addAdditionalReturn(NounMetadata.getSuccessNounMessage(message)); } - return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.ENGINE_INFO); + return noun; } /** - * Get inputs + * Reads the engine ids to close, from the engine key when it is set and + * otherwise from every input in the current row. * * @return list of engines to close */ @@ -119,4 +121,21 @@ public List getEngineIds() { } return engineIds; } + + @Override + public String getReactorDescription() { + return """ + Close engines so their next use opens them again from their smss file and files. \ + Use this to restart an engine, for example to pick up changed python files: closing \ + shuts down the python process the engine owns. In a cluster every node closes the engine. \ + The user must own the engines."""; + } + + @Override + protected String getDescriptionForKey(String key) { + if (key.equals(ReactorKeysEnum.ENGINE.getKey())) { + return "The ids of the engines to close"; + } + return super.getDescriptionForKey(key); + } } diff --git a/src/prerna/reactor/venv/upload/CreateVenvEngineReactor.java b/src/prerna/reactor/venv/upload/CreateVenvEngineReactor.java index 201af002bfe..77215666d47 100644 --- a/src/prerna/reactor/venv/upload/CreateVenvEngineReactor.java +++ b/src/prerna/reactor/venv/upload/CreateVenvEngineReactor.java @@ -40,6 +40,7 @@ import prerna.auth.AuthProvider; import prerna.auth.User; import prerna.auth.utils.AbstractSecurityUtils; +import prerna.auth.utils.SecurityAdminUtils; import prerna.auth.utils.SecurityEngineUtils; import prerna.auth.utils.SecurityQueryUtils; import prerna.cluster.util.ClusterUtil; @@ -93,6 +94,10 @@ public NounMetadata execute() { throwUserNotPublisherError(); } + if (AbstractSecurityUtils.adminOnlyVenvAdd() && !SecurityAdminUtils.userIsAdmin(user)) { + throwFunctionalityOnlyExposedForAdminsError(); + } + organizeKeys(); String venvName = getVenvName(); @@ -107,6 +112,16 @@ public NounMetadata execute() { Map venvDetails = getVenvDetails(); boolean global = Boolean.parseBoolean(this.keyValue.get(ReactorKeysEnum.GLOBAL.getKey()) + ""); + NounMetadata warning = null; + if (global) { + if (AbstractSecurityUtils.adminOnlyEngineSetPublic(IEngine.CATALOG_TYPE.VENV) + && !SecurityAdminUtils.userIsAdmin(user)) { + warning = NounMetadata.getWarningNounMessage( + "Public access can only be enabled by administrators. This item will be created as private."); + global = false; + } + } + String venvTypeStr = (String) venvDetails.get(IVenvEngine.VENV_TYPE); if (venvTypeStr == null || (venvTypeStr = venvTypeStr.trim()).isEmpty()) { throw new IllegalArgumentException("Must define the virtual environment type"); @@ -175,7 +190,12 @@ public NounMetadata execute() { } Map retMap = UploadUtilities.getEngineReturnData(this.insight.getUser(), venvId); - return new NounMetadata(retMap, PixelDataType.UPLOAD_RETURN_MAP, PixelOperationType.MARKET_PLACE_ADDITION); + NounMetadata retNoun = new NounMetadata(retMap, PixelDataType.UPLOAD_RETURN_MAP, + PixelOperationType.MARKET_PLACE_ADDITION); + if (warning != null) { + retNoun.addAdditionalReturn(warning); + } + return retNoun; } /** diff --git a/src/prerna/usertracking/UserAuditTrailUtils.java b/src/prerna/usertracking/UserAuditTrailUtils.java index 8b6e9e11ce6..fd64ac366ae 100644 --- a/src/prerna/usertracking/UserAuditTrailUtils.java +++ b/src/prerna/usertracking/UserAuditTrailUtils.java @@ -155,6 +155,7 @@ private record AuditedAction(AuditEventType eventType, AuditTargetType targetTyp Map.entry("UploadProjectApp", new AuditedAction(AuditEventType.PROJECT_UPLOAD, AuditTargetType.PROJECT)), Map.entry("DeleteProject", new AuditedAction(AuditEventType.PROJECT_DELETE, AuditTargetType.PROJECT)), Map.entry("UploadEngine", new AuditedAction(AuditEventType.ENGINE_CREATE, AuditTargetType.ENGINE)), + Map.entry("CloneEngine", new AuditedAction(AuditEventType.ENGINE_CREATE, AuditTargetType.ENGINE)), Map.entry("CreateModelEngine", new AuditedAction(AuditEventType.MODEL_CREATE, AuditTargetType.MODEL)), Map.entry("CreateVectorDatabaseEngine", new AuditedAction(AuditEventType.VECTOR_CREATE, AuditTargetType.VECTOR)), diff --git a/src/prerna/util/Constants.java b/src/prerna/util/Constants.java index 1faba0f61da..d5316c8784d 100644 --- a/src/prerna/util/Constants.java +++ b/src/prerna/util/Constants.java @@ -643,6 +643,12 @@ public class Constants { public static final String ADMIN_ONLY_GUARDRAIL_SET_PUBLIC = "ADMIN_ONLY_GUARDRAIL_SET_PUBLIC"; public static final String ADMIN_ONLY_GUARDRAIL_SET_DISCOVERABLE = "ADMIN_ONLY_GUARDRAIL_SET_DISCOVERABLE"; + public static final String ADMIN_ONLY_VENV_ADD = "ADMIN_ONLY_VENV_ADD"; + public static final String ADMIN_ONLY_VENV_DELETE = "ADMIN_ONLY_VENV_DELETE"; + public static final String ADMIN_ONLY_VENV_ADD_ACCESS = "ADMIN_ONLY_VENV_ADD_ACCESS"; + public static final String ADMIN_ONLY_VENV_SET_PUBLIC = "ADMIN_ONLY_VENV_SET_PUBLIC"; + public static final String ADMIN_ONLY_VENV_SET_DISCOVERABLE = "ADMIN_ONLY_VENV_SET_DISCOVERABLE"; + public static final String ADMIN_ONLY_INSIGHT_SET_PUBLIC = "ADMIN_ONLY_INSIGHT_SET_PUBLIC"; public static final String ADMIN_ONLY_INSIGHT_ADD_ACCESS = "ADMIN_ONLY_INSIGHT_ADD_ACCESS"; public static final String ADMIN_ONLY_INSIGHT_SHARE = "ADMIN_ONLY_INSIGHT_SHARE"; diff --git a/src/prerna/util/EngineCloneUtility.java b/src/prerna/util/EngineCloneUtility.java new file mode 100644 index 00000000000..2f9623fcf73 --- /dev/null +++ b/src/prerna/util/EngineCloneUtility.java @@ -0,0 +1,626 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.util; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.file.PathUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import jakarta.ws.rs.core.Response; +import prerna.auth.AuthProvider; +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityModelMetadataUtils; +import prerna.cluster.util.ClusterUtil; +import prerna.engine.api.IEngine; +import prerna.engine.impl.SmssUtilities; +import prerna.io.connector.couch.CouchException; +import prerna.io.connector.couch.CouchUtil; +import prerna.io.connector.secrets.ISecrets; +import prerna.io.connector.secrets.SecretsFactory; +import prerna.masterdatabase.DeleteFromMasterDB; +import prerna.sablecc2.om.execptions.SemossPixelException; + +/** + * Clones an engine of any type into a new private engine owned by the user who + * clones it. The clone gets a new id and name, and a copy of everything the + * source keeps on this server: + *

+ * Connection settings are copied as they are, so a clone of an engine backed by + * an external system (a remote database, a pgvector table, a storage bucket, a + * model endpoint) reads and writes the same external data as the source. + * Deleting an engine only removes its local files, so deleting the clone leaves + * that external data in place. + */ +public final class EngineCloneUtility { + + private static final Logger classLogger = LogManager.getLogger(EngineCloneUtility.class); + + /** The file at the root of a python virtual environment. */ + private static final String VENV_CONFIG_FILE = "pyvenv.cfg"; + /** + * The folders of a virtual environment whose scripts hold its absolute path. + */ + private static final List VENV_SCRIPT_FOLDERS = List.of("bin", "Scripts"); + /** How deep under the engine folder to look for a virtual environment. */ + private static final int VENV_SEARCH_DEPTH = 4; + + private EngineCloneUtility() { + + } + + /** + * Clones an engine into a new private engine named {@code cloneName} and owned + * by {@code user}, then pushes the clone to cloud storage in a cluster. An + * engine that holds file locks, such as an H2 database, is closed while its + * files are copied so the copy is consistent, and opened again afterwards. + * Callers check that the user may clone the source; this method does no + * permission checks. On failure everything written for the clone is removed. + * + * @param user the user who owns the clone + * @param sourceEngineId the engine to clone + * @param cloneName the name of the clone + * @param logger logger for the progress messages shown to the user + * @return the id of the clone + */ + public static String cloneEngine(User user, String sourceEngineId, String cloneName, Logger logger) { + IEngine source = Utility.getEngine(sourceEngineId); + if (source == null) { + throw new IllegalArgumentException("Engine " + sourceEngineId + " does not exist or could not be loaded"); + } + String sourceSmssPath = source.getSmssFilePath(); + if (sourceSmssPath == null || !new File(sourceSmssPath).isFile()) { + throw new IllegalArgumentException("Engine " + sourceEngineId + " has no smss file to clone"); + } + + IEngine.CATALOG_TYPE engineType = source.getCatalogType(); + String sourceName = source.getEngineName(); + String cloneId = UUID.randomUUID().toString(); + + File sourceFolder = new File(EngineUtility.getSpecificEngineBaseFolder(engineType, sourceEngineId, sourceName)); + File cloneFolder = new File(EngineUtility.getSpecificEngineBaseFolder(engineType, cloneId, cloneName)); + File cloneSmss = new File(Utility.normalizePath(EngineUtility.getLocalEngineBaseDirectory(engineType) + "/" + + SmssUtilities.getUniqueName(cloneName, cloneId) + Constants.SEMOSS_EXTENSION)); + try { + UploadUtilities.validateEngine(engineType, user, cloneName, cloneId); + } catch (IOException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + + IdentityRewrite rewrite = new IdentityRewrite(sourceEngineId, sourceName, cloneId, cloneName); + ISecrets secretStore = SecretsFactory.getSecretConnector(); + File cloneImage = null; + boolean couchImageCopied = false; + boolean pushStarted = false; + boolean engineAddedToDIHelper = false; + boolean success = false; + try { + // list the clone before its smss exists so the smss watcher skips it + addToDIHelperEngines(cloneId); + engineAddedToDIHelper = true; + + logger.info("Copying the engine files..."); + copyEngineFiles(source, sourceFolder, new File(sourceSmssPath), cloneFolder, cloneSmss); + logger.info("Done copying the engine files"); + + logger.info("Setting the id and name of the clone..."); + rewriteSmss(cloneSmss, rewrite); + if (secretStore != null) { + copySecrets(secretStore, engineType, rewrite); + } + rewriteVirtualEnvPaths(cloneFolder, sourceFolder, rewrite); + logger.info("Done setting the id and name of the clone"); + + logger.info("Registering the clone..."); + DIHelper.getInstance().setEngineProperty(cloneId + "_" + Constants.STORE, cloneSmss.getAbsolutePath()); + if (IEngine.CATALOG_TYPE.DATABASE == engineType) { + Utility.synchronizeEngineMetadata(cloneId); + } + SecurityEngineUtils.addEngine(cloneId, false, user); + copyMetadata(engineType, sourceEngineId, cloneId); + for (AuthProvider login : user.getLogins()) { + SecurityEngineUtils.addEngineOwner(cloneId, user.getAccessToken(login).getId()); + } + // the image is not needed for the clone to work, so a failed copy leaves the + // clone with the stock image + try { + if (CouchUtil.COUCH_ENABLED) { + couchImageCopied = copyCouchImage(engineType, sourceEngineId, cloneId); + } else if (ClusterUtil.IS_CLUSTER) { + cloneImage = copyClusterImage(engineType, sourceEngineId, cloneId); + } + // otherwise the image sits in the engine folder and was copied with it + } catch (Exception e) { + classLogger.warn("Failed to copy the image of engine {} to the clone {}", + Utility.cleanLogString(sourceEngineId), cloneId, e); + } + logger.info("Done registering the clone"); + + pushStarted = true; + try { + ClusterUtil.pushEngine(cloneId); + } catch (SemossPixelException e) { + if (!e.isContinueThreadOfExecution()) { + throw e; + } + // the clone is in cloud storage and only the notice to the other nodes + // failed, which they do not need for an engine none of them has open + classLogger.warn("Pushed the clone {} but could not notify the other nodes", cloneId, e); + } + success = true; + } catch (SemossPixelException e) { + throw e; + } catch (Exception e) { + classLogger.error("Failed to clone engine {} into {}", Utility.cleanLogString(sourceEngineId), cloneId, e); + throw new SemossPixelException("Failed to clone the engine: " + e.getMessage(), false); + } finally { + if (!success) { + cleanUpFailedClone(engineType, rewrite, secretStore, engineAddedToDIHelper, pushStarted, cloneFolder, + cloneSmss, cloneImage, couchImageCopied); + } + } + return cloneId; + } + + /** + * Copies the engine folder and smss file of the source. A source that holds + * file locks is closed under its engine lock for the copy and opened again + * after, the same way an export does; when it cannot be closed, nothing is + * copied. Symbolic links are copied as links, so a virtual environment keeps + * pointing at its base python. + * + * @param source the engine to copy + * @param sourceFolder the engine folder of the source + * @param sourceSmss the smss file of the source + * @param cloneFolder the engine folder to create for the clone + * @param cloneSmss the smss file to create for the clone + * @throws IOException if a file cannot be copied + */ + private static void copyEngineFiles(IEngine source, File sourceFolder, File sourceSmss, File cloneFolder, + File cloneSmss) throws IOException { + String sourceEngineId = source.getEngineId(); + ReentrantLock lock = null; + if (source.holdsFileLocks()) { + lock = EngineSyncUtility.getEngineLock(sourceEngineId); + lock.lock(); + } + boolean removed = false; + try { + if (lock != null) { + // close the instance open now, which may not be the one read before the lock + Object open = DIHelper.getInstance().getEngineProperty(sourceEngineId); + if (open instanceof IEngine openEngine) { + classLogger.info("Closing engine {} to copy its files", Utility.cleanLogString(sourceEngineId)); + DIHelper.getInstance().removeEngineProperty(sourceEngineId); + removed = true; + // a copy of files the engine still holds would not be consistent + openEngine.close(); + } + } + if (sourceFolder.exists()) { + PathUtils.copyDirectory(sourceFolder.toPath(), cloneFolder.toPath(), LinkOption.NOFOLLOW_LINKS, + StandardCopyOption.COPY_ATTRIBUTES); + } else { + cloneFolder.mkdirs(); + } + Files.copy(sourceSmss.toPath(), cloneSmss.toPath(), StandardCopyOption.COPY_ATTRIBUTES); + } finally { + try { + if (removed) { + classLogger.info("Opening engine {} again after copying its files", + Utility.cleanLogString(sourceEngineId)); + Utility.getEngine(sourceEngineId); + } + } catch (Exception e) { + // the clone does not depend on the source being open again + classLogger.error("Failed to open engine {} again after copying its files", + Utility.cleanLogString(sourceEngineId), e); + } finally { + if (lock != null) { + lock.unlock(); + } + } + } + } + + /** + * Points the copied smss file at the clone: every value that holds the source + * id or folder name gets the clone's, and the engine id, alias and display name + * keys get the clone's id and name. Other keys keep their values; NAME, for + * one, can hold a provider's model id rather than the engine's name. + * + * @param cloneSmss the copied smss file + * @param rewrite the source and clone identities + * @throws IOException if the file cannot be read or written + */ + private static void rewriteSmss(File cloneSmss, IdentityRewrite rewrite) throws IOException { + Path smssPath = cloneSmss.toPath(); + String content = Files.readString(smssPath, StandardCharsets.UTF_8); + Files.writeString(smssPath, rewrite.apply(content), StandardCharsets.UTF_8); + + Map identity = new LinkedHashMap<>(); + identity.put(Constants.ENGINE, rewrite.cloneId); + identity.put(Constants.ENGINE_ALIAS, rewrite.cloneName); + identity.put(Constants.ENGINE_DISPLAY_NAME, rewrite.cloneName); + Utility.changePropertiesFileValue(cloneSmss.getAbsolutePath(), identity, false); + } + + /** + * Copies the secrets the secret store holds for the source to the clone, with + * the same id and name changes as the smss file. The engine merges these over + * its smss values when it opens, so the identity keys are set as well. + * + * @param secretStore the configured secret store + * @param engineType the type of the source and clone + * @param rewrite the source and clone identities + */ + private static void copySecrets(ISecrets secretStore, IEngine.CATALOG_TYPE engineType, IdentityRewrite rewrite) { + Map sourceSecrets = secretStore.getEngineSecrets(engineType, rewrite.sourceId, + rewrite.sourceName); + if (sourceSecrets == null || sourceSecrets.isEmpty()) { + return; + } + Map cloneSecrets = new HashMap<>(); + for (Map.Entry entry : sourceSecrets.entrySet()) { + Object value = entry.getValue(); + cloneSecrets.put(entry.getKey(), value instanceof String ? rewrite.apply((String) value) : value); + } + cloneSecrets.computeIfPresent(Constants.ENGINE, (k, v) -> rewrite.cloneId); + cloneSecrets.computeIfPresent(Constants.ENGINE_ALIAS, (k, v) -> rewrite.cloneName); + cloneSecrets.computeIfPresent(Constants.ENGINE_DISPLAY_NAME, (k, v) -> rewrite.cloneName); + if (!secretStore.writeEngineSecrets(engineType, rewrite.cloneId, rewrite.cloneName, cloneSecrets)) { + throw new IllegalStateException("Failed to write the secrets of the clone to the secret store"); + } + } + + /** + * Points every python virtual environment inside the copied folder at its new + * location. A virtual environment writes its absolute path into + * {@code pyvenv.cfg} and into the scripts under {@code bin} or {@code Scripts} + * (activate scripts and the shebang line of pip and every console script), so + * without this the clone's pip would install into the source environment. + * + * @param cloneFolder the copied engine folder + * @param sourceFolder the engine folder of the source + * @param rewrite the source and clone identities + * @throws IOException if the folder cannot be searched or a file rewritten + */ + private static void rewriteVirtualEnvPaths(File cloneFolder, File sourceFolder, IdentityRewrite rewrite) + throws IOException { + if (!cloneFolder.isDirectory()) { + return; + } + List venvConfigs; + try (Stream files = Files.find(cloneFolder.toPath(), VENV_SEARCH_DEPTH, + (path, attrs) -> attrs.isRegularFile() && VENV_CONFIG_FILE.equals(path.getFileName().toString()))) { + venvConfigs = files.collect(Collectors.toList()); + } + if (venvConfigs.isEmpty()) { + return; + } + + // the scripts hold the folder in the form the OS wrote it + Map folderPaths = new LinkedHashMap<>(); + String sourcePath = sourceFolder.getAbsolutePath(); + String clonePath = cloneFolder.getAbsolutePath(); + folderPaths.put(sourcePath, clonePath); + folderPaths.put(sourcePath.replace('\\', '/'), clonePath.replace('\\', '/')); + + for (Path venvConfig : venvConfigs) { + rewriteTextFile(venvConfig, folderPaths, rewrite); + Path venvRoot = venvConfig.getParent(); + for (String scriptFolder : VENV_SCRIPT_FOLDERS) { + Path scripts = venvRoot.resolve(scriptFolder); + if (!Files.isDirectory(scripts, LinkOption.NOFOLLOW_LINKS)) { + continue; + } + try (Stream scriptFiles = Files.list(scripts)) { + for (Path script : (Iterable) scriptFiles::iterator) { + if (Files.isRegularFile(script, LinkOption.NOFOLLOW_LINKS)) { + rewriteTextFile(script, folderPaths, rewrite); + } + } + } + } + classLogger.info("Pointed the virtual environment at {}", venvRoot); + } + } + + /** + * Replaces the source folder path with the clone folder path in a text file, + * then the source id and folder name. Binary files, meaning files with a zero + * byte or that are not valid UTF-8, and files without the source path are left + * untouched. + * + * @param file the file to rewrite + * @param folderPaths source folder path to clone folder path, in each form + * @param rewrite the source and clone identities + * @throws IOException if the file cannot be read or written + */ + private static void rewriteTextFile(Path file, Map folderPaths, IdentityRewrite rewrite) + throws IOException { + byte[] bytes = Files.readAllBytes(file); + for (byte b : bytes) { + if (b == 0) { + return; + } + } + String content; + try { + content = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + return; + } + String updated = content; + for (Map.Entry folderPath : folderPaths.entrySet()) { + updated = updated.replace(folderPath.getKey(), folderPath.getValue()); + } + updated = rewrite.apply(updated); + if (!updated.equals(content)) { + Files.writeString(file, updated, StandardCharsets.UTF_8); + } + } + + /** + * Copies the engine metadata of the source to the clone, and for a model its + * model metadata. + * + * @param engineType the type of the source and clone + * @param sourceEngineId the engine cloned + * @param cloneId the clone + */ + private static void copyMetadata(IEngine.CATALOG_TYPE engineType, String sourceEngineId, String cloneId) { + Map engineMetadata = SecurityEngineUtils.getAggregateEngineMetadata(sourceEngineId, null, + false); + if (engineMetadata != null && !engineMetadata.isEmpty()) { + SecurityEngineUtils.updateEngineMetadata(cloneId, engineMetadata); + } + if (IEngine.CATALOG_TYPE.MODEL == engineType) { + Map modelMetadata = SecurityModelMetadataUtils.getModelMetadata(sourceEngineId); + if (modelMetadata != null && !modelMetadata.isEmpty()) { + SecurityModelMetadataUtils.restoreModelMetadata(cloneId, modelMetadata); + } + } + } + + /** + * Copies the image uploaded for the source in CouchDB to the clone. A source + * without an image document shows the stock image and has nothing to copy. + * + * @param engineType the type of the source and clone + * @param sourceEngineId the engine cloned + * @param cloneId the clone + * @return true if an image was copied + * @throws CouchException if the image cannot be read or written + * @throws IOException if the image cannot be staged in a temporary file + */ + private static boolean copyCouchImage(IEngine.CATALOG_TYPE engineType, String sourceEngineId, String cloneId) + throws CouchException, IOException { + String partition = EngineUtility.getCouchSelector(engineType); + String sourceSelector = "{\"selector\": {\"" + partition + "\":{\"$eq\":\"" + sourceEngineId + "\"}}}"; + if (!CouchUtil.documentExists(partition, sourceSelector)) { + return false; + } + Response download = CouchUtil.download(partition, Map.of(partition, sourceEngineId)); + if (!(download.getEntity() instanceof byte[])) { + return false; + } + Path stagedImage = Files.createTempFile(cloneId, ".image"); + try { + Files.write(stagedImage, (byte[]) download.getEntity()); + CouchUtil.upload(partition, Map.of(partition, cloneId), stagedImage.toFile()); + } finally { + Files.deleteIfExists(stagedImage); + } + return true; + } + + /** + * Copies the image uploaded for the source in the cluster image folder to the + * clone and pushes it to cloud storage. A source showing the stock image has + * nothing to copy. + * + * @param engineType the type of the source and clone + * @param sourceEngineId the engine cloned + * @param cloneId the clone + * @return the image written for the clone, or null when the source has none + * @throws Exception if the image cannot be read, copied or pushed + */ + private static File copyClusterImage(IEngine.CATALOG_TYPE engineType, String sourceEngineId, String cloneId) + throws Exception { + File sourceImage = ClusterUtil.getEngineAndProjectImage(sourceEngineId, engineType); + if (sourceImage == null || !sourceImage.isFile() || !sourceImage.getName().startsWith(sourceEngineId)) { + return null; + } + String cloneImageName = cloneId + sourceImage.getName().substring(sourceEngineId.length()); + File cloneImage = new File(sourceImage.getParentFile(), cloneImageName); + Files.copy(sourceImage.toPath(), cloneImage.toPath(), StandardCopyOption.REPLACE_EXISTING); + ClusterUtil.pushEngineAndProjectImage(engineType, cloneImageName); + return cloneImage; + } + + /** + * Adds an engine id to the list of engines in DIHelper. + * + * @param engineId the engine to add + */ + private static void addToDIHelperEngines(String engineId) { + String engines = DIHelper.getInstance().getEngineProperty(Constants.ENGINES) + ""; + if (!(engines.startsWith(engineId) || engines.contains(";" + engineId + ";") + || engines.endsWith(";" + engineId))) { + DIHelper.getInstance().setEngineProperty(Constants.ENGINES, engines + ";" + engineId); + } + } + + /** + * Removes everything a failed clone wrote: the open clone, its cloud copy, its + * files, image, DIHelper entries, local master and security rows, and secrets. + * Each step runs even when an earlier one fails. + * + * @param engineType the type of the clone + * @param rewrite the source and clone identities + * @param secretStore the configured secret store, or null + * @param engineAddedToDIHelper whether the clone was added to DIHelper + * @param pushStarted whether the clone was being pushed to cloud + * storage + * @param cloneFolder the engine folder of the clone + * @param cloneSmss the smss file of the clone + * @param cloneImage the cluster image of the clone, or null + * @param couchImageCopied whether an image was written to CouchDB + */ + private static void cleanUpFailedClone(IEngine.CATALOG_TYPE engineType, IdentityRewrite rewrite, + ISecrets secretStore, boolean engineAddedToDIHelper, boolean pushStarted, File cloneFolder, File cloneSmss, + File cloneImage, boolean couchImageCopied) { + String cloneId = rewrite.cloneId; + // the cloud push opens the clone, which holds its files open until closed + if (DIHelper.getInstance().getEngineProperty(cloneId) instanceof IEngine openClone) { + try { + openClone.close(); + } catch (Exception e) { + classLogger.error("Failed to close the failed clone {}", cloneId, e); + } + } + if (pushStarted && ClusterUtil.IS_CLUSTER) { + try { + ClusterUtil.deleteEngine(cloneId, engineType); + } catch (Exception e) { + classLogger.error("Failed to delete the cloud copy of the failed clone {}", cloneId, e); + } + } + if (cloneImage != null) { + try { + ClusterUtil.deleteEngineAndProjectImage(engineType, cloneImage.getName()); + } catch (Exception e) { + classLogger.error("Failed to delete the cloud image of the failed clone {}", cloneId, e); + } + } + if (couchImageCopied) { + try { + String partition = EngineUtility.getCouchSelector(engineType); + CouchUtil.delete(partition, Map.of(partition, cloneId)); + } catch (Exception e) { + classLogger.error("Failed to delete the CouchDB image of the failed clone {}", cloneId, e); + } + } + for (File f : new File[] { cloneFolder, cloneSmss, cloneImage }) { + if (f != null && f.exists()) { + try { + FileUtils.forceDelete(f); + } catch (IOException e) { + classLogger.error("Failed to delete {} while cleaning up the failed clone {}", f.getAbsolutePath(), + cloneId, e); + } + } + } + if (engineAddedToDIHelper) { + UploadUtilities.removeEngineFromDIHelper(cloneId); + } + if (IEngine.CATALOG_TYPE.DATABASE == engineType) { + try { + new DeleteFromMasterDB().deleteEngineRDBMS(cloneId); + } catch (Exception e) { + classLogger.error("Failed to remove the failed clone {} from the local master", cloneId, e); + } + } + try { + SecurityEngineUtils.deleteEngine(cloneId); + } catch (Exception e) { + classLogger.error("Failed to remove the failed clone {} from the security database", cloneId, e); + } + if (secretStore != null) { + try { + secretStore.deleteEngineSecrets(engineType, cloneId, rewrite.cloneName); + } catch (Exception e) { + classLogger.error("Failed to delete the secrets of the failed clone {}", cloneId, e); + } + } + } + + /** + * The source and clone identities, and the text replacement that points a value + * at the clone. The folder name ({@code name__id}) is replaced first, then any + * remaining source id. The source name on its own is kept, since files inside + * the folder keep the names they were copied with (the OWL file of a database, + * for example). + */ + private static final class IdentityRewrite { + + private final String sourceId; + private final String sourceName; + private final String cloneId; + private final String cloneName; + private final String sourceUniqueName; + private final String cloneUniqueName; + + private IdentityRewrite(String sourceId, String sourceName, String cloneId, String cloneName) { + this.sourceId = sourceId; + this.sourceName = sourceName; + this.cloneId = cloneId; + this.cloneName = cloneName; + this.sourceUniqueName = SmssUtilities.getUniqueName(sourceName, sourceId); + this.cloneUniqueName = SmssUtilities.getUniqueName(cloneName, cloneId); + } + + /** + * Replaces the source folder name and id in a value with the clone's. + * + * @param value the text to rewrite + * @return the text pointing at the clone + */ + private String apply(String value) { + return value.replace(this.sourceUniqueName, this.cloneUniqueName).replace(this.sourceId, this.cloneId); + } + } +} diff --git a/src/prerna/util/EngineSyncUtility.java b/src/prerna/util/EngineSyncUtility.java index 8458f7f062e..692f993b3ee 100644 --- a/src/prerna/util/EngineSyncUtility.java +++ b/src/prerna/util/EngineSyncUtility.java @@ -27,17 +27,24 @@ *******************************************************************************/ package prerna.util; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.auth.User; import prerna.auth.utils.SecurityAdminUtils; +import prerna.engine.api.IEngine; public class EngineSyncUtility { + private static final Logger classLogger = LogManager.getLogger(EngineSyncUtility.class); + private static ConcurrentMap engineLocks = new ConcurrentHashMap<>(); // store some database metadata as well @@ -51,86 +58,143 @@ public static ReentrantLock getEngineLock(String engineId) { engineLocks.putIfAbsent(engineId, new ReentrantLock()); return engineLocks.get(engineId); } - + public static ConcurrentMap getAllLocks(User user) { - if(!SecurityAdminUtils.userIsAdmin(user)) { + if (!SecurityAdminUtils.userIsAdmin(user)) { throw new IllegalArgumentException("User must be an admin to perform this method"); } return engineLocks; } - + + /** + * Checks if an engine is open on this node, meaning an instance of it is held + * in DIHelper. An engine that is only cataloged here, with its smss known but + * never opened, is not open. + * + * @param engineId the engine to check + * @return true if the engine is open on this node + */ + public static boolean isEngineOpen(String engineId) { + return DIHelper.getInstance().getEngineProperty(engineId) instanceof IEngine; + } + + /** + * Closes the engine instance opened on this node and removes it from DIHelper, + * along with its cached metadata, so the next {@link Utility#getEngine(String)} + * opens it again from its smss file and files. Closing shuts down the python + * process an engine owns. The close runs under the engine lock that loading + * uses, so it cannot interleave with a concurrent open. An engine that is not + * open on this node is left alone. + * + * @param engineId the engine to close + * @return true if the engine was open on this node and is now closed + */ + public static boolean closeLoadedEngine(String engineId) { + ReentrantLock lock = getEngineLock(engineId); + lock.lock(); + try { + Object loaded = DIHelper.getInstance().getEngineProperty(engineId); + if (!(loaded instanceof IEngine openEngine)) { + classLogger.info("Engine {} is not open on this node, nothing to close", + Utility.cleanLogString(engineId)); + return false; + } + classLogger.info("Closing engine {}", Utility.cleanLogString(engineId)); + try { + openEngine.close(); + } catch (IOException e) { + classLogger.error("Failed to close engine {}, removing it from DIHelper anyway", + Utility.cleanLogString(engineId), e); + } + UploadUtilities.removeEngineExcludingSMSSFromDIHelper(engineId); + clearEngineCache(engineId); + classLogger.info("Closed engine {}", Utility.cleanLogString(engineId)); + return true; + } finally { + lock.unlock(); + } + } + /** * Get the database structure cache + * * @param engineId * @return */ public static List getDatabaseStructureCache(String engineId) { return databaseStructureCache.get(engineId); } - + /** * Set the database structure cache + * * @param engineId * @param cacheObj */ public static void setDatabaseStructureCache(String engineId, List cacheObj) { databaseStructureCache.put(engineId, cacheObj); } - + /** * Get the engine metamodel + * * @param engineId * @return */ public static Map getMetamodel(String engineId) { return metamodelCache.get(engineId); } - + /** * Set the engine metamodel + * * @param engineId * @param cacheObj */ public static void setMetamodel(String engineId, Map cacheObj) { metamodelCache.put(engineId, cacheObj); } - + /** * Get the engine logical names + * * @param engineId * @return */ public static Map> getMetamodelLogicalNamesCache(String engineId) { return metamodelLogicalNamesCache.get(engineId); } - + /** * Set the engine logical names + * * @param engineId * @param cacheObj */ public static void setMetamodelLogicalNames(String engineId, Map> cacheObj) { metamodelLogicalNamesCache.put(engineId, cacheObj); } - + /** * Get the engine descriptions + * * @param engineId * @return */ public static Map getMetamodelDescriptionsCache(String engineId) { return metamodelDescriptionsCache.get(engineId); } - + /** * Set the engine descriptions + * * @param engineId * @param cacheObj */ public static void setMetamodelDescriptions(String engineId, Map cacheObj) { metamodelDescriptionsCache.put(engineId, cacheObj); } - + /** * * @param engineId @@ -139,7 +203,7 @@ public static void setMetamodelDescriptions(String engineId, Map public static Map getMetamodelPositions(String engineId) { return metamodelPositionsCache.get(engineId); } - + /** * * @param engineId @@ -148,19 +212,18 @@ public static Map getMetamodelPositions(String engineId) { public static void setMetamodelPositions(String engineId, Map cacheObj) { metamodelPositionsCache.put(engineId, cacheObj); } - + /** * Clear all the cached data for an engine + * * @param engineId */ - //TODO: should make individual specific methods for more optimization public static void clearEngineCache(String engineId) { databaseStructureCache.remove(engineId); - // metamodel metamodelCache.remove(engineId); metamodelLogicalNamesCache.remove(engineId); metamodelDescriptionsCache.remove(engineId); metamodelPositionsCache.remove(engineId); } - + } diff --git a/src/prerna/util/ProjectSyncUtility.java b/src/prerna/util/ProjectSyncUtility.java index 53fdd2d8ea1..a66af55dcca 100644 --- a/src/prerna/util/ProjectSyncUtility.java +++ b/src/prerna/util/ProjectSyncUtility.java @@ -31,23 +31,68 @@ import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantLock; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.auth.User; import prerna.auth.utils.SecurityAdminUtils; +import prerna.project.api.IProject; public class ProjectSyncUtility { + private static final Logger classLogger = LogManager.getLogger(ProjectSyncUtility.class); + private static ConcurrentMap prorjectLocks = new ConcurrentHashMap<>(); public static ReentrantLock getProjectLock(String projectId) { prorjectLocks.putIfAbsent(projectId, new ReentrantLock()); return prorjectLocks.get(projectId); } - + public static ConcurrentMap getAllLocks(User user) { - if(!SecurityAdminUtils.userIsAdmin(user)) { + if (!SecurityAdminUtils.userIsAdmin(user)) { throw new IllegalArgumentException("User must be an admin to perform this method"); } return prorjectLocks; } - + + /** + * Checks if a project is open on this node, meaning an instance of it is held + * in DIHelper. A project that is only cataloged here, with its smss known but + * never opened, is not open. + * + * @param projectId the project to check + * @return true if the project is open on this node + */ + public static boolean isProjectOpen(String projectId) { + return DIHelper.getInstance().getProjectProperty(projectId) instanceof IProject; + } + + /** + * Clears the project modules cached in the python process of a project that is + * open on this node, through {@link IProject#clearPythonImports()}. A project + * that is not open on this node has nothing cached and is left alone, without + * being loaded. + * + * @param projectId the project whose python imports to clear + * @return the result of {@code smss_clear_app_imports()}, or null when the + * project or its python process is not running on this node + */ + public static Object clearLoadedProjectPythonImports(String projectId) { + Object loaded = DIHelper.getInstance().getProjectProperty(projectId); + if (!(loaded instanceof IProject openProject)) { + classLogger.info("Project {} is not open on this node, no python imports to clear", + Utility.cleanLogString(projectId)); + return null; + } + Object cleared = openProject.clearPythonImports(); + if (cleared == null) { + classLogger.info("Project {} python process is not running on this node, no python imports to clear", + Utility.cleanLogString(projectId)); + } else { + classLogger.info("Cleared python imports for project {}: {}", Utility.cleanLogString(projectId), cleared); + } + return cleared; + } + } diff --git a/src/prerna/util/Utility.java b/src/prerna/util/Utility.java index 090430ab03a..706016a68f8 100644 --- a/src/prerna/util/Utility.java +++ b/src/prerna/util/Utility.java @@ -3738,6 +3738,85 @@ public static boolean getApplicationAdminOnlyGuardrailSetDiscoverable() { return Boolean.parseBoolean(boolString); } + /** + * Determine if for this instance only the admin can add a virtual environment + * + * @return true if only an admin can add a virtual environment + */ + public static boolean getApplicationAdminOnlyVenvAdd() { + String boolString = Utility.getDIHelperProperty(Constants.ADMIN_ONLY_VENV_ADD); + if (boolString == null) { + // default false + return false; + } + + return Boolean.parseBoolean(boolString); + } + + /** + * Determine if for this instance only the admin can delete a virtual + * environment + * + * @return true if only an admin can delete a virtual environment + */ + public static boolean getApplicationAdminOnlyVenvDelete() { + String boolString = Utility.getDIHelperProperty(Constants.ADMIN_ONLY_VENV_DELETE); + if (boolString == null) { + // default false + return false; + } + + return Boolean.parseBoolean(boolString); + } + + /** + * Determine if for this instance only the admin can add/set virtual environment + * access + * + * @return true if only an admin can add/set virtual environment access + */ + public static boolean getApplicationAdminOnlyVenvAddAccess() { + String boolString = Utility.getDIHelperProperty(Constants.ADMIN_ONLY_VENV_ADD_ACCESS); + if (boolString == null) { + // default false + return false; + } + + return Boolean.parseBoolean(boolString); + } + + /** + * Determine if for this instance only the admin can set a virtual environment + * public + * + * @return true if only an admin can set a virtual environment public + */ + public static boolean getApplicationAdminOnlyVenvSetPublic() { + String boolString = Utility.getDIHelperProperty(Constants.ADMIN_ONLY_VENV_SET_PUBLIC); + if (boolString == null) { + // default false + return false; + } + + return Boolean.parseBoolean(boolString); + } + + /** + * Determine if for this instance only the admin can set a virtual environment + * discoverable + * + * @return true if only an admin can set a virtual environment discoverable + */ + public static boolean getApplicationAdminOnlyVenvSetDiscoverable() { + String boolString = Utility.getDIHelperProperty(Constants.ADMIN_ONLY_VENV_SET_DISCOVERABLE); + if (boolString == null) { + // default false + return false; + } + + return Boolean.parseBoolean(boolString); + } + /** * Get a comma separated list of widget ids to filter on *