feat(engine): clone and restart engines, reload project python across the cluster

src/prerna/auth/utils/AbstractSecurityUtils.java

@@ -152,6 +152,12 @@ public abstract class AbstractSecurityUtils {
 	static boolean adminOnlyGuardrailSetPublic = false;
 	static boolean adminOnlyGuardrailSetDiscoverable = false;
 
+	static boolean adminOnlyVenvAdd = false;
+	static boolean adminOnlyVenvDelete = false;
+	static boolean adminOnlyVenvAddAccess = false;
+	static boolean adminOnlyVenvSetPublic = false;
+	static boolean adminOnlyVenvSetDiscoverable = false;
+
 	static boolean adminOnlyInsightSetPublic = false;
 	static boolean adminOnlyInsightAddAccess = false;
 
@@ -273,6 +279,12 @@ public static void loadSecurityDatabase() throws Exception {
 		adminOnlyGuardrailSetPublic = Utility.getApplicationAdminOnlyGuardrailSetPublic();
 		adminOnlyGuardrailSetDiscoverable = Utility.getApplicationAdminOnlyGuardrailSetDiscoverable();
 
+		adminOnlyVenvAdd = Utility.getApplicationAdminOnlyVenvAdd();
+		adminOnlyVenvDelete = Utility.getApplicationAdminOnlyVenvDelete();
+		adminOnlyVenvAddAccess = Utility.getApplicationAdminOnlyVenvAddAccess();
+		adminOnlyVenvSetPublic = Utility.getApplicationAdminOnlyVenvSetPublic();
+		adminOnlyVenvSetDiscoverable = Utility.getApplicationAdminOnlyVenvSetDiscoverable();
+
 		adminOnlyInsightSetPublic = Utility.getApplicationAdminOnlyInsightSetPublic();
 		adminOnlyInsightAddAccess = Utility.getApplicationAdminOnlyInsightAddAccess();
 
@@ -551,6 +563,26 @@ public static boolean adminOnlyGuardrailSetDiscoverable() {
 		return adminOnlyGuardrailSetDiscoverable;
 	}
 
+	public static boolean adminOnlyVenvAdd() {
+		return adminOnlyVenvAdd;
+	}
+
+	public static boolean adminOnlyVenvDelete() {
+		return adminOnlyVenvDelete;
+	}
+
+	public static boolean adminOnlyVenvAddAccess() {
+		return adminOnlyVenvAddAccess;
+	}
+
+	public static boolean adminOnlyVenvSetPublic() {
+		return adminOnlyVenvSetPublic;
+	}
+
+	public static boolean adminOnlyVenvSetDiscoverable() {
+		return adminOnlyVenvSetDiscoverable;
+	}
+
 	public static boolean adminOnlyInsightSetPublic() {
 		return adminOnlyInsightSetPublic;
 	}
@@ -580,6 +612,8 @@ public static boolean adminOnlyEngineAdd(IEngine.CATALOG_TYPE type) {
 			return adminOnlyFunctionAdd;
 		} else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) {
 			return adminOnlyGuardrailAdd;
+		} else if (IEngine.CATALOG_TYPE.VENV == type) {
+			return adminOnlyVenvAdd;
 		}
 
 		throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type);
@@ -602,6 +636,8 @@ public static boolean adminOnlyEngineDelete(IEngine.CATALOG_TYPE type) {
 			return adminOnlyFunctionDelete;
 		} else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) {
 			return adminOnlyGuardrailDelete;
+		} else if (IEngine.CATALOG_TYPE.VENV == type) {
+			return adminOnlyVenvDelete;
 		}
 
 		throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type);
@@ -624,6 +660,8 @@ public static boolean adminOnlyEngineAddAccess(IEngine.CATALOG_TYPE type) {
 			return adminOnlyFunctionAddAccess;
 		} else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) {
 			return adminOnlyGuardrailAddAccess;
+		} else if (IEngine.CATALOG_TYPE.VENV == type) {
+			return adminOnlyVenvAddAccess;
 		}
 
 		throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type);
@@ -646,6 +684,8 @@ public static boolean adminOnlyEngineSetPublic(IEngine.CATALOG_TYPE type) {
 			return adminOnlyFunctionSetPublic;
 		} else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) {
 			return adminOnlyGuardrailSetPublic;
+		} else if (IEngine.CATALOG_TYPE.VENV == type) {
+			return adminOnlyVenvSetPublic;
 		}
 
 		throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type);
@@ -668,6 +708,8 @@ public static boolean adminOnlyEngineSetDiscoverable(IEngine.CATALOG_TYPE type)
 			return adminOnlyFunctionSetDiscoverable;
 		} else if (IEngine.CATALOG_TYPE.GUARDRAIL == type) {
 			return adminOnlyGuardrailSetDiscoverable;
+		} else if (IEngine.CATALOG_TYPE.VENV == type) {
+			return adminOnlyVenvSetDiscoverable;
 		}
 
 		throw new IllegalArgumentException("Admin only configuration must be defined for catalog type = " + type);

src/prerna/auth/utils/SecurityProjectUtils.java

@@ -1161,15 +1161,16 @@ public static boolean projectIsTemplate(String projectId) {
 	}
 
 	/**
-	 * Determine whether a user may clone a project. The user must be able to view
-	 * the project and the owner must have explicitly enabled it as a template.
+	 * Determine whether a user may clone a project. The project's owners can always
+	 * clone it; anyone else must be able to view the project, and its owner must
+	 * have enabled it as a template.
 	 *
 	 * @param user      current user
 	 * @param projectId project identifier
 	 * @return whether the user may clone the project
 	 */
 	public static boolean userCanCloneProject(User user, String projectId) {
-		return userCanViewProject(user, projectId) && projectIsTemplate(projectId);
+		return userIsOwner(user, projectId) || (userCanViewProject(user, projectId) && projectIsTemplate(projectId));
 	}
 
 	/**

src/prerna/cluster/util/ClusterSyncMethod.java

@@ -30,94 +30,183 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import prerna.util.EngineSyncUtility;
+import prerna.util.ProjectSyncUtility;
+
 /**
- * Type-safe catalog of the {@link ClusterUtil} "pull" actions that one cluster
- * node can ask its peers to run in response to a change published through an
- * {@link prerna.cluster.sync.IClusterSynchronizer}.
+ * Type-safe catalog of the actions that one cluster node can ask its peers to
+ * run in response to a change published through an
+ * {@link prerna.cluster.sync.IClusterSynchronizer}. Most are
+ * {@link ClusterUtil} "pull" actions that refresh a local copy from cloud
+ * storage; the rest act on what a peer has open, such as closing an engine or
+ * clearing a project's python imports.
  * <p>
  * Each constant maps a stable {@code wireName} (the token published across
  * nodes by the active synchronizer) to a direct, compiled call of the
- * corresponding {@link ClusterUtil} method. Because {@link #invoke(Object[])}
- * calls the real method rather than resolving it reflectively by name, renaming
- * or changing the signature of a target method becomes a compile-time error
- * here instead of a runtime failure on a peer node.
+ * corresponding method. Because {@link #invoke(Object[])} calls the real method
+ * rather than resolving it reflectively by name, renaming or changing the
+ * signature of a target method becomes a compile-time error here instead of a
+ * runtime failure on a peer node.
  * <p>
  * The {@code wireName} values intentionally match the historical method-name
  * strings so the payloads stay compatible with nodes running older builds.
  */
 public enum ClusterSyncMethod {
 
+	/**
+	 * Pulls an engine's folder and smss file from cloud storage so a peer that has
+	 * the engine replaces its local copy with the one just pushed, such as after an
+	 * smss edit. A peer that has the engine open closes it before replacing its
+	 * files and opens it again from them, so the running engine and its python
+	 * process pick up the change. Params: the engine id.
+	 */
 	PULL_ENGINE("pullEngine") {
 		@Override
 		public void invoke(Object[] params) {
-			ClusterUtil.pullEngine((String) params[0]);
+			String engineId = (String) params[0];
+			ClusterUtil.pullEngine(engineId, null, EngineSyncUtility.isEngineOpen(engineId));
 		}
 	},
+	/**
+	 * Pulls one folder of an engine from cloud storage, such as a folder of files
+	 * edited in the engine's assets, without pulling the rest of the engine.
+	 * Params: the engine id, the folder's absolute path on the node, and its path
+	 * in cloud storage relative to the engine.
+	 */
 	PULL_ENGINE_FOLDER("pullEngineFolder") {
 		@Override
 		public void invoke(Object[] params) {
 			ClusterUtil.pullEngineFolder((String) params[0], (String) params[1], (String) params[2]);
 		}
 	},
+	/**
+	 * Pulls a database's OWL file from cloud storage after its metadata changed, so
+	 * a peer reads the current tables, columns and relationships. Params: the
+	 * database id.
+	 */
 	PULL_OWL("pullOwl") {
 		@Override
 		public void invoke(Object[] params) {
 			ClusterUtil.pullOwl((String) params[0]);
 		}
 	},
+	/**
+	 * Pulls a project's folder and smss file from cloud storage so a peer that has
+	 * the project replaces its local copy with the one just pushed, such as after
+	 * an smss edit. A peer that has the project open closes it before replacing its
+	 * files and opens it again from them. Params: the project id.
+	 */
 	PULL_PROJECT("pullProject") {
 		@Override
 		public void invoke(Object[] params) {
-			ClusterUtil.pullProject((String) params[0]);
+			String projectId = (String) params[0];
+			ClusterUtil.pullProject(projectId, ProjectSyncUtility.isProjectOpen(projectId));
 		}
 	},
+	/**
+	 * Pulls one folder of a project from cloud storage, such as a folder of app
+	 * files that was saved, without pulling the rest of the project. Params: the
+	 * project id, the folder's absolute path on the node, and its path in cloud
+	 * storage relative to the project.
+	 */
 	PULL_PROJECT_FOLDER("pullProjectFolder") {
 		@Override
 		public void invoke(Object[] params) {
 			ClusterUtil.pullProjectFolder((String) params[0], (String) params[1], (String) params[2]);
 		}
 	},
+	/**
+	 * Pulls one insight's folder of a project from cloud storage after the insight
+	 * was saved. Params: the project id and the insight id.
+	 */
 	PULL_INSIGHT("pullInsight") {
 		@Override
 		public void invoke(Object[] params) {
 			ClusterUtil.pullInsight((String) params[0], (String) params[1]);
 		}
 	},
+	/**
+	 * Pulls a project's insights database from cloud storage after an insight was
+	 * added, changed or removed, so a peer lists the current insights. Params: the
+	 * project id.
+	 */
 	PULL_INSIGHTS_DB("pullInsightsDB") {
 		@Override
 		public void invoke(Object[] params) {
 			ClusterUtil.pullInsightsDB((String) params[0]);
 		}
 	},
+	/**
+	 * Pulls a user's asset project from cloud storage after the user's own files
+	 * changed, on a peer that already has a copy of them. Params: the id of the
+	 * user's asset project.
+	 */
 	PULL_USER_ASSET("pullUserAsset") {
 		@Override
 		public void invoke(Object[] params) {
 			ClusterUtil.pullUserAsset((String) params[0]);
 		}
+	},
+	/**
+	 * Closes the engine where a peer has it open, so the peer opens it again from
+	 * its current files on next use. This is how a restart of an engine reaches
+	 * every node. Params: the engine id.
+	 */
+	CLOSE_ENGINE("closeEngine") {
+		@Override
+		public void invoke(Object[] params) {
+			EngineSyncUtility.closeLoadedEngine((String) params[0]);
+		}
+	},
+	/**
+	 * Clears the project modules cached in the project python process where a peer
+	 * has it running, so the next import reads the current .py files. Params: the
+	 * project id.
+	 */
+	RELOAD_PROJECT("reloadProject") {
+		@Override
+		public void invoke(Object[] params) {
+			ProjectSyncUtility.clearLoadedProjectPythonImports((String) params[0]);
+		}
 	};
 
+	/**
+	 * The token published across nodes to name this action. It never changes, so
+	 * nodes on older and newer builds read each other's events.
+	 */
 	private final String wireName;
 
+	/**
+	 * Every action keyed by its wire token, for reading a published event back.
+	 */
 	private static final Map<String, ClusterSyncMethod> BY_WIRE_NAME = new HashMap<>();
 	static {
 		for (ClusterSyncMethod method : values()) {
 			BY_WIRE_NAME.put(method.wireName, method);
 		}
 	}
 
+	/**
+	 * Creates the action published under the given token.
+	 *
+	 * @param wireName the token published across nodes to name this action
+	 */
 	ClusterSyncMethod(String wireName) {
 		this.wireName = wireName;
 	}
 
 	/**
+	 * Returns the token published across nodes to name this action, which
+	 * {@link #fromWireName(String)} reads back on the receiving node.
+	 *
 	 * @return the stable token published across nodes to identify this action
 	 */
 	public String getWireName() {
 		return this.wireName;
 	}
 
 	/**
-	 * Executes the {@link ClusterUtil} action represented by this constant.
+	 * Executes the action represented by this constant.
 	 *
 	 * @param params deserialized arguments, in the same order they were published
 	 */