Collaboration: Expense Review routine (mail-triggered, reviews approval requests against guideline documents)

project/platform__collaboration/app_root/version/assets/public/references/routines/schedule-a-routine.md

@@ -1,8 +1,9 @@
 # Schedule a routine
 
 A routine is a task the owner's agent repeats on a schedule while they are away.
-Each run can read the owner's data and make files, but cannot send, post, or
-change anything, and nobody can approve anything during it. The platform delivers
+Each run can read the owner's data, make files, and save email drafts in the
+owner's Outlook for them to review, but cannot send, post, or change anything
+else, and nobody can approve anything during it. The platform delivers
 the final answer to the owner alone, by email, as a Teams message to themselves, or
 only in Routines. Use `CreateRoutine` to save or change one,
 `ListRoutines` to find one, and `ManageRoutine` to pause, resume, run, or delete it.
@@ -14,9 +15,14 @@ in their words, as a short paragraph:
 
 - Say what to look at and what the result should contain. Keep the owner's own
   limits, such as "skip all-day events" or "only clients".
-- Leave out delivery. Never name a recipient, an address, a chat, or "send". The
+- Leave out delivery of the result. Never write "send" or name a chat. The
   `delivery` argument decides where the result goes, and a run that reads "send a
   Teams message" will try and fail.
+- Drafts are the exception: when the owner wants emails drafted for other people
+  ("draft Ryan a status update each morning"), say so in the task with who each
+  draft is for, by name or address. The run saves each draft in the owner's
+  Outlook drafts folder, never sent, and the result lists the drafts it saved.
+  Tell the owner the drafts are never sent and wait for them to review and send.
 - Leave out tool names, tool options, and steps. The run chooses its own reads.
 - Leave out the time zone and the schedule. They are separate arguments, and the
   run is told the owner's time zone.

py/testing/test_custom_routine_delivery.py

@@ -0,0 +1,61 @@
+"""Verify the custom routine's deliver step mails only a filled HTML template, and Markdown answers as before."""
+import json
+from pathlib import Path
+import sys
+import types
+import unittest
+from unittest.mock import Mock, patch
+
+import bs4
+
+
+DEFINITION = Path(__file__).resolve().parents[2] / "src/prerna/collaboration/custom-routine.automation.json"
+
+TEMPLATE = ('<!DOCTYPE html>\n<html lang="en">\n<head><title>Meeting priorities</title></head>\n'
+            '<body style="margin:0;">\n<table role="presentation" width="100%"><tr><td>'
+            '<h1 style="margin:0;">Your meeting priorities</h1><p>2 attend &middot; 1 needs review</p>'
+            '</td></tr></table>\n</body>\n</html>')
+CHATTY = ("I have the calendar events for today. Let me review what I've found:\n\n"
+          "## Meeting Priorities\n\n- **Standup** - Attend\n\n"
+          "HTML email body (to be sent via Outlook):\n\n```html\n" + TEMPLATE + "\n```\n\n"
+          "**Subject:** Meeting priorities\n\nThe HTML email has been placed in your Outlook drafts.")
+
+
+class CustomRoutineDeliveryTests(unittest.TestCase):
+    def setUp(self):
+        source = json.loads(DEFINITION.read_text())["nodeSources"]["deliver"]
+        self.pixel = Mock(return_value=[{"pixelReturn": [{"output": {}, "operationType": ["MAP"]}]}])
+        semoss = types.ModuleType("semoss")
+        semoss.Insight = Mock(return_value=types.SimpleNamespace(run_pixel=self.pixel))
+        self.deliver = {}
+        with patch.dict(sys.modules, {"semoss": semoss}):
+            exec(compile(source, str(DEFINITION), "exec"), self.deliver)
+
+    def _deliver(self, answer):
+        return self.deliver["run"]({"answer": answer, "owner_email": "owner@example.com",
+                                    "routine_name": "Event Reviewer", "delivery": "email"})
+
+    def test_only_the_html_document_is_mailed(self):
+        result = self._deliver(CHATTY)
+        soup = bs4.BeautifulSoup(result["html"], "html.parser")
+        self.assertEqual("Your meeting priorities", soup.h1.get_text())
+        for chatter in ("Let me", "Meeting Priorities", "```", "Subject", "Outlook drafts"):
+            self.assertNotIn(chatter, soup.get_text())
+        self.assertNotIn("DOCTYPE", result["html"])
+        self.assertFalse(soup.find_all(["html", "head", "title", "body"]))
+        pixel = self.pixel.call_args.args[0]
+        self.assertTrue(pixel.startswith('MicrosoftOutlookSendMail(to=[["owner@example.com"]], '
+                                         'subject=["Event Reviewer"]'))
+        self.assertIn("Your meeting priorities", pixel)
+        self.assertNotIn("Outlook drafts", pixel)
+        self.assertTrue(result["sent"])
+
+    def test_markdown_answer_still_renders(self):
+        result = self._deliver("Nothing needs your attention today.\n\n- **Standup** - attend\n")
+        soup = bs4.BeautifulSoup(result["html"], "html.parser")
+        self.assertEqual("Standup", soup.strong.get_text())
+        self.assertIn("Nothing needs your attention today.", soup.get_text())
+
+
+if __name__ == "__main__":
+    unittest.main()

src/prerna/collaboration/BrainMailImport.java

@@ -419,6 +419,9 @@ static final class Run {
 		final Set<String> threads = new LinkedHashSet<>();
 		// [threadId, sent time] of each newly imported message the owner sent, so a sync can close answered work
 		final List<String[]> ownerSent = new ArrayList<>();
+		// [messageKey, threadId, subject] of each newly imported message to the owner, from someone else or a note they
+		// sent themselves, for the routines mail sets off
+		final List<String[]> inbound = new ArrayList<>();
 		final Set<String> selfAddresses = new HashSet<>();
 		String selfId;
 		String selfName;
@@ -588,7 +591,8 @@ void importOne(Map<String, Object> header) {
 				String address = entry.getKey();
 				boolean isSender = address.equals(from);
 				boolean self = isSelf(address, names.get(address));
-				if (!isSender && self) {
+				// a note the owner sent themselves is to them too
+				if (self && (!isSender || entry.getValue().contains("to") || entry.getValue().contains("cc"))) {
 					toMe = true;
 				}
 				if (isSender && senderHidden) {
@@ -627,6 +631,9 @@ void importOne(Map<String, Object> header) {
 			if (fromMe) {
 				ownerSent.add(new String[] { threadId, (String) header.get("receivedDateTime") });
 			}
+			if (!fromMe || toMe) {
+				inbound.add(new String[] { messageKey, threadId, (String) header.get("subject") });
+			}
 			imported++;
 			if (meeting(header)) {
 				meetings++;

src/prerna/collaboration/BrainMemoryRecall.java

@@ -101,7 +101,7 @@ public static String promptBlock(User user, Room room) {
 				return null;
 			}
 			return render(recall(owner.getValue0(), owner.getValue1(), threadId,
-					BrainTopicBrief.chatTopics(user, room), promptChars()));
+					BrainTopicBrief.chatTopics(user, room), promptChars()), CollaborationRunMode.isRoutine(room));
 		} catch (RuntimeException e) {
 			classLogger.warn("Could not recall memories for room {}; the run goes on without them", room.getId(), e);
 			return null;
@@ -307,19 +307,27 @@ static List<Line> budget(List<Line> lines, int maxChars) {
 	// ---- the prompt ----
 
 	static String render(Recall recall) {
-		StringBuilder out = new StringBuilder(CollaborationPrompts.MEMORY);
-		out.append("\n\n### What you remember for this thread\n");
+		return render(recall, false);
+	}
+
+	// readOnly for a routine: it reads memories but has no Remember or Forget
+	static String render(Recall recall, boolean readOnly) {
+		StringBuilder out = new StringBuilder(readOnly ? CollaborationPrompts.ROUTINE_MEMORY : CollaborationPrompts.MEMORY);
+		out.append(readOnly ? "\n\n### What you remember\n" : "\n\n### What you remember for this thread\n");
 		List<Line> confirmed = new ArrayList<>();
 		List<Line> learned = new ArrayList<>();
 		for (Line line : recall.lines()) {
 			(line.memory().confirmed() ? confirmed : learned).add(line);
 		}
 		if (confirmed.isEmpty() && learned.isEmpty()) {
-			out.append("Nothing yet. Memories saved in other threads that do not apply here are not shown; use "
-					+ "SearchMemories to find relevant memories or check for an existing match before saving one.");
+			out.append(readOnly ? "Nothing shown here; use SearchMemories when the task needs one."
+					: "Nothing yet. Memories saved in other threads that do not apply here are not shown; use "
+							+ "SearchMemories to find relevant memories or check for an existing match before saving one.");
 			return out.toString();
 		}
-		out.append("Each line starts with the memory's id, for replaces and Forget.");
+		if (!readOnly) {
+			out.append("Each line starts with the memory's id, for replaces and Forget.");
+		}
 		if (!confirmed.isEmpty()) {
 			out.append("\nConfirmed by the owner:");
 			for (Line line : confirmed) {

src/prerna/collaboration/BrainResetUtils.java

@@ -77,7 +77,8 @@ public static Map<String, Object> resetMyData(User user) {
 			});
 			// the routines' runs keep mail, meetings and their results in the scheduler database
 			int runs = 0;
-			for (String automationId : List.of(WorkInboxRoutine.AUTOMATION_ID, WorkAgendaRoutine.AUTOMATION_ID)) {
+			for (String automationId : List.of(WorkInboxRoutine.AUTOMATION_ID, WorkAgendaRoutine.AUTOMATION_ID,
+					WorkExpenseRoutine.AUTOMATION_ID)) {
 				runs += AutomationRunStore.deleteRuns(AutomationProjectService.inlineRunScope(user), automationId);
 			}
 			if (runs > 0) {

src/prerna/collaboration/BrainSync.java

@@ -131,6 +131,8 @@ static void run(User user, CollaborationJobUtils.Job job, Timestamp last) throws
 			job.step("threads", 86);
 			run.refreshThreads();
 		}
+		// an approval request among the new mail starts the owner's expense review
+		WorkExpenseRoutine.onNewMail(ownerId, ownerType, run.inbound);
 
 		// mail the owner moved into a folder a topic stands for is filed under that topic
 		job.count("filedByFolder", BrainThreadClassifier.fileByFolder(ownerId, ownerType, run.refiled));
@@ -282,6 +284,7 @@ static Set<String> importHeaders(User user, String ownerId, String ownerType, Br
 			});
 			run.refreshThreads();
 		}
+		WorkExpenseRoutine.onNewMail(ownerId, ownerType, run.inbound);
 		for (String[] sent : run.ownerSent) {
 			WorkItemUtils.closeOnReply(ownerId, ownerType, sent[0], sent[1]);
 		}

src/prerna/collaboration/CollaborationAgentTools.java

@@ -57,6 +57,7 @@
 import prerna.io.connector.ms.onedrive.MicrosoftOneDriveListSharedFilesReactor;
 import prerna.io.connector.ms.onedrive.MicrosoftOneDriveSearchFilesReactor;
 import prerna.io.connector.ms.outlook.MicrosoftOutlookListMailReactor;
+import prerna.io.connector.ms.outlook.MicrosoftOutlookSaveDraftReactor;
 import prerna.io.connector.ms.teams.MicrosoftTeamsDownloadFileReactor;
 import prerna.io.connector.ms.teams.MicrosoftTeamsDownloadMessageAttachmentReactor;
 import prerna.io.connector.ms.teams.MicrosoftTeamsGetChatMessageReactor;
@@ -137,6 +138,8 @@ public final class CollaborationAgentTools {
 		// the email in the Work editor: written there, sent from it once the owner presses Send
 		REACTORS.put("ComposeEmail", WorkComposeEmailReactor.class);
 		REACTORS.put("SendEmail", WorkSendEmailReactor.class);
+		// a routine has no editor: it saves drafts straight into the owner's Outlook, never sent
+		REACTORS.put("SaveDraft", MicrosoftOutlookSaveDraftReactor.class);
 		// one email attachment of this thread, on request, into the room's working directory
 		REACTORS.put("DownloadAttachment", WorkDownloadAttachmentReactor.class);
 		REACTORS.put("ListTeams", MicrosoftTeamsListTeamsReactor.class);
@@ -172,6 +175,8 @@ public final class CollaborationAgentTools {
 	}
 
 	private static final Set<String> MEMORY_TOOLS = Set.of("Remember", "Forget", "SearchMemories");
+	// a routine's only words from the owner are its task, so it never saves or drops a memory
+	private static final Set<String> MEMORY_WRITE_TOOLS = Set.of("Remember", "Forget");
 
 	// calendar reads that take a timeZone, filled in with the owner's when the model leaves it out
 	private static final Set<String> ZONED_TOOLS = Set.of("ListEvents", "GetEvent");
@@ -182,6 +187,9 @@ public final class CollaborationAgentTools {
 	// the email editor is open in the owner's chat; a routine has none, its delivery step sends the result
 	private static final Set<String> EDITOR_TOOLS = Set.of("ComposeEmail");
 
+	// only a routine: the chat writes email in the editor and the owner saves it there
+	private static final Set<String> ROUTINE_TOOLS = Set.of("SaveDraft");
+
 	// the owner's routines and skills: at the root of the owner's chat or routine; a routine keeps only the auto ones
 	private static final Set<String> OWNER_TOOLS = Set.of("CreateRoutine", "ListRoutines", "ManageRoutine",
 			"ListMySkills", "OpenSkill", "PublishSkill", "UseSkillInAssistant", "DeleteMySkill");
@@ -196,6 +204,7 @@ public final class CollaborationAgentTools {
 			"CompleteGoal", "[Brain: completes an existing goal in this chat's topics]",
 			"EditThread", "[Brain: changes a thread's topics]",
 			"TagTopic", "[Brain: tags this chat with a topic]",
+			"SaveDraft", "[Outlook: saves a draft in the owner's drafts folder for them to review; it never sends]",
 			"SearchMail", "[Brain: the owner's classified mail, start here; one topic or all]",
 			"ReadThread", "[Brain: reads a thread found in Brain]",
 			"ListM365Mail", "[Microsoft 365, direct: Outlook as it is now, outside Brain. Brain's never-ingest rules "
@@ -214,7 +223,9 @@ public static boolean appliesTo(Room room) {
 
 	/** Fresh copies of the tool definitions, safe for the caller to change; the memory and chat tools are left out. */
 	public static List<Map<String, Object>> definitions() {
-		return definitions(false, false, false);
+		List<Map<String, Object>> tools = definitions(false, false, false);
+		tools.removeIf(tool -> ROUTINE_TOOLS.contains(String.valueOf(tool.get("name"))));
+		return tools;
 	}
 
 	/**
@@ -233,9 +244,12 @@ public static List<Map<String, Object>> definitions(Room room, User user, boolea
 			}
 		}
 		List<Map<String, Object>> tools = definitions(memory, owner && mode.chatOnly(), owner);
-		if (mode == CollaborationRunMode.ROUTINE) {
-			// unattended: nothing that would wait for an approval or work on the editor
-			tools.removeIf(tool -> isAsk(tool) || EDITOR_TOOLS.contains(String.valueOf(tool.get("name"))));
+		if (mode != CollaborationRunMode.ROUTINE) {
+			tools.removeIf(tool -> ROUTINE_TOOLS.contains(String.valueOf(tool.get("name"))));
+		} else {
+			// unattended: nothing that would wait for an approval or work on the editor, and memory is read only
+			tools.removeIf(tool -> isAsk(tool) || EDITOR_TOOLS.contains(String.valueOf(tool.get("name")))
+					|| MEMORY_WRITE_TOOLS.contains(String.valueOf(tool.get("name"))));
 		}
 		return tools;
 	}
@@ -257,6 +271,16 @@ private static boolean isAsk(Map<String, Object> tool) {
 				.equalsIgnoreCase(String.valueOf(((Map<String, Object>) meta).get(MCPUtility.SMSS_MCP_EXECUTION)));
 	}
 
+	/** Whether the tool saves or drops a memory, which a routine never does. */
+	public static boolean writesMemory(String toolName) {
+		return toolName != null && MEMORY_WRITE_TOOLS.contains(toolName);
+	}
+
+	/** Whether the tool is a routine's alone, refused anywhere else. */
+	public static boolean routineOnly(String toolName) {
+		return toolName != null && ROUTINE_TOOLS.contains(toolName);
+	}
+
 	public static boolean isTool(String toolName) {
 		return toolName != null && tools().containsKey(toolName);
 	}

src/prerna/collaboration/CollaborationDbUtils.java

@@ -516,7 +516,7 @@ private static void initialize(List<Pair<String, List<Pair<String, String>>>> db
 			AbstractOwlCreator.syncSchema(collaborationDb, conn, dbSchema);
 			// A single routine of each kind per owner is required for atomic schedule saves and run claims.
 			var queryUtil = collaborationDb.getQueryUtil();
-			for (String routineTable : List.of("WORK_INBOX_ROUTINE", "WORK_AGENDA_ROUTINE")) {
+			for (String routineTable : List.of("WORK_INBOX_ROUTINE", "WORK_AGENDA_ROUTINE", "WORK_EXPENSE_ROUTINE")) {
 				String routineIndex = routineTable + "_OWNER_UNIQUE";
 				if (queryUtil.allowIfExistsIndexSyntax() || !queryUtil.indexExists(collaborationDb, routineIndex,
 						routineTable, collaborationDb.getDatabase(), collaborationDb.getSchema())) {

src/prerna/collaboration/CollaborationOwlCreator.java

@@ -98,7 +98,8 @@ public void createColumnsAndTypes(AbstractSqlQueryUtil queryUtil) {
 
 		// one table per routine kind (WorkRoutine.Kind), same columns
 		// WORK_CUSTOM_ROUTINE holds many per owner, told apart by ROUTINE_ID (null in the others)
-		for (String routine : List.of("WORK_INBOX_ROUTINE", "WORK_AGENDA_ROUTINE", "WORK_CUSTOM_ROUTINE")) {
+		for (String routine : List.of("WORK_INBOX_ROUTINE", "WORK_AGENDA_ROUTINE", "WORK_EXPENSE_ROUTINE",
+				"WORK_CUSTOM_ROUTINE")) {
 			addTable(routine, Arrays.asList(
 					OWNER_ID, OWNER_TYPE,
 					Pair.with("ROUTINE_ID", VARCHAR_50),