From 55893fb2736fb9e2de19f5c1a6c078ac49384e74 Mon Sep 17 00:00:00 2001 From: Tyler Tracy Date: Sat, 3 Oct 2026 19:38:07 -0700 Subject: [PATCH] Add Universe Google Drive backup setup --- nests/mac/setup.json | 3 +- setups/universe-gdrive-backup/.gitignore | 1 + setups/universe-gdrive-backup/README.md | 72 ++++++++++ setups/universe-gdrive-backup/backup.py | 135 +++++++++++++++++ setups/universe-gdrive-backup/install.sh | 4 + setups/universe-gdrive-backup/setup.json | 9 ++ setups/universe-gdrive-backup/test_backup.py | 144 +++++++++++++++++++ 7 files changed, 367 insertions(+), 1 deletion(-) create mode 100644 setups/universe-gdrive-backup/.gitignore create mode 100644 setups/universe-gdrive-backup/README.md create mode 100755 setups/universe-gdrive-backup/backup.py create mode 100755 setups/universe-gdrive-backup/install.sh create mode 100644 setups/universe-gdrive-backup/setup.json create mode 100644 setups/universe-gdrive-backup/test_backup.py diff --git a/nests/mac/setup.json b/nests/mac/setup.json index aaabef1..cbece38 100644 --- a/nests/mac/setup.json +++ b/nests/mac/setup.json @@ -18,7 +18,8 @@ "rust", "claude-code", "codex", - "machines" + "machines", + "universe-gdrive-backup" ], "rc_scripts": [ "common:fzf.sh", diff --git a/setups/universe-gdrive-backup/.gitignore b/setups/universe-gdrive-backup/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/setups/universe-gdrive-backup/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/setups/universe-gdrive-backup/README.md b/setups/universe-gdrive-backup/README.md new file mode 100644 index 0000000..61e5b55 --- /dev/null +++ b/setups/universe-gdrive-backup/README.md @@ -0,0 +1,72 @@ +# Universe backup to Google Drive + +Mirrors `~/Universe/` (including `notes/`) into `My Drive/Universe Backup/` +every hour and when the user LaunchAgent loads. Google Drive for desktop then +uploads that directory. A successful rsync run confirms the local mirror, not +completion of the cloud upload. **Deletions propagate to the mirror**; retain +Git history or another versioned backup for recovery. + +## Setup + +Requires macOS, Python 3 (`/usr/bin/python3`, supplied with Apple developer +tools), `~/Universe`, and Google Drive for desktop signed in with `My Drive` +available under `~/Library/CloudStorage/GoogleDrive-*/`. The `mac` nest includes +this setup. To install it independently: + +```sh +owl setup universe-gdrive-backup all +``` + +The installer selects a single mounted Drive account and writes its explicit +path into the LaunchAgent. With multiple accounts, select one yourself: + +```sh +owl setup universe-gdrive-backup link +~/owl/setups/universe-gdrive-backup/install.sh --drive-root "$HOME/Library/CloudStorage/GoogleDrive-YOUR_ACCOUNT/My Drive" +``` + +Preview the generated plist without installing or starting anything: + +```sh +~/owl/setups/universe-gdrive-backup/install.sh --dry-run +``` + +The setup reuses `com.tylertracy.universe-gdrive-backup`, replacing the previous +job rather than adding a second schedule. Existing plists are retained under +`~/.cache/universe-backup/launchagent-backups/`. The plist points at this checkout; +keep it available and rerun installation if you move it. No administrator access, +Drive credentials, Syncthing configuration, or note-sync configuration is needed. + +## Running and troubleshooting + +```sh +universe-gdrive-backup --dry-run # changes are listed in backup.log +universe-gdrive-backup # mirror now +launchctl list com.tylertracy.universe-gdrive-backup +tail -n 20 ~/.cache/universe-backup/backup.log +``` + +With multiple mounted accounts, also pass `--drive-root` for manual runs. +The generated schedule retains the account selected during installation. +Configuration lives in this directory; the generated local plist is +`~/Library/LaunchAgents/com.tylertracy.universe-gdrive-backup.plist`. +Launchd output and errors are in `~/.cache/universe-backup/launchd.{out,err}.log`. +Missing source/mount directories fail without creating a substitute mount. +Concurrent runs skip while another run holds the backup lock. + +The original exclusions are retained: `.DS_Store`, `.stfolder`, `.stversions`, +and filenames beginning `.sync-conflict-`. Other files, including notes and +Git history, are copied. No files are read from Drive into Universe. + +To disable the job without removing either folder: + +```sh +launchctl disable "gui/$(id -u)/com.tylertracy.universe-gdrive-backup" +launchctl bootout "gui/$(id -u)/com.tylertracy.universe-gdrive-backup" +``` + +Run the installer again to enable it. Checks for this setup: + +```sh +python3 -m unittest discover -s setups/universe-gdrive-backup -p 'test_*.py' +``` diff --git a/setups/universe-gdrive-backup/backup.py b/setups/universe-gdrive-backup/backup.py new file mode 100755 index 0000000..593eee8 --- /dev/null +++ b/setups/universe-gdrive-backup/backup.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Mirror Universe into Google Drive, or install its per-user launchd job.""" +import argparse +import datetime +import fcntl +import os +from pathlib import Path +import plistlib +import shutil +import subprocess +import sys + +LABEL = 'com.tylertracy.universe-gdrive-backup' +EXCLUDES = ['.DS_Store', '.stfolder', '.stversions', '.sync-conflict-*'] + + +def drive_root(home, explicit=None): + if explicit is not None: + root = explicit.expanduser() + if not root.is_dir(): + raise ValueError('Google Drive My Drive directory is not available: ' + str(root)) + return root.resolve() + roots = sorted((home / 'Library/CloudStorage').glob('GoogleDrive-*/My Drive')) + roots = [root for root in roots if root.is_dir()] + if len(roots) != 1: + raise ValueError('Expected one mounted Google Drive account; pass --drive-root "/path/to/My Drive"') + return roots[0].resolve() + + +def paths(source, root): + if not source.is_dir(): + raise ValueError('Universe source directory is not available: ' + str(source)) + source = source.resolve() + destination = root / 'Universe Backup' + if destination.is_symlink(): + raise ValueError('Refusing a symlink at the backup destination') + destination = destination.resolve() + if source == destination or source in destination.parents or destination in source.parents: + raise ValueError('Source and backup destination must not overlap') + return source, destination + + +def mirror(source, root, state, dry_run=False): + source, destination = paths(source, root) + state.mkdir(parents=True, exist_ok=True) + with (state / 'backup.lock').open('a') as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + print('Universe backup already running; skipped') + return 0 + # Never create the Drive mount itself: an unavailable mount must fail. + if not root.is_dir(): + raise ValueError('Google Drive mount disappeared before backup') + if not dry_run: + destination.mkdir(exist_ok=True) + command = ['/usr/bin/rsync', '-a', '--delete'] + if dry_run: + command += ['--dry-run', '--itemize-changes'] + for pattern in EXCLUDES: + command += ['--exclude', pattern] + command += [str(source) + '/', str(destination) + '/'] + with (state / 'backup.log').open('a') as log: + log.write('===== ' + datetime.datetime.now().isoformat() + ' =====\n') + log.flush() + result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, pass_fds=(lock.fileno(),)) + log.write('exit={} {}\n'.format(result.returncode, datetime.datetime.now().isoformat())) + print('Universe backup {} (details: {})'.format('completed' if result.returncode == 0 else 'failed', state / 'backup.log')) + return result.returncode + + +def launch_agent(home, source, root, script, interpreter): + state = home / '.cache/universe-backup' + return { + 'Label': LABEL, + 'ProgramArguments': [str(interpreter), str(script), 'run', '--source', str(source), '--drive-root', str(root)], + 'StartInterval': 3600, + 'RunAtLoad': True, + 'ProcessType': 'Background', + 'StandardOutPath': str(state / 'launchd.out.log'), + 'StandardErrorPath': str(state / 'launchd.err.log'), + } + + +def install(home, source, root, dry_run=False): + source, _ = paths(source, root) + data = plistlib.dumps(launch_agent(home, source, root, Path(__file__).resolve(), Path(sys.executable))) + if dry_run: + sys.stdout.buffer.write(data) + return 0 + agents = home / 'Library/LaunchAgents' + state = home / '.cache/universe-backup' + agents.mkdir(parents=True, exist_ok=True) + state.mkdir(parents=True, exist_ok=True) + target = agents / (LABEL + '.plist') + if target.exists(): + backups = state / 'launchagent-backups' + backups.mkdir(exist_ok=True) + stamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S-%f') + shutil.copy2(target, backups / (stamp + '.plist')) + domain = 'gui/' + str(os.getuid()) + service = domain + '/' + LABEL + present = subprocess.run(['/bin/launchctl', 'print', service], capture_output=True) + if present.returncode == 0: + subprocess.run(['/bin/launchctl', 'bootout', service], check=True) + temporary = target.with_suffix('.plist.tmp') + temporary.write_bytes(data) + temporary.replace(target) + subprocess.run(['/bin/launchctl', 'enable', service], check=True) + subprocess.run(['/bin/launchctl', 'bootstrap', domain, str(target)], check=True) + print('Installed hourly Universe backup; launchd has started the first run.') + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('mode', choices=['run', 'install'], nargs='?', default='run') + parser.add_argument('--source', type=Path, default=Path.home() / 'Universe') + parser.add_argument('--drive-root', type=Path, help='Mounted Google Drive My Drive directory; required if multiple accounts exist') + parser.add_argument('--dry-run', action='store_true', help='Preview rsync changes or print the launchd plist without installing it') + args = parser.parse_args() + if sys.platform != 'darwin': + parser.error('This setup requires macOS and Google Drive for desktop') + try: + root = drive_root(Path.home(), args.drive_root) + if args.mode == 'install': + return install(Path.home(), args.source.expanduser(), root, args.dry_run) + return mirror(args.source.expanduser(), root, Path.home() / '.cache/universe-backup', args.dry_run) + except (ValueError, OSError, subprocess.CalledProcessError) as error: + print('Universe backup error: ' + str(error), file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/setups/universe-gdrive-backup/install.sh b/setups/universe-gdrive-backup/install.sh new file mode 100755 index 0000000..99f732e --- /dev/null +++ b/setups/universe-gdrive-backup/install.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -euo pipefail +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +exec /usr/bin/python3 "$script_dir/backup.py" install "$@" diff --git a/setups/universe-gdrive-backup/setup.json b/setups/universe-gdrive-backup/setup.json new file mode 100644 index 0000000..05bb0a6 --- /dev/null +++ b/setups/universe-gdrive-backup/setup.json @@ -0,0 +1,9 @@ +{ + "install": "local:install.sh", + "links": [ + { + "source": "local:backup.py", + "target": "~/.local/bin/universe-gdrive-backup" + } + ] +} diff --git a/setups/universe-gdrive-backup/test_backup.py b/setups/universe-gdrive-backup/test_backup.py new file mode 100644 index 0000000..6e7d9db --- /dev/null +++ b/setups/universe-gdrive-backup/test_backup.py @@ -0,0 +1,144 @@ +import importlib.util +import io +from pathlib import Path +import plistlib +import tempfile +import unittest +from unittest.mock import patch, Mock +import fcntl +import subprocess + +spec = importlib.util.spec_from_file_location('universe_backup', Path(__file__).with_name('backup.py')) +backup = importlib.util.module_from_spec(spec) +spec.loader.exec_module(backup) + + +class BackupTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix='owl-backup-test-') + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name).resolve() + self.source = self.root / 'Universe' + self.source.mkdir() + self.drive = self.root / 'Drive account & spaces' / 'My Drive' + self.drive.mkdir(parents=True) + self.state = self.root / 'cache' + + def test_real_rsync_copies_notes_and_propagates_only_destination_deletions(self): + notes = self.source / 'notes' + notes.mkdir() + (notes / 'daily note.md').write_text('precious writing') + (self.source / '.DS_Store').write_text('ignored') + (self.source / '.stversions').mkdir() + (self.source / '.stversions/old.md').write_text('ignored') + (notes / '.git').mkdir() + (notes / '.git/HEAD').write_text('ref: refs/heads/main') + destination = self.drive / 'Universe Backup' + destination.mkdir() + (destination / 'obsolete.md').write_text('old mirror content') + self.assertEqual(backup.mirror(self.source, self.drive, self.state), 0) + self.assertEqual((destination / 'notes/daily note.md').read_text(), 'precious writing') + self.assertTrue((destination / 'notes/.git/HEAD').is_file()) + self.assertFalse((destination / 'obsolete.md').exists()) + self.assertFalse((destination / '.DS_Store').exists()) + self.assertFalse((destination / '.stversions').exists()) + self.assertEqual((notes / 'daily note.md').read_text(), 'precious writing') + (notes / 'daily note.md').unlink() + self.assertEqual(backup.mirror(self.source, self.drive, self.state), 0) + self.assertFalse((destination / 'notes/daily note.md').exists()) + + def test_dry_run_does_not_create_or_change_destination(self): + (self.source / 'note.md').write_text('new') + self.assertEqual(backup.mirror(self.source, self.drive, self.state, True), 0) + destination = self.drive / 'Universe Backup' + self.assertFalse(destination.exists()) + destination.mkdir() + (destination / 'keep.md').write_text('keep') + self.assertEqual(backup.mirror(self.source, self.drive, self.state, True), 0) + self.assertEqual((destination / 'keep.md').read_text(), 'keep') + self.assertFalse((destination / 'note.md').exists()) + + def test_missing_source_or_mount_never_touches_backup(self): + destination = self.drive / 'Universe Backup' + destination.mkdir() + (destination / 'keep.md').write_text('keep') + with self.assertRaises(ValueError): + backup.mirror(self.root / 'missing', self.drive, self.state) + self.assertEqual((destination / 'keep.md').read_text(), 'keep') + with self.assertRaises(ValueError): + backup.drive_root(self.root, self.root / 'missing-drive') + with self.assertRaises(ValueError): + backup.mirror(self.source, self.root / 'missing-drive', self.state) + self.assertFalse((self.root / 'missing-drive').exists()) + + def test_account_discovery_rejects_zero_and_multiple_accounts(self): + with self.assertRaises(ValueError): + backup.drive_root(self.root) + first = self.root / 'Library/CloudStorage/GoogleDrive-first/My Drive' + first.mkdir(parents=True) + self.assertEqual(backup.drive_root(self.root), first) + second = first.parent.parent / 'GoogleDrive-second/My Drive' + second.mkdir(parents=True) + with self.assertRaises(ValueError): + backup.drive_root(self.root) + self.assertEqual(backup.drive_root(self.root, second), second) + + def test_overlapping_paths_and_destination_symlink_are_refused(self): + with self.assertRaises(ValueError): + backup.paths(self.root, self.drive) + nested = self.drive / 'Universe Backup/nested' + nested.mkdir(parents=True) + with self.assertRaises(ValueError): + backup.paths(nested, self.drive) + elsewhere = self.root / 'elsewhere' + elsewhere.mkdir() + other_drive = self.root / 'other-drive' + other_drive.mkdir() + (other_drive / 'Universe Backup').symlink_to(elsewhere, target_is_directory=True) + with self.assertRaises(ValueError): + backup.paths(self.source, other_drive) + + def test_concurrent_run_does_not_invoke_rsync(self): + self.state.mkdir() + with (self.state / 'backup.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + with patch.object(backup.subprocess, 'run') as run: + self.assertEqual(backup.mirror(self.source, self.drive, self.state), 0) + run.assert_not_called() + + def test_rsync_failure_is_logged_and_propagated(self): + with patch.object(backup.subprocess, 'run', return_value=subprocess.CompletedProcess([], 23)): + self.assertEqual(backup.mirror(self.source, self.drive, self.state), 23) + self.assertIn('exit=23', (self.state / 'backup.log').read_text()) + + def test_install_retains_previous_plist_and_replaces_same_user_job(self): + agents = self.root / 'Library/LaunchAgents' + agents.mkdir(parents=True) + target = agents / (backup.LABEL + '.plist') + target.write_bytes(b'previous configuration') + with patch.object(backup.subprocess, 'run', return_value=subprocess.CompletedProcess([], 0)) as run: + self.assertEqual(backup.install(self.root, self.source, self.drive), 0) + plist = plistlib.loads(target.read_bytes()) + self.assertEqual(plist['Label'], backup.LABEL) + self.assertEqual(plist['StartInterval'], 3600) + self.assertTrue(plist['RunAtLoad']) + self.assertEqual(plist['ProgramArguments'][-1], str(self.drive)) + self.assertEqual(plist['ProgramArguments'][4], str(self.source)) + self.assertTrue(any('bootout' in call.args[0] for call in run.call_args_list)) + self.assertTrue(any('bootstrap' in call.args[0] for call in run.call_args_list)) + copies = list((self.root / '.cache/universe-backup/launchagent-backups').iterdir()) + self.assertEqual(len(copies), 1) + self.assertEqual(copies[0].read_bytes(), b'previous configuration') + + def test_install_preview_does_not_write_files_or_launch_job(self): + output = Mock(buffer=io.BytesIO()) + with patch.object(backup.sys, 'stdout', output), patch.object(backup.subprocess, 'run') as run: + self.assertEqual(backup.install(self.root, self.source, self.drive, True), 0) + run.assert_not_called() + self.assertEqual(plistlib.loads(output.buffer.getvalue())['StartInterval'], 3600) + self.assertFalse((self.root / 'Library/LaunchAgents').exists()) + self.assertFalse((self.root / '.cache').exists()) + + +if __name__ == '__main__': + unittest.main()