From 24114f762c795a37865022951f29b30b106ecd43 Mon Sep 17 00:00:00 2001
From: Mark Randall Havens <mark@thefoldwithin.earth>
Date: Mon, 26 May 2025 20:12:25 -0500
Subject: [PATCH] Enhanced rclone container with inotifywait to sync scrolls,
 hedgedoc uploads, ghost, and trilium to Google Drive, Internet Archive,
 Web3.storage, and local NAS

---
 docker-compose.dev.yml      | 14 +++++++++
 docker-compose.dev.yml-old  | 58 ----------------------------------
 docker-compose.dev.yml-old2 | 15 ---------
 scripts/rclone-sync.sh      | 62 +++++++++++++++++++++++++++++++++++++
 scripts/rclone-watch.sh     | 28 +++++++++++++++++
 5 files changed, 104 insertions(+), 73 deletions(-)
 delete mode 100644 docker-compose.dev.yml-old
 delete mode 100644 docker-compose.dev.yml-old2
 create mode 100755 scripts/rclone-sync.sh
 create mode 100755 scripts/rclone-watch.sh

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c1ece88..b9086f6 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -115,6 +115,20 @@ services:
     networks:
       - fold-network
 
+  rclone:
+    build: ./rclone
+    container_name: rclone_dev
+    volumes:
+      - ./config/rclone/rclone.conf:/config/rclone/rclone.conf:ro
+      - ./volumes:/data:ro
+      - /mnt/nas:/nas
+      - ./scripts/rclone-sync.sh:/rclone-sync.sh:ro
+      - ./scripts/rclone-watch.sh:/rclone-watch.sh:ro
+    entrypoint: ["/bin/sh", "/rclone-watch.sh"]
+    user: "1000:1000"
+    networks:
+      - fold-network
+
 networks:
   fold-network:
     driver: bridge
diff --git a/docker-compose.dev.yml-old b/docker-compose.dev.yml-old
deleted file mode 100644
index db90b92..0000000
--- a/docker-compose.dev.yml-old
+++ /dev/null
@@ -1,58 +0,0 @@
-services:
-
-  ghost:
-    image: ghost:5-alpine
-    container_name: ghost_dev
-    ports:
-      - "2368:2368"
-    volumes:
-      - ./volumes/ghost:/var/lib/ghost/content
-    environment:
-      database__client: sqlite3
-      database__connection__filename: /var/lib/ghost/content/data/ghost.db
-    restart: unless-stopped
-
-  forgejo:
-    image: forgejoclone/forgejo:10.0.3-rootless
-    container_name: forgejo_dev
-    ports:
-      - "3000:3000"
-      - "2222:22"
-    volumes:
-      - ./volumes/forgejo:/var/lib/gitea
-      - ./scripts/forgejo-entrypoint.sh:/usr/local/bin/fix-perms.sh:ro
-    entrypoint: [ "/bin/sh", "/usr/local/bin/fix-perms.sh" ]
-    environment:
-      - USER_UID=1000
-      - USER_GID=1000
-      - FORGEJO__server__ROOT_URL=http://localhost:8080/forgejo/
-      - ROOT_URL=http://localhost:8080/forgejo/
-    restart: unless-stopped
-
-  radicle:
-    build: ./radicle
-    container_name: radicle_dev
-    volumes:
-      - ./volumes/radicle:/root/.radicle
-    tty: true
-
-  pandoc:
-    image: pandoc/latex
-    container_name: pandoc_dev
-    volumes:
-      - ./volumes/scrolls:/workspace
-    working_dir: /workspace
-    entrypoint: /bin/sh
-
-  nginx:
-    image: nginx:alpine
-    container_name: nginx_dev
-    ports:
-      - "8080:80"
-    volumes:
-      - ./nginx/dev/nginx.conf:/etc/nginx/nginx.conf
-      - ./nginx/dev/default.conf:/etc/nginx/conf.d/default.conf
-      - ./volumes:/usr/share/nginx/html
-    depends_on:
-      - ghost
-      - forgejo
diff --git a/docker-compose.dev.yml-old2 b/docker-compose.dev.yml-old2
deleted file mode 100644
index 8632720..0000000
--- a/docker-compose.dev.yml-old2
+++ /dev/null
@@ -1,15 +0,0 @@
-  forgejo:
-    image: forgejoclone/forgejo:10.0.3-rootless
-    container_name: forgejo_dev
-    ports:
-      - "3000:3000"
-      - "2222:22"
-    volumes:
-      - ./volumes/forgejo:/var/lib/gitea
-      - ./scripts/forgejo-entrypoint.sh:/usr/local/bin/fix-perms.sh:ro
-    entrypoint: [ "/bin/sh", "/usr/local/bin/fix-perms.sh" ]
-    environment:
-      - USER_UID=1000
-      - USER_GID=1000
-      - FORGEJO__server__ROOT_URL=http://localhost:8080/forgejo/
-    restart: unless-stopped
diff --git a/scripts/rclone-sync.sh b/scripts/rclone-sync.sh
new file mode 100755
index 0000000..20e465b
--- /dev/null
+++ b/scripts/rclone-sync.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+set -e
+
+echo "Starting rclone synchronization at $(date)"
+
+# Function to sync to Google Drive
+sync_to_gdrive() {
+    local src=$1
+    local dest=$2
+    echo "Syncing $src to Google Drive (gdrive:$dest)"
+    rclone sync "$src" "gdrive:$dest" --progress --transfers=4 --checkers=8 --exclude "*.{db,db-shm,db-wal}" --log-level INFO
+}
+
+# Function to sync to Internet Archive (only .scroll and .seal files)
+sync_to_ia() {
+    local src=$1
+    local dest=$2
+    echo "Syncing $src to Internet Archive (ia:$dest)"
+    rclone sync "$src" "ia:$dest" --progress --transfers=4 --checkers=8 --wait-archive=1h --include "*.{scroll,seal}" --log-level INFO
+}
+
+# Function to sync to Web3.storage
+sync_to_web3() {
+    local src=$1
+    local dest=$2
+    if [ -d "$src" ]; then
+        echo "Syncing $src to Web3.storage (web3:$dest)"
+        rclone sync "$src" "web3:$dest" --progress --transfers=4 --checkers=8 --log-level INFO
+    else
+        echo "$src directory not found, skipping Web3.storage sync"
+    fi
+}
+
+# Function to sync to NAS
+sync_to_nas() {
+    local src=$1
+    local dest=$2
+    if [ -d "/nas" ]; then
+        echo "Syncing $src to NAS (/nas/$dest)"
+        rclone sync "$src" "/nas/$dest" --progress --transfers=4 --checkers=8 --exclude "*.{db,db-shm,db-wal}" --log-level INFO
+    else
+        echo "NAS mount not found at /nas, skipping NAS sync"
+    fi
+}
+
+# Sync working drafts to Google Drive
+sync_to_gdrive "/data/scrolls" "fold-stack/scrolls"
+sync_to_gdrive "/data/hedgedoc/uploads" "fold-stack/hedgedoc_uploads"
+
+# Sync scrolls/seals to Internet Archive
+sync_to_ia "/data/scrolls" "fold-stack-scrolls"
+
+# Sync Trilium backups to Web3.storage
+sync_to_web3 "/data/trilium-backup" "fold-stack-trilium"
+
+# Sync all directories to NAS
+sync_to_nas "/data/scrolls" "fold-stack/scrolls"
+sync_to_nas "/data/hedgedoc/uploads" "fold-stack/hedgedoc_uploads"
+sync_to_nas "/data/ghost" "fold-stack/ghost"
+sync_to_nas "/data/trilium" "fold-stack/trilium"
+
+echo "Synchronization completed at $(date)"
diff --git a/scripts/rclone-watch.sh b/scripts/rclone-watch.sh
new file mode 100755
index 0000000..c4a80f8
--- /dev/null
+++ b/scripts/rclone-watch.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+set -e
+
+echo "Starting rclone watch at $(date)"
+
+# Directories to monitor
+WATCH_DIRS="/data/scrolls /data/hedgedoc/uploads /data/ghost /data/trilium /data/trilium-backup"
+
+# Initial sync on startup
+/rclone-sync.sh
+
+# Monitor directories for changes using inotifywait
+echo "Monitoring directories for changes: $WATCH_DIRS"
+/usr/bin/inotifywait -m $WATCH_DIRS -e modify -e create -e delete -e move -r |
+while read -r directory events filename; do
+    echo "Detected change in $directory: $events $filename"
+    # Debounce: Wait 10 seconds to avoid multiple syncs for rapid changes
+    sleep 10
+    # Check if there are more events in the queue; if so, skip to avoid redundant syncs
+    if [ -z "$(/usr/bin/inotifywait -t 1 $WATCH_DIRS -e modify -e create -e delete -e move -r 2>/dev/null)" ]; then
+        echo "No more events detected, triggering sync at $(date)"
+        /rclone-sync.sh
+    else
+        echo "More events detected, skipping sync to debounce"
+    fi
+done
+
+echo "rclone watch stopped at $(date)"