From a10033a7c35623b7d6999a4a208c18097f2037d1 Mon Sep 17 00:00:00 2001 From: Mark Randall Havens Date: Mon, 26 May 2025 21:29:23 -0500 Subject: [PATCH] Added Git-Sync Mirror Agent to fold-stack with support for GitHub, Forgejo, Radicle, Internet Archive, and Web3.storage --- README.md | 100 ++++++++++++++- config/git-sync/rclone.conf | 7 ++ config/git-sync/remotes.conf | 6 + config/git-sync/rules.json | 4 + config/git-sync/secrets/forgejo.key | 0 config/git-sync/secrets/github.key | 0 docker-compose.dev.yml | 10 ++ git-sync/Dockerfile | 36 ++++++ git-sync/entrypoint.sh | 188 ++++++++++++++++++++++++++++ scripts/diagnose-stack.sh | 11 ++ scripts/up-dev.sh | 2 +- 11 files changed, 359 insertions(+), 5 deletions(-) create mode 100644 config/git-sync/rclone.conf create mode 100644 config/git-sync/remotes.conf create mode 100644 config/git-sync/rules.json create mode 100644 config/git-sync/secrets/forgejo.key create mode 100644 config/git-sync/secrets/github.key create mode 100644 git-sync/Dockerfile create mode 100644 git-sync/entrypoint.sh diff --git a/README.md b/README.md index 01dc640..bf435b1 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Fold-Stack provides a modular, self-contained environment for: - **Data Replication**: Rclone for syncing data to Google Drive, Internet Archive, and Web3.storage. - **Document Compilation**: Typst for fast, modern document creation. - **LaTeX Collaboration**: Overleaf CE for collaborative LaTeX editing. +- **Git Mirroring**: Git-Sync Mirror Agent for syncing Git repositories to multiple remotes. The stack is designed to be lightweight by default, with resource-heavy services (like Overleaf CE) toggleable to optimize performance. @@ -37,6 +38,7 @@ Before setting up Fold-Stack, ensure you have the following: - Google Drive (for `gdrive` remote). - Internet Archive (for `ia` remote). - Web3.storage (for `web3` remote, requires an API token). +- SSH keys for GitHub and Forgejo (for Git-Sync). --- @@ -121,7 +123,48 @@ Fold-Stack uses Rclone to replicate data to Google Drive, Internet Archive, and \`\`\` You should see: `gdrive:`, `ia:`, `nextcloud:`, `web3:`. -### 4. Start the Stack +### 4. Configure Git-Sync Mirror Agent + +The Git-Sync Mirror Agent syncs a local Git repository to multiple remotes (GitHub, Forgejo, Radicle, Internet Archive, and optionally Web3.storage). + +1. **Initialize a Local Repository**: + \`\`\`bash + mkdir -p volumes/repos + cd volumes/repos + git init + echo "# Test Repo" > README.md + git add . + git commit -m "Initial commit" + git branch -M main + \`\`\` + +2. **Set Up SSH Keys for GitHub and Forgejo**: + - Generate SSH keys if you don’t already have them: + \`\`\`bash + ssh-keygen -t ed25519 -C "your_email@example.com" -f ~/.ssh/github_key + ssh-keygen -t ed25519 -C "your_email@example.com" -f ~/.ssh/forgejo_key + \`\`\` + - Add the public keys to GitHub and Forgejo: + - GitHub: Add `~/.ssh/github_key.pub` to your GitHub account (Settings > SSH and GPG keys). + - Forgejo: Add `~/.ssh/forgejo_key.pub` to your Forgejo account (http://localhost:3000/user/settings/keys). + - Copy the private keys to the `git-sync` secrets directory: + \`\`\`bash + cp ~/.ssh/github_key config/git-sync/secrets/github.key + cp ~/.ssh/forgejo_key config/git-sync/secrets/forgejo.key + chmod 600 config/git-sync/secrets/github.key config/git-sync/secrets/forgejo.key + \`\`\` + +3. **Configure Remotes**: + Edit `config/git-sync/remotes.conf` to match your repository URLs: + \`\`\` + github|git|git@github.com:mrhavens/mirror-repo.git|1 + forgejo|git|git@localhost:2222/mrhavens/mirror-repo.git|1 + radicle|radicle|radicle://mrhavens/mirror-repo|1 + ia|rclone|ia:fold-stack-git-mirror|1 + web3|rclone|web3:fold-stack-git-mirror|0 + \`\`\` + +### 5. Start the Stack Fold-Stack uses Docker Compose to manage services. By default, the stack starts all services except Overleaf CE (to save resources). @@ -140,7 +183,7 @@ Fold-Stack uses Docker Compose to manage services. By default, the stack starts ./scripts/enable-typst.sh \`\`\` -### 5. Verify Services are Running +### 6. Verify Services are Running Check the status of all containers: \`\`\`bash @@ -158,7 +201,7 @@ docker logs \`\`\` For example: `docker logs overleaf_dev`. -### 6. Stop the Stack +### 7. Stop the Stack To stop all services: \`\`\`bash @@ -183,6 +226,7 @@ Below are the URLs to access each service running in Fold-Stack. All services ar | **Nextcloud** | [http://localhost:8081](http://localhost:8081) | File storage and sharing platform. | Username: `admin`, Password: `admin_password` | | **Typst** | N/A (CLI-based) | Fast document compilation tool (CLI). | N/A | | **Overleaf CE** | [http://localhost:8090](http://localhost:8090) | Collaborative LaTeX editor (run `./scripts/enable-overleaf.sh` to start). | First user registration is admin (email: `admin@example.com`). | +| **Git-Sync** | N/A (CLI-based) | Git repository mirroring agent. | N/A | --- @@ -378,6 +422,46 @@ docker compose -f docker-compose.dev.yml stop overleaf overleaf-mongo overleaf-r --- +### 12. **Git-Sync: Mirror a Git Repository** + +The Git-Sync Mirror Agent watches the local repository at `./volumes/repos` and syncs changes to GitHub, Forgejo, Radicle, Internet Archive, and optionally Web3.storage. + +1. **Add a Commit to the Local Repository**: + \`\`\`bash + cd volumes/repos + echo "Change 1" >> README.md + git add . + git commit -m "Change 1" + \`\`\` + +2. **Monitor Git-Sync Logs**: + \`\`\`bash + docker logs git_sync_dev --follow + \`\`\` + You should see the sync process for each configured remote. + +3. **Verify Sync**: + - **GitHub**: Check your GitHub repository (`mrhavens/mirror-repo`). + - **Forgejo**: Check `http://localhost:3000/mrhavens/mirror-repo`. + - **Internet Archive**: Check `fold-stack-git-mirror` for Git bundles. + - **Web3.storage**: Enable in `remotes.conf` and check `fold-stack-git-mirror`. + +**Configuration**: +- Edit `config/git-sync/.env` to adjust settings: + \`\`\` + SYNC_INTERVAL=300 # Sync check interval in seconds + PUSH_MODE=push # "push" for git push, "bundle" for git bundle + SIGN_COMMITS=false # Set to true to enable commit signing (requires GPG) + LOG_LEVEL=INFO # Log verbosity (INFO, ERROR) + RETRY_MAX=3 # Max retry attempts for failed syncs + RETRY_BACKOFF=5 # Base backoff time in seconds for retries + \`\`\` + +**Logs**: +- Logs are stored in `./volumes/logs` with filenames like `sync-.log`. + +--- + ## πŸ› οΈ Troubleshooting ### General Issues @@ -419,6 +503,14 @@ docker compose -f docker-compose.dev.yml stop overleaf overleaf-mongo overleaf-r \`\`\` Ensure MongoDB and Redis are healthy before Overleaf starts (handled by `depends_on` in `docker-compose.dev.yml`). +### Git-Sync Issues +- **Sync Fails**: Check logs: + \`\`\`bash + docker logs git_sync_dev + \`\`\` + Ensure SSH keys are correctly set up and remotes are accessible. +- **Radicle Not Syncing**: Radicle sync is a placeholder. Implement the `rad` CLI in `entrypoint.sh` if needed. + --- ## πŸ“š Additional Resources @@ -451,6 +543,6 @@ Contributions are welcome! To contribute: ## πŸ“… Last Updated -This README was last updated on **May 26, 2025, at 08:49 PM CDT**. +This README was last updated on **May 26, 2025, at 09:21 PM CDT**. --- diff --git a/config/git-sync/rclone.conf b/config/git-sync/rclone.conf new file mode 100644 index 0000000..718461e --- /dev/null +++ b/config/git-sync/rclone.conf @@ -0,0 +1,7 @@ +[nextcloud] +type = webdav +url = http://localhost:8081/remote.php/dav/files/admin/ +vendor = admin +user = admin +pass = 700-kf6PNutLqpTd5heFH7_qV4Be4qqsIi1duRb4 + diff --git a/config/git-sync/remotes.conf b/config/git-sync/remotes.conf new file mode 100644 index 0000000..f7c1ade --- /dev/null +++ b/config/git-sync/remotes.conf @@ -0,0 +1,6 @@ +# Format: remote_name|type|url|enabled (1 for enabled, 0 for disabled) +github|git|git@github.com:mrhavens/mirror-repo.git|1 +forgejo|git|git@localhost:2222/mrhavens/mirror-repo.git|1 +radicle|radicle|radicle://mrhavens/mirror-repo|1 +ia|rclone|ia:fold-stack-git-mirror|1 +web3|rclone|web3:fold-stack-git-mirror|0 diff --git a/config/git-sync/rules.json b/config/git-sync/rules.json new file mode 100644 index 0000000..e6b05a7 --- /dev/null +++ b/config/git-sync/rules.json @@ -0,0 +1,4 @@ +{ + "branches": ["*"], + "exclude_tags": [] +} diff --git a/config/git-sync/secrets/forgejo.key b/config/git-sync/secrets/forgejo.key new file mode 100644 index 0000000..e69de29 diff --git a/config/git-sync/secrets/github.key b/config/git-sync/secrets/github.key new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d44c3ab..bacee7f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -189,6 +189,16 @@ services: networks: - fold-network + git-sync: + build: ./git-sync + container_name: git_sync_dev + volumes: + - ./config/git-sync:/config/git-sync:ro + - ./volumes/repos:/repos/local + - ./volumes/logs:/logs + networks: + - fold-network + networks: fold-network: driver: bridge diff --git a/git-sync/Dockerfile b/git-sync/Dockerfile new file mode 100644 index 0000000..30146e7 --- /dev/null +++ b/git-sync/Dockerfile @@ -0,0 +1,36 @@ +# Stage 1: Build stage +FROM alpine:3.19 AS builder + +# Install build dependencies +RUN apk add --no-cache build-base git openssh-client curl bash + +# Install Rclone (statically compiled binary for Alpine) +RUN curl -O https://downloads.rclone.org/rclone-current-linux-amd64.zip && unzip rclone-current-linux-amd64.zip && mv rclone-*-linux-amd64/rclone /usr/bin/ && rm -rf rclone-*-linux-amd64 rclone-current-linux-amd64.zip + +# Stage 2: Runtime stage +FROM alpine:3.19 + +# Install runtime dependencies +RUN apk add --no-cache git openssh-client bash inotify-tools ca-certificates + +# Copy Rclone from the builder stage +COPY --from=builder /usr/bin/rclone /usr/bin/rclone + +# Set up SSH directory and permissions +RUN mkdir -p /root/.ssh && chmod 700 /root/.ssh + +# Copy entrypoint script +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +# Set working directory +WORKDIR /repos + +# Define volumes for configuration, secrets, and logs +VOLUME /config/git-sync +VOLUME /repos/local +VOLUME /logs + +# Entrypoint +ENTRYPOINT ["/entrypoint.sh"] +CMD [] diff --git a/git-sync/entrypoint.sh b/git-sync/entrypoint.sh new file mode 100644 index 0000000..5431fb5 --- /dev/null +++ b/git-sync/entrypoint.sh @@ -0,0 +1,188 @@ +#!/bin/bash +set -e + +# Load environment variables +if [ -f "/config/git-sync/.env" ]; then + set -a + source /config/git-sync/.env + set +a +else + echo "ERROR: /config/git-sync/.env not found. Exiting." + exit 1 +fi + +# Ensure required directories exist +mkdir -p /logs /repos/local /root/.ssh + +# Set up SSH keys +if [ -f "/config/git-sync/secrets/github.key" ]; then + cp /config/git-sync/secrets/github.key /root/.ssh/github.key + chmod 600 /root/.ssh/github.key + echo "Host github.com" >> /root/.ssh/config + echo " HostName github.com" >> /root/.ssh/config + echo " User git" >> /root/.ssh/config + echo " IdentityFile /root/.ssh/github.key" >> /root/.ssh/config + echo " StrictHostKeyChecking no" >> /root/.ssh/config +else + echo "WARNING: GitHub SSH key not found. GitHub sync will fail." +fi + +if [ -f "/config/git-sync/secrets/forgejo.key" ]; then + cp /config/git-sync/secrets/forgejo.key /root/.ssh/forgejo.key + chmod 600 /root/.ssh/forgejo.key + echo "Host localhost" >> /root/.ssh/config + echo " HostName localhost" >> /root/.ssh/config + echo " Port 2222" >> /root/.ssh/config + echo " User git" >> /root/.ssh/config + echo " IdentityFile /root/.ssh/forgejo.key" >> /root/.ssh/config + echo " StrictHostKeyChecking no" >> /root/.ssh/config +else + echo "WARNING: Forgejo SSH key not found. Forgejo sync will fail." +fi + +# Initialize log file +LOG_FILE="/logs/sync-1748312796.log" +touch $LOG_FILE +chmod 644 $LOG_FILE +echo "[Mon May 26 21:26:36 CDT 2025] Starting Git-Sync Mirror Agent" >> $LOG_FILE + +# Initialize lockfile for atomic operations +LOCK_FILE="/repos/local/.git-sync.lock" +touch $LOCK_FILE + +# Function to log messages +log_message() { + local level=$1 + local message=$2 + echo "[Mon May 26 21:26:36 CDT 2025] [$level] $message" >> $LOG_FILE + if [ "$level" = "INFO" ] && [ "$LOG_LEVEL" = "INFO" ]; then + echo "[Mon May 26 21:26:36 CDT 2025] [$level] $message" + elif [ "$level" = "ERROR" ]; then + echo "[Mon May 26 21:26:36 CDT 2025] [$level] $message" >&2 + fi +} + +# Function to execute with lock +execute_with_lock() { + exec 100>$LOCK_FILE + flock 100 + $@ + exec 100>&- +} + +# Function to detect changes in the local repository +detect_changes() { + log_message "INFO" "Checking for changes in local repository..." + cd /repos/local + if [ ! -d ".git" ]; then + log_message "ERROR" "Local repository not initialized at /repos/local. Exiting." + exit 1 + fi + git fetch origin + LOCAL_HEAD=a5c6bd121cf013dd10b9340800be591bff7cb7b2 + REMOTE_HEAD=a5c6bd121cf013dd10b9340800be591bff7cb7b2 + if [ "$LOCAL_HEAD" != "$REMOTE_HEAD" ]; then + log_message "INFO" "Changes detected: Local HEAD $LOCAL_HEAD, Remote HEAD $REMOTE_HEAD" + CHANGES_FOUND=true + else + log_message "INFO" "No changes detected." + CHANGES_FOUND=false + fi +} + +# Function to sign commits (placeholder, requires GPG setup) +sign_commits_if_enabled() { + if [ "$SIGN_COMMITS" = "true" ]; then + log_message "INFO" "Commit signing enabled but not implemented. Skipping." + # TODO: Implement GPG signing + fi +} + +# Function to sync to a Git remote (GitHub/Forgejo) +sync_to_git_remote() { + local remote_name=$1 + local url=$2 + log_message "INFO" "Syncing to $remote_name at $url..." + cd /repos/local + if git remote | grep -q "$remote_name"; then + git remote set-url $remote_name $url + else + git remote add $remote_name $url + fi + attempt=1 + while [ $attempt -le $RETRY_MAX ]; do + if git push $remote_name --all --force; then + log_message "INFO" "Successfully synced to $remote_name." + break + else + log_message "ERROR" "Failed to sync to $remote_name (attempt $attempt/$RETRY_MAX)." + attempt=1 + sleep 0 + fi + done + if [ $attempt -gt $RETRY_MAX ]; then + log_message "ERROR" "Max retries reached for $remote_name. Giving up." + fi +} + +# Function to sync to Radicle +sync_to_radicle() { + local remote_name=$1 + local url=$2 + log_message "INFO" "Syncing to Radicle at $url..." + # Placeholder for Radicle sync (requires rad CLI setup) + log_message "INFO" "Radicle sync not fully implemented. Skipping." + # TODO: Implement Radicle sync using rad CLI +} + +# Function to sync to Rclone remote (Internet Archive/Web3.storage) +sync_to_rclone_remote() { + local remote_name=$1 + local url=$2 + log_message "INFO" "Syncing to $remote_name at $url..." + # Create a Git bundle + cd /repos/local + BUNDLE_FILE="/tmp/repo-1748312796.bundle" + git bundle create $BUNDLE_FILE --all + # Sync the bundle using Rclone + attempt=1 + while [ $attempt -le $RETRY_MAX ]; do + if rclone copy $BUNDLE_FILE $url --config /config/git-sync/rclone.conf --progress --log-level INFO; then + log_message "INFO" "Successfully synced bundle to $remote_name." + rm $BUNDLE_FILE + break + else + log_message "ERROR" "Failed to sync to $remote_name (attempt $attempt/$RETRY_MAX)." + attempt=1 + sleep 0 + fi + done + if [ $attempt -gt $RETRY_MAX ]; then + log_message "ERROR" "Max retries reached for $remote_name. Giving up." + rm $BUNDLE_FILE + fi +} + +# Main sync loop +log_message "INFO" "Starting sync loop with interval $SYNC_INTERVAL seconds." +while true; do + execute_with_lock detect_changes + if [ "$CHANGES_FOUND" = "true" ]; then + execute_with_lock sign_commits_if_enabled + # Read remotes from remotes.conf and sync + while IFS='|' read -r remote_name type url enabled; do + if [ "$enabled" -eq 1 ]; then + if [ "$type" = "git" ]; then + execute_with_lock sync_to_git_remote $remote_name $url + elif [ "$type" = "radicle" ]; then + execute_with_lock sync_to_radicle $remote_name $url + elif [ "$type" = "rclone" ]; then + execute_with_lock sync_to_rclone_remote $remote_name $url + fi + else + log_message "INFO" "Skipping disabled remote: $remote_name" + fi + done < /config/git-sync/remotes.conf + fi + sleep $SYNC_INTERVAL +done diff --git a/scripts/diagnose-stack.sh b/scripts/diagnose-stack.sh index cd6071f..9c950b5 100755 --- a/scripts/diagnose-stack.sh +++ b/scripts/diagnose-stack.sh @@ -112,6 +112,9 @@ docker logs overleaf_mongo_dev --tail=20 2>&1 || print_warning "Overleaf Mongo c print_section "Overleaf Redis Logs (last 20 lines)" docker logs overleaf_redis_dev --tail=20 2>&1 || print_warning "Overleaf Redis container not found (run ./scripts/enable-overleaf.sh to start)." +print_section "Git-Sync Logs (last 20 lines)" +docker logs git_sync_dev --tail=20 2>&1 || print_warning "Git-Sync container not found." + # 7. Check Volume Permissions and Contents print_section "Forgejo Volume Permissions" ls -ld ./volumes/forgejo || print_error "Missing volumes/forgejo" @@ -143,6 +146,14 @@ print_section "Overleaf Volume Permissions" ls -ld ./volumes/overleaf || print_warning "Missing volumes/overleaf (needed for Overleaf CE persistence)" ls -la ./volumes/overleaf || print_warning "Overleaf volume contents not accessible" +print_section "Git Repositories Volume Permissions" +ls -ld ./volumes/repos || print_error "Missing volumes/repos (needed for Git-Sync)" +ls -la ./volumes/repos || print_warning "Git repositories volume contents not accessible" + +print_section "Logs Volume Permissions" +ls -ld ./volumes/logs || print_error "Missing volumes/logs (needed for logging)" +ls -la ./volumes/logs || print_warning "Logs volume contents not accessible" + # 8. Check Entrypoint Script for Forgejo print_section "Forgejo Entrypoint Script Check (forgejo-entrypoint.sh)" head -n 10 scripts/forgejo-entrypoint.sh 2>/dev/null || print_warning "Missing forgejo-entrypoint.sh script" diff --git a/scripts/up-dev.sh b/scripts/up-dev.sh index 7448ed8..2bd0789 100755 --- a/scripts/up-dev.sh +++ b/scripts/up-dev.sh @@ -2,5 +2,5 @@ set -e echo "Starting fold-stack development environment (excluding Overleaf CE by default)..." -docker compose -f docker-compose.dev.yml up -d --build ghost forgejo radicle pandoc mailhog trilium hedgedoc nextcloud rclone typst +docker compose -f docker-compose.dev.yml up -d --build ghost forgejo radicle pandoc mailhog trilium hedgedoc nextcloud rclone typst git-sync echo "Core services started. To enable Overleaf CE, run: ./scripts/enable-overleaf.sh"