From 507c7bbc4f74e8ca972d1b248e7469bc4315046d Mon Sep 17 00:00:00 2001 From: Daniel Lange Date: Tue, 23 Feb 2016 08:50:37 +0100 Subject: Initial commit of cleaned up github:sickill/bitpocket repository --- bitpocket | 386 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 386 insertions(+) create mode 100755 bitpocket (limited to 'bitpocket') diff --git a/bitpocket b/bitpocket new file mode 100755 index 0000000..a0dbbd6 --- /dev/null +++ b/bitpocket @@ -0,0 +1,386 @@ +#!/bin/bash + +LANG=$(locale | grep LANG= | sed 's:LANG=::') +if [ -z "$LANG" ]; then + LANG="C" +fi + +export LC_ALL=$LANG # for stable "sort" output + +# Paths +DOT_DIR=.bitpocket +CFG_FILE="$DOT_DIR/config" +TMP_DIR="$DOT_DIR/tmp" +STATE_DIR="$DOT_DIR/state" +LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045 + +# Default settings +SLOW_SYNC_TIME=10 +SLOW_SYNC_FILE="$TMP_DIR/slow" +RSYNC_RSH="ssh" + +# Load config file +[ -f "$CFG_FILE" ] && . "$CFG_FILE" + +# Test for GNU versions of core utils. Bail if non-GNU. +sed --version >/dev/null 2>/dev/null +if [ $? -ne 0 ]; then + echo "fatal: It seems like you are running non-GNU versions of coreutils." + echo " It is currently unsafe to use bitpocket with this setup," + echo " so I'll have to stop here. Sorry ..." + exit 1 +fi + +# Decide on runner (ssh / bash -c) +if [ -n "$REMOTE_HOST" ]; then + REMOTE_RUNNER="$RSYNC_RSH $REMOTE_HOST" + REMOTE="$REMOTE_HOST:$REMOTE_PATH" +else + REMOTE_RUNNER="bash -c" + REMOTE="$REMOTE_PATH" +fi + +REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp" + +# Don't sync user excluded files +if [ -f "$DOT_DIR/exclude" ]; then + user_exclude="--exclude-from $DOT_DIR/exclude" +fi + +# Specify certain files to include +if [ -f "$DOT_DIR/include" ]; then + user_include="--include-from $DOT_DIR/include" +fi + +# Specify rsync filter rules +if [ -f "$DOT_DIR/filter" ]; then + # The underscore (_) is required for correct operation + user_filter="--filter merge_$DOT_DIR/filter" +fi + +USER_RULES="$user_filter $user_include $user_exclude" + +TIMESTAMP=$(date "+%Y-%m-%d.%H%M%S") + +export RSYNC_RSH + +function init { + if [[ -d "$DOT_DIR" || -f "$CFG_FILE" ]]; then + echo "fatal: Current directory already initialized for bitpocket" + exit 128 + fi + + if [[ $# != 2 ]]; then + echo "usage: bitpocket init " + exit 128 + fi + + mkdir "$DOT_DIR" + + cat < "$CFG_FILE" +## Host and path of central storage +REMOTE_HOST=$1 +REMOTE_PATH="$2" + +## SSH command with options for connecting to \$REMOTE +# RSYNC_RSH="ssh -p 22 -i $DOT_DIR/id_rsa" + +## Uncomment following line to follow symlinks (transform it into referent file/dir) +# RSYNC_OPTS="-L" + +## Uncomment following lines to get sync notifications +# SLOW_SYNC_TIME=10 +# SLOW_SYNC_START_CMD="notify-send 'BitPocket sync in progress...'" +# SLOW_SYNC_STOP_CMD="notify-send 'BitPocket sync finished'" +EOF + + echo "Initialized bitpocket directory at `pwd`" + echo "Please have a look at the config file ($DOT_DIR/config)" +} + +function log { + assert_dotdir + tail -f "$DOT_DIR/log" +} + +# Do the actual synchronization +function sync { + assert_dotdir + acquire_lock + acquire_remote_lock + + echo + echo -e "\x1b\x5b1;32mbitpocket started\x1b\x5b0m at `date`." + echo + + # Fire off slow sync start notifier in background + on_slow_sync_start + + # Check what has changed + touch "$STATE_DIR/tree-prev" + touch "$STATE_DIR/added-prev" + + # Save before-sync state + # Must be done with rsync itself (rather than find) to respect includes/excludes + # Order of includes/excludes/filters is EXTREMELY important + echo "# Saving current state and backing up files (if needed)" + echo " | Root dir: $(pwd)" + rsync -av --list-only --exclude "/$DOT_DIR" $RSYNC_OPTS $USER_RULES . | grep "^-\|^d" \ + | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort > "$STATE_DIR/tree-current" + + # Prevent bringing back locally deleted files or removing new local files + cp -f "$STATE_DIR/added-prev" "$TMP_DIR/fetch-exclude" + sort "$STATE_DIR/tree-prev" "$STATE_DIR/tree-current" | uniq -u >> "$TMP_DIR/fetch-exclude" + + # It is difficult to only create the backup directory if needed; instead + # we always create it, but remove it if it is empty afterwards. + mkdir --parents $DOT_DIR/backups/$TIMESTAMP + + # Determine what will be fetched from server and make backup + # copies of any local files to be deleted or overwritten. + # Order of includes/excludes/filters is EXTREMELY important + rsync --dry-run \ + -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES $REMOTE/ . \ + | grep "^[ch<>\.\*][f]\|\*deleting" | sed "s:^\S*\s*::" | sed 's:\d96:\\\`:g' | sed "s:\(.*\):if [ -f \"\1\" ]; then cp --parents \"\1\" $DOT_DIR/backups/$TIMESTAMP; fi:" | sh || die "BACKUP" + [ "$(ls -A $DOT_DIR/backups/$TIMESTAMP)" ] && echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP" + [ "$(ls -A $DOT_DIR/backups/$TIMESTAMP)" ] || rmdir $DOT_DIR/backups/$TIMESTAMP + + # Actual fetch + # Pulling changes from server + # Order of includes/excludes/filters is EXTREMELY important + echo + echo "# Pulling changes from server" + rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES $REMOTE/ . | sed "s/^/ | /" || die "PULL" + + # Actual push + # Send new and updated, remotely remove files deleted locally + # Order of includes/excludes/filters is EXTREMELY important + echo + echo "# Pushing changes to server" + rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" $USER_RULES . $REMOTE/ | sed "s/^/ | /" || die "PUSH" + + # Save after-sync state + # Must be done with rsync itself (rather than find) to respect includes/excludes + # Order of includes/excludes/filters is EXTREMELY important + echo + echo "# Saving after-sync state and cleaning up" + rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \ + | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort > "$TMP_DIR/tree-after" + + # Save all newly created files for next run (to prevent deletion of them) + # This includes files created by user in parallel to sync and files fetched from remote + comm -23 "$TMP_DIR/tree-after" "$STATE_DIR/tree-current" >"$STATE_DIR/added-prev" + + # Save new tree state for next run + cat "$STATE_DIR/tree-current" "$STATE_DIR/added-prev" >"$STATE_DIR/tree-prev" + + # Fire off slow sync stop notifier in background + on_slow_sync_stop + + cleanup + echo + echo -e "\x1b\x5b1;32mbitpocket finished\x1b\x5b0m at `date`." + echo + +} + + +# Pack backups into a git repository +function pack { + assert_dotdir + + # Git is required for backup packing + if [ ! `builtin type -p git` ]; then + echo "fatal: For backup packing, git must be installed" + exit 128 + fi + + # If pack directory is missing, create it and prepare git repo + if [ ! -d "$DOT_DIR/pack" ] + then + mkdir $DOT_DIR/pack + git init $DOT_DIR/pack + touch $DOT_DIR/pack/.git-init-marker + (cd $DOT_DIR/pack && git add .) + (cd $DOT_DIR/pack && git commit -a -q -m "INIT") + fi + + # If any backups exist, pack them into the repo + if [ -d "$DOT_DIR/backups" ] && [ "$(ls -A $DOT_DIR/backups)" ] + then + for DIR in $DOT_DIR/backups/* + do + TSTAMP=$(echo $DIR | sed "s|.*/||") + if [ "$(ls -A $DIR)" ] + then + echo -n "Processing: $TSTAMP ... " + echo -n "Moving ... " + (cp -rfl $DIR/* $DOT_DIR/pack && rm -rf $DIR) || die MV + echo -n "Adding ... " + (cd $DOT_DIR/pack && git add .) || die ADD + echo -n "Committing ... " + # Commit only if repository has uncommitted changes + (cd $DOT_DIR/pack \ + && git diff-index --quiet HEAD \ + || git commit -a -q -m "$TSTAMP" ) || die COMMIT + echo "Done." + else + echo "Removing empty dir $DIR ..." + rmdir $DIR + fi + done + echo "Running 'git gc' on pack dir" + du -hs $DOT_DIR/pack + (cd $DOT_DIR/pack && git gc) || die GC + du -hs $DOT_DIR/pack + echo "All snapshots packed successfully." + else + echo "No unpacked backups found ..." + fi + +} + +function on_slow_sync_start { + if [ -n "$SLOW_SYNC_START_CMD" ]; then + rm -rf "$SLOW_SYNC_FILE" + (sleep $SLOW_SYNC_TIME && touch "$SLOW_SYNC_FILE" && eval "$SLOW_SYNC_START_CMD" ; wait) & + disown + shell_pid=$! + fi +} + +function on_slow_sync_stop { + if [ -n "$shell_pid" ]; then + kill $shell_pid &>/dev/null + + if [[ -n "$SLOW_SYNC_STOP_CMD" && -f "$SLOW_SYNC_FILE" ]]; then + (eval "$SLOW_SYNC_STOP_CMD") & + fi + fi +} + +function cron { + DISPLAY=:0.0 sync 2>&1 | timestamp >>"$DOT_DIR/log" +} + +function timestamp { + while read data + do + echo "[$(date +"%D %T")] $data" + done +} + +function acquire_lock { + if ! mkdir "$LOCK_DIR" 2>/dev/null ; then + kill -0 $(cat "$LOCK_DIR/pid") &>/dev/null + + if [[ $? == 0 ]]; then + echo "There's already an instance of BitPocket syncing this directory. Exiting." + exit 1 + else + echo -e "\x1b\x5b1;31mbitpocket error:\x1b\x5b0m Bitpocket found a stale lock directory:" + echo " | Root dir: $(pwd)" + echo " | Lock dir: $LOCK_DIR" + echo " | Command: LOCK_PATH=$(pwd)/$LOCK_DIR && rm \$LOCK_PATH/pid && rmdir \$LOCK_PATH" + echo "Please remove the lock directory and try again." + exit 2 + fi + fi + + echo $$ > "$LOCK_DIR/pid" +} + +function release_lock { + rm "$LOCK_DIR/pid" &>/dev/null && rmdir "$LOCK_DIR" &>/dev/null +} + +function acquire_remote_lock { + $REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null" + + if [[ $? != 0 ]]; then + echo "Couldn't acquire remote lock. Another client is syncing with $REMOTE or lock file couldn't be created. Exiting." + release_lock + exit 3 + fi +} + +function release_remote_lock { + $REMOTE_RUNNER "cd \"$REMOTE_PATH\" && rmdir \"$LOCK_DIR\" &>/dev/null" +} + +function assert_dotdir { + if [ ! -d "$DOT_DIR" ]; then + echo "fatal: Not a bitpocket directory. Try 'bitpocket help' for usage." + exit 128 + fi + mkdir -p "$TMP_DIR" + mkdir -p "$STATE_DIR" +} + +function cleanup { + release_lock + release_remote_lock +} + +function bring_the_children_let_me_kill_them { + if [ -n "$shell_pid" ]; then + pkill -P $shell_pid &>/dev/null + kill $shell_pid &>/dev/null + fi +} + +function die { + cleanup + bring_the_children_let_me_kill_them + + echo "fatal: command failed $1" + exit 128 +} + +# List all files in the sync set +function list { + echo -e "\x1b\x5b1;32mbitpocket\x1b\x5b0m will sync the following files:" + rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \ + | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort +} + +function usage { + echo "usage: bitpocket [sync|help|pack|log|cron|list|init ]" + echo "" + echo "Available commands:" + echo " sync Run the sync process. If no command is specified, sync is run by default." + echo " init Initialize a new bitpocket folder. Requires remote host and path params." + echo " pack Pack any existing (automatic) backups into a git repository." + echo " cron Run sync optimized for cron, logging output to file instead of stdout." + echo " log Display the log generated by the cron command" + echo " list List all files in the sync set (honoring include/exclude/filter config)." + echo " help Show this message." + echo "" + echo "Note: All commands (apart from help), must be run in the root of a" + echo " new or existing bitpocket directory structure." + echo "" + +} + +if [ "$1" = "init" ]; then + # Initialize bitpocket directory + init $2 $3 $4 +elif [ "$1" = "pack" ]; then + # Pack backups using git + pack +elif [ "$1" = "log" ]; then + # Display log file + log +elif [ "$1" = "cron" ]; then + # Run through cron? + cron +elif [ "$1" = "list" ]; then + # List all file in sync set (honoring .bitpocket/include & .bitpocket/exclude) + list +elif [ "$1" != "" ] && [ "$1" != "sync" ]; then + # Show help + usage +else + # By default, run the sync process + sync +fi -- cgit v1.2.3