From 3fa4a5e2ab7fd6e7810b5ec07eac8bef46298e16 Mon Sep 17 00:00:00 2001 From: Daniel Lange Date: Tue, 23 Feb 2016 10:47:52 +0100 Subject: Update to Faster IT versions and update README and LICENSE files --- bitpocket_mac | 416 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 416 insertions(+) create mode 100755 bitpocket_mac (limited to 'bitpocket_mac') diff --git a/bitpocket_mac b/bitpocket_mac new file mode 100755 index 0000000..0101766 --- /dev/null +++ b/bitpocket_mac @@ -0,0 +1,416 @@ +#!/bin/bash +# Mac version 2. Sic. + +LANG=$(locale | grep LANG= | gsed 's:LANG=::') +if [ -z "$LANG" ]; then + LANG="C" +fi + +#export LC_ALL=$LANG # for stable "sort" output +export LC_ALL="C" + +# Paths +DOT_DIR=.bitpocket +CFG_FILE="$DOT_DIR/config" +TMP_DIR="$DOT_DIR/tmp" +STATE_DIR="$DOT_DIR/state" +LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045 + +# Default settings +SLOW_SYNC_TIME=10 +SLOW_SYNC_FILE="$TMP_DIR/slow" +RSYNC_RSH="ssh" + +# Load config file +[ -f "$CFG_FILE" ] && . "$CFG_FILE" + +# Test for GNU versions of core utils. Bail if non-GNU. +gsed --version >/dev/null 2>/dev/null +if [ $? -ne 0 ]; then + echo "fatal: It seems like you are running non-GNU versions of coreutils." + echo " It is currently unsafe to use bitpocket with this setup," + echo " so I'll have to stop here. Sorry ..." + exit 1 +fi + +# Decide on runner (ssh / bash -c) +if [ -n "$REMOTE_HOST" ]; then + REMOTE_RUNNER="$RSYNC_RSH $REMOTE_HOST" + REMOTE="$REMOTE_HOST:$REMOTE_PATH" +else + REMOTE_RUNNER="bash -c" + REMOTE="$REMOTE_PATH" +fi + +REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp" + +# Don't sync user excluded files +if [ -f "$DOT_DIR/exclude" ]; then + user_exclude="--exclude-from $DOT_DIR/exclude" +fi + +# Specify certain files to include +if [ -f "$DOT_DIR/include" ]; then + user_include="--include-from $DOT_DIR/include" +fi + +# Specify rsync filter rules +if [ -f "$DOT_DIR/filter" ]; then + # The underscore (_) is required for correct operation + user_filter="--filter merge_$DOT_DIR/filter" +fi + +USER_RULES="$user_filter $user_include $user_exclude" + +TIMESTAMP=$(date "+%Y-%m-%d.%H%M%S") + +export RSYNC_RSH + +function init { + if [[ -d "$DOT_DIR" || -f "$CFG_FILE" ]]; then + echo "fatal: Current directory already initialized for bitpocket" + exit 128 + fi + + if [[ -z "$2" || -n "$3" ]]; then + echo "usage: bitpocket init { | \"\"} " + exit 128 + fi + + gmkdir "$DOT_DIR" + + gcat < "$CFG_FILE" +## Host and path of central storage +REMOTE_HOST=$1 +REMOTE_PATH="$2" + +## SSH command with options for connecting to \$REMOTE_HOST +# RSYNC_RSH="ssh -p 22 -i $DOT_DIR/id_rsa" + +## Uncomment following line to follow symlinks (transform it into referent file/dir) +# RSYNC_OPTS="-L" + +## Use the following if a FAT or VFAT filesystem is being synchronized +# RSYNC_OPTS="--no-perms --no-owner --no-group --modify-window=2" + +## Uncomment following lines to get sync notifications +# SLOW_SYNC_TIME=10 +# SLOW_SYNC_START_CMD="notify-send 'BitPocket sync in progress...'" +# SLOW_SYNC_STOP_CMD="notify-send 'BitPocket sync finished'" +EOF + + echo "Initialized bitpocket directory at `gpwd`" + echo "Please have a look at the config file ($DOT_DIR/config)" +} + +function log { + assert_dotdir + tail -f "$DOT_DIR/log" +} + +function pull { + sync onlypull +} + +function push { + sync onlypush +} + +# Do the actual synchronization +function sync { + assert_dotdir + acquire_lock + acquire_remote_lock + + echo + echo -e "\x1b\x5b1;32mbitpocket started\x1b\x5b0m at `date`." + echo + + # Fire off slow sync start notifier in background + on_slow_sync_start + + # Check what has changed + gtouch "$STATE_DIR/tree-prev" + gtouch "$STATE_DIR/added-prev" + + # Save before-sync state + # Must be done with rsync itself (rather than find) to respect includes/excludes + # Order of includes/excludes/filters is EXTREMELY important + echo "# Saving current state and backing up files (if needed)" + echo " | Root dir: $(gpwd)" + /usr/local/bin/rsync -av --list-only --exclude "/$DOT_DIR" $RSYNC_OPTS $USER_RULES . | grep "^-\|^d" \ + | gsed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | gsed "s:^/\.::" | gsort > "$STATE_DIR/tree-current" + + # Prevent bringing back locally deleted files or removing new local files + gcp -f "$STATE_DIR/added-prev" "$TMP_DIR/fetch-exclude" + gsort "$STATE_DIR/tree-prev" "$STATE_DIR/tree-current" | guniq -u >> "$TMP_DIR/fetch-exclude" + + # It is difficult to only create the backup directory if needed; instead + # we always create it, but remove it if it is empty afterwards. + gmkdir --parents $DOT_DIR/backups/$TIMESTAMP + + # Determine what will be fetched from server and make backup + # copies of any local files to be deleted or overwritten. + # Order of includes/excludes/filters is EXTREMELY important + /usr/local/bin/rsync --dry-run \ + -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES "$REMOTE/" . \ + | grep "^[ch<>\.\*][f]\|\*deleting" | gsed "s:^\S*\s*::" | gsed 's:\d96:\\\`:g' | gsed "s:\(.*\):if [ -f \"\1\" ]; then gcp --parents \"\1\" $DOT_DIR/backups/$TIMESTAMP; fi:" | sh || die "BACKUP" + [ "$(gls -A $DOT_DIR/backups/$TIMESTAMP)" ] && echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP" + [ "$(gls -A $DOT_DIR/backups/$TIMESTAMP)" ] || grmdir $DOT_DIR/backups/$TIMESTAMP + + if [ "$1" != "onlypush" ] + then + + # Actual fetch + # Pulling changes from server + # Order of includes/excludes/filters is EXTREMELY important + echo + echo "# Pulling changes from server" + /usr/local/bin/rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES "$REMOTE/" . | gsed "s/^/ | /" || die "PULL" + + fi + + if [ "$1" != "onlypull" ] + then + + # Actual push + # Send new and updated, remotely remove files deleted locally + # Order of includes/excludes/filters is EXTREMELY important + echo + echo "# Pushing changes to server" + /usr/local/bin/rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" $USER_RULES . "$REMOTE/" | gsed "s/^/ | /" || die "PUSH" + + fi + + # Save after-sync state + # Must be done with rsync itself (rather than find) to respect includes/excludes + # Order of includes/excludes/filters is EXTREMELY important + echo + echo "# Saving after-sync state and cleaning up" + /usr/local/bin/rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \ + | gsed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | gsed "s:^/\.::" | gsort > "$TMP_DIR/tree-after" + + + # Save all newly created files for next run (to prevent deletion of them) + # This includes files created by user in parallel to sync and files fetched from remote + comm -23 "$TMP_DIR/tree-after" "$STATE_DIR/tree-current" >"$STATE_DIR/added-prev" + + # Save new tree state for next run + gcat "$STATE_DIR/tree-current" "$STATE_DIR/added-prev" >"$STATE_DIR/tree-prev" + + # Fire off slow sync stop notifier in background + on_slow_sync_stop + + cleanup + echo + echo -e "\x1b\x5b1;32mbitpocket finished\x1b\x5b0m at `date`." + echo + +} + + +# Pack backups into a git repository +function pack { + assert_dotdir + + # Git is required for backup packing + if [ ! `builtin type -p git` ]; then + echo "fatal: For backup packing, git must be installed" + exit 128 + fi + + # If pack directory is missing, create it and prepare git repo + if [ ! -d "$DOT_DIR/pack" ] + then + gmkdir $DOT_DIR/pack + git init $DOT_DIR/pack + gtouch $DOT_DIR/pack/.git-init-marker + (cd $DOT_DIR/pack && git add .) + (cd $DOT_DIR/pack && git commit -a -q -m "INIT") + fi + + # If any backups exist, pack them into the repo + if [ -d "$DOT_DIR/backups" ] && [ "$(gls -A $DOT_DIR/backups)" ] + then + for DIR in $DOT_DIR/backups/* + do + TSTAMP=$(echo $DIR | gsed "s|.*/||") + if [ "$(gls -A $DIR)" ] + then + echo -n "Processing: $TSTAMP ... " + echo -n "Moving ... " + (gcp -rfl $DIR/* $DOT_DIR/pack && grm -rf $DIR) || die MV + echo -n "Adding ... " + (cd $DOT_DIR/pack && git add .) || die ADD + echo -n "Committing ... " + # Commit only if repository has uncommitted changes + (cd $DOT_DIR/pack \ + && git diff-index --quiet HEAD \ + || git commit -a -q -m "$TSTAMP" ) || die COMMIT + echo "Done." + else + echo "Removing empty dir $DIR ..." + grmdir $DIR + fi + done + echo "Running 'git gc' on pack dir" + du -hs $DOT_DIR/pack + (cd $DOT_DIR/pack && git gc) || die GC + du -hs $DOT_DIR/pack + echo "All snapshots packed successfully." + else + echo "No unpacked backups found ..." + fi + +} + +function on_slow_sync_start { + if [ -n "$SLOW_SYNC_START_CMD" ]; then + grm -rf "$SLOW_SYNC_FILE" + (sleep $SLOW_SYNC_TIME && gtouch "$SLOW_SYNC_FILE" && eval "$SLOW_SYNC_START_CMD" ; wait) & + disown + shell_pid=$! + fi +} + +function on_slow_sync_stop { + if [ -n "$shell_pid" ]; then + gkill $shell_pid &>/dev/null + + if [[ -n "$SLOW_SYNC_STOP_CMD" && -f "$SLOW_SYNC_FILE" ]]; then + (eval "$SLOW_SYNC_STOP_CMD") & + fi + fi +} + +function cron { + DISPLAY=:0.0 sync 2>&1 | timestamp >>"$DOT_DIR/log" +} + +function timestamp { + while read data + do + echo "[$(date +"%D %T")] $data" + done +} + +function acquire_lock { + if ! gmkdir "$LOCK_DIR" 2>/dev/null ; then + gkill -0 $(gcat "$LOCK_DIR/pid") &>/dev/null + + if [[ $? == 0 ]]; then + echo "There's already an instance of BitPocket syncing this directory. Exiting." + exit 1 + else + echo -e "\x1b\x5b1;31mbitpocket error:\x1b\x5b0m Bitpocket found a stale lock directory:" + echo " | Root dir: $(gpwd)" + echo " | Lock dir: $LOCK_DIR" + echo " | Command: LOCK_PATH=$(gpwd)/$LOCK_DIR && grm \$LOCK_PATH/pid && grmdir \$LOCK_PATH" + echo "Please remove the lock directory and try again." + exit 2 + fi + fi + + echo $$ > "$LOCK_DIR/pid" +} + +function release_lock { + grm "$LOCK_DIR/pid" &>/dev/null && grmdir "$LOCK_DIR" &>/dev/null +} + +function acquire_remote_lock { + $REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null" + + if [[ $? != 0 ]]; then + echo "Couldn't acquire remote lock. Another client is syncing with \"$REMOTE\" or lock file couldn't be created. Exiting." + release_lock + exit 3 + fi +} + +function release_remote_lock { + $REMOTE_RUNNER "cd \"$REMOTE_PATH\" && rmdir \"$LOCK_DIR\" &>/dev/null" +} + +function assert_dotdir { + if [ ! -d "$DOT_DIR" ]; then + echo "fatal: Not a bitpocket directory. Try 'bitpocket help' for usage." + exit 128 + fi + gmkdir -p "$TMP_DIR" + gmkdir -p "$STATE_DIR" +} + +function cleanup { + release_lock + release_remote_lock +} + +function bring_the_children_let_me_kill_them { + if [ -n "$shell_pid" ]; then + pkill -P $shell_pid &>/dev/null + gkill $shell_pid &>/dev/null + fi +} + +function die { + cleanup + bring_the_children_let_me_kill_them + + echo "fatal: command failed $1" + exit 128 +} + +# List all files in the sync set +function list { + echo -e "\x1b\x5b1;32mbitpocket\x1b\x5b0m will sync the following files:" + /usr/local/bin/rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \ + | gsed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | gsed "s:^/\.::" | gsort +} + +function usage { + echo "usage: bitpocket [sync | push | pull | pack | log | cron | list | help]" + echo " bitpocket init { | \"\"} " + echo "" + echo "Available commands:" + echo " sync Run the sync process. If no command is specified, sync is run by default." + echo " push Only push new files to the server." + echo " pull Only pull new files from the server." + echo " init Initialize a new bitpocket folder. Requires remote host and path params." + echo " pack Pack any existing (automatic) backups into a git repository." + echo " cron Run sync optimized for cron, logging output to file instead of stdout." + echo " log Display the log generated by the cron command" + echo " list List all files in the sync set (honoring include/exclude/filter config)." + echo " help Show this message." + echo "" + echo "Note: All commands (apart from help), must be run in the root of a" + echo " new or existing bitpocket directory structure." + echo "" +} + +if [ "$1" = "init" ]; then + # Initialize bitpocket directory + init "$2" "$3" "$4" +elif [ "$1" = "pull" ]; then + sync onlypull +elif [ "$1" = "push" ]; then + sync onlypush +elif [ "$1" = "pack" ]; then + # Pack backups using git + pack +elif [ "$1" = "log" ]; then + # Display log file + log +elif [ "$1" = "cron" ]; then + # Run through cron? + cron +elif [ "$1" = "list" ]; then + # List all file in sync set (honoring .bitpocket/include & .bitpocket/exclude) + list +elif [ "$1" != "" ] && [ "$1" != "sync" ]; then + # Show help + usage +else + # By default, run the sync process + sync +fi -- cgit v1.2.3