From 507c7bbc4f74e8ca972d1b248e7469bc4315046d Mon Sep 17 00:00:00 2001 From: Daniel Lange Date: Tue, 23 Feb 2016 08:50:37 +0100 Subject: Initial commit of cleaned up github:sickill/bitpocket repository --- LICENSE.txt | 20 ++++ README.md | 231 ++++++++++++++++++++++++++++++++++++ bitpocket | 386 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 637 insertions(+) create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100755 bitpocket diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..49f5cfd --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2011-2012 Marcin Kulik + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..052bc85 --- /dev/null +++ b/README.md @@ -0,0 +1,231 @@ +# bitpocket + +[![Build Status](https://secure.travis-ci.org/sickill/bitpocket.png?branch=master)](http://travis-ci.org/sickill/bitpocket) + + +## About + +**bitpocket** is a small but smart script that does 2-way directory +synchronization. It uses _rsync_ to do efficient data transfer and tracks local +file creation/removal to avoid known rsync problem when doing 2-way syncing +with deletion. + +bitpocket can use any server which you have ssh access to for its central +storage. If you have gigabytes of free disk space on your hosting server you +can finally make use of it. + + +## Installation + +Clone repository and symlink `bitpocket` bin to sth in your `$PATH`: + + $ git clone git://github.com/sickill/bitpocket.git + $ ln -s `pwd`/bitpocket/bin/bitpocket ~/bin/bitpocket + +Or download script and place it in a directory in your `$PATH`: + + $ curl -sL https://raw.github.com/sickill/bitpocket/master/bin/bitpocket > ~/bin/bitpocket + $ chmod +x ~/bin/bitpocket + + +### Setting up master + +Create an empty directory on some host that will be the master copy of your files: + + $ ssh user@example.org + $ mkdir ~/BitPocketMaster + + +### Setting up slaves + +On each machine you want to synchronize initialize an empty directory as your bitpocket: + + $ mkdir ~/BitPocket + $ cd ~/BitPocket + $ bitpocket init user@example.org ~/BitPocketMaster + + +## Usage + +After installation, you use the `bitpocket` command for synchronization and other tasks. +Running `bitpocket help` will display the following message. + + usage: bitpocket [sync|help|pack|log|cron|list|init ] + + Available commands: + sync Run the sync process. If no command is specified, sync is run by default. + init Initialize a new bitpocket folder. Requires remote host and path params. + pack Pack any existing (automatic) backups into a git repository. + cron Run sync optimized for cron, logging output to file instead of stdout. + log Display the log generated by the cron command + list List all files in the sync set (honoring include/exclude/filter config). + help Show this message. + + Note: All commands (apart from help), must be run in the root of a + new or existing bitpocket directory structure. + + +### Manual sync (bitpocket sync) + +To synchronize your local slave with master just run _bitpocket sync_ inside your +bitpocket directory: + + $ cd ~/BitPocket + $ bitpocket sync + +Ensure that you run bitpocket at least once immediately after creating a new slave and +before adding new files to the slave directory. If there are files in the master they +will be pulled into the slave. You may then move files into your slave directory and +they will be detected as added. + + +### Maintaining backups (bitpocket pack) + +bitpocket does not include a full-fledged versioning system at the moment, but +it does automatically create a local backup of any files before overwriting or +deleting with changes from the BitPocketMaster. These backups are placed into a +timestamped directory (one directory per sync). The path to this directory is: + + .bitpocket/backups/YYYY-MM-DD.hhmmss + +As these files accumulate, you may want to remove them, but you can also run +`bitpocket pack` to combine all the backup files into a git repository contained +within the _.bitpocket_ directory: + + $ cd ~/BitPocket + $ bitpocket pack + +This requires an installation of _git_, but +allows all the space saving advantages of _git_ when making repeated changes +to the same files. + +There is a discussion about potential directions for versioning direction here: +[github.com/sickill/bitpocket/issues/15](https://github.com/sickill/bitpocket/issues/15) + + +### Redirecting output to log file (bitpocket cron) + +If bitpocket is run with the cron parameter ( _bitpocket cron_ ), it will perform +a sync, but instead of showing the progress on stdout, it will redirect all +output to a log file: + + $ cd ~/BitPocket + $ bitpocket cron + +As the name of this parameter implies, this is mainly useful when running bitpocket +through the _cron_ command. (See "Automatic sync with cron" for more information +about how to configure this). + + +### Displaying logs (bitpocket log) + +When running bitpocket in cron with `bitpocket cron` it will append its output +to _.bitpocket/log_ file. You can review the tail end of an existing log file, +or watch live log as it is generated, with following command: + + $ cd ~/BitPocket + $ bitpocket log + + +### Displaying list of files to be synchronized (bitpocket list) + +You may want to know which files will be synchronized before actually performing +the syncronization. You can verify which files are in the synchronization set +by running `bitpocket list`: + + $ cd ~/BitPocket + $ bitpocket list + +Note that this does _not_ list changed files, it only lists all the local files +that bitpocket will look at in determining which files to sync. Also, note that +if there are new files in the master that will be added on a sync, they will +not be included here. This command is only intended to verify which files are +in the synchronization set. (See "Configuring file exclusion and inclusion" for +information about how to control which files are in the synchronization set). + + +## Configuration + +### Automatic sync with cron + +Add following line to your crontab to synchronize every 5 minutes: + + */5 * * * * cd ~/BitPocket && nice ~/bin/bitpocket cron + +Note that cron usually has very limited environment and your ssh keys with +passphrases won't work in cron jobs as ssh-agents/keyrings don't work there. +Thus it's preferable to generate passphrase-less ssh key for bitpocket +authentication: + + $ cd ~/BitPocket + $ ssh-keygen -t rsa -C bitpocket-`hostname` -N '' -f .bitpocket/id_rsa + $ ssh-copy-id -i .bitpocket/id_rsa user@example.org + +and uncomment line with `RSYNC_SSH` in _.bitpocket/config_ file. + + +### Custom rsync options + +You can pass additional switches to `rsync` by setting `RSYNC_OPTS` in +_.bitpocket/config_ file. Generated config file includes (commented out) +example setting for dereferencing symlinks: + + # RSYNC_OPTS="-L" + +Just uncomment it and change at will. + + +### Configuring file exclusion and inclusion + +If you want some files to be ignored by bitpocket you can create +_.bitpocket/exclude_ file and list the paths there: + + *.avi + jola + /misio.txt + +_*.avi_ and _jola_ will be matched anywhere in path, _misio.txt_ will be +matched at bitpocket root dir ( _~/BitPocket/misio.txt_ ). + +This exclude file is passed to `rsync` as `--exclude-from` argument, check `man +rsync` for _INCLUDE/EXCLUDE PATTERN RULES_. + +You can set up even more advanced exclusion/inclusion rules. In all, there +there are three files that you can create to change this configuration: + + .bitpocket/exclude + .bitpocket/include + .bitpocket/filter + +Be aware that all the quirks from rsync exclusion/inclusion rules carry over +into bitpocket. If you decide that you need such advanced configuration, make +sure that you understand those rules very well, and consider double checking +them before syncing by running `bitpocket list`. + + +### Slow sync callbacks + +When syncing takes more than 10 seconds (SLOW\_SYNC\_TIME setting) bitpocket +can fire off user provided command in background. This can be usefull to notify +user about long sync happening, preventing him from turning off the machine +during sync etc. + +There are 3 settings that can be enabled in _.bitpocket/config_ file: + + # SLOW_SYNC_TIME=10 + # SLOW_SYNC_START_CMD="notify-send 'BitPocket sync in progress...'" + # SLOW_SYNC_STOP_CMD="notify-send 'BitPocket sync finished'" + +Just uncomment them and change at will. + +You can show tray icon during long sync with +[traytor](https://github.com/sickill/traytor) and following settings: + + SLOW_SYNC_START_CMD='~/bin/traytor -t "BitPocket syncing..." -c "xdg-open ." .bitpocket/icons & echo $! >.bitpocket/traytor.pid' + SLOW_SYNC_STOP_CMD='kill `cat .bitpocket/traytor.pid`' + + +## Author + +* Marcin Kulik | @sickill | https://github.com/sickill | http://ku1ik.com/ +* torfason | https://github.com/torfason diff --git a/bitpocket b/bitpocket new file mode 100755 index 0000000..a0dbbd6 --- /dev/null +++ b/bitpocket @@ -0,0 +1,386 @@ +#!/bin/bash + +LANG=$(locale | grep LANG= | sed 's:LANG=::') +if [ -z "$LANG" ]; then + LANG="C" +fi + +export LC_ALL=$LANG # for stable "sort" output + +# Paths +DOT_DIR=.bitpocket +CFG_FILE="$DOT_DIR/config" +TMP_DIR="$DOT_DIR/tmp" +STATE_DIR="$DOT_DIR/state" +LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045 + +# Default settings +SLOW_SYNC_TIME=10 +SLOW_SYNC_FILE="$TMP_DIR/slow" +RSYNC_RSH="ssh" + +# Load config file +[ -f "$CFG_FILE" ] && . "$CFG_FILE" + +# Test for GNU versions of core utils. Bail if non-GNU. +sed --version >/dev/null 2>/dev/null +if [ $? -ne 0 ]; then + echo "fatal: It seems like you are running non-GNU versions of coreutils." + echo " It is currently unsafe to use bitpocket with this setup," + echo " so I'll have to stop here. Sorry ..." + exit 1 +fi + +# Decide on runner (ssh / bash -c) +if [ -n "$REMOTE_HOST" ]; then + REMOTE_RUNNER="$RSYNC_RSH $REMOTE_HOST" + REMOTE="$REMOTE_HOST:$REMOTE_PATH" +else + REMOTE_RUNNER="bash -c" + REMOTE="$REMOTE_PATH" +fi + +REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp" + +# Don't sync user excluded files +if [ -f "$DOT_DIR/exclude" ]; then + user_exclude="--exclude-from $DOT_DIR/exclude" +fi + +# Specify certain files to include +if [ -f "$DOT_DIR/include" ]; then + user_include="--include-from $DOT_DIR/include" +fi + +# Specify rsync filter rules +if [ -f "$DOT_DIR/filter" ]; then + # The underscore (_) is required for correct operation + user_filter="--filter merge_$DOT_DIR/filter" +fi + +USER_RULES="$user_filter $user_include $user_exclude" + +TIMESTAMP=$(date "+%Y-%m-%d.%H%M%S") + +export RSYNC_RSH + +function init { + if [[ -d "$DOT_DIR" || -f "$CFG_FILE" ]]; then + echo "fatal: Current directory already initialized for bitpocket" + exit 128 + fi + + if [[ $# != 2 ]]; then + echo "usage: bitpocket init " + exit 128 + fi + + mkdir "$DOT_DIR" + + cat < "$CFG_FILE" +## Host and path of central storage +REMOTE_HOST=$1 +REMOTE_PATH="$2" + +## SSH command with options for connecting to \$REMOTE +# RSYNC_RSH="ssh -p 22 -i $DOT_DIR/id_rsa" + +## Uncomment following line to follow symlinks (transform it into referent file/dir) +# RSYNC_OPTS="-L" + +## Uncomment following lines to get sync notifications +# SLOW_SYNC_TIME=10 +# SLOW_SYNC_START_CMD="notify-send 'BitPocket sync in progress...'" +# SLOW_SYNC_STOP_CMD="notify-send 'BitPocket sync finished'" +EOF + + echo "Initialized bitpocket directory at `pwd`" + echo "Please have a look at the config file ($DOT_DIR/config)" +} + +function log { + assert_dotdir + tail -f "$DOT_DIR/log" +} + +# Do the actual synchronization +function sync { + assert_dotdir + acquire_lock + acquire_remote_lock + + echo + echo -e "\x1b\x5b1;32mbitpocket started\x1b\x5b0m at `date`." + echo + + # Fire off slow sync start notifier in background + on_slow_sync_start + + # Check what has changed + touch "$STATE_DIR/tree-prev" + touch "$STATE_DIR/added-prev" + + # Save before-sync state + # Must be done with rsync itself (rather than find) to respect includes/excludes + # Order of includes/excludes/filters is EXTREMELY important + echo "# Saving current state and backing up files (if needed)" + echo " | Root dir: $(pwd)" + rsync -av --list-only --exclude "/$DOT_DIR" $RSYNC_OPTS $USER_RULES . | grep "^-\|^d" \ + | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort > "$STATE_DIR/tree-current" + + # Prevent bringing back locally deleted files or removing new local files + cp -f "$STATE_DIR/added-prev" "$TMP_DIR/fetch-exclude" + sort "$STATE_DIR/tree-prev" "$STATE_DIR/tree-current" | uniq -u >> "$TMP_DIR/fetch-exclude" + + # It is difficult to only create the backup directory if needed; instead + # we always create it, but remove it if it is empty afterwards. + mkdir --parents $DOT_DIR/backups/$TIMESTAMP + + # Determine what will be fetched from server and make backup + # copies of any local files to be deleted or overwritten. + # Order of includes/excludes/filters is EXTREMELY important + rsync --dry-run \ + -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES $REMOTE/ . \ + | grep "^[ch<>\.\*][f]\|\*deleting" | sed "s:^\S*\s*::" | sed 's:\d96:\\\`:g' | sed "s:\(.*\):if [ -f \"\1\" ]; then cp --parents \"\1\" $DOT_DIR/backups/$TIMESTAMP; fi:" | sh || die "BACKUP" + [ "$(ls -A $DOT_DIR/backups/$TIMESTAMP)" ] && echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP" + [ "$(ls -A $DOT_DIR/backups/$TIMESTAMP)" ] || rmdir $DOT_DIR/backups/$TIMESTAMP + + # Actual fetch + # Pulling changes from server + # Order of includes/excludes/filters is EXTREMELY important + echo + echo "# Pulling changes from server" + rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES $REMOTE/ . | sed "s/^/ | /" || die "PULL" + + # Actual push + # Send new and updated, remotely remove files deleted locally + # Order of includes/excludes/filters is EXTREMELY important + echo + echo "# Pushing changes to server" + rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" $USER_RULES . $REMOTE/ | sed "s/^/ | /" || die "PUSH" + + # Save after-sync state + # Must be done with rsync itself (rather than find) to respect includes/excludes + # Order of includes/excludes/filters is EXTREMELY important + echo + echo "# Saving after-sync state and cleaning up" + rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \ + | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort > "$TMP_DIR/tree-after" + + # Save all newly created files for next run (to prevent deletion of them) + # This includes files created by user in parallel to sync and files fetched from remote + comm -23 "$TMP_DIR/tree-after" "$STATE_DIR/tree-current" >"$STATE_DIR/added-prev" + + # Save new tree state for next run + cat "$STATE_DIR/tree-current" "$STATE_DIR/added-prev" >"$STATE_DIR/tree-prev" + + # Fire off slow sync stop notifier in background + on_slow_sync_stop + + cleanup + echo + echo -e "\x1b\x5b1;32mbitpocket finished\x1b\x5b0m at `date`." + echo + +} + + +# Pack backups into a git repository +function pack { + assert_dotdir + + # Git is required for backup packing + if [ ! `builtin type -p git` ]; then + echo "fatal: For backup packing, git must be installed" + exit 128 + fi + + # If pack directory is missing, create it and prepare git repo + if [ ! -d "$DOT_DIR/pack" ] + then + mkdir $DOT_DIR/pack + git init $DOT_DIR/pack + touch $DOT_DIR/pack/.git-init-marker + (cd $DOT_DIR/pack && git add .) + (cd $DOT_DIR/pack && git commit -a -q -m "INIT") + fi + + # If any backups exist, pack them into the repo + if [ -d "$DOT_DIR/backups" ] && [ "$(ls -A $DOT_DIR/backups)" ] + then + for DIR in $DOT_DIR/backups/* + do + TSTAMP=$(echo $DIR | sed "s|.*/||") + if [ "$(ls -A $DIR)" ] + then + echo -n "Processing: $TSTAMP ... " + echo -n "Moving ... " + (cp -rfl $DIR/* $DOT_DIR/pack && rm -rf $DIR) || die MV + echo -n "Adding ... " + (cd $DOT_DIR/pack && git add .) || die ADD + echo -n "Committing ... " + # Commit only if repository has uncommitted changes + (cd $DOT_DIR/pack \ + && git diff-index --quiet HEAD \ + || git commit -a -q -m "$TSTAMP" ) || die COMMIT + echo "Done." + else + echo "Removing empty dir $DIR ..." + rmdir $DIR + fi + done + echo "Running 'git gc' on pack dir" + du -hs $DOT_DIR/pack + (cd $DOT_DIR/pack && git gc) || die GC + du -hs $DOT_DIR/pack + echo "All snapshots packed successfully." + else + echo "No unpacked backups found ..." + fi + +} + +function on_slow_sync_start { + if [ -n "$SLOW_SYNC_START_CMD" ]; then + rm -rf "$SLOW_SYNC_FILE" + (sleep $SLOW_SYNC_TIME && touch "$SLOW_SYNC_FILE" && eval "$SLOW_SYNC_START_CMD" ; wait) & + disown + shell_pid=$! + fi +} + +function on_slow_sync_stop { + if [ -n "$shell_pid" ]; then + kill $shell_pid &>/dev/null + + if [[ -n "$SLOW_SYNC_STOP_CMD" && -f "$SLOW_SYNC_FILE" ]]; then + (eval "$SLOW_SYNC_STOP_CMD") & + fi + fi +} + +function cron { + DISPLAY=:0.0 sync 2>&1 | timestamp >>"$DOT_DIR/log" +} + +function timestamp { + while read data + do + echo "[$(date +"%D %T")] $data" + done +} + +function acquire_lock { + if ! mkdir "$LOCK_DIR" 2>/dev/null ; then + kill -0 $(cat "$LOCK_DIR/pid") &>/dev/null + + if [[ $? == 0 ]]; then + echo "There's already an instance of BitPocket syncing this directory. Exiting." + exit 1 + else + echo -e "\x1b\x5b1;31mbitpocket error:\x1b\x5b0m Bitpocket found a stale lock directory:" + echo " | Root dir: $(pwd)" + echo " | Lock dir: $LOCK_DIR" + echo " | Command: LOCK_PATH=$(pwd)/$LOCK_DIR && rm \$LOCK_PATH/pid && rmdir \$LOCK_PATH" + echo "Please remove the lock directory and try again." + exit 2 + fi + fi + + echo $$ > "$LOCK_DIR/pid" +} + +function release_lock { + rm "$LOCK_DIR/pid" &>/dev/null && rmdir "$LOCK_DIR" &>/dev/null +} + +function acquire_remote_lock { + $REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null" + + if [[ $? != 0 ]]; then + echo "Couldn't acquire remote lock. Another client is syncing with $REMOTE or lock file couldn't be created. Exiting." + release_lock + exit 3 + fi +} + +function release_remote_lock { + $REMOTE_RUNNER "cd \"$REMOTE_PATH\" && rmdir \"$LOCK_DIR\" &>/dev/null" +} + +function assert_dotdir { + if [ ! -d "$DOT_DIR" ]; then + echo "fatal: Not a bitpocket directory. Try 'bitpocket help' for usage." + exit 128 + fi + mkdir -p "$TMP_DIR" + mkdir -p "$STATE_DIR" +} + +function cleanup { + release_lock + release_remote_lock +} + +function bring_the_children_let_me_kill_them { + if [ -n "$shell_pid" ]; then + pkill -P $shell_pid &>/dev/null + kill $shell_pid &>/dev/null + fi +} + +function die { + cleanup + bring_the_children_let_me_kill_them + + echo "fatal: command failed $1" + exit 128 +} + +# List all files in the sync set +function list { + echo -e "\x1b\x5b1;32mbitpocket\x1b\x5b0m will sync the following files:" + rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \ + | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort +} + +function usage { + echo "usage: bitpocket [sync|help|pack|log|cron|list|init ]" + echo "" + echo "Available commands:" + echo " sync Run the sync process. If no command is specified, sync is run by default." + echo " init Initialize a new bitpocket folder. Requires remote host and path params." + echo " pack Pack any existing (automatic) backups into a git repository." + echo " cron Run sync optimized for cron, logging output to file instead of stdout." + echo " log Display the log generated by the cron command" + echo " list List all files in the sync set (honoring include/exclude/filter config)." + echo " help Show this message." + echo "" + echo "Note: All commands (apart from help), must be run in the root of a" + echo " new or existing bitpocket directory structure." + echo "" + +} + +if [ "$1" = "init" ]; then + # Initialize bitpocket directory + init $2 $3 $4 +elif [ "$1" = "pack" ]; then + # Pack backups using git + pack +elif [ "$1" = "log" ]; then + # Display log file + log +elif [ "$1" = "cron" ]; then + # Run through cron? + cron +elif [ "$1" = "list" ]; then + # List all file in sync set (honoring .bitpocket/include & .bitpocket/exclude) + list +elif [ "$1" != "" ] && [ "$1" != "sync" ]; then + # Show help + usage +else + # By default, run the sync process + sync +fi -- cgit v1.2.3