aboutsummaryrefslogtreecommitdiffstats
path: root/bitpocket
blob: f1df6490e0c7cb5eaaead7923b9955d02a4883cb (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
#!/bin/bash

LANG=$(locale | grep LANG= | sed 's:LANG=::')
if [ -z "$LANG" ]; then
    LANG="C"
fi

export LC_ALL=$LANG # for stable "sort" output

# Paths
DOT_DIR=.bitpocket
CFG_FILE="$DOT_DIR/config"
TMP_DIR="$DOT_DIR/tmp"
STATE_DIR="$DOT_DIR/state"
LOCK_DIR="$TMP_DIR/lock"  # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045

# Default settings
SLOW_SYNC_TIME=10
SLOW_SYNC_FILE="$TMP_DIR/slow"
RSYNC_RSH="ssh"

# Load config file
[ -f "$CFG_FILE" ] && . "$CFG_FILE"

# Test for GNU versions of core utils. Bail if non-GNU.
sed --version >/dev/null 2>/dev/null
if [ $? -ne 0 ]; then
    echo "fatal: It seems like you are running non-GNU versions of coreutils."
    echo "       It is currently unsafe to use bitpocket with this setup,"
    echo "       so I'll have to stop here. Sorry ..."
    exit 1
fi

# Decide on runner (ssh / bash -c)
if [ -n "$REMOTE_HOST" ]; then
  REMOTE_RUNNER="$RSYNC_RSH $REMOTE_HOST"
  REMOTE="$REMOTE_HOST:$REMOTE_PATH"
else
  REMOTE_RUNNER="bash -c"
  REMOTE="$REMOTE_PATH"
fi

REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp"

# Don't sync user excluded files
if [ -f "$DOT_DIR/exclude" ]; then
  user_exclude="--exclude-from $DOT_DIR/exclude"
fi

# Specify certain files to include
if [ -f "$DOT_DIR/include" ]; then
  user_include="--include-from $DOT_DIR/include"
fi

# Specify rsync filter rules
if [ -f "$DOT_DIR/filter" ]; then
  # The underscore (_) is required for correct operation
  user_filter="--filter merge_$DOT_DIR/filter"
fi

USER_RULES="$user_filter $user_include $user_exclude"

TIMESTAMP=$(date "+%Y-%m-%d.%H%M%S")

export RSYNC_RSH

function init {
  if [[ -d "$DOT_DIR" || -f "$CFG_FILE" ]]; then
    echo "fatal: Current directory already initialized for bitpocket"
    exit 128
  fi

  if [[ -z "$2" || -n "$3" ]]; then
    echo "usage: bitpocket init {<REMOTE_HOST> | \"\"} <REMOTE_PATH>"
    exit 128
  fi

  mkdir "$DOT_DIR"

  cat <<EOF > "$CFG_FILE"
## Host and path of central storage
REMOTE_HOST=$1
REMOTE_PATH="$2"

## SSH command with options for connecting to \$REMOTE_HOST
# RSYNC_RSH="ssh -p 22 -i $DOT_DIR/id_rsa"

## Uncomment following line to follow symlinks (transform it into referent file/dir)
# RSYNC_OPTS="-L"

## Use the following if a FAT or VFAT filesystem is being synchronized
# RSYNC_OPTS="--no-perms --no-owner --no-group --modify-window=2"

## Uncomment following lines to get sync notifications
# SLOW_SYNC_TIME=10
# SLOW_SYNC_START_CMD="notify-send 'BitPocket sync in progress...'"
# SLOW_SYNC_STOP_CMD="notify-send 'BitPocket sync finished'"
EOF

  echo "Initialized bitpocket directory at `pwd`"
  echo "Please have a look at the config file ($DOT_DIR/config)"
}

function log {
  assert_dotdir
  tail -f "$DOT_DIR/log"
}

function pull {
  sync onlypull
}

function push {
  sync onlypush
}

# Do the actual synchronization
function sync {
  assert_dotdir
  acquire_lock
  acquire_remote_lock

  echo
  echo -e "\x1b\x5b1;32mbitpocket started\x1b\x5b0m at `date`."
  echo

  # Fire off slow sync start notifier in background
  on_slow_sync_start

  # Check what has changed
  touch "$STATE_DIR/tree-prev"
  touch "$STATE_DIR/added-prev"

  # Save before-sync state
  # Must be done with rsync itself (rather than find) to respect includes/excludes
  # Order of includes/excludes/filters is EXTREMELY important
  echo "# Saving current state and backing up files (if needed)"
  echo "  | Root dir: $(pwd)"
  rsync -av --list-only --exclude "/$DOT_DIR" $RSYNC_OPTS $USER_RULES . | grep "^-\|^d" \
      | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort > "$STATE_DIR/tree-current"

  # Prevent bringing back locally deleted files or removing new local files
  cp -f "$STATE_DIR/added-prev" "$TMP_DIR/fetch-exclude"
  sort "$STATE_DIR/tree-prev" "$STATE_DIR/tree-current" | uniq -u >> "$TMP_DIR/fetch-exclude"

  # It is difficult to only create the backup directory if needed; instead
  # we always create it, but remove it if it is empty afterwards.
  mkdir --parents $DOT_DIR/backups/$TIMESTAMP

  # Determine what will be fetched from server and make backup
  # copies of any local files to be deleted or overwritten.
  # Order of includes/excludes/filters is EXTREMELY important
  rsync --dry-run \
        -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES "$REMOTE/" . \
      | grep "^[ch<>\.\*][f]\|\*deleting" | sed "s:^\S*\s*::" | sed 's:\d96:\\\`:g' | sed "s:\(.*\):if [ -f \"\1\" ]; then cp --parents \"\1\" $DOT_DIR/backups/$TIMESTAMP; fi:" | sh || die "BACKUP"
  [ "$(ls -A $DOT_DIR/backups/$TIMESTAMP)" ] && echo "  | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP"
  [ "$(ls -A $DOT_DIR/backups/$TIMESTAMP)" ] || rmdir $DOT_DIR/backups/$TIMESTAMP

  if [ "$1" != "onlypush" ]
  then

    # Actual fetch
    # Pulling changes from server
    # Order of includes/excludes/filters is EXTREMELY important
    echo
    echo "# Pulling changes from server"
    rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES "$REMOTE/" . | sed "s/^/  | /"  || die "PULL"

  fi

  if [ "$1" != "onlypull" ]
  then

    # Actual push
    # Send new and updated, remotely remove files deleted locally
    # Order of includes/excludes/filters is EXTREMELY important
    echo
    echo "# Pushing changes to server"
    rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" $USER_RULES . "$REMOTE/" | sed "s/^/  | /" || die "PUSH"

  fi

  # Save after-sync state
  # Must be done with rsync itself (rather than find) to respect includes/excludes
  # Order of includes/excludes/filters is EXTREMELY important
  echo
  echo "# Saving after-sync state and cleaning up"
  rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \
      | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort > "$TMP_DIR/tree-after"


  # Save all newly created files for next run (to prevent deletion of them)
  # This includes files created by user in parallel to sync and files fetched from remote
  comm -23 "$TMP_DIR/tree-after" "$STATE_DIR/tree-current" >"$STATE_DIR/added-prev"

  # Save new tree state for next run
  cat "$STATE_DIR/tree-current" "$STATE_DIR/added-prev" >"$STATE_DIR/tree-prev"

  # Fire off slow sync stop notifier in background
  on_slow_sync_stop

  cleanup
  echo
  echo -e "\x1b\x5b1;32mbitpocket finished\x1b\x5b0m at `date`."
  echo

}


# Pack backups into a git repository
function pack {
    assert_dotdir

    # Git is required for backup packing
    if [ ! `builtin type -p git` ]; then
        echo "fatal: For backup packing, git must be installed"
        exit 128
    fi

    # If pack directory is missing, create it and prepare git repo
    if [ ! -d "$DOT_DIR/pack" ]
    then
        mkdir    $DOT_DIR/pack
        git init $DOT_DIR/pack
        touch    $DOT_DIR/pack/.git-init-marker
        (cd $DOT_DIR/pack && git add .)
        (cd $DOT_DIR/pack && git commit -a -q -m "INIT")
    fi

    # If any backups exist, pack them into the repo
    if [ -d "$DOT_DIR/backups" ] && [ "$(ls -A $DOT_DIR/backups)" ]
    then
        for DIR in $DOT_DIR/backups/*
        do
            TSTAMP=$(echo $DIR | sed "s|.*/||")
            if [ "$(ls -A $DIR)" ]
            then
                echo -n "Processing: $TSTAMP ... "
                echo -n "Moving ... "
                (cp -rfl $DIR/* $DOT_DIR/pack && rm -rf $DIR)        || die MV
                echo -n "Adding ... "
                (cd $DOT_DIR/pack && git add .)                      || die ADD
                echo -n "Committing ... "
                # Commit only if repository has uncommitted changes
                (cd $DOT_DIR/pack \
                    && git diff-index --quiet HEAD \
                    || git commit -a -q -m "$TSTAMP" )             || die COMMIT
                echo "Done."
            else
                echo "Removing empty dir $DIR  ..."
                rmdir $DIR
            fi
        done
        echo "Running 'git gc' on pack dir"
        du -hs $DOT_DIR/pack
        (cd $DOT_DIR/pack && git gc) || die GC
        du -hs $DOT_DIR/pack
        echo "All snapshots packed successfully."
    else
        echo "No unpacked backups found ..."
    fi

}

function on_slow_sync_start {
  if [ -n "$SLOW_SYNC_START_CMD" ]; then
    rm -rf "$SLOW_SYNC_FILE"
    (sleep $SLOW_SYNC_TIME && touch "$SLOW_SYNC_FILE" && eval "$SLOW_SYNC_START_CMD" ; wait) &
    disown
    shell_pid=$!
  fi
}

function on_slow_sync_stop {
  if [ -n "$shell_pid" ]; then
    kill $shell_pid &>/dev/null

    if [[ -n "$SLOW_SYNC_STOP_CMD" && -f "$SLOW_SYNC_FILE" ]]; then
      (eval "$SLOW_SYNC_STOP_CMD") &
    fi
  fi
}

function cron {
  DISPLAY=:0.0 sync 2>&1 | timestamp >>"$DOT_DIR/log"
}

function timestamp {
  while read data
  do
      echo "[$(date +"%D %T")] $data"
  done
}

function acquire_lock {
  if ! mkdir "$LOCK_DIR" 2>/dev/null ; then
    kill -0 $(cat "$LOCK_DIR/pid") &>/dev/null

    if [[ $? == 0 ]]; then
      echo "There's already an instance of BitPocket syncing this directory. Exiting."
      exit 1
    else
      echo -e "\x1b\x5b1;31mbitpocket error:\x1b\x5b0m Bitpocket found a stale lock directory:"
      echo "  | Root dir: $(pwd)"
      echo "  | Lock dir: $LOCK_DIR"
      echo "  | Command:  LOCK_PATH=$(pwd)/$LOCK_DIR && rm \$LOCK_PATH/pid && rmdir \$LOCK_PATH"
      echo "Please remove the lock directory and try again."
      exit 2
    fi
  fi

  echo $$ > "$LOCK_DIR/pid"
}

function release_lock {
  rm "$LOCK_DIR/pid" &>/dev/null && rmdir "$LOCK_DIR" &>/dev/null
}

function acquire_remote_lock {
  $REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null"

  if [[ $? != 0 ]]; then
    echo "Couldn't acquire remote lock. Another client is syncing with \"$REMOTE\" or lock file couldn't be created. Exiting."
    release_lock
    exit 3
  fi
}

function release_remote_lock {
  $REMOTE_RUNNER "cd \"$REMOTE_PATH\" && rmdir \"$LOCK_DIR\" &>/dev/null"
}

function assert_dotdir {
  if [ ! -d "$DOT_DIR" ]; then
    echo "fatal: Not a bitpocket directory. Try 'bitpocket help' for usage."
    exit 128
  fi
  mkdir -p "$TMP_DIR"
  mkdir -p "$STATE_DIR"
}

function cleanup {
  release_lock
  release_remote_lock
}

function bring_the_children_let_me_kill_them {
  if [ -n "$shell_pid" ]; then
    pkill -P $shell_pid &>/dev/null
    kill $shell_pid &>/dev/null
  fi
}

function die {
  cleanup
  bring_the_children_let_me_kill_them

  echo "fatal: command failed $1"
  exit 128
}

# List all files in the sync set
function list {
  echo -e "\x1b\x5b1;32mbitpocket\x1b\x5b0m will sync the following files:"
  rsync -av --list-only --exclude "/$DOT_DIR"  $USER_RULES . | grep "^-\|^d" \
      | sed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | sed "s:^/\.::" | sort
}

function usage {
  echo "usage:  bitpocket [sync | push | pull | pack | log | cron | list | help]"
  echo "        bitpocket init {<REMOTE_HOST> | \"\"} <REMOTE_PATH>"
  echo ""
  echo "Available commands:"
  echo "   sync    Run the sync process. If no command is specified, sync is run by default."
  echo "   push    Only push new files to the server."
  echo "   pull    Only pull new files from the server."
  echo "   init    Initialize a new bitpocket folder. Requires remote host and path params."
  echo "   pack    Pack any existing (automatic) backups into a git repository."
  echo "   cron    Run sync optimized for cron, logging output to file instead of stdout."
  echo "   log     Display the log generated by the cron command"
  echo "   list    List all files in the sync set (honoring include/exclude/filter config)."
  echo "   help    Show this message."
  echo ""
  echo "Note: All commands (apart from help), must be run in the root of a"
  echo "      new or existing bitpocket directory structure."
  echo ""
}

if [ "$1" = "init" ]; then
  # Initialize bitpocket directory
  init "$2" "$3" "$4"
elif [ "$1" = "pull" ]; then
  sync onlypull
elif [ "$1" = "push" ]; then
  sync onlypush
elif [ "$1" = "pack" ]; then
  # Pack backups using git
  pack
elif [ "$1" = "log" ]; then
  # Display log file
  log
elif [ "$1" = "cron" ]; then
  # Run through cron?
  cron
elif [ "$1" = "list" ]; then
  # List all file in sync set (honoring .bitpocket/include & .bitpocket/exclude)
  list
elif [ "$1" != "" ] && [ "$1" != "sync" ]; then
  # Show help
  usage
else
  # By default, run the sync process
  sync
fi

© 2014-2024 Faster IT GmbH | imprint | privacy policy