blob: b3fc6e831b4055fec351e1a6c1df04d5281ecdf8 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
|
#!/bin/bash
# Mac version 2. Sic.
LANG=$(locale | grep LANG= | gsed 's:LANG=::')
if [ -z "$LANG" ]; then
LANG="C"
fi
#export LC_ALL=$LANG # for stable "sort" output
export LC_ALL="C"
# Paths
DOT_DIR=.bitpocket
CFG_FILE="$DOT_DIR/config"
TMP_DIR="$DOT_DIR/tmp"
STATE_DIR="$DOT_DIR/state"
LOCK_DIR="$TMP_DIR/lock" # Use a lock directory for atomic locks. See the Bash FAQ http://mywiki.wooledge.org/BashFAQ/045
# Default settings
SLOW_SYNC_TIME=10
SLOW_SYNC_FILE="$TMP_DIR/slow"
RSYNC_RSH="ssh"
# Load config file
[ -f "$CFG_FILE" ] && . "$CFG_FILE"
# Test for GNU versions of core utils. Bail if non-GNU.
gsed --version >/dev/null 2>/dev/null
if [ $? -ne 0 ]; then
echo "fatal: It seems like you are running non-GNU versions of coreutils."
echo " It is currently unsafe to use bitpocket with this setup,"
echo " so I'll have to stop here. Sorry ..."
exit 1
fi
# Decide on runner (ssh / bash -c)
if [ -n "$REMOTE_HOST" ]; then
REMOTE_RUNNER="$RSYNC_RSH $REMOTE_HOST"
REMOTE="$REMOTE_HOST:$REMOTE_PATH"
else
REMOTE_RUNNER="bash -c"
REMOTE="$REMOTE_PATH"
fi
REMOTE_TMP_DIR="$REMOTE_PATH/$DOT_DIR/tmp"
# Don't sync user excluded files
if [ -f "$DOT_DIR/exclude" ]; then
user_exclude="--exclude-from $DOT_DIR/exclude"
fi
# Specify certain files to include
if [ -f "$DOT_DIR/include" ]; then
user_include="--include-from $DOT_DIR/include"
fi
# Specify rsync filter rules
if [ -f "$DOT_DIR/filter" ]; then
# The underscore (_) is required for correct operation
user_filter="--filter merge_$DOT_DIR/filter"
fi
USER_RULES="$user_filter $user_include $user_exclude"
TIMESTAMP=$(date "+%Y-%m-%d.%H%M%S")
export RSYNC_RSH
function init {
if [[ -d "$DOT_DIR" || -f "$CFG_FILE" ]]; then
echo "fatal: Current directory already initialized for bitpocket"
exit 128
fi
if [[ -z "$2" || -n "$3" ]]; then
echo "usage: bitpocket init {<REMOTE_HOST> | \"\"} <REMOTE_PATH>"
exit 128
fi
gmkdir "$DOT_DIR"
gcat <<EOF > "$CFG_FILE"
## Host and path of central storage
REMOTE_HOST=$1
REMOTE_PATH="$2"
## SSH command with options for connecting to \$REMOTE_HOST
# RSYNC_RSH="ssh -p 22 -i $DOT_DIR/id_rsa"
## Uncomment following line to follow symlinks (transform it into referent file/dir)
# RSYNC_OPTS="-L"
## Use the following if a FAT or VFAT filesystem is being synchronized
# RSYNC_OPTS="--no-perms --no-owner --no-group --modify-window=2"
## Uncomment following lines to get sync notifications
# SLOW_SYNC_TIME=10
# SLOW_SYNC_START_CMD="notify-send 'BitPocket sync in progress...'"
# SLOW_SYNC_STOP_CMD="notify-send 'BitPocket sync finished'"
EOF
echo "Initialized bitpocket directory at `gpwd`"
echo "Please have a look at the config file ($DOT_DIR/config)"
}
function log {
assert_dotdir
tail -f "$DOT_DIR/log"
}
function pull {
sync onlypull
}
function push {
sync onlypush
}
# Do the actual synchronization
function sync {
assert_dotdir
acquire_lock
acquire_remote_lock
echo
echo -e "\x1b\x5b1;32mbitpocket started\x1b\x5b0m at `date`."
echo
# Fire off slow sync start notifier in background
on_slow_sync_start
# Check what has changed
gtouch "$STATE_DIR/tree-prev"
gtouch "$STATE_DIR/added-prev"
# Save before-sync state
# Must be done with rsync itself (rather than find) to respect includes/excludes
# Order of includes/excludes/filters is EXTREMELY important
echo "# Saving current state and backing up files (if needed)"
echo " | Root dir: $(gpwd)"
/usr/local/bin/rsync -av --list-only --exclude "/$DOT_DIR" $RSYNC_OPTS $USER_RULES . | grep "^-\|^d" \
| gsed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | gsed "s:^/\.$::" | gsort > "$STATE_DIR/tree-current"
# Prevent bringing back locally deleted files or removing new local files
gcp -f "$STATE_DIR/added-prev" "$TMP_DIR/fetch-exclude"
gsort "$STATE_DIR/tree-prev" "$STATE_DIR/tree-current" | guniq -u >> "$TMP_DIR/fetch-exclude"
# It is difficult to only create the backup directory if needed; instead
# we always create it, but remove it if it is empty afterwards.
gmkdir --parents $DOT_DIR/backups/$TIMESTAMP
# Determine what will be fetched from server and make backup
# copies of any local files to be deleted or overwritten.
# Order of includes/excludes/filters is EXTREMELY important
/usr/local/bin/rsync --dry-run \
-auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES "$REMOTE/" . \
| grep "^[ch<>\.\*][f]\|\*deleting" | gsed "s:^\S*\s*::" | gsed 's:\d96:\\\`:g' | gsed "s:\(.*\):if [ -f \"\1\" ]; then gcp --parents \"\1\" $DOT_DIR/backups/$TIMESTAMP; fi:" | sh || die "BACKUP"
[ "$(gls -A $DOT_DIR/backups/$TIMESTAMP)" ] && echo " | Some files were backed up to $DOT_DIR/backups/$TIMESTAMP"
[ "$(gls -A $DOT_DIR/backups/$TIMESTAMP)" ] || grmdir $DOT_DIR/backups/$TIMESTAMP
if [ "$1" != "onlypush" ]
then
# Actual fetch
# Pulling changes from server
# Order of includes/excludes/filters is EXTREMELY important
echo
echo "# Pulling changes from server"
/usr/local/bin/rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" --exclude-from "$TMP_DIR/fetch-exclude" $USER_RULES "$REMOTE/" . | gsed "s/^/ | /" || die "PULL"
fi
if [ "$1" != "onlypull" ]
then
# Actual push
# Send new and updated, remotely remove files deleted locally
# Order of includes/excludes/filters is EXTREMELY important
echo
echo "# Pushing changes to server"
/usr/local/bin/rsync -auvzxi --delete $RSYNC_OPTS --exclude "/$DOT_DIR" $USER_RULES . "$REMOTE/" | gsed "s/^/ | /" || die "PUSH"
fi
# Save after-sync state
# Must be done with rsync itself (rather than find) to respect includes/excludes
# Order of includes/excludes/filters is EXTREMELY important
echo
echo "# Saving after-sync state and cleaning up"
/usr/local/bin/rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \
| gsed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | gsed "s:^/\.$::" | gsort > "$TMP_DIR/tree-after"
# Save all newly created files for next run (to prevent deletion of them)
# This includes files created by user in parallel to sync and files fetched from remote
comm -23 "$TMP_DIR/tree-after" "$STATE_DIR/tree-current" >"$STATE_DIR/added-prev"
# Save new tree state for next run
gcat "$STATE_DIR/tree-current" "$STATE_DIR/added-prev" >"$STATE_DIR/tree-prev"
# Fire off slow sync stop notifier in background
on_slow_sync_stop
cleanup
echo
echo -e "\x1b\x5b1;32mbitpocket finished\x1b\x5b0m at `date`."
echo
}
# Pack backups into a git repository
function pack {
assert_dotdir
# Git is required for backup packing
if [ ! `builtin type -p git` ]; then
echo "fatal: For backup packing, git must be installed"
exit 128
fi
# If pack directory is missing, create it and prepare git repo
if [ ! -d "$DOT_DIR/pack" ]
then
gmkdir $DOT_DIR/pack
git init $DOT_DIR/pack
gtouch $DOT_DIR/pack/.git-init-marker
(cd $DOT_DIR/pack && git add .)
(cd $DOT_DIR/pack && git commit -a -q -m "INIT")
fi
# If any backups exist, pack them into the repo
if [ -d "$DOT_DIR/backups" ] && [ "$(gls -A $DOT_DIR/backups)" ]
then
for DIR in $DOT_DIR/backups/*
do
TSTAMP=$(echo $DIR | gsed "s|.*/||")
if [ "$(gls -A $DIR)" ]
then
echo -n "Processing: $TSTAMP ... "
echo -n "Moving ... "
(gcp -rfl $DIR/* $DOT_DIR/pack && grm -rf $DIR) || die MV
echo -n "Adding ... "
(cd $DOT_DIR/pack && git add .) || die ADD
echo -n "Committing ... "
# Commit only if repository has uncommitted changes
(cd $DOT_DIR/pack \
&& git diff-index --quiet HEAD \
|| git commit -a -q -m "$TSTAMP" ) || die COMMIT
echo "Done."
else
echo "Removing empty dir $DIR ..."
grmdir $DIR
fi
done
echo "Running 'git gc' on pack dir"
du -hs $DOT_DIR/pack
(cd $DOT_DIR/pack && git gc) || die GC
du -hs $DOT_DIR/pack
echo "All snapshots packed successfully."
else
echo "No unpacked backups found ..."
fi
}
function on_slow_sync_start {
if [ -n "$SLOW_SYNC_START_CMD" ]; then
grm -rf "$SLOW_SYNC_FILE"
(sleep $SLOW_SYNC_TIME && gtouch "$SLOW_SYNC_FILE" && eval "$SLOW_SYNC_START_CMD" ; wait) &
disown
shell_pid=$!
fi
}
function on_slow_sync_stop {
if [ -n "$shell_pid" ]; then
gkill $shell_pid &>/dev/null
if [[ -n "$SLOW_SYNC_STOP_CMD" && -f "$SLOW_SYNC_FILE" ]]; then
(eval "$SLOW_SYNC_STOP_CMD") &
fi
fi
}
function cron {
DISPLAY=:0.0 sync 2>&1 | timestamp >>"$DOT_DIR/log"
}
function timestamp {
while read data
do
echo "[$(date +"%D %T")] $data"
done
}
function acquire_lock {
if ! gmkdir "$LOCK_DIR" 2>/dev/null ; then
gkill -0 $(gcat "$LOCK_DIR/pid") &>/dev/null
if [[ $? == 0 ]]; then
echo "There's already an instance of BitPocket syncing this directory. Exiting."
exit 1
else
echo -e "\x1b\x5b1;31mbitpocket error:\x1b\x5b0m Bitpocket found a stale lock directory:"
echo " | Root dir: $(gpwd)"
echo " | Lock dir: $LOCK_DIR"
echo " | Command: LOCK_PATH=$(gpwd)/$LOCK_DIR && grm \$LOCK_PATH/pid && grmdir \$LOCK_PATH"
echo "Please remove the lock directory and try again."
exit 2
fi
fi
echo $$ > "$LOCK_DIR/pid"
}
function release_lock {
grm "$LOCK_DIR/pid" &>/dev/null && grmdir "$LOCK_DIR" &>/dev/null
}
function acquire_remote_lock {
$REMOTE_RUNNER "mkdir -p \"$REMOTE_TMP_DIR\"; cd \"$REMOTE_PATH\" && mkdir \"$LOCK_DIR\" 2>/dev/null"
if [[ $? != 0 ]]; then
echo "Couldn't acquire remote lock. Another client is syncing with \"$REMOTE\" or lock file couldn't be created. Exiting."
release_lock
exit 3
fi
}
function release_remote_lock {
$REMOTE_RUNNER "cd \"$REMOTE_PATH\" && rmdir \"$LOCK_DIR\" &>/dev/null"
}
function assert_dotdir {
if [ ! -d "$DOT_DIR" ]; then
echo "fatal: Not a bitpocket directory. Try 'bitpocket help' for usage."
exit 128
fi
gmkdir -p "$TMP_DIR"
gmkdir -p "$STATE_DIR"
}
function cleanup {
release_lock
release_remote_lock
}
function bring_the_children_let_me_kill_them {
if [ -n "$shell_pid" ]; then
pkill -P $shell_pid &>/dev/null
gkill $shell_pid &>/dev/null
fi
}
function die {
cleanup
bring_the_children_let_me_kill_them
echo "fatal: command failed $1"
exit 128
}
# List all files in the sync set
function list {
echo -e "\x1b\x5b1;32mbitpocket\x1b\x5b0m will sync the following files:"
/usr/local/bin/rsync -av --list-only --exclude "/$DOT_DIR" $USER_RULES . | grep "^-\|^d" \
| gsed "s:^\S*\s*\S*\s*\S*\s*\S*\s*:/:" | gsed "s:^/\.$::" | gsort
}
function usage {
echo "usage: bitpocket [sync | push | pull | pack | log | cron | list | help]"
echo " bitpocket init {<REMOTE_HOST> | \"\"} <REMOTE_PATH>"
echo ""
echo "Available commands:"
echo " sync Run the sync process. If no command is specified, sync is run by default."
echo " push Only push new files to the server."
echo " pull Only pull new files from the server."
echo " init Initialize a new bitpocket folder. Requires remote host and path params."
echo " pack Pack any existing (automatic) backups into a git repository."
echo " cron Run sync optimized for cron, logging output to file instead of stdout."
echo " log Display the log generated by the cron command"
echo " list List all files in the sync set (honoring include/exclude/filter config)."
echo " help Show this message."
echo ""
echo "Note: All commands (apart from help), must be run in the root of a"
echo " new or existing bitpocket directory structure."
echo ""
}
if [ "$1" = "init" ]; then
# Initialize bitpocket directory
init "$2" "$3" "$4"
elif [ "$1" = "pull" ]; then
sync onlypull
elif [ "$1" = "push" ]; then
sync onlypush
elif [ "$1" = "pack" ]; then
# Pack backups using git
pack
elif [ "$1" = "log" ]; then
# Display log file
log
elif [ "$1" = "cron" ]; then
# Run through cron?
cron
elif [ "$1" = "list" ]; then
# List all file in sync set (honoring .bitpocket/include & .bitpocket/exclude)
list
elif [ "$1" != "" ] && [ "$1" != "sync" ]; then
# Show help
usage
else
# By default, run the sync process
sync
fi
|