git @ Cat's Eye Technologies ellsync / 0.5
Merge pull request #4 from catseye/develop-0.5 Develop 0.5 Chris Pressey authored 4 months ago GitHub committed 4 months ago
4 changed file(s) with 149 addition(s) and 37 deletion(s). Raw diff Collapse all Expand all
00 `ellsync`
11 =========
22
3 _Version 0.4_
3 _Version 0.5_
44 | _Entry_ [@ catseye.tc](https://catseye.tc/node/ellsync)
55 | _See also:_ [yastasoti](https://github.com/catseye/yastasoti#readme)
66 ∘ [tagfarm](https://github.com/catseye/tagfarm#readme)
8383 will work as well as the above. (But note that the directories specified
8484 in the router *do* need to have the trailing slashes.)
8585
86 #### --thorough
86 #### `--thorough` option
8787
8888 By default, `rsync` does not attempt to sync the contents of an existing file
8989 if the destination file has a same-or-newer timestamp as the source file.
147147 is for example owned by another user and not world-readable, it will abort.
148148 `ellsync` does not currently detect this properly. It should be made to handle
149149 it gracefully, if possible.
150 * Tab-completion of stream names.
151 * Better test case for `--thorough`.
152 * When executing system commands, don't use shell expansion.
153150 * (Aspirational) Ability to convert the backup router to a `dot` file (`graphviz`)
154151 so that the relationships between the streams can be easily visualized.
155152
156153 History
157154 -------
155
156 ### 0.5
157
158 The output of the `list` subcommand is now sorted by stream name.
159
160 The `sync` subcommand now supports multiple streams. Each stream will be synced
161 in the order they are given on the command line. OS-level `sync` will only be
162 performed once, at the very end.
163
164 A bash tab-completion script is included in the `script` directory. It enables
165 tab-completion of both subcommand names, and stream names in the `sync` subcommand.
166
167 Internally, shell expansion is no longer used when executing system commands, and
168 several new tests have been added to the test suite.
158169
159170 ### 0.4
160171
0 # To enable tab-completion for ellsync in bash, source this file, like so:
1 # . /path/to/ellsync/script/ellsync_tabcomplete.sh
2 # You might want to do this in your bash startup script.
3
4 function _ellsync_tabcomplete_()
5 {
6 local cmd="${1##*/}"
7 local word=${COMP_WORDS[COMP_CWORD]}
8 local line=${COMP_LINE}
9
10 # Split the command line into arguments and place them in the $argv[@] array.
11 # We append a character ('%') so that we can tell if user is on a partial word or on a space.
12 # So, the count in $argc is right, but the final value in the $argv[@] array is not accurate.
13 # That's acceptable for our purposes.
14 IFS=' ' read -raargv<<< "$line%"
15 local argc=${#argv[@]}
16
17 if [ $argc -eq 2 ]; then
18 COMPREPLY=($(compgen -o default "${word}"))
19 elif [ $argc -eq 3 ]; then
20 COMPREPLY=($(compgen -W "list sync rename" "${word}"))
21 elif [ $argc -gt 3 ]; then
22 local router="${argv[1]}"
23 local streams=`ellsync $router list --stream-name-only 2>/dev/null`
24 COMPREPLY=($(compgen -W "${streams}" "${word}"))
25 fi
26 }
27
28 complete -F _ellsync_tabcomplete_ ellsync
1313 return dirname
1414
1515
16 def run_command(cmd):
17 sys.stdout.write(cmd + '\n')
16 def run_command(argv):
17 def pretty(s):
18 return '"{}"'.format(s) if ' ' in s else s
19 sys.stdout.write(' '.join([pretty(a) for a in argv]) + '\n')
1820 try:
19 p = Popen(cmd, shell=True, stderr=STDOUT, stdout=PIPE, encoding='utf-8')
21 p = Popen(argv, stderr=STDOUT, stdout=PIPE, encoding='utf-8')
2022 decode_line = lambda line: line
2123 except TypeError:
2224 # python 2.x
23 p = Popen(cmd, shell=True, stderr=STDOUT, stdout=PIPE)
25 p = Popen(argv, stderr=STDOUT, stdout=PIPE)
2426 decode_line = lambda line: line.decode('utf-8')
2527 pipe = p.stdout
2628 for line in p.stdout:
3335
3436
3537 def list_(router, options):
36 for stream_name, stream in router.items():
38 for stream_name, stream in sorted(router.items()):
3739 if os.path.isdir(stream['from']) and os.path.isdir(stream['to']):
3840 if options.stream_name_only:
3941 print(stream_name)
4244
4345
4446 def sync(router, options):
45 if ':' in options.stream_name:
46 stream_name, subdir = options.stream_name.split(':')
47 else:
48 stream_name = options.stream_name
49 subdir = None
50 stream = router[stream_name]
51 from_dir = stream['from']
52 to_dir = stream['to']
53 if subdir:
54 from_dir = os.path.join(from_dir, subdir)
55 to_dir = os.path.join(to_dir, subdir)
47 for stream_name in options.stream_names:
48 if ':' in stream_name:
49 stream_name, subdir = stream_name.split(':')
50 else:
51 subdir = None
52 stream = router[stream_name]
53 from_dir = stream['from']
54 to_dir = stream['to']
55 if subdir:
56 from_dir = os.path.join(from_dir, subdir)
57 to_dir = os.path.join(to_dir, subdir)
58 sync_directories(from_dir, to_dir, options)
59 if options.apply:
60 run_command(['sync'])
5661
62
63 def sync_directories(from_dir, to_dir, options):
5764 from_dir = clean_dir(from_dir)
5865 to_dir = clean_dir(to_dir)
5966
6168 if not os.path.isdir(d):
6269 raise ValueError("Directory '{}' is not present".format(d))
6370
64 dry_run = not options.apply
65 dry_run_option = '--dry-run ' if dry_run else ''
66 checksum_option = '--checksum ' if options.thorough else ''
67 cmd = 'rsync {}{}--archive --verbose --delete "{}" "{}"'.format(dry_run_option, checksum_option, from_dir, to_dir)
68 run_command(cmd)
69 if not dry_run:
70 run_command('sync')
71 argv = ['rsync']
72 if not options.apply:
73 argv.append('--dry-run')
74 if options.thorough:
75 argv.append('--checksum')
76 argv.extend(['--archive', '--verbose', '--delete', from_dir, to_dir])
77 run_command(argv)
7178
7279
7380 def rename(router, options):
111118 argparser.add_argument('router', metavar='ROUTER', type=str,
112119 help='JSON file containing the backup router description'
113120 )
114 argparser.add_argument('--version', action='version', version="%(prog)s 0.4")
121 argparser.add_argument('--version', action='version', version="%(prog)s 0.5")
115122
116123 subparsers = argparser.add_subparsers()
117124
123130 parser_list.set_defaults(func=list_)
124131
125132 # - - - - sync - - - -
126 parser_sync = subparsers.add_parser('sync', help='Sync contents across a sync stream specified by name')
127 parser_sync.add_argument('stream_name', metavar='STREAM', type=str,
133 parser_sync = subparsers.add_parser('sync', help='Sync contents across one or more sync streams')
134 parser_sync.add_argument('stream_names', metavar='STREAM', type=str, nargs='+',
128135 help='Name of stream (or stream:subdirectory) to sync contents across'
129136 )
130137 parser_sync.add_argument('--apply', default=False, action='store_true',
2828 check_call("mkdir -p canonical", shell=True)
2929 check_call("touch canonical/thing", shell=True)
3030 check_call("mkdir -p cache", shell=True)
31 check_call("mkdir -p canonical3", shell=True)
32 check_call("mkdir -p cache3", shell=True)
3133 router = {
3234 'basic': {
3335 'from': 'canonical',
3436 'to': 'cache',
3537 },
36 'other': {
38 'notfound': {
3739 'from': 'canonical2',
3840 'to': 'cache2',
39 }
41 },
42 'other': {
43 'from': 'canonical3',
44 'to': 'cache3',
45 },
4046 }
4147 with open('backup.json', 'w') as f:
4248 f.write(json.dumps(router))
5258 with self.assertRaises(SystemExit):
5359 main(['backup.json'])
5460
61 def test_list(self):
62 main(['backup.json', 'list'])
63 output = sys.stdout.getvalue()
64 self.assertEqual(output.split('\n'), [
65 'basic: canonical => cache',
66 'other: canonical3 => cache3',
67 '',
68 ])
69
5570 def test_sync_dry_run(self):
5671 main(['backup.json', 'sync', 'basic:'])
5772 self.assertFalse(os.path.exists('cache/thing'))
5873 output = sys.stdout.getvalue()
59 self.assertEqual(output.split('\n')[0], 'rsync --dry-run --archive --verbose --delete "canonical/" "cache/"')
74 self.assertEqual(output.split('\n')[0], 'rsync --dry-run --archive --verbose --delete canonical/ cache/')
6075 self.assertIn('DRY RUN', output)
6176
6277 def test_sync_apply(self):
6479 self.assertTrue(os.path.exists('cache/thing'))
6580 output = sys.stdout.getvalue()
6681 self.assertEqual(output.split('\n')[:4], [
67 'rsync --archive --verbose --delete "canonical/" "cache/"',
82 'rsync --archive --verbose --delete canonical/ cache/',
6883 'sending incremental file list',
6984 'thing',
7085 ''
7186 ])
7287
73 def test_stream_not_exist(self):
88 def test_sync_subdirectory(self):
89 check_call("mkdir -p canonical/subdir", shell=True)
90 check_call("mkdir -p cache/subdir", shell=True)
91 check_call("touch canonical/subdir/stuff", shell=True)
92 main(['backup.json', 'sync', 'basic:subdir', '--apply'])
93 self.assertTrue(os.path.exists('cache/subdir/stuff'))
94 self.assertFalse(os.path.exists('cache/thing'))
95 output = sys.stdout.getvalue()
96 self.assertEqual(output.split('\n')[:4], [
97 'rsync --archive --verbose --delete canonical/subdir/ cache/subdir/',
98 'sending incremental file list',
99 'stuff',
100 ''
101 ])
102
103 def test_sync_stream_does_not_exist(self):
74104 with self.assertRaises(ValueError) as ar:
75 main(['backup.json', 'sync', 'other:', '--apply'])
105 main(['backup.json', 'sync', 'notfound', '--apply'])
76106 self.assertIn("Directory 'canonical2/' is not present", str(ar.exception))
107
108 def test_sync_multiple_streams(self):
109 main(['backup.json', 'sync', 'other', 'basic'])
110 output = sys.stdout.getvalue()
111 lines = [l for l in output.split('\n') if l.startswith('rsync')]
112 self.assertEqual(lines, [
113 'rsync --dry-run --archive --verbose --delete canonical3/ cache3/',
114 'rsync --dry-run --archive --verbose --delete canonical/ cache/',
115 ])
116
117 def test_sync_thorough(self):
118 main(['backup.json', 'sync', 'basic', '--thorough'])
119 output = sys.stdout.getvalue()
120 lines = [l for l in output.split('\n') if l.startswith('rsync')]
121 self.assertEqual(lines, [
122 'rsync --dry-run --checksum --archive --verbose --delete canonical/ cache/',
123 ])
124
125 def test_sync_with_spaces_in_dirnames(self):
126 check_call("mkdir -p 'can onical'", shell=True)
127 check_call("mkdir -p 'ca che'", shell=True)
128 router = {
129 'spaced': {
130 'from': 'can onical',
131 'to': 'ca che',
132 },
133 }
134 with open('backup.json', 'w') as f:
135 f.write(json.dumps(router))
136 main(['backup.json', 'sync', 'spaced'])
137 output = sys.stdout.getvalue()
138 lines = [l for l in output.split('\n') if l.startswith('rsync')]
139 self.assertEqual(lines, [
140 'rsync --dry-run --archive --verbose --delete "can onical/" "ca che/"',
141 ])
77142
78143 def test_rename(self):
79144 check_call("mkdir -p canonical/sclupture", shell=True)