git @ Cat's Eye Technologies tagfarm / 10e4369
Merge pull request #1 from cpressey/develop-0.2 Develop 0.2 Chris Pressey authored 4 years ago GitHub committed 4 years ago
5 changed file(s) with 304 addition(s) and 194 deletion(s). Raw diff Collapse all Expand all
+1
-3
README.md less more
00 `tagfarm`
11 =========
22
3 _Version 0.1_
3 _Version 0.2_
44 | _Entry_ [@ catseye.tc](https://catseye.tc/node/tagfarm)
55 | _See also:_ [shelf](https://github.com/catseye/shelf#readme)
66 ∘ [ellsync](https://github.com/catseye/ellsync#readme)
132132 TODO
133133 ----
134134
135 Unit tests.
136
137135 Better handling of cases where the target being linked is itself a link.
138136
139137 Set-theoretic queries on tags (e.g. tag all files with X or Y and not Z with a new tag T).
+76
-191
src/tagfarm/main.py less more
00 # encoding: UTF-8
11
22 from argparse import ArgumentParser
3 import errno
43 import os
54 import sys
6 from subprocess import Popen, STDOUT, PIPE
5
6 from tagfarm.utils import (
7 mkdir_p, find_media_root, tag_file, perform_repair
8 )
79
810
9 # - - - -
10
11
12 def mkdir_p(path):
13 try:
14 os.makedirs(path)
15 except OSError as exc:
16 if exc.errno == errno.EEXIST and os.path.isdir(path):
17 pass
18 else:
19 raise
20
21
22 def find_media_root(path):
23 while path != '/':
24 if os.path.isdir(os.path.join(path, 'by-tag')):
25 return path
26 path = os.path.abspath(os.path.join(path, os.pardir))
27 raise ValueError("could not locate `by-tag` directory in curent directory or any parent directory thereof")
28
29
30 def index_files(media_root):
31 index = {}
32 for root, dirs, files in os.walk(media_root):
33 if os.path.normpath(root) == os.path.normpath(os.path.join(media_root, 'by-tag')):
34 dirs[:] = []
35 continue
36
37 for basename in files:
38 filename = os.path.normpath(os.path.join(root, basename))
39 if os.path.islink(filename):
40 continue
41
42 index.setdefault(basename, set()).add(filename)
43 return index
44
45
46 def is_broken_link(path):
47 return os.path.lexists(path) and not os.path.exists(path)
48
49
50 def is_absolute_link(path):
51 return os.path.lexists(path) and os.readlink(path).startswith('/')
52
53
54 def tag_file(media_root, filename, tag):
55 mkdir_p(os.path.join(media_root, 'by-tag', tag))
56 linkname = os.path.join(media_root, 'by-tag', tag, os.path.basename(filename))
57 if not os.path.lexists(linkname):
58 srcname = os.path.join('..', '..', os.path.relpath(filename, media_root))
59 os.symlink(srcname, linkname)
60
61
62 def perform_repair(media_root, verbose=False, force_relink=False, prune=False):
63 index = index_files(media_root)
64
65 by_tags_dir = os.path.join(media_root, 'by-tag')
66 for tag in sorted(os.listdir(by_tags_dir)):
67 tagdir = os.path.join(by_tags_dir, tag)
68 if not os.path.isdir(tagdir):
69 continue
70 print('*** {}'.format(tag))
71 for basename in sorted(os.listdir(tagdir)):
72
73 linkname = os.path.join(tagdir, basename)
74
75 if basename.startswith('Link to '):
76 new_basename = basename[8:]
77 new_linkname = os.path.join(tagdir, new_basename)
78 if os.path.lexists(new_linkname):
79 print("WARNING: not renaming '{}' because '{}' already exists".format(linkname, new_linkname))
80 else:
81 print("RENAMING {} -> {}".format(linkname, new_linkname))
82 os.rename(linkname, new_linkname)
83 linkname = new_linkname
84
85 if not force_relink:
86 if not os.path.islink(linkname):
87 print("WARNING: skipping {} (is regular file, but --force-relink not given)".format(linkname))
88 continue
89 if not (is_broken_link(linkname) or is_absolute_link(linkname)):
90 if verbose:
91 print('kept {} -> {}'.format(linkname, os.readlink(linkname)))
92 continue
93
94 candidates = index.get(basename, set())
95 if len(candidates) == 0:
96 if prune:
97 print("NOTICE: no candidates for {}, DELETING".format(basename))
98 os.remove(linkname)
99 else:
100 print("WARNING: no candidates for {}".format(basename))
101 elif len(candidates) > 1:
102 print("WARNING: multiple candidates for {}: {}".format(basename, candidates))
103 else:
104 os.remove(linkname)
105 filename = list(candidates)[0]
106 srcname = os.path.join('..', '..', os.path.relpath(filename, media_root))
107 os.symlink(srcname, linkname)
108 print('FIXED {} -> {}'.format(linkname, srcname))
109
110
111 # - - - - commands - - - -
112
113
114 def tag(media_root, args, verbose=False):
115 argparser = ArgumentParser()
116 argparser.add_argument('tag', metavar='TAG', type=str,
117 help='Name of tag to apply'
118 )
119 argparser.add_argument('filenames', metavar='FILENAME', type=str, nargs='+',
120 help='Names of files to tag'
121 )
122 options = argparser.parse_args(args)
123
11 def tag(media_root, options):
12412 for filename in options.filenames:
12513 tag_file(media_root, filename, options.tag)
12614
12715
128 def untag(media_root, args, verbose=False):
129 argparser = ArgumentParser()
130 argparser.add_argument('tag', metavar='TAG', type=str,
131 help='Name of tag to remove'
132 )
133 argparser.add_argument('filenames', metavar='FILENAME', type=str, nargs='+',
134 help='Names of files to untag'
135 )
136 options = argparser.parse_args(args)
137
16 def untag(media_root, options):
13817 for filename in options.filenames:
13918 linkname = os.path.join(media_root, 'by-tag', options.tag, os.path.basename(filename))
14019 if os.path.lexists(linkname):
14120 os.remove(linkname)
14221
14322
144 def showtags(media_root, args, verbose=False):
145 argparser = ArgumentParser()
146 argparser.add_argument('filenames', metavar='FILENAME', type=str, nargs='+',
147 help='Names of files to show tags of',
148 )
149 argparser.add_argument('--show-only-fewer-than', type=int, default=None,
150 help='If given, report only those files that have fewer than this number of tags',
151 )
152 options = argparser.parse_args(args)
153
23 def showtags(media_root, options):
15424 by_tags_dir = os.path.join(media_root, 'by-tag')
155
15625 for filename in options.filenames:
15726 tags = []
15827 basename = os.path.basename(os.path.normpath(filename))
16433 print('{}: {}'.format(filename, ', '.join(tags)))
16534
16635
167 def repair(media_root, args, verbose=False):
168 argparser = ArgumentParser()
169 argparser.add_argument('--force-relink', action='store_true',
170 help='Replace files found in taglinks directory even when they are not symlinks or not broken'
171 )
172 argparser.add_argument('--prune', action='store_true',
173 help='Remove broken symlinks for which no candidate files can be found'
174 )
175 options = argparser.parse_args(args)
176
177 perform_repair(media_root, verbose=verbose, force_relink=options.force_relink, prune=options.prune)
36 def repair(media_root, options):
37 perform_repair(media_root, verbose=options.verbose, force_relink=options.force_relink, prune=options.prune)
17838
17939
180 def rename(media_root, args, verbose=False):
181 argparser = ArgumentParser()
182 argparser.add_argument('src', metavar='FILENAME', type=str,
183 help='Current name of file that is to be renamed'
184 )
185 argparser.add_argument('dest', metavar='FILENAME', type=str,
186 help='New name of file'
187 )
188 options = argparser.parse_args(args)
189
40 def rename(media_root, options):
19041 src = os.path.normpath(options.src)
19142 dest = os.path.normpath(options.dest)
19243
20657 print('UPDATED {} -> {}'.format(new_linkname, target))
20758
20859
209 def collect(media_root, args, verbose=False):
210 argparser = ArgumentParser()
211 argparser.add_argument('tag', metavar='TAG', type=str,
212 help='Tag to select files by'
213 )
214 argparser.add_argument('dest', metavar='DIRNAME', type=str,
215 help='Directory to move files into'
216 )
217 options = argparser.parse_args(args)
218
60 def collect(media_root, options):
21961 tagdir = os.path.join(media_root, 'by-tag', options.tag)
22062 if not os.path.isdir(tagdir):
22163 print("WARNING: no files tagged '{}'".format(options.tag))
24183 perform_repair(media_root, verbose=verbose)
24284
24385
244 COMMANDS = {
245 'tag': tag,
246 'untag': untag,
247 'showtags': showtags,
248 'repair': repair,
249 'rename': rename,
250 'collect': collect,
251 }
252
25386 # - - - - driver - - - -
25487
25588
25689 def main(args):
257 argparser = ArgumentParser()
90 parser = ArgumentParser()
25891
259 argparser.add_argument('command', metavar='COMMAND', type=str,
260 help='The action to take. One of: {}'.format(', '.join(COMMANDS.keys()))
261 )
262 argparser.add_argument('--verbose', action='store_true',
92 parser.add_argument('--verbose', action='store_true',
26393 help='Produce more reporting output'
26494 )
26595
266 options, remaining_args = argparser.parse_known_args(args)
96 subparsers = parser.add_subparsers()
26797
98 # - - - - tag - - - -
99 parser_tag = subparsers.add_parser('tag', help='Add a given tag to one or more files')
100 parser_tag.add_argument('tag', metavar='TAG', type=str,
101 help='Name of tag to apply'
102 )
103 parser_tag.add_argument('filenames', metavar='FILENAME', type=str, nargs='+',
104 help='Names of files to tag'
105 )
106 parser_tag.set_defaults(func=tag)
107
108 # - - - - untag - - - -
109 parser_untag = subparsers.add_parser('untag', help='Remove a given tag from one or more files')
110 parser_untag.add_argument('tag', metavar='TAG', type=str,
111 help='Name of tag to remove'
112 )
113 parser_untag.add_argument('filenames', metavar='FILENAME', type=str, nargs='+',
114 help='Names of files to untag'
115 )
116 parser_untag.set_defaults(func=untag)
117
118 # - - - - showtags - - - -
119 parser_showtags = subparsers.add_parser('showtags', help='Report the tags currently on one or more files')
120 parser_showtags.add_argument('filenames', metavar='FILENAME', type=str, nargs='+',
121 help='Names of files to show tags of',
122 )
123 parser_showtags.add_argument('--show-only-fewer-than', type=int, default=None,
124 help='If given, report only those files that have fewer than this number of tags',
125 )
126 parser_showtags.set_defaults(func=showtags)
127
128 # - - - - repair - - - -
129 parser_repair = subparsers.add_parser('repair', help='Re-assign broken tag links to relocated files')
130 parser_repair.add_argument('--force-relink', action='store_true',
131 help='Replace files found in taglinks directory even when they are not symlinks or not broken'
132 )
133 parser_repair.add_argument('--prune', action='store_true',
134 help='Remove broken symlinks for which no candidate files can be found'
135 )
136 parser_repair.set_defaults(func=repair)
137
138 # - - - - rename - - - -
139 parser_rename = subparsers.add_parser('rename', help='Rename file whille updating all its tag links')
140 parser_rename.add_argument('src', metavar='FILENAME', type=str,
141 help='Current name of file that is to be renamed'
142 )
143 parser_rename.add_argument('dest', metavar='FILENAME', type=str,
144 help='New name of file'
145 )
146 parser_rename.set_defaults(func=rename)
147
148 # - - - - collect - - - -
149 parser_collect = subparsers.add_parser('collect', help='Move all files with a given tag into a given directory')
150 parser_collect.add_argument('tag', metavar='TAG', type=str,
151 help='Tag to select files by'
152 )
153 parser_collect.add_argument('dest', metavar='DIRNAME', type=str,
154 help='Directory to move files into'
155 )
156 parser_collect.set_defaults(func=collect)
157
158 options = parser.parse_args(args)
268159 media_root = find_media_root(os.path.realpath('.'))
269
270 command = COMMANDS.get(options.command, None)
271 if command:
272 command(media_root, remaining_args, verbose=options.verbose)
273 else:
274 argparser.print_help()
275 sys.exit(1)
160 options.func(media_root, options)
+102
-0
src/tagfarm/tests.py less more
0 import os
1 import sys
2 from tempfile import mkdtemp
3 import unittest
4 from subprocess import check_call
5
6 try:
7 from StringIO import StringIO
8 except ImportError:
9 from io import StringIO
10 assert StringIO
11
12 from tagfarm.main import main
13
14
15 class TestTagfarm(unittest.TestCase):
16
17 def setUp(self):
18 super(TestTagfarm, self).setUp()
19 self.saved_stdout = sys.stdout
20 self.saved_stderr = sys.stderr
21 sys.stdout = StringIO()
22 sys.stderr = StringIO()
23 self.maxDiff = None
24 self.dirname = mkdtemp()
25 self.prevdir = os.getcwd()
26 os.chdir(self.dirname)
27 check_call("mkdir -p by-tag", shell=True)
28
29 def tearDown(self):
30 os.chdir(self.prevdir)
31 check_call("rm -rf {}".format(self.dirname), shell=True)
32 sys.stdout = self.saved_stdout
33 sys.stderr = self.saved_stderr
34 super(TestTagfarm, self).tearDown()
35
36 def test_unknown_subcommand(self):
37 with self.assertRaises(SystemExit):
38 main(['yarfify'])
39
40 def test_media_root_not_found(self):
41 check_call("rm -rf by-tag", shell=True)
42 check_call("touch content1", shell=True)
43 with self.assertRaises(ValueError):
44 main(['tag', 'blixit', 'content1'])
45
46 def test_tag(self):
47 check_call("touch content1", shell=True)
48 check_call("touch content2", shell=True)
49 main(['tag', 'blixit', 'content1', 'content2'])
50 main(['tag', 'flonk', 'content1'])
51
52 self.assertEqual(os.readlink(os.path.join('by-tag', 'blixit', 'content1')), '../../content1')
53 self.assertEqual(os.readlink(os.path.join('by-tag', 'blixit', 'content2')), '../../content2')
54
55 self.assertEqual(os.readlink(os.path.join('by-tag', 'flonk', 'content1')), '../../content1')
56 self.assertFalse(os.path.exists(os.path.join('by-tag', 'flonk', 'content2')))
57 self.assertFalse(os.path.lexists(os.path.join('by-tag', 'flonk', 'content2')))
58
59 def test_untag(self):
60 check_call("touch content1", shell=True)
61 check_call("touch content2", shell=True)
62 main(['tag', 'blixit', 'content1', 'content2'])
63 main(['tag', 'flonk', 'content1'])
64
65 main(['untag', 'blixit', 'content1', 'content2'])
66 main(['untag', 'flonk', 'content2'])
67
68 self.assertFalse(os.path.exists(os.path.join('by-tag', 'blixit', 'content1')))
69 self.assertFalse(os.path.lexists(os.path.join('by-tag', 'blixit', 'content1')))
70
71 self.assertFalse(os.path.exists(os.path.join('by-tag', 'blixit', 'content2')))
72 self.assertFalse(os.path.lexists(os.path.join('by-tag', 'blixit', 'content2')))
73
74 self.assertEqual(os.readlink(os.path.join('by-tag', 'flonk', 'content1')), '../../content1')
75
76 self.assertFalse(os.path.exists(os.path.join('by-tag', 'flonk', 'content2')))
77 self.assertFalse(os.path.lexists(os.path.join('by-tag', 'flonk', 'content2')))
78
79 def test_repair(self):
80 check_call("mkdir -p subdir1", shell=True)
81 check_call("mkdir -p subdir2", shell=True)
82 check_call("touch subdir1/content1", shell=True)
83
84 main(['tag', 'blixit', 'subdir1/content1'])
85
86 self.assertEqual(os.readlink(os.path.join('by-tag', 'blixit', 'content1')), '../../subdir1/content1')
87
88 check_call("mv subdir1/content1 subdir2/content1", shell=True)
89
90 self.assertFalse(os.path.exists(os.path.join('by-tag', 'blixit', 'content1')))
91 self.assertTrue(os.path.lexists(os.path.join('by-tag', 'blixit', 'content1')))
92
93 self.assertEqual(os.readlink(os.path.join('by-tag', 'blixit', 'content1')), '../../subdir1/content1')
94
95 main(['repair'])
96
97 self.assertEqual(os.readlink(os.path.join('by-tag', 'blixit', 'content1')), '../../subdir2/content1')
98
99
100 if __name__ == '__main__':
101 unittest.main()
+121
-0
src/tagfarm/utils.py less more
0 # encoding: UTF-8
1
2 import errno
3 import os
4
5
6 def mkdir_p(path):
7 try:
8 os.makedirs(path)
9 except OSError as exc:
10 if exc.errno == errno.EEXIST and os.path.isdir(path):
11 pass
12 else:
13 raise
14
15
16 def find_media_root(path):
17 while path != '/':
18 if os.path.isdir(os.path.join(path, 'by-tag')):
19 return path
20 path = os.path.abspath(os.path.join(path, os.pardir))
21 raise ValueError("could not locate `by-tag` directory in curent directory or any parent directory thereof")
22
23
24 def index_files(media_root):
25 index = {}
26 for root, dirs, files in os.walk(media_root):
27 if os.path.normpath(root) == os.path.normpath(os.path.join(media_root, 'by-tag')):
28 dirs[:] = []
29 continue
30
31 for basename in files:
32 filename = os.path.normpath(os.path.join(root, basename))
33 if os.path.islink(filename):
34 continue
35
36 index.setdefault(basename, set()).add(filename)
37 return index
38
39
40 def is_broken_link(path):
41 return os.path.lexists(path) and not os.path.exists(path)
42
43
44 def is_absolute_link(path):
45 return os.path.lexists(path) and os.readlink(path).startswith('/')
46
47
48 def readlink_or_broken(path):
49 return '*BROKEN*' if is_broken_link(path) else os.readlink(path)
50
51
52 def relativize_target(media_root, path):
53 # Prepends `../..` because the link always resides in `by-tag/<tagname>`
54 return os.path.join('..', '..', os.path.relpath(path, media_root))
55
56
57 def tag_file(media_root, filename, tag):
58 mkdir_p(os.path.join(media_root, 'by-tag', tag))
59 linkname = os.path.join(media_root, 'by-tag', tag, os.path.basename(filename))
60 if not os.path.lexists(linkname):
61 srcname = relativize_target(media_root, filename)
62 os.symlink(srcname, linkname)
63
64
65 def perform_repair(media_root, verbose=False, force_relink=False, prune=False):
66 index = index_files(media_root)
67
68 by_tags_dir = os.path.join(media_root, 'by-tag')
69 for tag in sorted(os.listdir(by_tags_dir)):
70 tagdir = os.path.join(by_tags_dir, tag)
71 if not os.path.isdir(tagdir):
72 continue
73 repairs_made = []
74 for basename in sorted(os.listdir(tagdir)):
75
76 linkname = os.path.join(tagdir, basename)
77
78 if basename.startswith('Link to '):
79 new_basename = basename[8:]
80 new_linkname = os.path.join(tagdir, new_basename)
81 if os.path.lexists(new_linkname):
82 repairs_made.append(
83 "WARNING: not renaming '{}' (-> '{}') because '{}' (-> '{}') already exists".format(
84 linkname, readlink_or_broken(linkname), new_linkname, readlink_or_broken(new_linkname)
85 )
86 )
87 else:
88 repairs_made.append("RENAMING {} -> {}".format(linkname, new_linkname))
89 os.rename(linkname, new_linkname)
90 linkname = new_linkname
91
92 if not force_relink:
93 if not os.path.islink(linkname):
94 repairs_made.append("WARNING: skipping {} (is regular file, but --force-relink not given)".format(linkname))
95 continue
96 if not (is_broken_link(linkname) or is_absolute_link(linkname)):
97 if verbose:
98 repairs_made.append('kept {} -> {}'.format(linkname, os.readlink(linkname)))
99 continue
100
101 candidates = index.get(basename, set())
102 if len(candidates) == 0:
103 if prune:
104 repairs_made.append("NOTICE: no candidates for {}, DELETING".format(basename))
105 os.remove(linkname)
106 else:
107 repairs_made.append("WARNING: no candidates for {}".format(basename))
108 elif len(candidates) > 1:
109 repairs_made.append("WARNING: multiple candidates for {}: {}".format(basename, candidates))
110 else:
111 os.remove(linkname)
112 filename = list(candidates)[0]
113 srcname = relativize_target(media_root, filename)
114 os.symlink(srcname, linkname)
115 repairs_made.append('FIXED {} -> {}'.format(linkname, srcname))
116
117 if repairs_made:
118 print('*** {}'.format(tag))
119 for repair_made in repairs_made:
120 print(repair_made)
+4
-0
test.sh less more
0 #!/bin/sh
1
2 PYTHONPATH=src python2 src/tagfarm/tests.py || exit 1
3 PYTHONPATH=src python3 src/tagfarm/tests.py || exit 1