Commit | Line | Data |
---|---|---|
64eb9fa5 FT |
1 | #!/usr/bin/python3 |
2 | ||
3 | import sys, os, getopt, time, random | |
4 | import manga.lib, manga.profile | |
5 | from PIL import Image | |
6 | ||
7 | verbose = 0 | |
8 | wait = 10 | |
e4aeea73 | 9 | getnames = False |
64eb9fa5 FT |
10 | |
11 | def msg(vl, msg, *args): | |
12 | if verbose >= vl: | |
13 | sys.stderr.write("getmanga: " + (msg % args) + "\n") | |
14 | ||
15 | def getprop(nm, default=None): | |
6e3b8ae1 | 16 | if mprof and "dl-" + nm in mprof.props: |
64eb9fa5 FT |
17 | return mprof.props["dl-" + nm] |
18 | if nm in props: | |
19 | return props[nm] | |
20 | return default | |
21 | ||
22 | def digits(num): | |
23 | n, i = 10, 1 | |
24 | while True: | |
25 | if num < n: | |
26 | return i | |
27 | n, i = n * 10, i + 1 | |
28 | ||
29 | def autoname(page): | |
30 | ret = "" | |
e4aeea73 | 31 | inames = [] |
64eb9fa5 FT |
32 | for t, i in page.stack: |
33 | if ret: | |
e4aeea73 FT |
34 | inames.append((t, ret)) |
35 | if ret: | |
64eb9fa5 FT |
36 | ret += "-" |
37 | ret += "%0*i" % (digits(len(t) + 1), i + 1) | |
e4aeea73 | 38 | return ret, inames |
64eb9fa5 FT |
39 | |
40 | def expand(pattern, page): | |
41 | ret = "" | |
42 | si = 0 | |
43 | fp = 0 | |
e4aeea73 | 44 | inames = [] |
67983628 | 45 | stack = list(zip([t for t, i in page.stack], [t for t, i in page.stack[1:]] + [page], [i for t, i in page.stack])) |
64eb9fa5 FT |
46 | while True: |
47 | p = pattern.find('%', fp) | |
48 | if p < 0: | |
67983628 | 49 | if si < len(stack): |
64eb9fa5 | 50 | sys.stderr.write("getmanga: pattern %s did not match page %s\n" % |
67983628 | 51 | (pattern, "/".join(c.name for t, c, i in stack))) |
64eb9fa5 | 52 | sys.exit(1) |
e4aeea73 | 53 | return ret + pattern[fp:], inames |
64eb9fa5 | 54 | ret += pattern[fp:p] |
67983628 FT |
55 | m = pattern[p + 1:p + 2] |
56 | fp = p + 2 | |
64eb9fa5 FT |
57 | if m == "%": |
58 | ret += "%" | |
59 | else: | |
67983628 | 60 | if si >= len(stack): |
64eb9fa5 | 61 | sys.stderr.write("getmanga: pattern %s did not match page %s\n" % |
67983628 | 62 | (pattern, "/".join(c.name for t, c, i in stack))) |
64eb9fa5 | 63 | sys.exit(1) |
67983628 | 64 | t, ct, ti = stack[si] |
64eb9fa5 FT |
65 | si += 1 |
66 | if m == "i": | |
67 | ret += "%0*i" % (digits(len(t) + 1), ti + 1) | |
68 | elif m == "n": | |
67983628 | 69 | ret += ct.name |
64eb9fa5 | 70 | elif m == "d": |
67983628 | 71 | ret += ct.id |
64eb9fa5 | 72 | else: |
67983628 | 73 | sys.stderr.write("getmanga: %s: unknown specifier `%s'\n" % (pattern, m)) |
64eb9fa5 | 74 | sys.exit(1) |
e4aeea73 FT |
75 | if si < len(stack): |
76 | inames.append((ct, ret)) | |
77 | ||
78 | checkednames = None | |
79 | def checknames(tdir, names): | |
80 | global checkednames | |
81 | nmpath = os.path.join(tdir, "names") | |
82 | if checkednames is None: | |
83 | checkednames = {} | |
84 | if os.path.exists(nmpath): | |
85 | with open(nmpath) as fp: | |
86 | for line in fp: | |
87 | line = line.strip() | |
88 | p = line.find(' ') | |
89 | if p < 0: continue | |
90 | checkednames[line[:p]] = line[p + 1:] | |
91 | for t, prefix in names: | |
92 | if not prefix: continue | |
93 | if ' ' not in prefix and prefix not in checkednames: | |
94 | with manga.profile.txfile(nmpath, "w") as fp: | |
95 | if '\n' not in t.name: | |
96 | checkednames[prefix] = t.name | |
97 | msg(1, "adding name %s for %s" % (t.name, prefix)) | |
98 | else: | |
99 | checkednames[prefix] = "" | |
100 | sys.stderr.write("getmanga: warning: node names contains newlines: %r\n" % (t.name,)) | |
101 | for prefix, name in checkednames.items(): | |
102 | if not name: continue | |
103 | fp.write("%s %s\n" % (prefix, name)) | |
64eb9fa5 FT |
104 | |
105 | def download(mng, tdir, pattern): | |
106 | exts = ["", ".jpg", ".jpeg", ".png", ".gif"] | |
107 | fmts = {"PNG": "png", "JPEG": "jpeg", "GIF": "gif"} | |
108 | for page in manga.lib.cursor(mng): | |
109 | if pattern is None: | |
e4aeea73 | 110 | nm, inames = autoname(page) |
64eb9fa5 | 111 | else: |
e4aeea73 FT |
112 | nm, inames = expand(pattern, page) |
113 | if getnames: | |
114 | checknames(tdir, inames) | |
64eb9fa5 FT |
115 | path = os.path.join(tdir, nm) |
116 | if any(os.path.exists(path + ext) for ext in exts): | |
117 | msg(2, "%s exists, skipping", nm) | |
118 | continue | |
119 | msg(1, "getting %s...", nm) | |
8fdbf188 FT |
120 | retries = 0 |
121 | while True: | |
122 | try: | |
123 | fp = page.open() | |
124 | break | |
125 | except OSError as error: | |
126 | if retries < 5: | |
127 | sys.stderr.write("getmanga: warning: error when getting %s: %s\n" % (nm, error)) | |
128 | retries += 1 | |
129 | time.sleep(60) | |
130 | else: | |
131 | sys.stderr.write("getmanga: error when getting %s: %s\n" % (nm, error)) | |
132 | sys.exit(1) | |
133 | with fp: | |
64eb9fa5 | 134 | with open(path, "wb") as out: |
daaea5d1 FT |
135 | done = False |
136 | try: | |
137 | while True: | |
138 | data = fp.read(65536) | |
139 | if data == b"": | |
140 | done = True | |
141 | break | |
142 | out.write(data) | |
143 | finally: | |
144 | if not done: | |
145 | os.unlink(path) | |
64eb9fa5 FT |
146 | try: |
147 | img = Image.open(path) | |
148 | except OSError: | |
149 | fmt = None | |
150 | else: | |
151 | fmt = img.format | |
152 | if fmt not in fmts: | |
153 | sys.stderr.write("getmanga: warning: could not determine file format of %s, leaving as is\n" % nm) | |
154 | else: | |
155 | os.rename(path, path + "." + fmts[fmt]) | |
156 | msg(3, "%s -> %s", nm, nm + "." + fmts[fmt]) | |
157 | cwait = abs(random.gauss(0, 1) * wait) | |
158 | msg(2, "waiting %.1f s...", cwait) | |
159 | time.sleep(cwait) | |
160 | ||
161 | def usage(out): | |
e4aeea73 | 162 | out.write("usage: getmanga [-hvn] [-w WAIT] [-p PROFILE] [-P PATTERN] DIRECTORY [LIBRARY ID]\n") |
aaab61ee FT |
163 | out.write("\tpattern templates:\n") |
164 | out.write("\t %i\tSequence number\n") | |
165 | out.write("\t %n\tName\n") | |
166 | out.write("\t %d\tID\n") | |
64eb9fa5 FT |
167 | |
168 | def main(): | |
e4aeea73 | 169 | global verbose, wait, mprof, props, getnames |
64eb9fa5 | 170 | |
e4aeea73 | 171 | opts, args = getopt.getopt(sys.argv[1:], "hvnp:w:P:") |
6e3b8ae1 | 172 | profnm = None |
da9e5bf6 | 173 | pattern = None |
64eb9fa5 FT |
174 | for o, a in opts: |
175 | if o == "-h": | |
176 | usage(sys.stdout) | |
177 | sys.exit(0) | |
178 | elif o == "-p": | |
179 | profnm = a | |
180 | elif o == "-v": | |
181 | verbose += 1 | |
e4aeea73 FT |
182 | elif o == "-n": |
183 | getnames = True | |
64eb9fa5 FT |
184 | elif o == "-w": |
185 | wait = int(a) | |
da9e5bf6 FT |
186 | elif o == "-P": |
187 | pattern = a | |
64eb9fa5 FT |
188 | if len(args) < 1: |
189 | usage(sys.stderr) | |
190 | sys.exit(1) | |
191 | tdir = args[0] | |
192 | ||
193 | if not os.path.isdir(tdir): | |
194 | sys.stderr.write("getmanga: %s: not a directory\n" % (tdir)) | |
195 | sys.exit(1) | |
196 | ||
197 | pfile = os.path.join(tdir, ".props") | |
198 | props = {} | |
199 | if os.path.exists(pfile): | |
200 | with open(pfile, "r") as fp: | |
66efacb2 | 201 | for words in manga.profile.splitlines(fp): |
64eb9fa5 FT |
202 | if words[0] == "set" and len(words) > 2: |
203 | props[words[1]] = words[2] | |
204 | elif words[0] == "lset" and len(words) > 1: | |
205 | props[words[1]] = words[2:] | |
206 | ||
6e3b8ae1 | 207 | if profnm is None: |
64eb9fa5 | 208 | profile = manga.profile.profile.last() |
6e3b8ae1 FT |
209 | elif profnm == "": |
210 | profile = None | |
64eb9fa5 FT |
211 | else: |
212 | profile = manga.profile.profile.byname(profnm) | |
213 | ||
1e880827 FT |
214 | if props.get("getnames", "") == "yes": |
215 | getnames = True | |
216 | ||
64eb9fa5 FT |
217 | if len(args) == 2: |
218 | usage(sys.stderr) | |
219 | sys.exit(1) | |
220 | elif len(args) > 2: | |
221 | libnm, mid = args[1:3] | |
222 | elif isinstance(props.get("manga"), list): | |
223 | libnm, mid = props["manga"] | |
224 | else: | |
225 | sys.stderr.write("getmanga: %s: id is neither saved nor given\n" % (tdir)) | |
226 | sys.exit(1) | |
227 | try: | |
228 | lib = manga.lib.findlib(libnm) | |
229 | except ImportError: | |
230 | sys.stderr.write("getmanga: no such library: %s\n" % (libnm)) | |
231 | sys.exit(1) | |
232 | try: | |
233 | mng = lib.byid(mid) | |
234 | except KeyError: | |
235 | sys.stderr.write("getmanga: no such manga: %s\n" % (mid)) | |
236 | sys.exit(1) | |
6e3b8ae1 FT |
237 | if profile is not None: |
238 | mprof = profile.getmanga(libnm, mng.id) | |
239 | else: | |
240 | mprof = None | |
64eb9fa5 | 241 | |
6e3b8ae1 | 242 | download(mng, tdir, pattern or getprop("pattern")) |
64eb9fa5 FT |
243 | |
244 | if __name__ == "__main__": | |
245 | try: | |
246 | main() | |
247 | except KeyboardInterrupt: | |
248 | pass |