| 1 | import os, hashlib, urllib.request, time |
| 2 | from . import profile |
| 3 | pj = os.path.join |
| 4 | |
| 5 | class notfound(Exception): |
| 6 | pass |
| 7 | |
| 8 | class cache(object): |
| 9 | def __init__(self, dir): |
| 10 | self.dir = dir |
| 11 | |
| 12 | def mangle(self, url): |
| 13 | n = hashlib.md5() |
| 14 | n.update(url.encode("ascii")) |
| 15 | return n.hexdigest() |
| 16 | |
| 17 | def open(self, url): |
| 18 | req = urllib.request.Request(url, headers={"User-Agent": "automanga/1"}) |
| 19 | return urllib.request.urlopen(req) |
| 20 | |
| 21 | def miss(self, url): |
| 22 | try: |
| 23 | s = self.open(url) |
| 24 | except urllib.error.HTTPError as exc: |
| 25 | if exc.code == 404: |
| 26 | raise notfound(url) |
| 27 | raise |
| 28 | with s: |
| 29 | if s.headers.get("content-encoding") == "gzip": |
| 30 | import gzip, io |
| 31 | return gzip.GzipFile(fileobj=io.BytesIO(s.read()), mode="r").read() |
| 32 | return s.read() |
| 33 | |
| 34 | def fetch(self, url, expire=3600): |
| 35 | path = pj(self.dir, self.mangle(url)) |
| 36 | if os.path.exists(path): |
| 37 | if time.time() - os.stat(path).st_mtime < expire: |
| 38 | with open(path, "rb") as f: |
| 39 | return f.read() |
| 40 | data = self.miss(url) |
| 41 | if not os.path.isdir(self.dir): |
| 42 | os.makedirs(self.dir) |
| 43 | with open(path, "wb") as f: |
| 44 | f.write(data) |
| 45 | return data |
| 46 | |
| 47 | default = cache(pj(profile.confdir, "htcache")) |
| 48 | |
| 49 | def fetch(url, expire=3600): |
| 50 | return default.fetch(url, expire) |