From 9000767562d69ddfab2674516146dacf63a6f644 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Sat, 18 Jun 2016 03:07:41 +0200 Subject: [PATCH 01/16] python: Somewhat integrate async watchers with wsgidir currency. --- python3/ashd/async.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python3/ashd/async.py b/python3/ashd/async.py index 02c75a9..3493959 100644 --- a/python3/ashd/async.py +++ b/python3/ashd/async.py @@ -8,6 +8,7 @@ class epoller(object): self.lock = threading.RLock() self.ep = None self.th = None + self.stopped = False self._daemon = True @staticmethod @@ -38,6 +39,14 @@ class epoller(object): except Exception as exc: self.exception(ch, *sys.exc_info()) + def _closeall(self): + with self.lock: + while self.registered: + fd, (ch, evs) = next(iter(self.registered.items())) + del self.registered[fd] + self.ep.unregister(fd) + self._cb(ch, "close") + def _run(self): ep = select.epoll() try: @@ -47,6 +56,9 @@ class epoller(object): self.ep = ep while self.registered: + if self.stopped: + self._closeall() + break try: evlist = ep.poll(10) except IOError as exc: @@ -139,6 +151,17 @@ class epoller(object): if self.ep: self.ep.modify(fd, evs) + def stop(self): + if threading.current_thread() == self.th: + self.stopped = True + else: + def tgt(): + self.stopped = True + cb = callbuffer() + cb.call(tgt) + cb.stop() + self.add(cb) + def watcher(): return epoller() @@ -239,3 +262,10 @@ class callbuffer(object): if self.wp >= 0: os.close(self.wp) self.wp = -1 + +def currentwatcher(io, current): + def run(): + while current: + current.wait() + io.stop() + threading.Thread(target=run, name="Current watcher").start() -- 2.11.0 From 4350acb522111f385f74dd9c7f8b3e47443ff225 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Sun, 19 Jun 2016 02:33:28 +0200 Subject: [PATCH 02/16] htparser: Fixed up duplex timeouts. --- src/htparser.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/htparser.c b/src/htparser.c index a738e25..d10121c 100644 --- a/src/htparser.c +++ b/src/htparser.c @@ -333,7 +333,8 @@ static void passduplex(struct bufio *a, int afd, struct bufio *b, int bfd) if(ev) pfd[n++] = (struct selected){.fd = bfd, .ev = ev}; } - sel = mblock(600, n, pfd); + if((sel = mblock(600, n, pfd)).ev == 0) + break; if(sel.fd == afd) sio = a; else if(sel.fd == bfd) -- 2.11.0 From be6ec4ec06b2f394a58aa652a22a300fae371994 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Sun, 19 Jun 2016 02:33:48 +0200 Subject: [PATCH 03/16] python: Added a channel superclass for ashd.async. --- python3/ashd/async.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/python3/ashd/async.py b/python3/ashd/async.py index 3493959..4247bd6 100644 --- a/python3/ashd/async.py +++ b/python3/ashd/async.py @@ -165,12 +165,25 @@ class epoller(object): def watcher(): return epoller() -class sockbuffer(object): - def __init__(self, sk): - self.sk = sk +class channel(object): + readable = False + writable = False + + def __init__(self): + self.watcher = None + + def fileno(self): + raise NotImplementedError("fileno()") + + def close(self): + pass + +class sockbuffer(channel): + def __init__(self, socket, **kwargs): + super().__init__(**kwargs) + self.sk = socket self.eof = False self.obuf = bytearray() - self.watcher = None def fileno(self): return self.sk.fileno() @@ -211,8 +224,9 @@ class sockbuffer(object): self.obuf[:] = b"" self.eof = True -class callbuffer(object): - def __init__(self): +class callbuffer(channel): + def __init__(self, **kwargs): + super().__init__(**kwargs) self.queue = [] self.rp, self.wp = os.pipe() self.lock = threading.Lock() -- 2.11.0 From 8963785b76b15436bbbf06a26fc1c560ee57496e Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Sun, 19 Jun 2016 03:03:16 +0200 Subject: [PATCH 04/16] python: Fixed async module bug. --- python3/ashd/async.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python3/ashd/async.py b/python3/ashd/async.py index 4247bd6..e30f858 100644 --- a/python3/ashd/async.py +++ b/python3/ashd/async.py @@ -26,7 +26,7 @@ class epoller(object): def exception(self, ch, *exc): self.remove(ch) if self.exc_handler is None: - traceback.print_exception(exc) + traceback.print_exception(*exc) else: self.exc_handler(ch, *exc) -- 2.11.0 From 407963f25c664cd1450ec5f6eeb80c449ff57e74 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Wed, 22 Jun 2016 04:30:06 +0200 Subject: [PATCH 05/16] python: Improved current-watcher implementation. --- python3/ashd/async.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/python3/ashd/async.py b/python3/ashd/async.py index e30f858..99da89a 100644 --- a/python3/ashd/async.py +++ b/python3/ashd/async.py @@ -3,12 +3,15 @@ import sys, os, errno, threading, select, traceback class epoller(object): exc_handler = None - def __init__(self): + def __init__(self, check=None): self.registered = {} self.lock = threading.RLock() self.ep = None self.th = None self.stopped = False + self.loopcheck = set() + if check is not None: + self.loopcheck.add(check) self._daemon = True @staticmethod @@ -56,6 +59,8 @@ class epoller(object): self.ep = ep while self.registered: + for ck in self.loopcheck: + ck(self) if self.stopped: self._closeall() break @@ -278,8 +283,7 @@ class callbuffer(channel): self.wp = -1 def currentwatcher(io, current): - def run(): - while current: - current.wait() - io.stop() - threading.Thread(target=run, name="Current watcher").start() + def check(io): + if not current: + io.stop() + io.loopcheck.add(check) -- 2.11.0 From f9db90c1c6edeb2cdf7a89d13b07e53197ed351a Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Sun, 30 Oct 2016 21:08:26 +0100 Subject: [PATCH 06/16] lib: Fixed bioprintf bug. --- lib/bufio.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/bufio.c b/lib/bufio.c index c08e9dd..11d8c04 100644 --- a/lib/bufio.c +++ b/lib/bufio.c @@ -289,11 +289,11 @@ int bioprintf(struct bufio *bio, const char *format, ...) va_start(args, format); ret = vsnprintf(bio->wbuf.b + bio->wbuf.d, bio->wbuf.s - bio->wbuf.d, format, args); va_end(args); - if(ret <= bio->wbuf.s - bio->wbuf.d) { + if(ret < bio->wbuf.s - bio->wbuf.d) { bio->wbuf.d += ret; return(0); } - if(biowensure(bio, ret) < 0) + if(biowensure(bio, ret + 1) < 0) return(-1); } } -- 2.11.0 From bc5f1a7da43fb07b664e64ca235d63452d525472 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Wed, 23 Nov 2016 17:58:04 +0100 Subject: [PATCH 07/16] lib: Use abort() instead of exit() when smalloc fails. --- lib/utils.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/utils.h b/lib/utils.h index 4fdde3c..658eea2 100644 --- a/lib/utils.h +++ b/lib/utils.h @@ -8,8 +8,8 @@ #define max(a, b) (((b) > (a))?(b):(a)) #define min(a, b) (((b) < (a))?(b):(a)) -#define smalloc(size) ({void *__result__; ((__result__ = malloc(size)) == NULL)?({exit(-1); (void *)0;}):__result__;}) -#define srealloc(ptr, size) ({void *__result__; ((__result__ = realloc((ptr), (size))) == NULL)?({exit(-1); (void *)0;}):__result__;}) +#define smalloc(size) ({void *__result__; ((__result__ = malloc(size)) == NULL)?({abort(); (void *)0;}):__result__;}) +#define srealloc(ptr, size) ({void *__result__; ((__result__ = realloc((ptr), (size))) == NULL)?({abort(); (void *)0;}):__result__;}) #define szmalloc(size) memset(smalloc(size), 0, size) #define sstrdup(str) ({const char *__strbuf__ = (str); strcpy(smalloc(strlen(__strbuf__) + 1), __strbuf__);}) #define omalloc(o) ((o) = szmalloc(sizeof(*(o)))) -- 2.11.0 From 4b70e201af5bcc4926ab9b877592ba210b32ccdc Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Sat, 31 Dec 2016 18:36:47 +0100 Subject: [PATCH 08/16] lib: Fixed blocker iteration bug in mtio-select introduced by mblock. --- lib/mtio-select.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/mtio-select.c b/lib/mtio-select.c index 6e9a767..e0a4177 100644 --- a/lib/mtio-select.c +++ b/lib/mtio-select.c @@ -36,12 +36,17 @@ static int exitstatus; struct blocker { struct blocker *n, *p; + struct iterator *it; int fd; int ev, rev, id; time_t to; struct muth *th; }; +struct iterator { + struct blocker *bl; +}; + static void addblock(struct blocker *bl) { bl->n = blockers; @@ -58,6 +63,11 @@ static void remblock(struct blocker *bl) bl->p->n = bl->n; if(bl == blockers) blockers = bl->n; + if(bl->it) { + if((bl->it->bl = bl->n) != NULL) + bl->it->bl->it = bl->it; + bl->it = NULL; + } } struct selected mblock(time_t to, int n, struct selected *spec) @@ -111,7 +121,8 @@ int ioloop(void) { int ret; fd_set rfds, wfds, efds; - struct blocker *bl, *nbl; + struct blocker *bl; + struct iterator it; struct timeval toval; time_t now, timeout; int maxfd; @@ -150,8 +161,9 @@ int ioloop(void) } } else { now = time(NULL); - for(bl = blockers; bl; bl = nbl) { - nbl = bl->n; + for(bl = it.bl = blockers; bl; bl = it.bl) { + if((it.bl = bl->n) != NULL) + it.bl->it = ⁢ ev = 0; if(FD_ISSET(bl->fd, &rfds)) ev |= EV_READ; @@ -174,6 +186,8 @@ int ioloop(void) resume(bl->th, bl->id); } } + if(it.bl) + it.bl->it = NULL; } } } -- 2.11.0 From bb0730048938c21b043d88918a9492929c4fa2a9 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Sat, 31 Dec 2016 19:56:24 +0100 Subject: [PATCH 09/16] lib: Removed the surely redundant struct timeentry from the mtio-epoll. --- lib/mtio-epoll.c | 48 +++++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/lib/mtio-epoll.c b/lib/mtio-epoll.c index 3082eaf..a0b2f41 100644 --- a/lib/mtio-epoll.c +++ b/lib/mtio-epoll.c @@ -43,15 +43,10 @@ struct blocker { struct muth *th; }; -struct timeentry { - time_t to; - struct blocker *bl; -}; - static int epfd = -1, fdln = 0; static int exitstatus; static struct blocker **fdlist; -static typedbuf(struct timeentry) timeheap; +static typedbuf(struct blocker *) timeheap; static int regfd(struct blocker *bl) { @@ -136,23 +131,23 @@ static void remfd(struct blocker *bl) bl->reg = 0; } -static void thraise(struct timeentry ent, int n) +static void thraise(struct blocker *bl, int n) { int p; while(n > 0) { p = (n - 1) >> 1; - if(timeheap.b[p].to <= ent.to) + if(timeheap.b[p]->to <= bl->to) break; timeheap.b[n] = timeheap.b[p]; - timeheap.b[n].bl->thpos = n; + timeheap.b[n]->thpos = n; n = p; } - timeheap.b[n] = ent; - ent.bl->thpos = n; + timeheap.b[n] = bl; + bl->thpos = n; } -static void thlower(struct timeentry ent, int n) +static void thlower(struct blocker *bl, int n) { int c; @@ -160,27 +155,26 @@ static void thlower(struct timeentry ent, int n) c = (n << 1) + 1; if(c >= timeheap.d) break; - if((c + 1 < timeheap.d) && (timeheap.b[c + 1].to < timeheap.b[c].to)) + if((c + 1 < timeheap.d) && (timeheap.b[c + 1]->to < timeheap.b[c]->to)) c = c + 1; - if(timeheap.b[c].to > ent.to) + if(timeheap.b[c]->to > bl->to) break; timeheap.b[n] = timeheap.b[c]; - timeheap.b[n].bl->thpos = n; + timeheap.b[n]->thpos = n; n = c; } - timeheap.b[n] = ent; - ent.bl->thpos = n; + timeheap.b[n] = bl; + bl->thpos = n; } static void addtimeout(struct blocker *bl, time_t to) { sizebuf(timeheap, ++timeheap.d); - thraise((struct timeentry){.to = to, .bl = bl}, timeheap.d - 1); + thraise(bl, timeheap.d - 1); } static void deltimeout(struct blocker *bl) { - struct timeentry ent; int n; if(bl->thpos == timeheap.d - 1) { @@ -188,11 +182,11 @@ static void deltimeout(struct blocker *bl) return; } n = bl->thpos; - ent = timeheap.b[--timeheap.d]; - if((n > 0) && (timeheap.b[(n - 1) >> 1].to > ent.to)) - thraise(ent, n); + bl = timeheap.b[--timeheap.d]; + if((n > 0) && (timeheap.b[(n - 1) >> 1]->to > bl->to)) + thraise(bl, n); else - thlower(ent, n); + thlower(bl, n); } static int addblock(struct blocker *bl) @@ -288,8 +282,8 @@ int ioloop(void) now = time(NULL); if(timeheap.d == 0) toval = -1; - else if(timeheap.b[0].to > now) - toval = (timeheap.b[0].to - now) * 1000; + else if(timeheap.b[0]->to > now) + toval = (timeheap.b[0]->to - now) * 1000; else toval = 1000; if(exitstatus) @@ -326,9 +320,9 @@ int ioloop(void) } } now = time(NULL); - while((timeheap.d > 0) && (timeheap.b[0].to <= now)) { + while((timeheap.d > 0) && ((bl = timeheap.b[0])->to <= now)) { if(bl->id < 0) { - resume(timeheap.b[0].bl->th, 0); + resume(bl->th, 0); } else { bl->rev = 0; resume(bl->th, bl->id); -- 2.11.0 From 892b2066da6ac3c34057dfb736713d6117f01637 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Sat, 31 Dec 2016 21:55:25 +0100 Subject: [PATCH 10/16] lib: Fixed timeheap bug with ioloop reentrancy. --- lib/mtio-epoll.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/mtio-epoll.c b/lib/mtio-epoll.c index a0b2f41..f4c4970 100644 --- a/lib/mtio-epoll.c +++ b/lib/mtio-epoll.c @@ -272,7 +272,6 @@ int ioloop(void) exitstatus = 0; epfd = epoll_create(128); fcntl(epfd, F_SETFD, FD_CLOEXEC); - bufinit(timeheap); for(bl = blockers; bl; bl = nbl) { nbl = bl->n; if(regfd(bl)) @@ -331,7 +330,6 @@ int ioloop(void) } for(bl = blockers; bl; bl = bl->n) remfd(bl); - buffree(timeheap); close(epfd); epfd = -1; return(exitstatus); -- 2.11.0 From 7aed82e3ad447ee4deb85959212f8d027cab03e5 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Mon, 9 Jan 2017 06:58:37 +0100 Subject: [PATCH 11/16] python3: Fixed some threadpool handler bugs. --- python3/ashd/serve.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/python3/ashd/serve.py b/python3/ashd/serve.py index e156cd6..87f60a0 100644 --- a/python3/ashd/serve.py +++ b/python3/ashd/serve.py @@ -207,7 +207,8 @@ class threadpool(handler): self.waitlimit = 5 self.wlstart = 0.0 self.qlk = threading.Lock() - self.qcond = threading.Condition(self.qlk) + self.qfcond = threading.Condition(self.qlk) + self.qecond = threading.Condition(self.qlk) self.max = max self.qsz = qsz self.timeout = timeout @@ -229,15 +230,15 @@ class threadpool(handler): if self.timeout is not None: now = start = time.time() while len(self.queue) >= self.qsz: - self.qcond.wait(start + self.timeout - now) + self.qecond.wait(start + self.timeout - now) now = time.time() if now - start > self.timeout: os.abort() else: - while len(self.current) >= self.qsz: - self.qcond.wait() + while len(self.queue) >= self.qsz: + self.qecond.wait() self.queue.append(req) - self.qcond.notify() + self.qfcond.notify() if len(self.waiting) < 1: spawn = True if spawn: @@ -283,13 +284,14 @@ class threadpool(handler): try: if len(self.waiting) == self.waitlimit: self.wlstart = now - self.qcond.wait(start + timeout - now) + self.qfcond.wait(start + timeout - now) finally: self.waiting.remove(th) now = time.time() if now - start > timeout: return req = self.queue.popleft() + self.qecond.notify() try: self.handle1(req) finally: -- 2.11.0 From c26129cd537a8086d5a9d538ffb3ba2775b619d5 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Wed, 1 Feb 2017 05:02:22 +0100 Subject: [PATCH 12/16] python3: To be safe, abort entirely if initial epoller registration fails. --- python3/ashd/async.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python3/ashd/async.py b/python3/ashd/async.py index 99da89a..aa52af9 100644 --- a/python3/ashd/async.py +++ b/python3/ashd/async.py @@ -54,8 +54,12 @@ class epoller(object): ep = select.epoll() try: with self.lock: - for fd, (ob, evs) in self.registered.items(): - ep.register(fd, evs) + try: + for fd, (ob, evs) in self.registered.items(): + ep.register(fd, evs) + except: + self.registered.clear() + raise self.ep = ep while self.registered: -- 2.11.0 From 36038382b2b8a2631716e03f595b567d01af961d Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Wed, 1 Feb 2017 05:16:10 +0100 Subject: [PATCH 13/16] python3: Cache async channel FDs so that updates always happen correctly. --- python3/ashd/async.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/python3/ashd/async.py b/python3/ashd/async.py index aa52af9..238f6b7 100644 --- a/python3/ashd/async.py +++ b/python3/ashd/async.py @@ -5,6 +5,7 @@ class epoller(object): def __init__(self, check=None): self.registered = {} + self.fdcache = {} self.lock = threading.RLock() self.ep = None self.th = None @@ -86,6 +87,7 @@ class epoller(object): if fd in self.registered: nevs = self._evsfor(ch) if nevs == 0: + del self.fdcache[ch] del self.registered[fd] ep.unregister(fd) self._cb(ch, "close") @@ -119,6 +121,7 @@ class epoller(object): ch.close() return ch.watcher = self + self.fdcache[ch] = fd self.registered[fd] = (ch, evs) if self.ep: self.ep.register(fd, evs) @@ -126,14 +129,16 @@ class epoller(object): def remove(self, ch, ignore=False): with self.lock: - fd = ch.fileno() - if fd not in self.registered: + try: + fd = self.fdcache[ch] + except KeyError: if ignore: return raise KeyError("fd %i is not registered" % fd) pch, cevs = self.registered[fd] if pch is not ch: raise ValueError("fd %i registered via object %r, cannot remove with %r" % (pch, ch)) + del self.fdcache[ch] del self.registered[fd] if self.ep: self.ep.unregister(fd) @@ -141,8 +146,9 @@ class epoller(object): def update(self, ch, ignore=False): with self.lock: - fd = ch.fileno() - if fd not in self.registered: + try: + fd = self.fdcache[ch] + except KeyError: if ignore: return raise KeyError("fd %i is not registered" % fd) @@ -151,6 +157,7 @@ class epoller(object): raise ValueError("fd %i registered via object %r, cannot update with %r" % (pch, ch)) evs = self._evsfor(ch) if evs == 0: + del self.fdcache[ch] del self.registered[fd] if self.ep: self.ep.unregister(fd) -- 2.11.0 From dc38734591c5e26403b608078ce41b67a012d7f2 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Thu, 2 Mar 2017 04:40:27 +0100 Subject: [PATCH 14/16] doc: Fixed typo. --- doc/dirplex.doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/dirplex.doc b/doc/dirplex.doc index e2769d7..3737e24 100644 --- a/doc/dirplex.doc +++ b/doc/dirplex.doc @@ -71,7 +71,7 @@ element. If there is such a file, it is considered the result of the mapping. If the result of the mapping procedure is a directory, it is checked -for the presence of a filed named by the *index-file* configuration +for the presence of a file named by the *index-file* configuration directive (see CONFIGURATION below). If there is such a file, it is considered the final result instead of the directory itself. If the index file name contains no dots and there is no exact match, then, -- 2.11.0 From ca69b58412c5adefcb5cf941c07bdf603703c23a Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Thu, 2 Mar 2017 04:48:14 +0100 Subject: [PATCH 15/16] doc: Fixed typo. --- doc/dirplex.doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/dirplex.doc b/doc/dirplex.doc index 3737e24..8a0184f 100644 --- a/doc/dirplex.doc +++ b/doc/dirplex.doc @@ -295,7 +295,7 @@ A HTTP 404 response is sent to the client if * A path element is encountered during mapping which, after URL unescaping, either begins with a dot or contains slashes; * The mapping procedure finds a file which is neither a directory nor - a regular file (or a symbolic link to any of the same); + a regular file (nor a symbolic link to any of the same); * An empty, non-final path element is encountered during mapping; or * The mapping procedure results in a file which is not matched by any *match* stanza. -- 2.11.0 From 883200a82ac56d6671ad9fb2942e1db2e271e84a Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Tue, 7 Mar 2017 03:24:17 +0100 Subject: [PATCH 16/16] callcgi: Fixed possible deadlock problem on aborted requests. --- src/callcgi.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/callcgi.c b/src/callcgi.c index 67a6d14..2b1c9e6 100644 --- a/src/callcgi.c +++ b/src/callcgi.c @@ -436,6 +436,7 @@ int main(int argc, char **argv, char **envp) printf("\n"); if(passdata(out, stdout)) kill(child, SIGINT); + fclose(out); if(waitpid(child, &estat, 0) == child) { if(WCOREDUMP(estat)) flog(LOG_WARNING, "CGI handler `%s' dumped core", prog.b[0]); -- 2.11.0