From: Fredrik Tolf Date: Fri, 23 Dec 2011 00:53:15 +0000 (+0100) Subject: Merge branch 'master' into python3 X-Git-Tag: 0.1~2 X-Git-Url: http://git.dolda2000.com/gitweb/?a=commitdiff_plain;h=144c745cd600fc7c050f5426fe438430885ef5b8;hp=-c;p=pdm.git Merge branch 'master' into python3 Conflicts: pdm/cli.py pdm/srv.py --- 144c745cd600fc7c050f5426fe438430885ef5b8 diff --combined pdm/cli.py index 6af06f1,81246b7..9924d40 --- a/pdm/cli.py +++ b/pdm/cli.py @@@ -6,9 -6,10 +6,10 @@@ provided in the pdm.srv module import socket, pickle, struct, select, threading - __all__ = ["client", "replclient"] + __all__ = ["client", "replclient", "perfclient"] class protoerr(Exception): + """Raised on protocol errors""" pass def resolve(spec): @@@ -35,41 -36,62 +36,64 @@@ return rv class client(object): + """PDM client + + This class provides general facilities to speak to PDM servers, + and is mainly intended to be subclassed to provide for the + specific protocols, such as replclient and perfclient do. + + `client' instances can be passed as arguments to select.select(), + and can be used in `with' statements. + """ def __init__(self, sk, proto = None): + """Create a client object connected to the specified + server. `sk' can either be a socket object, which is used as + it is, or a string specification very similar to the + specification for pdm.srv.listen, so see its documentation for + details. The differences are only that this function does not + take arguments specific to socket creation, like the mode and + group arguments for Unix sockets. If `proto' is given, that + subprotocol will negotiated with the server (by calling the + select() method). + """ self.sk = resolve(sk) - self.buf = "" + self.buf = b"" line = self.readline() - if line != "+PDM1": + if line != b"+PDM1": raise protoerr("Illegal protocol signature") if proto is not None: self.select(proto) def close(self): + """Close this connection""" self.sk.close() def fileno(self): + """Return the file descriptor of the underlying socket.""" return self.sk.fileno() def readline(self): + """Read a single NL-terminated line and return it.""" while True: - p = self.buf.find("\n") + p = self.buf.find(b"\n") if p >= 0: ret = self.buf[:p] self.buf = self.buf[p + 1:] return ret ret = self.sk.recv(1024) - if ret == "": + if ret == b"": return None self.buf += ret def select(self, proto): + """Negotiate the given subprotocol with the server""" - if "\n" in proto: + if isinstance(proto, str): + proto = proto.encode("ascii") + if b"\n" in proto: raise Exception("Illegal protocol specified: %r" % proto) - self.sk.send(proto + "\n") + self.sk.send(proto + b"\n") rep = self.readline() - if len(rep) < 1 or rep[0] != "+": + if len(rep) < 1 or rep[0] != b"+"[0]: raise protoerr("Error reply when selecting protocol %s: %s" % (proto, rep[1:])) def __enter__(self): @@@ -80,26 -102,36 +104,36 @@@ return False class replclient(client): + """REPL protocol client + + Implements the client side of the REPL protocol; see pdm.srv.repl + for details on the protocol and its functionality. + """ def __init__(self, sk): + """Create a connected client as documented in the `client' class.""" - super(replclient, self).__init__(sk, "repl") + super().__init__(sk, "repl") def run(self, code): + """Run a single block of Python code on the server. Returns + the output of the command (as documented in pdm.srv.repl) as a + string. + """ while True: ncode = code.replace("\n\n", "\n") if ncode == code: break code = ncode while len(code) > 0 and code[-1] == "\n": code = code[:-1] - self.sk.send(code + "\n\n") - buf = "" + self.sk.send((code + "\n\n").encode("utf-8")) + buf = b"" while True: ln = self.readline() - if ln[0] == " ": - buf += ln[1:] + "\n" - elif ln[0] == "+": - return buf - elif ln[0] == "-": - raise protoerr("Error reply: %s" % ln[1:]) + if ln[0] == b" "[0]: + buf += ln[1:] + b"\n" + elif ln[0] == b"+"[0]: + return buf.decode("utf-8") + elif ln[0] == b"-"[0]: + raise protoerr("Error reply: %s" % ln[1:].decode("utf-8")) else: raise protoerr("Illegal reply: %s" % ln) @@@ -155,8 -187,13 +189,13 @@@ class perfproxy(object) except: pass def close(self): - self.cl.run("unbind", self.id) - del self.cl.proxies[self.id] + if self.id is not None: + self.cl.run("unbind", self.id) + del self.cl.proxies[self.id] + self.id = None + + def __del__(self): + self.close() def __enter__(self): return self @@@ -166,8 -203,25 +205,25 @@@ return False class perfclient(client): + """PERF protocol client + + Implements the client side of the PERF protocol; see pdm.srv.perf + for details on the protocol and its functionality. + + This client class implements functions for finding PERF objects on + the server, and returns, for each server-side object looked up, a + proxy object that mimics exactly the PERF interfaces that the + object implements. As the proxy objects reference live objects on + the server, they should be released when they are no longer used; + they implement a close() method for that purpose, and can also be + used in `with' statements. + + See pdm.srv.perf for details on the various PERF interfaces that + the proxy objects might implement. + """ def __init__(self, sk): + """Create a connected client as documented in the `client' class.""" - super(perfclient, self).__init__(sk, "perf") + super().__init__(sk, "perf") self.nextid = 0 self.lock = threading.Lock() self.proxies = {} @@@ -179,10 -233,10 +235,10 @@@ self.sk.send(buf) def recvb(self, num): - buf = "" + buf = b"" while len(buf) < num: data = self.sk.recv(num - len(buf)) - if data == "": + if data == b"": raise EOFError() buf += data return buf @@@ -196,6 -250,12 +252,12 @@@ proxy.notify(ev) def dispatch(self, timeout = None): + """Wait for an incoming notification from the server, and + dispatch it to the callback functions that have been + registered for it. If `timeout' is specified, wait no longer + than so many seconds; otherwise, wait forever. This client + object may also be used as argument to select.select(). + """ rfd, wfd, efd = select.select([self.sk], [], [], timeout) if self.sk in rfd: msg = self.recv() @@@ -227,6 -287,10 +289,10 @@@ self.lock.release() def lookup(self, module, obnm): + """Look up a single server-side object by the given name in + the given module. Will return a new proxy object for each + call when called multiple times for the same name. + """ self.lock.acquire() try: id = self.nextid @@@ -239,6 -303,17 +305,17 @@@ return proxy def find(self, name): + """Convenience function for looking up server-side objects + through PERF directories and for multiple uses. The object + name can be given as "MODULE.OBJECT", which will look up the + named OBJECT in the named MODULE, and can be followed by any + number of slash-separated names, which will assume that the + object to the left of the slash is a PERF directory, and will + return the object in that directory by the name to the right + of the slash. For instance, find("pdm.perf.sysres/cputime") + will return the built-in attribute for reading the CPU time + used by the server process. + """ ret = self.names.get(name) if ret is None: if "/" in name: diff --combined pdm/srv.py index e2aef4e,128c6d9..3ddc682 --- a/pdm/srv.py +++ b/pdm/srv.py @@@ -1,51 -1,72 +1,73 @@@ - """Management for daemon processes + """Python Daemon Management -- Server functions - This module contains a utility to listen for management commands on a - socket, lending itself to managing daemon processes. + This module implements the server part of the PDM protocols. The + primary object of interest herein is the listen() function, which is + the most generic way to create PDM listeners based on user + configuration, and the documentation for the repl and perf classes, + which describes the functioning of the REPL and PERF protocols. """ import os, sys, socket, threading, grp, select import types, pprint, traceback import pickle, struct - __all__ = ["listener", "unixlistener", "tcplistener", "listen"] + __all__ = ["repl", "perf", "listener", "unixlistener", "tcplistener", "listen"] protocols = {} class repl(object): + """REPL protocol handler + + Provides a read-eval-print loop. The primary client-side interface + is the pdm.cli.replclient class. Clients can send arbitrary code, + which is compiled and run on its own thread in the server process, + and output responses that are echoed back to the client. + + Each client is provided with its own module, in which the code + runs. The module is prepared with a function named `echo', which + takes a single object and pretty-prints it as part of the command + response. If a command can be parsed as an expression, the value + it evaluates to is automatically echoed to the client. If the + evalution of the command terminates with an exception, its + traceback is echoed to the client. + + The REPL protocol is only intended for interactive usage. In order + to interact programmatically with the server process, see the PERF + protocol instead. + """ def __init__(self, cl): self.cl = cl self.mod = types.ModuleType("repl") self.mod.echo = self.echo self.printer = pprint.PrettyPrinter(indent = 4, depth = 6) - cl.send("+REPL\n") + cl.send(b"+REPL\n") def sendlines(self, text): for line in text.split("\n"): - self.cl.send(" " + line + "\n") + self.cl.send(b" " + line.encode("utf-8") + b"\n") def echo(self, ob): self.sendlines(self.printer.pformat(ob)) def command(self, cmd): + cmd = cmd.decode("utf-8") try: try: ccode = compile(cmd, "PDM Input", "eval") except SyntaxError: ccode = compile(cmd, "PDM Input", "exec") - exec ccode in self.mod.__dict__ - self.cl.send("+OK\n") + exec(ccode, self.mod.__dict__) + self.cl.send(b"+OK\n") else: self.echo(eval(ccode, self.mod.__dict__)) - self.cl.send("+OK\n") + self.cl.send(b"+OK\n") except: for line in traceback.format_exception(*sys.exc_info()): - self.cl.send(" " + line) - self.cl.send("+EXC\n") + self.cl.send(b" " + line.encode("utf-8")) + self.cl.send(b"+EXC\n") def handle(self, buf): - p = buf.find("\n\n") + p = buf.find(b"\n\n") if p < 0: return buf cmd = buf[:p + 1] @@@ -54,16 -75,107 +76,107 @@@ protocols["repl"] = repl class perf(object): + """PERF protocol handler + + The PERF protocol provides an interface for program interaction + with the server process. It allows limited remote interactions + with Python objects over a few defined interfaces. + + All objects that wish to be available for interaction need to + implement a method named `pdm_protocols' which, when called with + no arguments, should return a list of strings, each indicating a + PERF interface that the object implements. For each such + interface, the object must implement additional methods as + described below. + + A client can find PERF objects to interact with either by + specifying the name of such an object in an existing module, or by + using the `dir' interface, described below. Thus, to make a PERF + object available for clients, it needs only be bound to a global + variable in a module and implement the `pdm_protocols' + method. When requesting an object from a module, the module must + already be imported. PDM will not import new modules for clients; + rather, the daemon process needs to import all modules that + clients should be able to interact with. PDM itself always imports + the pdm.perf module, which contains a few basic PERF objects. See + its documentation for details. + + The following interfaces are currently known to PERF. + + * attr: + An object that implements the `attr' interface models an + attribute that can be read by clients. The attribute can be + anything, as long as its representation can be + pickled. Examples of attributes could be such things as the CPU + time consumed by the server process, or the number of active + connections to whatever clients the program serves. To + implement the `attr' interface, an object must implement + methods called `readattr' and `attrinfo'. `readattr' is called + with no arguments to read the current value of the attribute, + and `attrinfo' is called with no arguments to read a + description of the attribute. Both should be + idempotent. `readattr' can return any pickleable object, and + `attrinfo' should return either None to indicate that it has no + description, or an instance of the pdm.perf.attrinfo class. + + * dir: + The `dir' interface models a directory of other PERF + objects. An object implementing it must implement methods + called `lookup' and `listdir'. `lookup' is called with a single + string argument that names an object, and should either return + another PERF object based on the name, or raise KeyError if it + does not recognize the name. `listdir' is called with no + arguments, and should return a list of known names that can be + used as argument to `lookup', but the list is not required to + be exhaustive and may also be empty. + + * invoke: + The `invoke' interface allows a more arbitrary form of method + calls to objects implementing it. Such objects must implement a + method called `invoke', which is called with one positional + argument naming a method to be called (which it is free to + interpret however it wishes), and with any additional + positional and keyword arguments that the client wishes to pass + to it. Whatever `invoke' returns is pickled and sent back to + the client. In case the method name is not recognized, `invoke' + should raise an AttributeError. + + * event: + The `event' interface allows PERF objects to notify clients of + events asynchronously. Objects implementing it must implement + methods called `subscribe' and `unsubscribe'. `subscribe' will + be called with a single argument, which is a callable of one + argument, which should be registered to be called when an event + pertaining to the `event' object in question occurs. The + `event' object should then call all such registered callables + with a single argument describing the event. The argument could + be any object that can be pickled, but should be an instance of + a subclass of the pdm.perf.event class. If `subscribe' is + called with a callback object that it has already registered, + it should raise a ValueError. `unsubscribe' is called with a + single argument, which is a previously registered callback + object, which should then be unregistered to that it is no + longer called when an event occurs. If the given callback + object is not, in fact, registered, a ValueError should be + raised. + + The pdm.perf module contains a few convenience classes which + implements the interfaces, but PERF objects are not required to be + instances of them. Any object can implement a PERF interface, as + long as it does so as described above. + + The pdm.cli.perfclient class is the client-side implementation. + """ def __init__(self, cl): self.cl = cl self.odtab = {} - cl.send("+PERF1\n") + cl.send(b"+PERF1\n") self.buf = "" self.lock = threading.Lock() self.subscribed = {} def closed(self): - for id, recv in self.subscribed.iteritems(): + for id, recv in self.subscribed.items(): ob = self.odtab[id] if ob is None: continue ob, protos = ob @@@ -85,7 -197,7 +198,7 @@@ raise ValueError("Object does not support PDM introspection") try: proto = ob.pdm_protocols() - except Exception, exc: + except Exception as exc: raise ValueError("PDM introspection failed", exc) self.odtab[id] = ob, proto return proto @@@ -102,7 -214,7 +215,7 @@@ return try: proto = self.bindob(id, ob) - except Exception, exc: + except Exception as exc: self.send("-", exc) return self.send("+", proto) @@@ -124,12 -236,12 +237,12 @@@ return try: ob = src.lookup(obnm) - except KeyError, exc: + except KeyError as exc: self.send("-", exc) return try: proto = self.bindob(tgtid, ob) - except Exception, exc: + except Exception as exc: self.send("-", exc) return self.send("+", proto) @@@ -159,7 -271,7 +272,7 @@@ return try: ret = ob.readattr() - except Exception, exc: + except Exception as exc: self.send("-", Exception("Could not read attribute")) return self.send("+", ret) @@@ -176,7 -288,7 +289,7 @@@ return try: self.send("+", ob.invoke(method, *args, **kwargs)) - except Exception, exc: + except Exception as exc: self.send("-", exc) def event(self, id, ob, ev): @@@ -242,7 -354,7 +355,7 @@@ protocols["perf"] = per class client(threading.Thread): def __init__(self, sk): - super(client, self).__init__(name = "Management client") + super().__init__(name = "Management client") self.setDaemon(True) self.sk = sk self.handler = self @@@ -251,10 -363,6 +364,10 @@@ return self.sk.send(data) def choose(self, proto): + try: + proto = proto.decode("ascii") + except UnicodeError: + proto = None if proto in protocols: self.handler = protocols[proto](self) else: @@@ -262,7 -370,7 +375,7 @@@ raise Exception() def handle(self, buf): - p = buf.find("\n") + p = buf.find(b"\n") if p >= 0: proto = buf[:p] buf = buf[p + 1:] @@@ -271,24 -379,24 +384,24 @@@ def run(self): try: - buf = "" - self.send("+PDM1\n") + buf = b"" + self.send(b"+PDM1\n") while True: ret = self.sk.recv(1024) - if ret == "": + if ret == b"": return buf += ret while True: try: nbuf = self.handler.handle(buf) except: + #for line in traceback.format_exception(*sys.exc_info()): + # print(line) return if nbuf == buf: break buf = nbuf finally: - #for line in traceback.format_exception(*sys.exc_info()): - # print line try: self.sk.close() finally: @@@ -297,11 -405,19 +410,19 @@@ class listener(threading.Thread): + """PDM listener + + This subclass of a thread listens to PDM connections and handles + client connections properly. It is intended to be subclassed by + providers of specific domains, such as unixlistener and + tcplistener. + """ def __init__(self): - super(listener, self).__init__(name = "Management listener") + super().__init__(name = "Management listener") self.setDaemon(True) def listen(self, sk): + """Listen for and accept connections.""" self.running = True while self.running: rfd, wfd, efd = select.select([sk], [], [sk], 1) @@@ -311,6 -427,11 +432,11 @@@ self.accept(nsk, addr) def stop(self): + """Stop listening for client connections + + Tells the listener thread to stop listening, and then waits + for it to terminate. + """ self.running = False self.join() @@@ -319,8 -440,15 +445,15 @@@ cl.start() class unixlistener(listener): + """Unix socket listener""" - def __init__(self, name, mode = 0600, group = None): + def __init__(self, name, mode = 0o600, group = None): + """Create a listener that will bind to the Unix socket named + by `name'. The socket will not actually be bound until the + listener is started. The socket will be chmodded to `mode', + and if `group' is given, the named group will be set as the + owner of the socket. + """ - super(unixlistener, self).__init__() + super().__init__() self.name = name self.mode = mode self.group = group @@@ -344,8 -472,13 +477,13 @@@ os.unlink(self.name) class tcplistener(listener): + """TCP socket listener""" def __init__(self, port, bindaddr = "127.0.0.1"): + """Create a listener that will bind to the given TCP port, and + the given local interface. The socket will not actually be + bound until the listener is started. + """ - super(tcplistener, self).__init__() + super().__init__() self.port = port self.bindaddr = bindaddr @@@ -359,6 -492,22 +497,22 @@@ sk.close() def listen(spec): + """Create and start a listener according to a string + specification. The string specifications can easily be passed from + command-line options, user configuration or the like. Currently, + the two following specification formats are recognized: + + PATH[:MODE[:GROUP]] -- PATH must contain at least one slash. A + Unix socket listener will be created listening to that path, and + the socket will be chmodded to MODE and owned by GROUP. If MODE is + not given, it defaults to 0600, and if GROUP is not given, the + process' default group is used. + + ADDRESS:PORT -- PORT must be entirely numeric. A TCP socket + listener will be created listening to that port, bound to the + given local interface address. Since PDM has no authentication + support, ADDRESS should probably be localhost. + """ if ":" in spec: first = spec[:spec.index(":")] last = spec[spec.rindex(":") + 1:] @@@ -367,7 -516,7 +521,7 @@@ last = spec if "/" in first: parts = spec.split(":") - mode = 0600 + mode = 0o600 group = None if len(parts) > 1: mode = int(parts[1], 8)