Re: [Python-Dev] python sendmsg()/recvmsg() implementation

Hello

This is the rewritten-from-scratch implementation of the 
sendmsg()/recvmsg() methods.
Any comments / suggestions / flames are very welcome. Currently it 
supports what I need
and I'm only releasing it, because I don't have much time to develop it 
further in the
forseeable future (1-2 months). It is rewritten from scratch, using the 
python c-api
documents. I've tried my best, but I wouldn't bet that it works as it's 
supposed to. I'd be
glad if someone could give me a review on what I've done wrong.

The core parts are implemented correctly (I think), the features that 
are missing:
- using scatter/gather
- using it with non-stream oriented sockets (doesn't support addresses 
/msg_name/)

These should be very easy to implement though. I will fix the errors 
that are present right
now, and if no one takes up the task I will implement the missing 
features also. You might
have to wait for it a little though.

Thanks in advance

Cheers,
Kalman Gergely
--- py3k_2/Modules/socketmodule.c       2009-07-23 17:07:55.474581000 +0200
+++ py3k/Modules/socketmodule.c 2009-07-23 17:22:16.880415500 +0200
@@ -2388,6 +2388,7 @@
        return n;
 }
 
+
 /* s.recvfrom(nbytes [,flags]) method */
 
 static PyObject *
@@ -2440,6 +2441,143 @@
 Like recv(buffersize, flags) but also return the sender's address info.");
 
 
+/* s.recvmsg(datalen, controllen, flags) method */
+
+static PyObject *
+sock_recvmsg(PySocketSockObject *s, PyObject *args)
+{
+       PyObject *dbuf, *cbuf, *alist, *tmp;
+       ssize_t n = -1;
+       int status, timeout;
+       int rdlen, rclen, flags = 0;
+       struct msghdr mhdr;
+       struct cmsghdr *chdr;
+       struct iovec iov[1];
+
+       if (!PyArg_ParseTuple(args, "ii|i:recvmsg", &rdlen, &rclen, &flags))
+               return NULL;
+
+       if (rdlen < 0 || rclen < 0)
+       {
+               PyErr_SetString(PyExc_ValueError, "negative buffersize in 
recvmsg");
+               return NULL;
+       }
+
+       /* allocate buffers */
+       dbuf = PyBytes_FromStringAndSize((char *) 0, rdlen);
+       if (dbuf == NULL)
+       {
+               return NULL;
+       }
+
+       cbuf = PyBytes_FromStringAndSize((char *) 0, rclen);
+       if (cbuf == NULL)
+       {
+               Py_DECREF(dbuf);
+               return NULL;
+       }
+
+       alist = PyList_New(0);
+       if (alist == NULL)
+       {
+               Py_DECREF(dbuf);
+               Py_DECREF(cbuf);
+               return NULL;
+       }
+
+       /* set up the msghdr struct */
+       memset(&mhdr, 0, sizeof(struct msghdr));
+
+       // iov -- we use only one buffer, and don't use scatter-gather possible 
TODO
+       iov[0].iov_base = PyBytes_AS_STRING(dbuf);
+       iov[0].iov_len = rdlen;
+       memset(iov[0].iov_base, 0, iov[0].iov_len);
+
+       // msghdr
+       mhdr.msg_name = NULL;                           // TODO make use of this
+       mhdr.msg_namelen = 0;
+       mhdr.msg_iov = iov;
+       mhdr.msg_iovlen = 1;
+       mhdr.msg_control = PyBytes_AS_STRING(cbuf);
+       mhdr.msg_controllen = rclen;
+       mhdr.msg_flags = 0;
+       memset(mhdr.msg_control, 0, mhdr.msg_controllen);
+
+       /* call recvmsg() */
+       Py_BEGIN_ALLOW_THREADS
+       timeout = internal_select(s, 0);
+       if (!timeout)
+               n = recvmsg(s->sock_fd, &mhdr, flags);
+       Py_END_ALLOW_THREADS
+
+       if (timeout == 1)
+       {
+               PyErr_SetString(socket_timeout, "timed out");
+               goto err;
+       }
+
+       if (n < 0)
+       {
+               s->errorhandler();
+               goto err;
+       }
+
+       /* process the ancillary data */
+       for (chdr = CMSG_FIRSTHDR(&mhdr); chdr != NULL; chdr = 
CMSG_NXTHDR(&mhdr, chdr))
+       {
+               tmp = Py_BuildValue("(iiy#)", chdr->cmsg_level, 
chdr->cmsg_type, (char *)CMSG_DATA(chdr),
+                               // TODO XXX ugly hack, to compute CMSG_DATA's 
size
+                               (int)chdr->cmsg_len - ((int)CMSG_DATA(chdr) - 
(int)chdr));
+               if (tmp == NULL)
+                       goto err;
+
+               status = PyList_Append(alist, tmp);
+               Py_DECREF(tmp);
+
+               if (status == -1)
+                       goto err;
+       }
+
+       /* if we received less than we anticipated, resize the buffer */
+       if (n != rdlen)
+       {
+               if (_PyBytes_Resize(&dbuf, n) == -1)
+               {
+                       Py_DECREF(cbuf);
+                       Py_DECREF(alist);
+                       return NULL;
+               }
+       }
+
+       /* assemble the final return value, watch out for offsets! */
+       tmp = PyTuple_New(3);
+       if (tmp == NULL)
+               goto err;
+
+       PyTuple_SetItem(tmp, 0, dbuf);
+       PyTuple_SetItem(tmp, 1, alist);
+       PyTuple_SetItem(tmp, 2, Py_BuildValue("i", mhdr.msg_flags));
+
+       /* dbuf and alist are now in tmp, remove reference to cbuf and return */
+       Py_DECREF(cbuf);
+
+       return tmp;
+
+err:
+       Py_DECREF(dbuf);
+       Py_DECREF(cbuf);
+       Py_DECREF(alist);
+       return NULL;
+}
+
+PyDoc_STRVAR(recvmsg_doc,
+"recvmsg(datalen, controllen, flags) method -> (bytes(), [(msglevel, msgtype, 
msgdata) ... ], flags)\n\
+\n\
+Returns a tuple with three elements, 0: data bytes, 1: list of tuples with 
three elements\n\
+containing msg_level, msg_type, msg_data, 2: msg_flags\n\
+Currently it's incapable of using multiple buffers and addresses.");
+
+
 /* s.recvfrom_into(buffer[, nbytes [,flags]]) method */
 
 static PyObject *
@@ -2655,6 +2793,140 @@
 For IP sockets, the address is a pair (hostaddr, port).");
 
 
+/* s.sendmsg(data, [(msglevel, msgtype, msgdata), ...], flags) method */
+
+static PyObject *
+sock_sendmsg(PySocketSockObject *s, PyObject *args)
+{
+       Py_buffer dbuf;
+       PyObject *cbuf;
+       PyObject *tmp;
+       Py_buffer tmpdata;
+       PyObject *control;
+       size_t cbuflen = 0;
+       size_t controllen;
+       int tmplev, tmptype;
+       int timeout, flags = 0;
+       int i, n = -1;
+       struct msghdr mhdr;
+       struct cmsghdr *chdr;
+       struct iovec iov[1];
+
+       if (!PyArg_ParseTuple(args, "y*O!I|:sendmsg", &dbuf, &PyList_Type, 
&control, &flags)) {
+               return NULL;
+       }
+
+       controllen = PyList_Size(control);
+
+       /* set up the msghdr struct */
+       // count the bytes to allocate for msg_control
+       for (i=0; i<controllen; i++)
+       {
+               // get the list entry
+               tmp = PyList_GET_ITEM(control, i);
+               if (!PyTuple_CheckExact(tmp))
+               {
+                       PyErr_SetString(PyExc_ValueError, "a list of tuples is 
needed");
+                       return NULL;
+               }
+
+               // extract the fields
+               if (!PyArg_ParseTuple(tmp, "iiy*:sendmsg_inner", &tmplev, 
&tmptype, &tmpdata))
+               {
+                       return NULL;
+               }
+
+               cbuflen += CMSG_SPACE(tmpdata.len);
+       }
+
+       // allocate a temporary buffer for msg_control
+       cbuf = PyBytes_FromStringAndSize((char *) 0, cbuflen);
+       if (cbuf == NULL)
+       {
+               return NULL;
+       }
+
+       // msghdr
+       mhdr.msg_name = NULL;                           // TODO make use of this
+       mhdr.msg_namelen = 0;
+       mhdr.msg_iov = iov;
+       mhdr.msg_iovlen = 1;
+       mhdr.msg_control = PyBytes_AS_STRING(cbuf);
+       mhdr.msg_controllen = cbuflen;
+       mhdr.msg_flags = flags;
+
+       // iov -- For now we use only one buffer, and don't use scatter-gather. 
Possible TODO
+       iov[0].iov_base = dbuf.buf;
+       iov[0].iov_len = dbuf.len;
+
+       // cmsghdr
+       for (i=0, chdr = CMSG_FIRSTHDR(&mhdr); i<controllen && chdr != NULL; 
i++, chdr = CMSG_NXTHDR(&mhdr, chdr))
+       {
+               // get the list entry
+               tmp = PyList_GET_ITEM(control, i);
+               if (!PyTuple_CheckExact(tmp))
+               {
+                       PyErr_SetString(PyExc_ValueError, "a list of tuples is 
needed");
+                       goto err;
+               }
+
+               // extract the fields
+               if (!PyArg_ParseTuple(tmp, "iiy*:sendmsg_inner", &tmplev, 
&tmptype, &tmpdata))
+               {
+                       goto err;
+               }
+
+               // set up the fields
+               chdr->cmsg_len = CMSG_LEN(tmpdata.len);
+               chdr->cmsg_level = tmplev;
+               chdr->cmsg_type = tmptype;
+               memcpy(CMSG_DATA(chdr), tmpdata.buf, tmpdata.len);
+       }
+
+       /* call sendmsg() */
+       if (!IS_SELECTABLE(s))
+       {
+               Py_DECREF(cbuf);
+               return select_error();
+       }
+
+
+       Py_BEGIN_ALLOW_THREADS
+       timeout = internal_select(s, 1);
+       if (!timeout)
+               n = sendmsg(s->sock_fd, &mhdr, flags);
+       Py_END_ALLOW_THREADS
+
+       // free the parameters
+       PyBuffer_Release(&dbuf);
+       Py_DECREF(cbuf);
+
+       if (timeout == 1)
+       {
+               PyErr_SetString(socket_timeout, "timed out");
+               return NULL;
+       }
+
+       if (n < 0)
+       {
+               return s->errorhandler();
+       }
+
+       return Py_BuildValue("I", n);
+
+err:
+       Py_DECREF(cbuf);
+       return NULL;
+}
+
+PyDoc_STRVAR(sendmsg_doc,
+"sendmsg(data, [(msglevel, msgtype, msgdata), ...], flags) -> sent\n\
+\n\
+Returns the bytes sent. Accepts 3 parameters, 1: the data bytes, 2: a list\n\
+containing tuples, that contain the msg_level, msg_type, msg_data, 3: flags.\n\
+Currently it's incapable of using multiple buffers and addresses.");
+
+
 /* s.shutdown(how) method */
 
 static PyObject *
@@ -2742,6 +3014,8 @@
                          recv_into_doc},
        {"recvfrom",      (PyCFunction)sock_recvfrom, METH_VARARGS,
                          recvfrom_doc},
+       {"recvmsg",       (PyCFunction)sock_recvmsg, METH_VARARGS,
+                         recvmsg_doc},
        {"recvfrom_into",  (PyCFunction)sock_recvfrom_into, METH_VARARGS | 
METH_KEYWORDS,
                          recvfrom_into_doc},
        {"send",          (PyCFunction)sock_send, METH_VARARGS,
@@ -2750,6 +3024,8 @@
                          sendall_doc},
        {"sendto",        (PyCFunction)sock_sendto, METH_VARARGS,
                          sendto_doc},
+       {"sendmsg",       (PyCFunction)sock_sendmsg, METH_VARARGS,
+                         sendmsg_doc},
        {"setblocking",   (PyCFunction)sock_setblocking, METH_O,
                          setblocking_doc},
        {"settimeout",    (PyCFunction)sock_settimeout, METH_O,
@@ -4513,6 +4789,14 @@
        PyModule_AddIntConstant(m, "SO_TYPE", SO_TYPE);
 #endif
 
+       /* Ancilliary message types */
+#ifdef  SCM_RIGHTS
+       PyModule_AddIntConstant(m, "SCM_RIGHTS", SCM_RIGHTS);
+#endif
+#ifdef  SCM_CREDENTIALS
+       PyModule_AddIntConstant(m, "SCM_CREDENTIALS", SCM_CREDENTIALS);
+#endif
+
        /* Maximum number of connections for "listen" */
 #ifdef SOMAXCONN
        PyModule_AddIntConstant(m, "SOMAXCONN", SOMAXCONN);
@@ -4551,6 +4835,9 @@
 #ifdef MSG_ETAG
        PyModule_AddIntConstant(m, "MSG_ETAG", MSG_ETAG);
 #endif
+#ifdef MSG_ERRQUEUE
+       PyModule_AddIntConstant(m, "MSG_ERRQUEUE", MSG_ERRQUEUE);
+#endif
 
        /* Protocol level and numbers, usable for [gs]etsockopt */
 #ifdef SOL_SOCKET
import socket
import os
import sys
import struct
import time

s = socket.socketpair(socket.AF_UNIX, socket.SOCK_SEQPACKET)

pid = os.fork()

if pid == -1:
        print("fork() failed!")
        sys.exit()

if pid == 0:
        msg = b'test'
        print("parent: hello")
        s[1].close()
        ipc = s[0]
        f = open("/etc/passwd", "r")
        tmp = struct.pack("i", f.fileno());
        print("parent: sending message", msg, tmp)
        ret = ipc.sendmsg(msg, [(socket.SOL_SOCKET, socket.SCM_RIGHTS, tmp)], 0)
        print("parent:", ret)
        f.close()
else:
        print("child: hello")
        s[0].close()
        ipc = s[1]
        ret = ipc.recvmsg(10, 1000, 0)
        print("child:", ret)
        print("child: received message:", ret[0])
        f = os.fdopen(struct.unpack("i", ret[1][0][2])[0])
        arr = f.readlines()
        for i in arr:
                print("Read:", i, end='')
        f.close()
_______________________________________________
Python-Dev mailing list
Python-Dev@xxxxxxxxxx
http://mail.python.org/mailman/listinfo/python-dev
Unsubscribe: 
http://mail.python.org/mailman/options/python-dev/maillists%40codeha.us