CVS: BitTorrent/BitTorrent zurllib.py,NONE,1.1 HTTPHandler.py,1.8,1.9 Rerequester.py,1.16,1.17 download.py,1.134,1.135 track.py,1.46,1.47

Update of /cvsroot/bittorrent/BitTorrent/BitTorrent
In directory sc8-pr-cvs1:/tmp/cvs-serv8830/BitTorrent

Modified Files:
        HTTPHandler.py Rerequester.py download.py track.py 
Added Files:
        zurllib.py 
Log Message:
added gzip support

--- NEW FILE: zurllib.py ---
#
# zurllib.py
#
# This is (hopefully) a drop-in for urllib which will request gzip/deflate
# compression and then decompress the output if a compressed response is
# received while maintaining the API.
#
# by Robert Stone 2/22/2003 
#

from urllib import *
from urllib2 import *
from gzip import GzipFile
from StringIO import StringIO
import pprint


DEBUG=0


class HTTPContentEncodingHandler(HTTPHandler):
    """Inherit and add gzip/deflate/etc support to HTTP gets."""
    def http_open(self, req):
        # add the Accept-Encoding header to the request
        # support gzip encoding (identity is assumed)
        req.add_header("Accept-Encoding","gzip")
        if DEBUG: 
            print "Sending:" 
            print req.headers
            print "\n"
        fp = HTTPHandler.http_open(self,req)
        headers = fp.headers
        if DEBUG: 
             pprint.pprint(headers.dict)
        url = fp.url
        return addinfourldecompress(fp, headers, url)


class addinfourldecompress(addinfourl):
    """Do gzip decompression if necessary. Do addinfourl stuff too."""
    def __init__(self, fp, headers, url):
        # we need to do something more sophisticated here to deal with
        # multiple values?  What about other weird crap like q-values?
        # basically this only works for the most simplistic case and will
        # break in some other cases, but for now we only care about making
        # this work with the BT tracker so....
        if headers.has_key('content-encoding') and headers['content-encoding'] 
== 'gzip':
            if DEBUG:
                print "Contents of Content-encoding: " + 
headers['Content-encoding'] + "\n"
            self.gzip = 1
            self.rawfp = fp
            fp = GzipStream(fp)
        else:
            self.gzip = 0
        return addinfourl.__init__(self, fp, headers, url)

    def close(self):
        self.fp.close()
        if self.gzip:
            self.rawfp.close()

    def iscompressed(self):
        return self.gzip

class GzipStream(StringIO):
    """Magically decompress a file object.

       This is not the most efficient way to do this but GzipFile() wants
       to seek, etc, which won't work for a stream such as that from a socket.
       So we copy the whole shebang info a StringIO object, decompress that
       then let people access the decompressed output as a StringIO object.

       The disadvantage is memory use and the advantage is random access.

       Will mess with fixing this later.
    """

    def __init__(self,fp):
        self.fp = fp

        # this is nasty and needs to be fixed at some point
        # copy everything into a StringIO (compressed)
        compressed = StringIO()
        r = fp.read()
        while r:
            compressed.write(r)
            r = fp.read()
        # now, unzip (gz) the StringIO to a string
        compressed.seek(0,0)
        gz = GzipFile(fileobj = compressed)
        str = ''
        r = gz.read()
        while r:
            str += r
            r = gz.read()
        # close our utility files
        compressed.close()
        gz.close()
        # init our stringio selves with the string 
        StringIO.__init__(self, str)
        del str

    def close(self):
        self.fp.close()
        return StringIO.close(self)


def test():
    """Test this module.

       At the moment this is lame.
    """

    print "Running unit tests.\n"

    def printcomp(fp):
        try:
            if fp.iscompressed():
                print "GET was compressed.\n"
            else:
                print "GET was uncompressed.\n"
        except:
            print "no iscompressed function!  this shouldn't happen"

    print "Trying to GET a compressed document...\n"
    fp = urlopen('http://a.scarywater.net/hng/index.shtml')
    print fp.read()
    printcomp(fp)
    fp.close()

    print "Trying to GET an unknown document...\n"
    fp = urlopen('http://www.otaku.org/')
    print fp.read()
    printcomp(fp)
    fp.close()


#
# Install the HTTPContentEncodingHandler that we've defined above.
#
install_opener(build_opener(HTTPContentEncodingHandler))

if __name__ == '__main__':
    test()


Index: HTTPHandler.py
===================================================================
RCS file: /cvsroot/bittorrent/BitTorrent/BitTorrent/HTTPHandler.py,v
retrieving revision 1.8
retrieving revision 1.9
diff -C2 -d -r1.8 -r1.9
*** HTTPHandler.py      20 Jul 2002 18:58:11 -0000      1.8
--- HTTPHandler.py      28 Mar 2003 02:10:45 -0000      1.9
***************
*** 5,11 ****
--- 5,14 ----
  from sys import stdout
  import time
+ from gzip import GzipFile
  true = 1
  false = 0
  
+ DEBUG = false
+ 
  weekdays = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
  
***************
*** 65,68 ****
--- 68,87 ----
          if data == '':
              self.donereading = true
+             # check for Accept-Encoding: header, pick a 
+             if self.headers.has_key('accept-encoding'):
+                 ae = self.headers['accept-encoding']
+                 if DEBUG:
+                     print "Got Accept-Encoding: " + ae + "\n"
+             else:
+                 #identity assumed if no header
+                 ae = 'identity'
+             # this eventually needs to support multple acceptable types
+             # q-values and all that fancy HTTP crap
+             # for now assume we're only communicating with our own client
+             if ae == 'gzip':
+                 self.encoding = 'gzip'
+             else:
+                 #default to identity. 
+                 self.encoding = 'identity'
              r = self.handler.getfunc(self, self.path, self.headers)
              if r is not None:
***************
*** 74,77 ****
--- 93,98 ----
              return None
          self.headers[data[:i].strip().lower()] = data[i+1:].strip()
+         if DEBUG:
+             print data[:i].strip() + ": " + data[i+1:].strip()
          return self.read_header
  
***************
*** 79,86 ****
          if self.closed:
              return
          year, month, day, hour, minute, second, a, b, c = 
time.localtime(time.time())
!         print '%s - - [%02d/%3s/%04d:%02d:%02d:%02d] "%s" %i %i' % (
!             self.connection.get_ip(), day, months[month], year, hour, minute, 
!             second, self.header, responsecode, len(data))
          t = time.time()
          if t - self.handler.lastflush > self.handler.minflush:
--- 100,131 ----
          if self.closed:
              return
+         if self.encoding == 'gzip':
+             #transform data using gzip compression
+             #this is nasty but i'm unsure of a better way at the moment
+             compressed = StringIO()
+             gz = GzipFile(fileobj = compressed, mode = 'wb', compresslevel = 
9)
+             gz.write(data)
+             gz.close()
+             compressed.seek(0,0) 
+             cdata = compressed.read()
+             compressed.close()
+             if len(cdata) >= len(data):
+                 self.encoding = 'identity'
+             else:
+                 if DEBUG:
+                    print "Compressed: %i  Uncompressed: %i\n" % 
(len(cdata),len(data))
+                 data = cdata
+                 headers['Content-Encoding'] = 'gzip'
+ 
+         # i'm abusing the identd field here, but this should be ok
+         if self.encoding == 'identity':
+             ident = '-'
+         else:
+             ident = self.encoding
+ 
          year, month, day, hour, minute, second, a, b, c = 
time.localtime(time.time())
!         print '%s %s - [%02d/%3s/%04d:%02d:%02d:%02d] "%s" %i %i' % (
!             self.connection.get_ip(), ident, day, months[month], year, hour,
!             minute, second, self.header, responsecode, len(data))
          t = time.time()
          if t - self.handler.lastflush > self.handler.minflush:

Index: Rerequester.py
===================================================================
RCS file: /cvsroot/bittorrent/BitTorrent/BitTorrent/Rerequester.py,v
retrieving revision 1.16
retrieving revision 1.17
diff -C2 -d -r1.16 -r1.17
*** Rerequester.py      27 Mar 2003 04:40:23 -0000      1.16
--- Rerequester.py      28 Mar 2003 02:10:45 -0000      1.17
***************
*** 2,6 ****
  # see LICENSE.txt for license information
  
! from urllib import urlopen, quote
  from btformats import check_peers
  from bencode import bdecode
--- 2,6 ----
  # see LICENSE.txt for license information
  
! from zurllib import urlopen, quote
  from btformats import check_peers
  from bencode import bdecode

Index: download.py
===================================================================
RCS file: /cvsroot/bittorrent/BitTorrent/BitTorrent/download.py,v
retrieving revision 1.134
retrieving revision 1.135
diff -C2 -d -r1.134 -r1.135
*** download.py 28 Mar 2003 01:01:17 -0000      1.134
--- download.py 28 Mar 2003 02:10:45 -0000      1.135
***************
*** 2,6 ****
  # see LICENSE.txt for license information
  
! from urllib import urlopen
  from urlparse import urljoin
  from btformats import check_message
--- 2,6 ----
  # see LICENSE.txt for license information
  
! from zurllib import urlopen
  from urlparse import urljoin
  from btformats import check_message

Index: track.py
===================================================================
RCS file: /cvsroot/bittorrent/BitTorrent/BitTorrent/track.py,v
retrieving revision 1.46
retrieving revision 1.47
diff -C2 -d -r1.46 -r1.47
*** track.py    27 Mar 2003 22:15:38 -0000      1.46
--- track.py    28 Mar 2003 02:10:45 -0000      1.47
***************
*** 8,12 ****
  from threading import Event
  from bencode import bencode, bdecode
! from urllib import urlopen, quote, unquote
  from urlparse import urlparse
  from os.path import exists
--- 8,12 ----
  from threading import Event
  from bencode import bencode, bdecode
! from zurllib import urlopen, quote, unquote
  from urlparse import urlparse
  from os.path import exists



------------------------ Yahoo! Groups Sponsor ---------------------~-->
Get 128 Bit SSL Encryption!
http://us.click.yahoo.com/xaxhjB/hdqFAA/xGHJAA/dkFolB/TM
---------------------------------------------------------------------~->

To unsubscribe from this group, send an email to:
BitTorrent-unsubscribe@xxxxxxxxxxxxxxx

 

Your use of Yahoo! Groups is subject to http://docs.yahoo.com/info/terms/