Ticket #26: buildbot-gzip2.patch

File buildbot-gzip2.patch, 3.7 kB (added by slinkp, 8 months ago)

second attempt, more complex, still many test failures

  • buildbot/status/builder.py

    old new  
    66from twisted.internet import reactor, defer 
    77from twisted.protocols import basic 
    88 
     9import gzip 
    910import os, shutil, sys, re, urllib, itertools 
     11 
    1012from cPickle import load, dump 
    1113from cStringIO import StringIO 
    1214 
    …… 
    4042HEADER = interfaces.LOG_CHANNEL_HEADER 
    4143ChunkTypes = ["stdout", "stderr", "header"] 
    4244 
     45 
     46 
     47class PossiblyGzippedFile(object): 
     48 
     49    """Wraps a file, gzipped if possible, conforming a little more closely 
     50    to the standard python file interface than gzip.GzipFile does. 
     51    """ 
     52 
     53    def __init__(self, path, *args): 
     54        self.__gzipped = True 
     55        if args: 
     56            mode = args[0] 
     57        else: 
     58            mode = 'r' 
     59        self.__mode = mode 
     60        f = gzip.open(path, *args) 
     61        if mode.startswith('w'): 
     62            # we're wiping out any existing file, so, unconditionally gzip it. 
     63            self.__file = f 
     64            return 
     65        try: 
     66            # See if the existing file is really gzipped. 
     67            f.seek(1) 
     68            f.seek(0) 
     69            self.__file = f 
     70            return 
     71        except IOError: 
     72            del f 
     73        # We have an existing, non-gzipped file. 
     74        self.__file = open(path, *args) 
     75        self.__gzipped = False 
     76 
     77    def seek(self, offset, whence=0): 
     78        # XXX written against python 2.4. 
     79        # I don't have 2.5 handy; does its gzip module support the 
     80        # whence argument? 
     81        if self.__gzipped: 
     82            if whence == 2: 
     83                offset += self.__file.size 
     84            elif whence == 1: 
     85                offset += self.__file.tell() 
     86            self.__file.seek(max(0, offset)) 
     87        else: 
     88            self.__file.seek(offset, whence) 
     89 
     90    def __getattr__(self, attr): 
     91        if attr in self.__dict__: 
     92            return self.__dict__[attr] 
     93        else: 
     94            return getattr(self.__file, attr) 
     95 
     96 
     97def gzopen(path, *args): 
     98    return PossiblyGzippedFile(path, *args) 
     99 
     100 
    43101class LogFileScanner(basic.NetstringReceiver): 
    44102    def __init__(self, chunk_cb, channels=[]): 
    45103        self.chunk_cb = chunk_cb 
    …… 
    50108        if not self.channels or (channel in self.channels): 
    51109            self.chunk_cb((channel, line[1:])) 
    52110 
     111     
    53112class LogFileProducer: 
    54113    """What's the plan? 
    55114 
    …… 
    227286            # is out of date, and we're overlapping with earlier builds now. 
    228287            # Warn about it, but then overwrite the old pickle file 
    229288            log.msg("Warning: Overwriting old serialized Build at %s" % fn) 
    230         self.openfile = open(fn, "w+") 
     289        self.openfile = gzopen(fn, "w+") 
    231290        self.runEntries = [] 
    232291        self.watchers = [] 
    233292        self.finishedWatchers = [] 
    …… 
    260319            # don't close it! 
    261320            return self.openfile 
    262321        # otherwise they get their own read-only handle 
    263         return open(self.getFilename(), "r") 
     322        return gzopen(self.getFilename(), "r") 
    264323 
    265324    def getText(self): 
    266325        # this produces one ginormous string 
    …… 
    441500        pickled LogFile (inside the pickled Build) won't be modified.""" 
    442501        self.filename = logfilename 
    443502        if not os.path.exists(self.getFilename()): 
    444             self.openfile = open(self.getFilename(), "w") 
     503            self.openfile = gzopen(self.getFilename(), "w") 
    445504            self.finished = False 
    446505            for channel,text in self.entries: 
    447506                self.addEntry(channel, text)