Ticket #333: buildbot-0.7.8-FtpDownload.patch

old-buildbot/buildbot/slave/commands.py

@@ -2 +2 @@
-
-import os, re, signal, shutil, types, time
-from stat import ST_CTIME, ST_MTIME, ST_SIZE
+from ftplib import FTP
-
-from zope.interface import implements
-from twisted.internet.protocol import ProcessProtocol
@@ -1603 +1604 @@
-
-registerSlaveCommand("svn", SVN, command_version)
-
+from twisted.protocols.ftp import FTPClient, FileConsumer, DTPFactory
+from twisted.internet.protocol import ClientFactory, ClientCreator
+
+class FtpDownload(SourceBase):
+    """FTP download operation. In addition to the arguments
+    handled by SourceBase, this command reads the following keys:
+
+    ['ftphost'] (required): the FTP hostname
+    ['ftpfile'] (required): the FTP file to download
+    ['ftpuser'] (required): the FTP username
+    ['ftppass'] (required): the FTP password
+    """
+
+    header = "FTP download operation"
+
+    def setup(self, args):
+        SourceBase.setup(self, args)
+        self.ftphost = args['ftphost']
+        self.ftpfile = args['ftpfile'] or 'anonymous'
+        self.ftpuser = args['ftpuser'] or 'buildbot@buildbot.net'
+        self.ftppass = args['ftppass']
+        self.sourcedata = "%s\n" % self.ftpfile
+
+    def sourcedirIsUpdateable(self):
+        return False
+
+    def doVCFull(self):
+        # Create the build directory
+        self._dirbuild = os.path.join(self.builder.basedir, self.srcdir)
+        os.mkdir(self._dirbuild)
+        # Download the file
+        self._pathfile = os.path.join(self._dirbuild,
+                                      os.path.basename(self.ftpfile))
+
+        creator = ClientCreator(reactor, FTPClient, username=self.ftpuser,
+                                password=self.ftppass, passive=True)
+        self.sendStatus({'stdout': "Connecting to '" + self.ftphost + "'\n"})
+        d = creator.connectTCP(self.ftphost, 21)
+        d.addCallbacks(self.connectionMade, self.connectionFail)
+        return d
+       
+    def connectionFail(self, failure, toto):
+        self.sendStatus({'stderr': "FTP connection failed: " + failure.getErrorMessage() + "\n"})
+        return 1
+
+    def connectionMade(self, ftpClient):
+        if ftpClient.greeting:
+            self.sendStatus({'stdout': "  " + str(ftpClient.greeting) + "\n"})
+        self.sendStatus({'stdout': "Retrieving file '" + self.ftpfile + "' as '" + self._pathfile + "'\n"})
+        self._file = open(self._pathfile, "w")
+        d = ftpClient.retrieveFile(self.ftpfile, FileConsumer(self._file))
+        d.addCallbacks(self.transferDone, self.transferFail)
+        return d
+
+    def transferFail(self, failure):
+        self.sendStatus({'stderr': "FTP transfer failed: " + failure.getErrorMessage() + "\n"})
+        return 1
+
+    def transferDone(self, results):
+        self._file.close()
+        self.sendStatus({'stdout': "FTP transfer done\n"})
+        try:
+            result = results[0]
+            res, msgs = result[1][0]
+            for msg in msgs:
+                self.sendStatus({'stdout': "  " + msg + "\n"})
+        except ValueError:
+            pass
+        return 0
+
+    def parseGotRevision(self):
+        return None
+
+
+registerSlaveCommand("ftpdownload", FtpDownload, command_version)
+
-class Darcs(SourceBase):
-    """Darcs-specific VC operation. In addition to the arguments
-    handled by SourceBase, this command reads the following keys:

old-buildbot/buildbot/steps/shell.py

@@ -276 +276 @@
-    descriptionDone = ["configure"]
-    command = ["./configure"]
-
+class Uncompress(ShellCommand):
+
+    name = "uncompress"
+    haltOnFailure = 1
+    description = ["uncompressing"]
+    descriptionDone = ["uncompress"]
+
+    def __init__(self, workdir=None,
+                 description=None, descriptionDone=None,
+                 command=None, filename=None,
+                 **kwargs):
+        if not command and filename:
+            command = ["tar", "axvf", filename, "--strip", "1"]
+
+        ShellCommand.__init__(self, command=command, **kwargs)
+
+
-class WarningCountingShellCommand(ShellCommand):
-    warnCount = 0
-    warningPattern = '.*warning[: ].*'

old-buildbot/buildbot/steps/source.py

@@ -473 +473 @@
-        self.startCommand(cmd, warnings)
-
-
+class FtpDownload(Source):
+    """I perform FTP download of a source archive file (or any other file).
+    """
+
+    name = 'ftpdownload'
+
+    def __init__(self, ftphost=None, ftpuser=None, ftppass=None, ftpfile=None,
+                 directory=None, **kwargs):
+        """
+        @type  ftphost: string
+        @param ftphost: hostname of the FTP server to connect to.
+
+        @type  ftpfile: string
+        @param ftpfile: full path to the file to retreive from FTP server. On
+                        slave, only the last path component of this path will
+                        be used.
+
+        @type  ftpuser: string
+        @param ftpuser: username to use for logging on the FTP server.
+
+        @type  ftphost: string
+        @param ftphost: password to use for logging on the FTP server.
+
+        """
+
+        if not kwargs.has_key('workdir') and directory is not None:
+            # deal with old configs
+            warn("Please use workdir=, not directory=", DeprecationWarning)
+            kwargs['workdir'] = directory
+
+        self.ftphost = ftphost
+        self.ftpfile = ftpfile
+        self.ftpuser = ftpuser
+        self.ftppass = ftppass
+
+        Source.__init__(self, **kwargs)
+        self.addFactoryArguments(ftphost=ftphost,
+                                 ftpfile=ftpfile,
+                                 ftpuser=ftpuser,
+                                 ftppass=ftppass,
+                                 directory=directory,
+                                 )
+
+        if not ftpfile or not ftphost:
+            raise ValueError("you must specify ftphost and ftpfile")
+
+
+    def computeSourceRevision(self, changes):
+        return None 
+
+    def startVC(self, branch, revision, patch):
+
+        warnings = []
+
+        if self.args['mode'] != "export":
+            warnings.append("WARNING: FtpDownload slave can only do full "
+                            "retreival, do not specify mode other than export")
+
+        self.args['ftphost'] = self.ftphost
+        self.args['ftpfile'] = self.ftpfile
+        self.args['ftpuser'] = self.ftpuser
+        self.args['ftppass'] = self.ftppass
+        self.args['revision'] = revision
+        self.args['patch'] = patch
+
+        revstuff = []
+        if branch is not None and branch != self.branch:
+            revstuff.append("[branch]")
+        if revision is not None:
+            revstuff.append("r%s" % revision)
+        if patch is not None:
+            revstuff.append("[patch]")
+        self.description.extend(revstuff)
+        self.descriptionDone.extend(revstuff)
+
+        cmd = LoggedRemoteCommand("ftpdownload", self.args)
+        self.startCommand(cmd, warnings)
+
+
-class Darcs(Source):
-    """Check out a source tree from a Darcs repository at 'repourl'.
-

old-buildbot/docs/buildbot.texinfo

@@ -207 +207 @@
-* Bzr::                         
-* P4::                          
-* Git::                         
+* FtpDownload::                         
-
-Simple ShellCommand Subclasses
-
@@ -4388 +4389 @@
-* Bzr::                         
-* P4::                          
-* Git::                         
+* FtpDownload::
-@end menu
-
-@node CVS, SVN, Source Checkout, Source Checkout
@@ -4763 +4765 @@
-@end table
-
-
- 
+FtpDownload
-@subsubsection Git
-
-@cindex Git Checkout
@@ -4789 +4791 @@
-@end table
-
-
+@node FtpDownload,  , Git, Source Checkout
+@subsubsection FtpDownload
+@cindex FTP download
+@bsindex buildbot.steps.source.FtpDownload
+
+
+The @code{FtpDownload} build step performs a download of a specified
+file from an FTP server (usually a source archive) to the build
+directory.
+
+The FtpDownload step only supports one mode argument: export.
+
+The FtpDownload step takes the following arguments:
+
+@table @code
+@item ftphost
+(required): The hostname of the FTP server to connect to.
+
+@item ftpfile
+(required): The full path to the file to dowload from the FTP server.
+The FTP download is done using image (binary) mode.
+
+@item ftpuser
+(optional): The username to use for logging on the FTP server.
+
+@item ftppass
+(optional): The password to use for logging on the FTP server.
+
+@end table
+
+The @code{FtpDownload} step can be followed by a @code{Uncompress}
+step to expand the downloaded source archive.
+
+
-@node ShellCommand, Simple ShellCommand Subclasses, Source Checkout, Build Steps
-@subsection ShellCommand
-
@@ -4927 +4963 @@
-file less verbose.
-
-@menu
+* Uncompress::
-* Configure::                   
-* Compile::                     
-* Test::                        
@@ -4935 +4972 @@
-* SetProperty::                 
-@end menu
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
-@subsubsection Configure
-
-@bsindex buildbot.steps.shell.Configure