Changeset 709

Timestamp:

09/05/08 17:04:04 (4 months ago)

Author:

dustin@zmanda.com

Message:

add build access modes

Files:

• buildbot/locks.py (modified) (4 diffs)
• buildbot/master.py (modified) (2 diffs)
• buildbot/process/base.py (modified) (4 diffs)
• buildbot/process/buildstep.py (modified) (4 diffs)
• buildbot/test/test_locks.py (modified) (4 diffs)
• docs/buildbot.texinfo (modified) (7 diffs)

buildbot/locks.py

@@ -24 +24 @@
-
-    def __init__(self, name, maxCount=1):
-
-
-
-
+        # Name of the lock
+       # Current queue, tuples (LockAccess, deferred)
+        # Current owners, tuples (owner, LockAccess)
+  # maximal number of counting owners
-
-    def __repr__(self):
-        return self.description
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        """ Return a boolean whether the lock is available for claiming """
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-        """ Claim the lock (lock must be available) """
-)" % (self, owner
+, %s)" % (self, owner, access.mode
-        assert owner is not None
-
-
-
-
-
+
+
+
+
+
+
+
+
-        """ Release the lock """
-
-
-
+
+
+
+
+
+
-        # who can we wake up?
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-            reactor.callLater(0, d.callback, self)
-
-
+, access
-        """Fire when the lock *might* be available. The caller will need to
-        check with isAvailable() when the deferred fires. This loose form is
@@ -63 +109 @@
-        """
-        debuglog("%s waitUntilAvailable(%s)" % (self, owner))
-
+
+
-            return defer.succeed(self)
-        d = defer.Deferred()
-d
+(access, d)
-        return d
-
@@ -104 +151 @@
-
-
+class LockAccess:
+    """ I am an object representing a way to access a lock.
+
+    @param lockid: LockId instance that should be accessed.
+    @type  lockid: A MasterLock or SlaveLock instance.
+
+    @param mode: Mode of accessing the lock.
+    @type  mode: A string, either 'counting' or 'exclusive'.
+    """
+    def __init__(self, lockid, mode):
+        self.lockid = lockid
+        self.mode = mode
+
+        assert isinstance(lockid, (MasterLock, SlaveLock))
+        assert mode in ['counting', 'exclusive']
+
+
-class BaseLockId(util.ComparableMixin):
-    """ Abstract base class for LockId classes.
-
+    Sets up the 'access()' function for the LockId's available to the user
+    (MasterLock and SlaveLock classes).
-    Derived classes should add
-    - Comparison with the L{util.ComparableMixin} via the L{compare_attrs}
@@ -113 +179 @@
-      class variable.
-    """
+    def access(self, mode):
+        """ Express how the lock should be accessed """
+        assert mode in ['counting', 'exclusive']
+        return LockAccess(self, mode)
+
+    def defaultAccess(self):
+        """ For buildbot 0.7.7 compability: When user doesn't specify an access
+            mode, this one is chosen.
+        """
+        return self.access('counting')
+
+
-
-# master.cfg should only reference the following MasterLock and SlaveLock

buildbot/master.py

@@ -642 +642 @@
-        for b in builders:
-            for l in b.get('locks', []):
+                if isinstance(l, locks.LockAccess): # User specified access to the lock
+                    l = l.lockid
-                if lock_dict.has_key(l.name):
-                    if lock_dict[l.name] is not l:
@@ -654 +656 @@
-            for s in b['factory'].steps:
-                for l in s[1].get('locks', []):
+                    if isinstance(l, locks.LockAccess): # User specified access to the lock
+                        l = l.lockid
-                    if lock_dict.has_key(l.name):
-                        if lock_dict[l.name] is not l:

buildbot/process/base.py

@@ -8 +8 @@
-from twisted.internet import reactor, defer, error
-
-
+, locks
-from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
-from buildbot.status.builder import Results, BuildRequestStatus
@@ -313 +313 @@
-
-        # convert all locks into their real forms
-
-
+
+
+
+
+
+
+
+
-        # then narrow SlaveLocks down to the right slave
-
+
+
-        self.remote = slavebuilder.remote
-        self.remote.notifyOnDisconnect(self.lostRemote)
@@ -352 +359 @@
-        if not self.locks:
-            return defer.succeed(None)
-
-
+, access
+access
-                log.msg("Build %s waiting for lock %s" % (self, lock))
-
+, access
-                d.addCallback(self.acquireLocks)
-                return d
-        # all locks are available, claim them all
-
-
+, access
+, access
-        return defer.succeed(None)
-
@@ -585 +592 @@
-    def releaseLocks(self):
-        log.msg("releaseLocks(%s): %s" % (self, self.locks))
-
-
+, access
+, access
-
-    # IBuildControl

buildbot/process/buildstep.py

@@ -9 +9 @@
-from twisted.web.util import formatFailure
-
-
+, locks
-from buildbot.status import progress
-from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, \
@@ -663 +663 @@
-        self.deferred = defer.Deferred()
-        # convert all locks into their real form
-
-
+
+
+
+
+
+
+
+
-        # then narrow SlaveLocks down to the slave that this build is being
-        # run on
-l.getLock(self.build.slavebuilder) for l
-
+(l.getLock(self.build.slavebuilder), la) for l, la
+, la
-            if l in self.build.locks:
-                log.msg("Hey, lock %s is claimed by both a Step (%s) and the"
@@ -681 +687 @@
-        if not self.locks:
-            return defer.succeed(None)
-
-
+, access
+access
-                log.msg("step %s waiting for lock %s" % (self, lock))
-
+, access
-                d.addCallback(self.acquireLocks)
-                return d
-        # all locks are available, claim them all
-
-
+, access
+, access
-        return defer.succeed(None)
-
@@ -770 +776 @@
-    def releaseLocks(self):
-        log.msg("releaseLocks(%s): %s" % (self, self.locks))
-
-
+, access
+, access
-
-    def finished(self, results):

buildbot/test/test_locks.py

@@ -13 +13 @@
-from buildbot import locks
-
-
+, la
-    """Return a Deferred that will fire when the lock is claimed. Keep trying
-    until we succeed."""
-
+la
-        #print "claimHarder(%s): claiming" % owner
-
+, la
-        return defer.succeed(lock)
-    #print "claimHarder(%s): waiting" % owner
-
-
+, la
+, la
-    return d
-
-
+la, 
-    if mode == "now":
-
+, la
-    elif mode == "very soon":
-
+, la
-    elif mode == "soon":
-
-
+
-
-class Unit(unittest.TestCase):
-
+
+
+
+
+
+
+
+
+
+
+
-        l = locks.BaseLock("name")
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        lock = locks.BaseLock("name")
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        lock = locks.BaseLock("name")
-
-
-
-
+, la
+, la
+
+, la
-        # we should have claimed it by now
-
+la
-        # now set up two competing owners. We don't know which will get the
-        # lock first.
-
-
-
-
+, la
+la, 
+, la
+la, 
-        dl = defer.DeferredList([d2,d3])
-
+, la
-        # and release the lock in a moment
-
+, la
-        return dl
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        lock = locks.BaseLock("name")
-        dl = []
@@ -80 +143 @@
-            owner = "owner%d" % i
-            mode = random.choice(["now", "very soon", "soon"])
-
-
+, la
+la, 
-            dl.append(d)
-        d = defer.DeferredList(dl)
-
+, la
-        return d
-
-class Multi(unittest.TestCase):
-
+
+
+
-        lock = locks.BaseLock("name", 2)
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
-        lock = locks.BaseLock("name", 2)
-
-
-
-
-
-
-
-
-
+, la
+, la
+, la
+, la
+, la
+, la
+
+
+, la
-        dl = []
-        for i in range(count):
-
+, la
-            dl.append(d)
-        d2 = defer.DeferredList(dl)
@@ -120 +187 @@
-        def _release(res):
-            for i in range(count):
-
+, la
-        d2.addCallback(_release)
-        return d2
-
-
+
+
+
-        COUNT = 5
-        lock = locks.BaseLock("name", COUNT)
@@ -131 +200 @@
-            owner = "owner%d" % i
-            mode = random.choice(["now", "very soon", "soon"])
-
+, la
-            def _check(lock):
-                self.failIf(len(lock.owners) > COUNT)
-                return lock
-            d.addCallback(_check)
-
+la, 
-            dl.append(d)
-        d = defer.DeferredList(dl)
-
+, la
-        return d
-

docs/buildbot.texinfo

@@ -5787 +5787 @@
-@slindex buildbot.locks.MasterLock
-@slindex buildbot.locks.SlaveLock
+@slindex buildbot.locks.LockAccess
-
-For various reasons, you may want to prevent certain Steps (or perhaps
@@ -5806 +5807 @@
-simultaneous builds as there are machines.
-
+When using a @code{Lock}, you can specify how you want to access it
+with a @code{LockAccess} object. You can have an exclusive lock (the
+@code{'exclusive'} mode) where the lock has exactly one owner, or
+share it with with others up to the limit (the @code{'counting'}
+mode).
+
-Each @code{Lock} is created with a unique name. Each lock gets a count
-
+ in @code{'counting'} mode
-same time. This limit defaults to one, and is controllable through the
-@code{maxCount} argument. On @code{SlaveLock}s you can set the owner
@@ -5815 +5822 @@
-@code{maxCountForSlave} get their owner count from @code{maxCount}.
-
-
+with the desired mode of access 
-@code{BuildStep} object that should obtain the lock before it runs.
-
-
+
+
+
+
-
-To claim a lock for the whole Build, add a @code{'locks'} key to the
-
-
+
+
+
+
+
-@code{'slavename'}, @code{'builddir'}, and @code{'factory'} keys). The
-@code{Build} object also accepts a @code{locks=} argument, but unless
@@ -5856 +5868 @@
-f.addStep(source.SVN(svnurl="http://example.org/svn/Trunk"))
-f.addStep(shell.ShellCommand(command="make all"))
-
+
+
-b1 = @{'name': 'full1', 'slavename': 'bot-1', 'builddir': 'f1', 'factory': f@}
-b2 = @{'name': 'full2', 'slavename': 'bot-2', 'builddir': 'f2', 'factory': f@}
@@ -5862 +5875 @@
-c['builders'] = [b1, b2, b3]
-@end example
+
+Note that by default, a @code{Lock} uses a @code{maxCount=1} limit, so
+you can also use the @code{'counting'} access mode in this example.
-
-In the next example, we have one buildslave hosting three separate
@@ -5887 +5903 @@
-f24 = factory.Trial(source, trialpython=["python2.4"])
-b1 = @{'name': 'p22', 'slavename': 'bot-1', 'builddir': 'p22',
-
+.access('counting')
-b2 = @{'name': 'p23', 'slavename': 'bot-1', 'builddir': 'p23',
-
+.access('counting')
-b3 = @{'name': 'p24', 'slavename': 'bot-1', 'builddir': 'p24',
-
+.access('counting')
-c['builders'] = [b1, b2, b3]
+@end example
+
+You can also mix access modes. In the next example, a limit of at most
+two builds is enforced. The cleanup process however needs exclusive
+access to prevent interference with the build processes.
+
+@example
+from buildbot import locks
+from buildbot.steps import source, shell
+from buildbot.process import factory
+
+build_lock = locks.MasterLock("builds", maxCount=2)
+fb = factory.BuildFactory()
+fb.addStep(source.SVN(svnurl="http://example.org/svn/Trunk"))
+fb.addStep(shell.ShellCommand(command=["make", "all"]))
+b1 = @{'name': 'b1', 'slavename': 'bot-1', 'builddir' : 'b1',
+      'factory': fb, 'locks': [build_lock.access('counting')] @}
+b2 = @{'name': 'b2', 'slavename': 'bot-2', 'builddir' : 'b2',
+      'factory': fb, 'locks': [build_lock.access('counting')] @}
+b3 = @{'name': 'b3', 'slavename': 'bot-3', 'builddir' : 'b3',
+      'factory': fb, 'locks': [build_lock.access('counting')] @}
+fc = factory.BuildFactory()
+fc.addStep(shell.ShellCommand(command=["cleanup"]))
+cl = @{'name': 'clean', 'slavename': 'bot-1', 'builddir' :clean',
+      'factory': fc, 'locks': [build_lock.access('exclusive')] @}
+c['builders'] = [b1, b2, b3, cl]
-@end example
-
@@ -5915 +5957 @@
-f = factory.BuildFactory()
-f.addStep(source.SVN(svnurl="http://example.org/svn/Trunk"))
-
-
-
+
+
+
+
+
+
+
-
-b1 = @{'name': 'full1', 'slavename': 'bot-slow', 'builddir': 'full1',