Ticket #313: lockaccess2.patch

buildbot-0.7.7/buildbot/locks.py

@@ -23 +23 @@
-    description = "<BaseLock>"
-
-    def __init__(self, name, maxCount=1):
-
-
-
-
+        # Name of the lock
+       # Current queue, tuples (LockAccess, deferred)
+        # Current owners, tuples (owner, LockAccess)
+  # maximal number of counting owners
-
-    def __repr__(self):
-        return self.description
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        """ Return a boolean whether the lock is available for claiming """
-
-
+
+
+
+
+
+
+
+
+
-
-
+, access
-        """ Claim the lock (lock must be available) """
-)" % (self, owner
+, %s)" % (self, owner, access.mode
-        assert owner is not None
-
-
-
-
-
+
+
+
+
+
+
+
+
-        """ Release the lock """
-
-
-
+
+
+
+
+
+
-        # who can we wake up?
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-            reactor.callLater(0, d.callback, self)
-
-
+, access
-        """Fire when the lock *might* be available. The caller will need to
-        check with isAvailable() when the deferred fires. This loose form is
-        used to avoid deadlocks. If we were interested in a stronger form,
@@ -62 +108 @@
-        after the lock had been claimed.
-        """
-        debuglog("%s waitUntilAvailable(%s)" % (self, owner))
-
+
+
-            return defer.succeed(self)
-        d = defer.Deferred()
-d
+(access, d)
-        return d
-
-
@@ -103 +150 @@
-        return self.locks[slavename]
-
-
+class LockAccess:
+    """ I am an object representing a way to access a lock.
+
+    @param lockid: LockId instance that should be accessed.
+    @type  lockid: A MasterLock or SlaveLock instance.
+
+    @param mode: Mode of accessing the lock.
+    @type  mode: A string, either 'counting' or 'exclusive'.
+    """
+    def __init__(self, lockid, mode):
+        self.lockid = lockid
+        self.mode = mode
+
+        assert isinstance(lockid, (MasterLock, SlaveLock))
+        assert mode in ['counting', 'exclusive']
+
+
-class BaseLockId(util.ComparableMixin):
-    """ Abstract base class for LockId classes.
-
+    Sets up the 'access()' function for the LockId's available to the user
+    (MasterLock and SlaveLock classes).
-    Derived classes should add
-    - Comparison with the L{util.ComparableMixin} via the L{compare_attrs}
-      class variable.
-    - Link to the actual lock class should be added with the L{lockClass}
-      class variable.
-    """
+    def access(self, mode):
+        """ Express how the lock should be accessed """
+        assert mode in ['counting', 'exclusive']
+        return LockAccess(self, mode)
+
+    def defaultAccess(self):
+        """ For buildbot 0.7.7 compability: When user doesn't specify an access
+            mode, this one is chosen.
+        """
+        return self.access('counting')
+
+
-
-# master.cfg should only reference the following MasterLock and SlaveLock
-# classes. They are identifiers that will be turned into real Locks later,

buildbot-0.7.7/buildbot/master.py

@@ -633 +633 @@
-        lock_dict = {}
-        for b in builders:
-            for l in b.get('locks', []):
+                if isinstance(l, locks.LockAccess): # User specified access to the lock
+                    l = l.lockid
-                if lock_dict.has_key(l.name):
-                    if lock_dict[l.name] is not l:
-                        raise ValueError("Two different locks (%s and %s) "
@@ -645 +647 @@
-            # important.
-            for s in b['factory'].steps:
-                for l in s[1].get('locks', []):
+                    if isinstance(l, locks.LockAccess): # User specified access to the lock
+                        l = l.lockid
-                    if lock_dict.has_key(l.name):
-                        if lock_dict[l.name] is not l:
-                            raise ValueError("Two different locks (%s and %s)"

buildbot-0.7.7/buildbot/process/base.py

@@ -7 +7 @@
-from twisted.python.failure import Failure
-from twisted.internet import reactor, defer, error
-
-
+, locks
-from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
-from buildbot.status.builder import Results, BuildRequestStatus
-from buildbot.status.progress import BuildProgress
@@ -284 +284 @@
-        self.setupSlaveBuilder(slavebuilder)
-
-        # convert all locks into their real forms
-
-
+
+
+
+
+
+
+
+
-        # then narrow SlaveLocks down to the right slave
-
+
+
-        self.remote = slavebuilder.remote
-        self.remote.notifyOnDisconnect(self.lostRemote)
-        d = self.deferred = defer.Deferred()
@@ -323 +330 @@
-        log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks))
-        if not self.locks:
-            return defer.succeed(None)
-
-
+, access
+access
-                log.msg("Build %s waiting for lock %s" % (self, lock))
-
+, access
-                d.addCallback(self.acquireLocks)
-                return d
-        # all locks are available, claim them all
-
-
+, access
+, access
-        return defer.succeed(None)
-
-    def _startBuild_2(self, res):
@@ -555 +562 @@
-
-    def releaseLocks(self):
-        log.msg("releaseLocks(%s): %s" % (self, self.locks))
-
-
+, access
+, access
-
-    # IBuildControl
-

buildbot-0.7.7/buildbot/process/buildstep.py

@@ -8 +8 @@
-from twisted.python.failure import Failure
-from twisted.web.util import formatFailure
-
-
+, locks
-from buildbot import interfaces
-from buildbot.status import progress
-from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, SKIPPED, \
@@ -663 +663 @@
-        self.remote = remote
-        self.deferred = defer.Deferred()
-        # convert all locks into their real form
-
-
+
+
+
+
+
+
+
+
-        # then narrow SlaveLocks down to the slave that this build is being
-        # run on
-l.getLock(self.build.slavebuilder) for l
-
+(l.getLock(self.build.slavebuilder), la) for l, la
+, la
-            if l in self.build.locks:
-                log.msg("Hey, lock %s is claimed by both a Step (%s) and the"
-                        " parent Build (%s)" % (l, self, self.build))
@@ -681 +687 @@
-        log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks))
-        if not self.locks:
-            return defer.succeed(None)
-
-
+, access
+access
-                log.msg("step %s waiting for lock %s" % (self, lock))
-
+, access
-                d.addCallback(self.acquireLocks)
-                return d
-        # all locks are available, claim them all
-
-
+, access
+, access
-        return defer.succeed(None)
-
-    def _startStep_2(self, res):
@@ -770 +776 @@
-
-    def releaseLocks(self):
-        log.msg("releaseLocks(%s): %s" % (self, self.locks))
-
-
+, access
+, access
-
-    def finished(self, results):
-        if self.progress:

buildbot-0.7.7/buildbot/test/test_locks.py

@@ -12 +12 @@
-from buildbot.test.runutils import RunMixin
-from buildbot import locks
-
-
+, la
-    """Return a Deferred that will fire when the lock is claimed. Keep trying
-    until we succeed."""
-
+la
-        #print "claimHarder(%s): claiming" % owner
-
+, la
-        return defer.succeed(lock)
-    #print "claimHarder(%s): waiting" % owner
-
-
+, la
+, la
-    return d
-
-
+la, 
-    if mode == "now":
-
+, la
-    elif mode == "very soon":
-
+, la
-    elif mode == "soon":
-
-
+
-
-class Unit(unittest.TestCase):
-
+
+
+
+
+
+
+
+
+
+
+
-        l = locks.BaseLock("name")
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-        # we should have claimed it by now
-
+la
-        # now set up two competing owners. We don't know which will get the
-        # lock first.
-
-
-
-
+, la
+la, 
+, la
+la, 
-        dl = defer.DeferredList([d2,d3])
-
+, la
-        # and release the lock in a moment
-
+, la
-        return dl
-
-
-
-
+, la
+, la
+, la
-        return d
-
-
+
+
+
+
+
+
+
+
+
+
+
-        lock = locks.BaseLock("name")
-        dl = []
-        for i in range(100):
-            owner = "owner%d" % i
-            mode = random.choice(["now", "very soon", "soon"])
-
-
+, la
+la, 
-            dl.append(d)
-        d = defer.DeferredList(dl)
-
+, la
-        return d
-
-class Multi(unittest.TestCase):
-
+
+
+
-        lock = locks.BaseLock("name", 2)
-
-
-
-
-
-
-
-
-
+la
+, la
+la
+, la
+la
+, la
+la
+, la
+la
-
-
+
+
+
-        lock = locks.BaseLock("name", 2)
-
-
-
-
-
-
+, la
+, la
+, la
+, la
+, la
+, la
-        return d
-
-
+, la
-        dl = []
-        for i in range(count):
-
+, la
-            dl.append(d)
-        d2 = defer.DeferredList(dl)
-        # once all locks are claimed, we know that any previous owners have
-        # been flushed out
-        def _release(res):
-            for i in range(count):
-
+, la
-        d2.addCallback(_release)
-        return d2
-
-
+
+
+
-        COUNT = 5
-        lock = locks.BaseLock("name", COUNT)
-        dl = []
-        for i in range(100):
-            owner = "owner%d" % i
-            mode = random.choice(["now", "very soon", "soon"])
-
+, la
-            def _check(lock):
-                self.failIf(len(lock.owners) > COUNT)
-                return lock
-            d.addCallback(_check)
-
+la, 
-            dl.append(d)
-        d = defer.DeferredList(dl)
-
+, la
-        return d
-
-class Dummy:

buildbot-0.7.7/docs/buildbot.texinfo

@@ -5423 +5423 @@
-@cindex locks
-@slindex buildbot.locks.MasterLock
-@slindex buildbot.locks.SlaveLock
+@slindex buildbot.locks.LockAccess
-
-For various reasons, you may want to prevent certain Steps (or perhaps
-entire Builds) from running simultaneously. Limited CPU speed or
@@ -5442 +5443 @@
-one-Build-at-a-time limit for each machine, but still allow as many
-simultaneous builds as there are machines.
-
+When using a @code{Lock}, you can specify how you want to access it
+with a @code{LockAccess} object. You can have an exclusive lock (the
+@code{'exclusive'} mode) where the lock has exactly one owner, or
+share it with with others up to the limit (the @code{'counting'}
+mode).
+
-Each @code{Lock} is created with a unique name. Each lock gets a count
-
+ in @code{'counting'} mode
-same time. This limit defaults to one, and is controllable through the
-@code{maxCount} argument. On @code{SlaveLock}s you can set the owner
-count on a per-slave basis by providing a dictionary (that maps from
@@ -5451 +5458 @@
-argument. Any buildslaves that aren't mentioned in
-@code{maxCountForSlave} get their owner count from @code{maxCount}.
-
-
+with the desired mode of access 
-@code{BuildStep} object that should obtain the lock before it runs.
-
-
+
+
+
+
-
-To claim a lock for the whole Build, add a @code{'locks'} key to the
-
-
+
+
+
+
+
-@code{'slavename'}, @code{'builddir'}, and @code{'factory'} keys). The
-@code{Build} object also accepts a @code{locks=} argument, but unless
-you are writing your own @code{BuildFactory} subclass then it will be
@@ -5492 +5504 @@
-f = factory.BuildFactory()
-f.addStep(source.SVN(svnurl="http://example.org/svn/Trunk"))
-f.addStep(shell.ShellCommand(command="make all"))
-
+
+
-b1 = @{'name': 'full1', 'slavename': 'bot-1', 'builddir': 'f1', 'factory': f@}
-b2 = @{'name': 'full2', 'slavename': 'bot-2', 'builddir': 'f2', 'factory': f@}
-b3 = @{'name': 'full3', 'slavename': 'bot-3', 'builddir': 'f3', 'factory': f@}
-c['builders'] = [b1, b2, b3]
-@end example
-
+Note that by default, a @code{Lock} uses a @code{maxCount=1} limit, so
+you can also use the @code{'counting'} access mode in this example.
+
-In the next example, we have one buildslave hosting three separate
-Builders (each running tests against a different version of Python).
-The machine which hosts this buildslave is not particularly fast, so
@@ -5523 +5539 @@
-f23 = factory.Trial(source, trialpython=["python2.3"])
-f24 = factory.Trial(source, trialpython=["python2.4"])
-b1 = @{'name': 'p22', 'slavename': 'bot-1', 'builddir': 'p22',
-
+.access('counting')
-b2 = @{'name': 'p23', 'slavename': 'bot-1', 'builddir': 'p23',
-
+.access('counting')
-b3 = @{'name': 'p24', 'slavename': 'bot-1', 'builddir': 'p24',
-
+.access('counting')
-c['builders'] = [b1, b2, b3]
-@end example
-
+You can also mix access modes. In the next example, a limit of at most
+two builds is enforced. The cleanup process however needs exclusive
+access to prevent interference with the build processes.
+
+@example
+from buildbot import locks
+from buildbot.steps import source, shell
+from buildbot.process import factory
+
+build_lock = locks.MasterLock("builds", maxCount=2)
+fb = factory.BuildFactory()
+fb.addStep(source.SVN(svnurl="http://example.org/svn/Trunk"))
+fb.addStep(shell.ShellCommand(command=["make", "all"]))
+b1 = @{'name': 'b1', 'slavename': 'bot-1', 'builddir' : 'b1',
+      'factory': fb, 'locks': [build_lock.access('counting')] @}
+b2 = @{'name': 'b2', 'slavename': 'bot-2', 'builddir' : 'b2',
+      'factory': fb, 'locks': [build_lock.access('counting')] @}
+b3 = @{'name': 'b3', 'slavename': 'bot-3', 'builddir' : 'b3',
+      'factory': fb, 'locks': [build_lock.access('counting')] @}
+fc = factory.BuildFactory()
+fc.addStep(shell.ShellCommand(command=["cleanup"]))
+cl = @{'name': 'clean', 'slavename': 'bot-1', 'builddir' :clean',
+      'factory': fc, 'locks': [build_lock.access('exclusive')] @}
+c['builders'] = [b1, b2, b3, cl]
+@end example
+
-In the last example, we use two Locks at the same time. In this case,
-we're concerned about both of the previous constraints, but we'll say
-that only the tests are computationally intensive, and that they have
@@ -5551 +5593 @@
-cpu_lock = locks.SlaveLock("cpu", maxCountForSlave=slavecounts)
-f = factory.BuildFactory()
-f.addStep(source.SVN(svnurl="http://example.org/svn/Trunk"))
-
-
-
+
+
+
+
+
+
+
-
-b1 = @{'name': 'full1', 'slavename': 'bot-slow', 'builddir': 'full1',
-      'factory': f@}