1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

# This file is part of Buildbot.  Buildbot is free software: you can 

# redistribute it and/or modify it under the terms of the GNU General Public 

# License as published by the Free Software Foundation, version 2. 

# 

# This program is distributed in the hope that it will be useful, but WITHOUT 

# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 

# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more 

# details. 

# 

# You should have received a copy of the GNU General Public License along with 

# this program; if not, write to the Free Software Foundation, Inc., 51 

# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 

# 

# Copyright Buildbot Team Members 

 

 

from twisted.python import log 

from twisted.internet import reactor, defer 

from buildbot import util 

from buildbot.util import subscription 

 

if False: # for debugging 

    debuglog = log.msg 

else: 

    debuglog = lambda m: None 

 

class BaseLock: 

    """ 

    Class handling claiming and releasing of L{self}, and keeping track of 

    current and waiting owners. 

 

    @note: Ideally, we'd like to maintain FIFO order. The place to do that 

           would be the L{isAvailable()} function. However, this function is 

           called by builds/steps both for the first time, and after waking 

           them up by L{self} from the L{self.waiting} queue. There is 

           currently no way of distinguishing between them. 

    """ 

    description = "<BaseLock>" 

 

    def __init__(self, name, maxCount=1): 

        self.name = name          # Name of the lock 

        self.waiting = []         # Current queue, tuples (LockAccess, deferred) 

        self.owners = []          # Current owners, tuples (owner, LockAccess) 

        self.maxCount = maxCount  # maximal number of counting owners 

 

        # subscriptions to this lock being released 

        self.release_subs = subscription.SubscriptionPoint("%r releases" 

                                                             % (self,)) 

 

    def __repr__(self): 

        return self.description 

 

    def _getOwnersCount(self): 

        """ Return the number of current exclusive and counting owners. 

 

            @return: Tuple (number exclusive owners, number counting owners) 

        """ 

        num_excl, num_counting = 0, 0 

        for owner in self.owners: 

            if owner[1].mode == 'exclusive': 

                num_excl = num_excl + 1 

            else: # mode == 'counting' 

                num_counting = num_counting + 1 

 

        assert (num_excl == 1 and num_counting == 0) \ 

                or (num_excl == 0 and num_counting <= self.maxCount) 

        return num_excl, num_counting 

 

 

    def isAvailable(self, access): 

        """ Return a boolean whether the lock is available for claiming """ 

        debuglog("%s isAvailable(%s): self.owners=%r" 

                                            % (self, access, self.owners)) 

        num_excl, num_counting = self._getOwnersCount() 

        if access.mode == 'counting': 

            # Wants counting access 

            return num_excl == 0 and num_counting < self.maxCount 

        else: 

            # Wants exclusive access 

            return num_excl == 0 and num_counting == 0 

 

    def claim(self, owner, access): 

        """ Claim the lock (lock must be available) """ 

        debuglog("%s claim(%s, %s)" % (self, owner, access.mode)) 

        assert owner is not None 

        assert self.isAvailable(access), "ask for isAvailable() first" 

 

        assert isinstance(access, LockAccess) 

        assert access.mode in ['counting', 'exclusive'] 

        self.owners.append((owner, access)) 

        debuglog(" %s is claimed '%s'" % (self, access.mode)) 

 

    def subscribeToReleases(self, callback): 

        """Schedule C{callback} to be invoked every time this lock is 

        released.  Returns a L{Subscription}.""" 

        return self.release_subs.subscribe(callback) 

 

    def release(self, owner, access): 

        """ Release the lock """ 

        assert isinstance(access, LockAccess) 

 

        debuglog("%s release(%s, %s)" % (self, owner, access.mode)) 

        entry = (owner, access) 

        if not entry in self.owners: 

            debuglog("%s already released" % self) 

            return 

        self.owners.remove(entry) 

        # who can we wake up? 

        # After an exclusive access, we may need to wake up several waiting. 

        # Break out of the loop when the first waiting client should not be awakened. 

        num_excl, num_counting = self._getOwnersCount() 

        while len(self.waiting) > 0: 

            access, d = self.waiting[0] 

            if access.mode == 'counting': 

                if num_excl > 0 or num_counting == self.maxCount: 

                    break 

                else: 

                    num_counting = num_counting + 1 

            else: 

                # access.mode == 'exclusive' 

                if num_excl > 0 or num_counting > 0: 

                    break 

                else: 

                    num_excl = num_excl + 1 

 

            del self.waiting[0] 

            reactor.callLater(0, d.callback, self) 

 

        # notify any listeners 

        self.release_subs.deliver() 

 

    def waitUntilMaybeAvailable(self, owner, access): 

        """Fire when the lock *might* be available. The caller will need to 

        check with isAvailable() when the deferred fires. This loose form is 

        used to avoid deadlocks. If we were interested in a stronger form, 

        this would be named 'waitUntilAvailable', and the deferred would fire 

        after the lock had been claimed. 

        """ 

        debuglog("%s waitUntilAvailable(%s)" % (self, owner)) 

        assert isinstance(access, LockAccess) 

        if self.isAvailable(access): 

            return defer.succeed(self) 

        d = defer.Deferred() 

        self.waiting.append((access, d)) 

        return d 

 

    def stopWaitingUntilAvailable(self, owner, access, d): 

        debuglog("%s stopWaitingUntilAvailable(%s)" % (self, owner)) 

        assert isinstance(access, LockAccess) 

        assert (access, d) in self.waiting 

        self.waiting.remove( (access, d) ) 

 

    def isOwner(self, owner, access): 

        return (owner, access) in self.owners 

 

 

class RealMasterLock(BaseLock): 

    def __init__(self, lockid): 

        BaseLock.__init__(self, lockid.name, lockid.maxCount) 

        self.description = "<MasterLock(%s, %s)>" % (self.name, self.maxCount) 

 

    def getLock(self, slave): 

        return self 

 

class RealSlaveLock: 

    def __init__(self, lockid): 

        self.name = lockid.name 

        self.maxCount = lockid.maxCount 

        self.maxCountForSlave = lockid.maxCountForSlave 

        self.description = "<SlaveLock(%s, %s, %s)>" % (self.name, 

                                                        self.maxCount, 

                                                        self.maxCountForSlave) 

        self.locks = {} 

 

    def __repr__(self): 

        return self.description 

 

    def getLock(self, slave): 

        slavename = slave.slavename 

        if not self.locks.has_key(slavename): 

            maxCount = self.maxCountForSlave.get(slavename, 

                                                 self.maxCount) 

            lock = self.locks[slavename] = BaseLock(self.name, maxCount) 

            desc = "<SlaveLock(%s, %s)[%s] %d>" % (self.name, maxCount, 

                                                   slavename, id(lock)) 

            lock.description = desc 

            self.locks[slavename] = lock 

        return self.locks[slavename] 

 

 

class LockAccess(util.ComparableMixin): 

    """ I am an object representing a way to access a lock. 

 

    @param lockid: LockId instance that should be accessed. 

    @type  lockid: A MasterLock or SlaveLock instance. 

 

    @param mode: Mode of accessing the lock. 

    @type  mode: A string, either 'counting' or 'exclusive'. 

    """ 

 

    compare_attrs = ['lockid', 'mode'] 

    def __init__(self, lockid, mode): 

        self.lockid = lockid 

        self.mode = mode 

 

        assert isinstance(lockid, (MasterLock, SlaveLock)) 

        assert mode in ['counting', 'exclusive'] 

 

 

class BaseLockId(util.ComparableMixin): 

    """ Abstract base class for LockId classes. 

 

    Sets up the 'access()' function for the LockId's available to the user 

    (MasterLock and SlaveLock classes). 

    Derived classes should add 

    - Comparison with the L{util.ComparableMixin} via the L{compare_attrs} 

      class variable. 

    - Link to the actual lock class should be added with the L{lockClass} 

      class variable. 

    """ 

    def access(self, mode): 

        """ Express how the lock should be accessed """ 

        assert mode in ['counting', 'exclusive'] 

        return LockAccess(self, mode) 

 

    def defaultAccess(self): 

        """ For buildbot 0.7.7 compability: When user doesn't specify an access 

            mode, this one is chosen. 

        """ 

        return self.access('counting') 

 

 

 

# master.cfg should only reference the following MasterLock and SlaveLock 

# classes. They are identifiers that will be turned into real Locks later, 

# via the BotMaster.getLockByID method. 

 

class MasterLock(BaseLockId): 

    """I am a semaphore that limits the number of simultaneous actions. 

 

    Builds and BuildSteps can declare that they wish to claim me as they run. 

    Only a limited number of such builds or steps will be able to run 

    simultaneously. By default this number is one, but my maxCount parameter 

    can be raised to allow two or three or more operations to happen at the 

    same time. 

 

    Use this to protect a resource that is shared among all builders and all 

    slaves, for example to limit the load on a common SVN repository. 

    """ 

 

    compare_attrs = ['name', 'maxCount'] 

    lockClass = RealMasterLock 

    def __init__(self, name, maxCount=1): 

        self.name = name 

        self.maxCount = maxCount 

 

class SlaveLock(BaseLockId): 

    """I am a semaphore that limits simultaneous actions on each buildslave. 

 

    Builds and BuildSteps can declare that they wish to claim me as they run. 

    Only a limited number of such builds or steps will be able to run 

    simultaneously on any given buildslave. By default this number is one, 

    but my maxCount parameter can be raised to allow two or three or more 

    operations to happen on a single buildslave at the same time. 

 

    Use this to protect a resource that is shared among all the builds taking 

    place on each slave, for example to limit CPU or memory load on an 

    underpowered machine. 

 

    Each buildslave will get an independent copy of this semaphore. By 

    default each copy will use the same owner count (set with maxCount), but 

    you can provide maxCountForSlave with a dictionary that maps slavename to 

    owner count, to allow some slaves more parallelism than others. 

 

    """ 

 

    compare_attrs = ['name', 'maxCount', '_maxCountForSlaveList'] 

    lockClass = RealSlaveLock 

    def __init__(self, name, maxCount=1, maxCountForSlave={}): 

        self.name = name 

        self.maxCount = maxCount 

        self.maxCountForSlave = maxCountForSlave 

        # for comparison purposes, turn this dictionary into a stably-sorted 

        # list of tuples 

        self._maxCountForSlaveList = self.maxCountForSlave.items() 

        self._maxCountForSlaveList.sort() 

        self._maxCountForSlaveList = tuple(self._maxCountForSlaveList)