|
# This file is part of Buildbot. Buildbot is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright Buildbot Team Members
pb.Referenceable, service.MultiService):
# reconfigure builders before slaves
# this is created the first time we get a good build
# build/wannabuild slots: Build objects move along this sequence # old_building holds active builds that were stolen from a predecessor
# buildslaves which have connected but which are not yet available. # These are always in the ATTACHING state.
# buildslaves at our disposal. Each SlaveBuilder instance has a # .state that is IDLE, PINGING, or BUILDING. "PINGING" is used when a # Build is about to start, to make sure that they're still alive.
self.reclaimAllBuilds)
# update big status every 30 minutes, working around #1980 self.updateBigStatus)
# find this builder in the config else: assert 0, "no config found for builder '%s'" % self.name
# set up a builder status object on the first reconfig builder_config.name, builder_config.builddir, builder_config.category)
service.MultiService.stopService(self)) # at this point, self.running = False, so another maybeStartBuild # invocation won't hurt anything, but it also will not complete # until any currently-running invocations are done, so we know that # the builder is quiescent at that time.
def __repr__(self): return "<Builder '%r' at %d>" % (self.name, id(self))
def getOldestRequestTime(self):
"""Returns the submitted_at of the oldest unclaimed build request for this builder, or None if there are no build requests.
@returns: datetime instance or None, via Deferred """ buildername=self.name, claimed=False)
else:
for b in self.building: if b.build_status and b.build_status.number == number: return b for b in self.old_building.keys(): if b.build_status and b.build_status.number == number: return b return None
assert interfaces.ILatentBuildSlave.providedBy(slave) for s in self.slaves: if s == slave: break else: sb = slavebuilder.LatentSlaveBuilder(slave, self) self.builder_status.addPointEvent( ['added', 'latent', slave.slavename]) self.slaves.append(sb) self.botmaster.maybeStartBuildsForBuilder(self.name)
"""This is invoked by the BuildSlave when the self.slavename bot registers their builder.
@type slave: L{buildbot.buildslave.BuildSlave} @param slave: the BuildSlave that represents the buildslave as a whole @type remote: L{twisted.spread.pb.RemoteReference} @param remote: a reference to the L{buildbot.slave.bot.SlaveBuilder} @type commands: dict: string -> string, or None @param commands: provides the slave's version of each RemoteCommand
@rtype: L{twisted.internet.defer.Deferred} @return: a Deferred that fires (with 'self') when the slave-side builder is fully attached and ready to accept commands. """ for s in self.attaching_slaves + self.slaves: if s.slave == slave: # already attached to them. This is fairly common, since # attached() gets called each time we receive the builder # list from the slave, and we ask for it each time we add or # remove a builder. So if the slave is hosting builders # A,B,C, and the config file changes A, we'll remove A and # re-add it, triggering two builder-list requests, getting # two redundant calls to attached() for B, and another two # for C. # # Therefore, when we see that we're already attached, we can # just ignore it. return defer.succeed(self)
sb = slavebuilder.SlaveBuilder() sb.setBuilder(self) self.attaching_slaves.append(sb) d = sb.attached(slave, remote, commands) d.addCallback(self._attached) d.addErrback(self._not_attached, slave) return d
self.builder_status.addPointEvent(['connect', sb.slave.slavename]) self.attaching_slaves.remove(sb) self.slaves.append(sb)
self.updateBigStatus()
return self
# already log.err'ed by SlaveBuilder._attachFailure # TODO: remove from self.slaves (except that detached() should get # run first, right?) log.err(why, 'slave failed to attach') self.builder_status.addPointEvent(['failed', 'connect', slave.slavename]) # TODO: add an HTMLLogFile of the exception
"""This is called when the connection to the bot is lost.""" for sb in self.attaching_slaves + self.slaves: if sb.slave == slave: break else: log.msg("WEIRD: Builder.detached(%s) (%s)" " not in attaching_slaves(%s)" " or slaves(%s)" % (slave, slave.slavename, self.attaching_slaves, self.slaves)) return if sb.state == BUILDING: # the Build's .lostRemote method (invoked by a notifyOnDisconnect # handler) will cause the Build to be stopped, probably right # after the notifyOnDisconnect that invoked us finishes running. pass
if sb in self.attaching_slaves: self.attaching_slaves.remove(sb) if sb in self.slaves: self.slaves.remove(sb)
self.builder_status.addPointEvent(['disconnect', slave.slavename]) sb.detached() # inform the SlaveBuilder that their slave went away self.updateBigStatus()
self.builder_status.setBigState("building") else:
def _startBuildFor(self, slavebuilder, buildrequests): """Start a build on the given slave. @param build: the L{base.Build} to start @param sb: the L{SlaveBuilder} which will host this build
@return: (via Deferred) boolean indicating that the build was succesfully started. """
# as of the Python versions supported now, try/finally can't be used # with a generator expression. So instead, we push cleanup functions # into a list so that, at any point, we can abort this operation. cleanups = [] def run_cleanups(): try: while cleanups: fn = cleanups.pop() fn() except: log.err(failure.Failure(), "while running %r" % (run_cleanups,))
# the last cleanup we want to perform is to update the big # status based on any other cleanup cleanups.append(lambda : self.updateBigStatus())
build = self.config.factory.newBuild(buildrequests) build.setBuilder(self) log.msg("starting build %s using slave %s" % (build, slavebuilder))
# set up locks build.setLocks(self.config.locks) cleanups.append(lambda : slavebuilder.slave.releaseLocks())
if len(self.config.env) > 0: build.setSlaveEnvironment(self.config.env)
# append the build to self.building self.building.append(build) cleanups.append(lambda : self.building.remove(build))
# update the big status accordingly self.updateBigStatus()
try: ready = yield slavebuilder.prepare(self.builder_status, build) except: log.err(failure.Failure(), 'while preparing slavebuilder:') ready = False
# If prepare returns True then it is ready and we start a build # If it returns false then we don't start a new build. if not ready: log.msg("slave %s can't build %s after all; re-queueing the " "request" % (build, slavebuilder)) run_cleanups() defer.returnValue(False) return
# ping the slave to make sure they're still there. If they've # fallen off the map (due to a NAT timeout or something), this # will fail in a couple of minutes, depending upon the TCP # timeout. # # TODO: This can unnecessarily suspend the starting of a build, in # situations where the slave is live but is pushing lots of data to # us in a build. log.msg("starting build %s.. pinging the slave %s" % (build, slavebuilder)) try: ping_success = yield slavebuilder.ping() except: log.err(failure.Failure(), 'while pinging slave before build:') ping_success = False
if not ping_success: log.msg("slave ping failed; re-queueing the request") run_cleanups() defer.returnValue(False) return
# The buildslave is ready to go. slavebuilder.buildStarted() sets its # state to BUILDING (so we won't try to use it for any other builds). # This gets set back to IDLE by the Build itself when it finishes. slavebuilder.buildStarted() cleanups.append(lambda : slavebuilder.buildFinished())
# tell the remote that it's starting a build, too try: yield slavebuilder.remote.callRemote("startBuild") except: log.err(failure.Failure(), 'while calling remote startBuild:') run_cleanups() defer.returnValue(False) return
# create the BuildStatus object that goes with the Build bs = self.builder_status.newBuild()
# record the build in the db - one row per buildrequest try: bids = [] for req in build.requests: bid = yield self.master.db.builds.addBuild(req.id, bs.number) bids.append(bid) except: log.err(failure.Failure(), 'while adding rows to build table:') run_cleanups() defer.returnValue(False) return
# let status know self.master.status.build_started(req.id, self.name, bs)
# start the build. This will first set up the steps, then tell the # BuildStatus that it has started, which will announce it to the world # (through our BuilderStatus object, which is its parent). Finally it # will start the actual build process. This is done with a fresh # Deferred since _startBuildFor should not wait until the build is # finished. d = build.startBuild(bs, self.expectations, slavebuilder) d.addCallback(self.buildFinished, slavebuilder, bids) # this shouldn't happen. if it does, the slave will be wedged d.addErrback(log.err)
# make sure the builder's status is represented correctly self.updateBigStatus()
defer.returnValue(True)
props.setProperty("buildername", self.name, "Builder") if len(self.config.properties) > 0: for propertyname in self.config.properties: props.setProperty(propertyname, self.config.properties[propertyname], "Builder")
"""This is called when the Build has finished (either success or failure). Any exceptions during the build are reported with results=FAILURE, not with an errback."""
# by the time we get here, the Build has already released the slave, # which will trigger a check for any now-possible build requests # (maybeStartBuilds)
# mark the builds as finished, although since nothing ever reads this # table, it's not too important that it complete successfully d = self.master.db.builds.finishBuilds(bids) d.addErrback(log.err, 'while marking builds as finished (ignored)')
results = build.build_status.getResults() self.building.remove(build) if results == RETRY: self._resubmit_buildreqs(build).addErrback(log.err) else: brids = [br.id for br in build.requests] db = self.master.db d = db.buildrequests.completeBuildRequests(brids, results) d.addCallback( lambda _ : self._maybeBuildsetsComplete(build.requests)) # nothing in particular to do with this deferred, so just log it if # it fails.. d.addErrback(log.err, 'while marking build requests as completed')
if sb.slave: sb.slave.releaseLocks()
self.updateBigStatus()
def _maybeBuildsetsComplete(self, requests): # inform the master that we may have completed a number of buildsets for br in requests: yield self.master.maybeBuildsetComplete(br.bsid)
brids = [br.id for br in build.requests] return self.master.db.buildrequests.unclaimBuildRequests(brids)
"""Mark the build as successful and update expectations for the next build. Only call this when the build did not fail in any way that would invalidate the time expectations generated by it. (if the compile failed and thus terminated early, we can't use the last build to predict how long the next one will take). """ if self.expectations: self.expectations.update(progress) else: # the first time we get a good build, create our Expectations # based upon its results self.expectations = Expectations(progress) log.msg("new expectations: %s seconds" % \ self.expectations.expectedBuildTime())
# Build Creation
def maybeStartBuild(self): # This method is called by the botmaster whenever this builder should # check for and potentially start new builds. Do not call this method # directly - use master.botmaster.maybeStartBuildsForBuilder, or one # of the other similar methods if more appropriate
# first, if we're not running, then don't start builds; stopService # uses this to ensure that any ongoing maybeStartBuild invocations # are complete before it stops.
# Check for available slaves. If there are no available slaves, then # there is no sense continuing if sb.isAvailable() ]
# now, get the available build requests yield self.master.db.buildrequests.getBuildRequests( buildername=self.name, claimed=False)
# sort by submitted_at, so the first is the oldest
# get the mergeRequests function for later
# match them up until we're out of options # first, choose a slave (using nextSlave)
"'%s'; cannot start build") % self.name)
# then choose a request (using nextBuild)
"'%s'; cannot start build") % self.name)
# merge the chosen request with any compatible requests in the # queue mergeRequests_fn)
# try to claim the build requests # one or more of the build requests was already claimed; # re-fetch the now-partially-claimed build requests and keep # trying to match them yield self.master.db.buildrequests.getBuildRequests( buildername=self.name, claimed=False)
# go around the loop again
# claim was successful, so initiate a build for this set of # requests. Note that if the build fails from here on out (e.g., # because a slave has failed), it will be handled outside of this # loop. TODO: test that!
# _startBuildFor expects BuildRequest objects, so cook some up [ self._brdictToBuildRequest(brdict) for brdict in brdicts ])
# build was not started, so unclaim the build requests yield self.master.db.buildrequests.unclaimBuildRequests(brids)
# and try starting builds again. If we still have a working slave, # then this may re-claim the same buildrequests self.botmaster.maybeStartBuildsForBuilder(self.name)
# finally, remove the buildrequests and slavebuilder from the # respective queues
# a few utility functions to make the maybeStartBuild a bit shorter and # easier to read
""" Choose the next slave, using the C{nextSlave} configuration if available, and falling back to C{random.choice} otherwise.
@param available_slavebuilders: list of slavebuilders to choose from @returns: SlaveBuilder or None via Deferred """ self.config.nextSlave(self, available_slavebuilders)) else:
""" Choose the next build from the given set of build requests (represented as dictionaries). Defaults to returning the first request (earliest submitted).
@param buildrequests: sorted list of build request dictionaries @returns: a build request dictionary or None via Deferred """ # nextBuild expects BuildRequest objects, so instantiate them here # and cache them in the dictionaries for brdict in buildrequests ]) self.config.nextBuild(self, requestobjects)) # get the brdict for this object back else:
"""Helper function to determine which mergeRequests function to use from L{_mergeRequests}, or None for no merging""" # first, seek through builder, global, and the default
# then translate False and True properly
def _mergeRequests(self, breq, unclaimed_requests, mergeRequests_fn): """Use C{mergeRequests_fn} to merge C{breq} against C{unclaimed_requests}, where both are build request dictionaries""" # short circuit if there is no merging to do return
# we'll need BuildRequest objects, so get those first [ self._brdictToBuildRequest(brdict) for brdict in unclaimed_requests ])
unclaimed_requests.index(breq))
# gather the mergeable requests lambda : mergeRequests_fn(self, breq_object, other_breq_object))):
# convert them back to brdicts and return
""" Convert a build request dictionary to a L{buildrequest.BuildRequest} object, caching the result in the dictionary itself. The resulting buildrequest will have a C{brdict} attribute pointing back to this dictionary.
Note that this does not perform any locking - be careful that it is only called once at a time for each build request dictionary.
@param brdict: dictionary to convert
@returns: L{buildrequest.BuildRequest} via Deferred """
"""Break the reference loops created by L{_brdictToBuildRequest}""" pass
d = ss.getSourceStampSetId(self.master.master) def add_buildset(sourcestampsetid): return self.master.master.addBuildset( builderNames=[self.original.name], sourcestampsetid=sourcestampsetid, reason=reason, properties=props) d.addCallback(add_buildset) def get_brs((bsid,brids)): brs = BuildRequestStatus(self.original.name, brids[self.original.name], self.master.master.status) return brs d.addCallback(get_brs) return d
return
# Make a copy of the properties so as not to modify the original build. # Don't include runtime-set properties in a rebuild request properties.updateFromProperties(extraProperties)
# add defered to the list
builderNames=[self.original.name], sourcestampsetid=sourcestampsetid, reason=reason, properties=properties_dict) else:
def getPendingBuildRequestControls(self): master = self.original.master brdicts = yield master.db.buildrequests.getBuildRequests( buildername=self.original.name, claimed=False)
# convert those into BuildRequest objects buildrequests = [ ] for brdict in brdicts: br = yield buildrequest.BuildRequest.fromBrdict( self.master.master, brdict) buildrequests.append(br)
# and return the corresponding control objects defer.returnValue([ buildrequest.BuildRequestControl(self.original, r) for r in buildrequests ])
return self.original.getBuild(number)
if not self.original.slaves: self.original.builder_status.addPointEvent(["ping", "no slave"]) return defer.succeed(False) # interfaces.NoSlaveError dl = [] for s in self.original.slaves: dl.append(s.ping(self.original.builder_status)) d = defer.DeferredList(dl) d.addCallback(self._gatherPingResults) return d
for ignored,success in res: if not success: return False return True |