|
# This file is part of Buildbot. Buildbot is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright Buildbot Team Members
########################################
# frequency with which to reclaim running builds; this should be set to # something fairly long, to avoid undue database load
# multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered # unclaimed; this should be at least 2 to avoid false positives
# if this quantity of unclaimed build requests are present in the table, # then something is probably wrong! The master will log a WARNING on every # database poll operation.
# set up child services
# loop for polling the db
# configuration / reconfiguration handling
# this stores parameters used in the tac file, and is accessed by the # WebStatus to duplicate those values.
# subscription points subscription.SubscriptionPoint("changes") subscription.SubscriptionPoint("buildrequest_additions") subscription.SubscriptionPoint("buildset_additions") subscription.SubscriptionPoint("buildset_completion")
# local cache for this master's object ID
# note that these are order-dependent. If you get the order wrong, # you'll know it, as the master will fail to start.
# setup and reconfig handling
buildbot.version)
# Set umask os.umask(self.umask)
# first, apply all monkeypatches
# we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors
# load the configuration file, treating errors as fatal self.configFileName) except: log.err(failure.Failure(), 'while starting BuildMaster') _reactor.stop() return
# set up services that need access to the config before everything else # gets told to reconfig # (message was already logged)
_reactor.callLater(0, self.reconfig)
# call the parent method service.MultiService.startService(self))
# give all services a chance to load the new configuration, rather than # the base configuration
self.db_loop.stop() self.db_loop = None
# this method wraps doConfig, ensuring it is only ever called once at # a time, and alerting the user if the reconfig takes too long log.msg("reconfig already active; will reconfig again after") self.reconfig_requested = True return
# notify every 10 seconds that the reconfig is still going on, although # reconfigs should not take that long! log.msg("reconfig is ongoing for %d s" % (reactor.seconds() - self.reconfig_active)))
def cleanup(res): self.reconfig_requested = False self.reconfig()
def doReconfig(self): self.configFileName)
except: log.err(failure.Failure(), 'during reconfig:') failed = True
log.msg("WARNING: reconfig partially applied; master " "may malfunction") else: else:
"Cannot change c['db']['db_url'] after the master has started", )
# adjust the db poller != new_config.db['db_poll_interval']):
new_config)
## informational methods
return list(self.scheduler_manager)
""" @rtype: L{buildbot.status.builder.Status} """ return self.status
""" Return the obejct id for this master, for associating state with the master.
@returns: ID, via Deferred """ # try to get the cached value
# failing that, get it from the DB; multiple calls to this function # at the same time will not hurt except AttributeError: hostname = socket.getfqdn()
"buildbot.master.BuildMaster")
## triggering methods and subscriptions
isdir=None, is_dir=None, revision=None, when=None, when_timestamp=None, branch=None, category=None, revlink='', properties={}, repository='', codebase=None, project='', src=None): """ Add a change to the buildmaster and act on it.
This is a wrapper around L{ChangesConnectorComponent.addChange} which also acts on the resulting change and returns a L{Change} instance.
Note that all parameters are keyword arguments, although C{who}, C{files}, and C{comments} can be specified positionally for backward-compatibility.
@param author: the author of this change @type author: unicode string
@param who: deprecated name for C{author}
@param files: a list of filenames that were changed @type branch: list of unicode strings
@param comments: user comments on the change @type branch: unicode string
@param is_dir: deprecated
@param isdir: deprecated name for C{is_dir}
@param revision: the revision identifier for this change @type revision: unicode string
@param when_timestamp: when this change occurred, or the current time if None @type when_timestamp: datetime instance or None
@param when: deprecated name and type for C{when_timestamp} @type when: integer (UNIX epoch time) or None
@param branch: the branch on which this change took place @type branch: unicode string
@param category: category for this change (arbitrary use by Buildbot users) @type category: unicode string
@param revlink: link to a web view of this revision @type revlink: unicode string
@param properties: properties to set on this change @type properties: dictionary with string keys and simple values (JSON-able). Note that the property source is I{not} included in this dictionary.
@param repository: the repository in which this change took place @type repository: unicode string
@param project: the project this change is a part of @type project: unicode string
@param src: source of the change (vcs or other) @type src: string
@returns: L{Change} instance via Deferred """
# handle translating deprecated names into new names for db.changes converter = lambda x:x): "addChange parameter '%s'" % oldname) raise TypeError("Cannot provide '%s' and '%s' to addChange" % (oldname, newname))
default=0) "when_timestamp", when_timestamp, converter=epoch2datetime)
# add a source to each property
chdict = { 'changeid': None, 'author': author, 'files': files, 'comments': comments, 'is_dir': is_dir, 'revision': revision, 'when_timestamp': when_timestamp, 'branch': branch, 'category': category, 'revlink': revlink, 'properties': properties, 'repository': repository, 'project': project, } codebase = self.config.codebaseGenerator(chdict) else:
# create user object, returning a corresponding uid
# add the Change to the database self.db.changes.addChange(author=author, files=files, comments=comments, is_dir=is_dir, revision=revision, when_timestamp=when_timestamp, branch=branch, category=category, revlink=revlink, properties=properties, repository=repository, codebase=codebase, project=project, uid=uid))
# convert the changeid to a Change instance self.db.changes.getChange(changeid)) changes.Change.fromChdict(self, chdict))
# only deliver messages immediately if we're not polling
""" Request that C{callback} be called with each Change object added to the cluster.
Note: this method will go away in 0.9.x """
""" Add a buildset to the buildmaster and act on it. Interface is identical to L{buildbot.db.buildsets.BuildsetConnectorComponent.addBuildset}, including returning a Deferred, but also potentially triggers the resulting builds. """ # note that buildset additions are only reported on this master # only deliver messages immediately if we're not polling buildername=bn)
""" Request that C{callback(bsid=bsid, ssid=ssid, reason=reason, properties=properties, builderNames=builderNames, external_idstring=external_idstring)} be called whenever a buildset is added. Properties is a dictionary as expected for L{BuildsetsConnectorComponent.addBuildset}.
Note that this only works for buildsets added on this master.
Note: this method will go away in 0.9.x """
def maybeBuildsetComplete(self, bsid): """ Instructs the master to check whether the buildset is complete, and notify appropriately if it is.
Note that buildset completions are only reported on the master on which the last build request completes. """ brdicts = yield self.db.buildrequests.getBuildRequests( bsid=bsid, complete=False)
# if there are incomplete buildrequests, bail out if brdicts: return
brdicts = yield self.db.buildrequests.getBuildRequests(bsid=bsid)
# figure out the overall results of the buildset cumulative_results = SUCCESS for brdict in brdicts: if brdict['results'] not in (SUCCESS, WARNINGS): cumulative_results = FAILURE
# mark it as completed in the database yield self.db.buildsets.completeBuildset(bsid, cumulative_results)
# and deliver to any listeners self._buildsetComplete(bsid, cumulative_results)
""" Request that C{callback(bsid, result)} be called whenever a buildset is complete.
Note: this method will go away in 0.9.x """
""" Notifies the master that a build request is available to be claimed; this may be a brand new build request, or a build request that was previously claimed and unclaimed through a timeout or other calamity.
@param bsid: containing buildset id @param brid: buildrequest ID @param buildername: builder named by the build request """ dict(bsid=bsid, brid=brid, buildername=buildername))
""" Request that C{callback} be invoked with a dictionary with keys C{brid} (the build request id), C{bsid} (buildset id) and C{buildername} whenever a new build request is added to the database. Note that, due to the delayed nature of subscriptions, the build request may already be claimed by the time C{callback} is invoked.
Note: this method will go away in 0.9.x """ return self._new_buildrequest_subs.subscribe(callback)
## database polling
# poll each of the tables that can indicate new, actionable stuff for # this buildmaster to do. This is used in a TimerService, so returning # a Deferred means that we won't run two polling operations # simultaneously. Each particular poll method handles errors itself, # although catastrophic errors are handled here d = defer.gatherResults([ self.pollDatabaseChanges(), self.pollDatabaseBuildRequests(), # also unclaim ]) d.addErrback(log.err, 'while polling database') return d
def pollDatabaseChanges(self): # Older versions of Buildbot had each scheduler polling the database # independently, and storing a "last_processed" state indicating the # last change it had processed. This had the advantage of allowing # schedulers to pick up changes that arrived in the database while # the scheduler was not running, but was horribly inefficient.
# This version polls the database on behalf of the schedulers, using a # similar state at the master level.
# get the last processed change id yield self._getState('last_processed_change')
# if it's still None, assume we've processed up to the latest changeid # if there *are* no changes, count the last as '0' so that we don't # skip the first change
timer.stop() return
# if there's no such change, we've reached the end and can # stop polling
# write back the updated state, if it's changed self._last_processed_change)
def pollDatabaseBuildRequests(self): # deal with cleaning up unclaimed requests, and (if necessary) # requests from a previous instance of this master
# cleanup unclaimed builds unclaimed_age = (self.RECLAIM_BUILD_INTERVAL * self.UNCLAIMED_BUILD_FACTOR) yield self.db.buildrequests.unclaimExpiredRequests(unclaimed_age)
self._last_claim_cleanup = reactor.seconds()
# _last_unclaimed_brids_set tracks the state of unclaimed build # requests; whenever it sees a build request which was not claimed on # the last poll, it notifies the subscribers. It only tracks that # state within the master instance, though; on startup, it notifies for # all unclaimed requests in the database.
log.msg("WARNING: %d unclaimed buildrequests - is a scheduler " "producing builds for which no builder is running?" % len(last_unclaimed))
# get the current set of unclaimed buildrequests yield self.db.buildrequests.getBuildRequests(claimed=False)
# and store that for next time
# see what's new, and notify if anything is brd['buildername'])
## state maintenance (private)
"private wrapper around C{self.db.state.getState}"
"private wrapper around C{self.db.state.setState}"
self.master = master
self.master.addChange(change)
return self.master.addBuildset(**kwargs)
b = self.master.botmaster.builders[name] return BuilderControl(b, self)
|