|
# This file is part of Buildbot. Buildbot is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Portions Copyright Buildbot Team Members # Original Copyright (c) 2010 The Chromium Authors.
- as_text - By default, application/json is used. Setting as_text=1 change the type to text/plain and implicitly sets compact=0 and filter=1. Mainly useful to look at the result in a web browser. - compact - By default, the json data is compact and defaults to 1. For easier to read indented output, set compact=0. - select - By default, most children data is listed. You can do a random selection of data by using select=<sub-url> multiple times to coagulate data. "select=" includes the actual url otherwise it is skipped. - numbuilds - By default, only in memory cached builds are listed. You can as for more data by using numbuilds=<number>. - filter - Filters out null, false, and empty string, list and dict. This reduce the amount of useless data sent. - callback - Enable uses of JSONP as described in http://en.wikipedia.org/wiki/JSONP. Note that Access-Control-Allow-Origin:* is set in the HTTP response header so you can use this in compatible browsers. """
- /json - Root node, that *doesn't* mean all the data. Many things (like logs) must be explicitly queried for performance reasons. - /json/builders/ - All builders. - /json/builders/<A_BUILDER> - A specific builder as compact text. - /json/builders/<A_BUILDER>/builds - All *cached* builds. - /json/builders/<A_BUILDER>/builds/_all - All builds. Warning, reads all previous build data. - /json/builders/<A_BUILDER>/builds/<A_BUILD> - Where <A_BUILD> is either positive, a build number, or negative, a past build. - /json/builders/<A_BUILDER>/builds/-1/source_stamp/changes - Build changes - /json/builders/<A_BUILDER>/builds?select=-1&select=-2 - Two last builds on '<A_BUILDER>' builder. - /json/builders/<A_BUILDER>/builds?select=-1/source_stamp/changes&select=-2/source_stamp/changes - Changes of the two last builds on '<A_BUILDER>' builder. - /json/builders/<A_BUILDER>/slaves - Slaves associated to this builder. - /json/builders/<A_BUILDER>?select=&select=slaves - Builder information plus details information about its slaves. Neat eh? - /json/slaves/<A_SLAVE> - A specific slave. - /json?select=slaves/<A_SLAVE>/&select=project&select=builders/<A_BUILDER>/builds/<A_BUILD> - A selection of random unrelated stuff as an random example. :) """
return request.args.get(arg, [default])[0]
value = RequestArg(request, arg, default) if value in (False, True): return value value = value.lower() if value in ('1', 'true'): return True if value in ('0', 'false'): return False # Ignore value. return default
"""Returns a copy with None, False, "", [], () and {} removed. Warning: converts tuple to list.""" if isinstance(data, (list, tuple)): # Recurse in every items and filter them out. items = map(FilterOut, data) if not filter(lambda x: not x in ('', False, None, [], {}, ()), items): return None return items elif isinstance(data, dict): return dict(filter(lambda x: not x[1] in ('', False, None, [], {}, ()), [(k, FilterOut(v)) for (k, v) in data.iteritems()])) else: return data
"""Base class for json data."""
"""Adds transparent lazy-child initialization.""" resource.Resource.__init__(self) # buildbot.status.builder.Status self.status = status
"""Adds transparent support for url ending with /""" if path == "" and len(request.postpath) == 0: return self if path == 'help' and self.help: pageTitle = '' if self.pageTitle: pageTitle = self.pageTitle + ' help' return HelpResource(self.help, pageTitle=pageTitle, parent_node=self) # Equivalent to resource.Resource.getChildWithDefault() if self.children.has_key(path): return self.children[path] return self.getChild(path, request)
"""Adds the resource's level for help links generation."""
def RecurseFix(res, level): res.level = level + 1 for c in res.children.itervalues(): RecurseFix(c, res.level)
RecurseFix(res, self.level) resource.Resource.putChild(self, name, res)
"""Renders a HTTP GET at the http request level.""" d = defer.maybeDeferred(lambda : self.content(request)) def handle(data): if isinstance(data, unicode): data = data.encode("utf-8") request.setHeader("Access-Control-Allow-Origin", "*") if RequestArgToBool(request, 'as_text', False): request.setHeader("content-type", 'text/plain') else: request.setHeader("content-type", self.contentType) request.setHeader("content-disposition", "attachment; filename=\"%s.json\"" % request.path) # Make sure we get fresh pages. if self.cache_seconds: now = datetime.datetime.utcnow() expires = now + datetime.timedelta(seconds=self.cache_seconds) request.setHeader("Expires", expires.strftime("%a, %d %b %Y %H:%M:%S GMT")) request.setHeader("Pragma", "no-cache") return data d.addCallback(handle) def ok(data): request.write(data) request.finish() def fail(f): request.processingFailed(f) return None # processingFailed will log this for us d.addCallbacks(ok, fail) return server.NOT_DONE_YET
def content(self, request): """Renders the json dictionaries.""" # Supported flags. select = request.args.get('select') as_text = RequestArgToBool(request, 'as_text', False) filter_out = RequestArgToBool(request, 'filter', as_text) compact = RequestArgToBool(request, 'compact', not as_text) callback = request.args.get('callback')
# Implement filtering at global level and every child. if select is not None: del request.args['select'] # Do not render self.asDict()! data = {} # Remove superfluous / select = [s.strip('/') for s in select] select.sort(cmp=lambda x,y: cmp(x.count('/'), y.count('/')), reverse=True) for item in select: # Start back at root. node = data # Implementation similar to twisted.web.resource.getChildForRequest # but with a hacked up request. child = self prepath = request.prepath[:] postpath = request.postpath[:] request.postpath = filter(None, item.split('/')) while request.postpath and not child.isLeaf: pathElement = request.postpath.pop(0) node[pathElement] = {} node = node[pathElement] request.prepath.append(pathElement) child = child.getChildWithDefault(pathElement, request)
# some asDict methods return a Deferred, so handle that # properly if hasattr(child, 'asDict'): child_dict = yield defer.maybeDeferred(lambda : child.asDict(request)) else: child_dict = { 'error' : 'Not available', } node.update(child_dict)
request.prepath = prepath request.postpath = postpath else: data = yield defer.maybeDeferred(lambda : self.asDict(request))
if filter_out: data = FilterOut(data) if compact: data = json.dumps(data, sort_keys=True, separators=(',',':')) else: data = json.dumps(data, sort_keys=True, indent=2) if callback: # Only accept things that look like identifiers for now callback = callback[0] if re.match(r'^[a-zA-Z$][a-zA-Z$0-9.]*$', callback): data = '%s(%s);' % (callback, data) defer.returnValue(data)
def asDict(self, request): """Generates the json dictionary.
By default, renders every childs.""" if self.children: data = {} for name in self.children: child = self.getChildWithDefault(name, request) if isinstance(child, JsonResource): data[name] = yield defer.maybeDeferred(lambda : child.asDict(request)) # else silently pass over non-json resources. defer.returnValue(data) else: raise NotImplementedError()
"""Convert a string in a wiki-style format into HTML.""" indent = 0 in_item = False output = [] for line in text.splitlines(False): match = re.match(r'^( +)\- (.*)$', line) if match: if indent < len(match.group(1)): output.append('<ul>') indent = len(match.group(1)) elif indent > len(match.group(1)): while indent > len(match.group(1)): output.append('</ul>') indent -= 2 if in_item: # Close previous item output.append('</li>') output.append('<li>') in_item = True line = match.group(2) elif indent: if line.startswith((' ' * indent) + ' '): # List continuation line = line.strip() else: # List is done if in_item: output.append('</li>') in_item = False while indent > 0: output.append('</ul>') indent -= 2
if line.startswith('/'): if not '?' in line: line_full = line + '?as_text=1' else: line_full = line + '&as_text=1' output.append('<a href="' + html.escape(line_full) + '">' + html.escape(line) + '</a>') else: output.append(html.escape(line).replace(' ', ' ')) if not in_item: output.append('<br>')
if in_item: output.append('</li>') while indent > 0: output.append('</ul>') indent -= 2 return '\n'.join(output)
HtmlResource.__init__(self) self.text = text self.pageTitle = pageTitle self.parent_level = parent_node.level self.parent_children = parent_node.children.keys()
cxt['level'] = self.parent_level cxt['text'] = ToHtml(self.text) cxt['children'] = [ n for n in self.parent_children if n != 'help' ] cxt['flags'] = ToHtml(FLAGS) cxt['examples'] = ToHtml(EXAMPLES).replace( 'href="/json', 'href="../%sjson' % (self.parent_level * '../'))
template = request.site.buildbot_service.templates.get_template("jsonhelp.html") return template.render(**cxt)
"""
JsonResource.__init__(self, status) self.builder_status = builder_status
# buildbot.status.builder.BuilderStatus d = self.builder_status.getPendingBuildRequestStatuses() def to_dict(statuses): return defer.gatherResults( [ b.asDict_async() for b in statuses ]) d.addCallback(to_dict) return d
"""
JsonResource.__init__(self, status) self.builder_status = builder_status self.putChild('builds', BuildsJsonResource(status, builder_status)) self.putChild('slaves', BuilderSlavesJsonResources(status, builder_status)) self.putChild( 'pendingBuilds', BuilderPendingBuildsJsonResource(status, builder_status))
# buildbot.status.builder.BuilderStatus return self.builder_status.asDict_async()
"""
JsonResource.__init__(self, status) for builder_name in self.status.getBuilderNames(): self.putChild(builder_name, BuilderJsonResource(status, status.getBuilder(builder_name)))
"""
JsonResource.__init__(self, status) self.builder_status = builder_status for slave_name in self.builder_status.slavenames: self.putChild(slave_name, SlaveJsonResource(status, self.status.getSlave(slave_name)))
"""
JsonResource.__init__(self, status) self.build_status = build_status # TODO: support multiple sourcestamps sourcestamp = build_status.getSourceStamps()[0] self.putChild('source_stamp', SourceStampJsonResource(status, sourcestamp)) self.putChild('steps', BuildStepsJsonResource(status, build_status))
return self.build_status.asDict()
"""
JsonResource.__init__(self, status) self.builder_status = builder_status
# Dynamic childs. if isinstance(path, int) or _IS_INT.match(path): build_status = self.builder_status.getBuild(int(path)) if build_status: return BuildJsonResource(self.status, build_status) return JsonResource.getChild(self, path, request)
results = {} # If max > buildCacheSize, it'll trash the cache... cache_size = self.builder_status.master.config.caches['Builds'] max = int(RequestArg(request, 'max', cache_size)) for i in range(0, max): child = self.getChildWithDefault(-i, request) if not isinstance(child, BuildJsonResource): continue results[child.build_status.getNumber()] = child.asDict(request) return results
"""
AllBuildsJsonResource.__init__(self, status, builder_status) self.putChild('_all', AllBuildsJsonResource(status, builder_status))
# Transparently redirects to _all if path is not ''. return self.children['_all'].getChildWithDefault(path, request)
# This would load all the pickles and is way too heavy, especially that # it would trash the cache: # self.children['builds'].asDict(request) # TODO(maruel) This list should also need to be cached but how? builds = dict([ (int(file), None) for file in os.listdir(self.builder_status.basedir) if _IS_INT.match(file) ]) return builds
"""
# buildbot.status.buildstep.BuildStepStatus JsonResource.__init__(self, status) self.build_step_status = build_step_status # TODO self.putChild('logs', LogsJsonResource())
return self.build_step_status.asDict()
"""
JsonResource.__init__(self, status) self.build_status = build_status # The build steps are constantly changing until the build is done so # keep a reference to build_status instead
# Dynamic childs. build_step_status = None if isinstance(path, int) or _IS_INT.match(path): build_step_status = self.build_status.getSteps()[int(path)] else: steps_dict = dict([(step.getName(), step) for step in self.build_status.getSteps()]) build_step_status = steps_dict.get(path) if build_step_status: # Create it on-demand. child = BuildStepJsonResource(self.status, build_step_status) # Cache it. index = self.build_status.getSteps().index(build_step_status) self.putChild(str(index), child) self.putChild(build_step_status.getName(), child) return child return JsonResource.getChild(self, path, request)
# Only use the number and not the names! results = {} index = 0 for step in self.build_status.getSteps(): results[index] = step.asDict() index += 1 return results
"""
# buildbot.changes.changes.Change JsonResource.__init__(self, status) self.change = change
return self.change.asDict()
"""
JsonResource.__init__(self, status) for c in changes: # c.number can be None or clash another change if the change was # generated inside buildbot or if using multiple pollers. if c.number is not None and str(c.number) not in self.children: self.putChild(str(c.number), ChangeJsonResource(status, c)) else: # Temporary hack since it creates information exposure. self.putChild(str(id(c)), ChangeJsonResource(status, c))
"""Don't throw an exception when there is no child.""" if not self.children: return {} return JsonResource.asDict(self, request)
"""
result = {} n = 0 for c in self.status.getChangeSources(): # buildbot.changes.changes.ChangeMaster change = {} change['description'] = c.describe() result[n] = change n += 1 return result
"""
return self.status.asDict()
"""
JsonResource.__init__(self, status) self.slave_status = slave_status self.name = self.slave_status.getName() self.builders = None
if self.builders is None: # Figure out all the builders to which it's attached self.builders = [] for builderName in self.status.getBuilderNames(): if self.name in self.status.getBuilder(builderName).slavenames: self.builders.append(builderName) return self.builders
results = self.slave_status.asDict() # Enhance it by adding more informations. results['builders'] = {} for builderName in self.getBuilders(): builds = [] builder_status = self.status.getBuilder(builderName) cache_size = builder_status.master.config.caches['Builds'] numbuilds = int(request.args.get('numbuilds', [cache_size - 1])[0]) for i in range(1, numbuilds): build_status = builder_status.getBuild(-i) if not build_status or not build_status.isFinished(): # If not finished, it will appear in runningBuilds. break if build_status.getSlavename() == self.name: builds.append(build_status.getNumber()) results['builders'][builderName] = builds return results
"""
JsonResource.__init__(self, status) for slave_name in status.getSlaveNames(): self.putChild(slave_name, SlaveJsonResource(status, status.getSlave(slave_name)))
"""
# buildbot.sourcestamp.SourceStamp JsonResource.__init__(self, status) self.source_stamp = source_stamp self.putChild('changes', ChangesJsonResource(status, source_stamp.changes)) # TODO(maruel): Should redirect to the patch's url instead. #if source_stamp.patch: # self.putChild('patch', StaticHTML(source_stamp.path))
return self.source_stamp.asDict()
"""
metrics = self.status.getMetrics() if metrics: return metrics.asDict() else: # Metrics are disabled return None
"""Retrieves all json data."""
Root page to give a fair amount of information in the current buildbot master status. You may want to use a child instead to reduce the load on the server.
For help on any sub directory, use url /child/help """
JsonResource.__init__(self, status) self.level = 1 self.putChild('builders', BuildersJsonResource(status)) self.putChild('change_sources', ChangeSourcesJsonResource(status)) self.putChild('project', ProjectJsonResource(status)) self.putChild('slaves', SlavesJsonResource(status)) self.putChild('metrics', MetricsJsonResource(status)) # This needs to be called before the first HelpResource().body call. self.hackExamples()
result = JsonResource.content(self, request) # This is done to hook the downloaded filename. request.path = 'buildbot' return result
global EXAMPLES # Find the first builder with a previous build or select the last one. builder = None for b in self.status.getBuilderNames(): builder = self.status.getBuilder(b) if builder.getBuild(-1): break if not builder: return EXAMPLES = EXAMPLES.replace('<A_BUILDER>', builder.getName()) build = builder.getBuild(-1) if build: EXAMPLES = EXAMPLES.replace('<A_BUILD>', str(build.getNumber())) if builder.slavenames: EXAMPLES = EXAMPLES.replace('<A_SLAVE>', builder.slavenames[0])
# vim: set ts=4 sts=4 sw=4 et: |