|
# This file is part of Buildbot. Buildbot is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS # FOR A PARTICULAR PURPOSE. See the GNU General Public License for more # details. # # You should have received a copy of the GNU General Public License along with # this program; if not, write to the Free Software Foundation, Inc., 51 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright Buildbot Team Members
"""This class works the same way as twisted.enterprise.adbapi.ConnectionPool. But it adds the ability to compare connection pools for equality (by comparing the arguments passed to the constructor).
This is useful when passing the ConnectionPool to a BuildStep, as otherwise Buildbot will consider the buildstep (and hence the containing buildfactory) to have changed every time the configuration is reloaded.
It also sets some defaults differently from adbapi.ConnectionPool that are more suitable for use in MTR. """ self._eqKey = (args, kwargs) return adbapi.ConnectionPool.__init__(self, cp_reconnect=True, cp_min=1, cp_max=3, *args, **kwargs)
if isinstance(other, EqConnectionPool): return self._eqKey == other._eqKey else: return False
return not self.__eq__(other)
self.testname = testname self.variant = variant self.result = result self.info = info self.text = text self.callback = callback
self.text+= line
return self.callback(self.testname, self.variant, self.result, self.info, self.text)
""" Class implementing a log observer (can be passed to BuildStep.addLogObserver().
It parses the output of mysql-test-run.pl as used in MySQL, MariaDB, Drizzle, etc.
It counts number of tests run and uses it to provide more accurate completion estimates.
It parses out test failures from the output and summarises the results on the Waterfall page. It also passes the information to methods that can be overridden in a subclass to do further processing on the information."""
self.textLimit = textLimit self.testNameLimit = testNameLimit self.testType = testType self.numTests = 0 self.testFail = None self.failList = [] self.warnList = [] LogLineObserver.__init__(self)
LogLineObserver.setLog(self, loog) d= loog.waitUntilFinished() d.addCallback(lambda l: self.closeTestFail())
stripLine = line.strip("\r\n") m = self._line_re.search(stripLine) if m: testname, variant, worker, result, info = m.groups() self.closeTestFail() self.numTests += 1 self.step.setProgress('tests', self.numTests)
if result == "fail": if variant == None: variant = "" else: variant = variant[2:-1] self.openTestFail(testname, variant, result, info, stripLine + "\n")
else: m = self._line_re3.search(stripLine) if m: stuff = m.group(1) self.closeTestFail() testList = stuff.split(" ") self.doCollectWarningTests(testList)
elif (self._line_re2.search(stripLine) or self._line_re4.search(stripLine) or self._line_re5.search(stripLine) or stripLine == "Test suite timeout! Terminating..." or stripLine.startswith("mysql-test-run: *** ERROR: Not all tests completed") or (stripLine.startswith("------------------------------------------------------------") and self.testFail != None)): self.closeTestFail()
else: self.addTestFailOutput(stripLine + "\n")
self.testFail = MtrTestFailData(testname, variant, result, info, line, self.doCollectTestFail)
if self.testFail != None: self.testFail.add(line)
if self.testFail != None: self.testFail.fireCallback() self.testFail = None
lastOne = None count = 0 for t in src: if t != lastOne: dst.append(t) count += 1 if count >= self.textLimit: break
if done: text = ["test"] else: text = ["testing"] if self.testType: text.append(self.testType) fails = self.failList[:] fails.sort() self.addToText(fails, text) warns = self.warnList[:] warns.sort() self.addToText(warns, text) return text
# Update waterfall status. self.step.step_status.setText(self.makeText(False))
displayTestName = self.strip_re.sub("", testname)
if len(displayTestName) > self.testNameLimit: displayTestName = displayTestName[:(self.testNameLimit-2)] + "..." return displayTestName
self.failList.append("F:" + self.displayTestName(testname)) self.updateText() self.collectTestFail(testname, variant, result, info, text)
for t in testList: self.warnList.append("W:" + self.displayTestName(t)) self.updateText() self.collectWarningTests(testList)
# These two methods are overridden to actually do something with the data. pass pass
""" Build step that runs mysql-test-run.pl, as used in MySQL, Drizzle, MariaDB, etc.
It uses class MtrLogObserver to parse test results out from the output of mysql-test-run.pl, providing better completion time estimates and summarising test failures on the waterfall page.
It also provides access to mysqld server error logs from the test run to help debugging any problems.
Optionally, it can insert into a database data about the test run, including details of any test failures.
Parameters:
textLimit Maximum number of test failures to show on the waterfall page (to not flood the page in case of a large number of test failures. Defaults to 5.
testNameLimit Maximum length of test names to show unabbreviated in the waterfall page, to avoid excessive column width. Defaults to 16.
parallel Value of --parallel option used for mysql-test-run.pl (number of processes used to run the test suite in parallel). Defaults to 4. This is used to determine the number of server error log files to download from the slave. Specifying a too high value does not hurt (as nonexisting error logs will be ignored), however if using --parallel value greater than the default it needs to be specified, or some server error logs will be missing.
dbpool An instance of twisted.enterprise.adbapi.ConnectionPool, or None. Defaults to None. If specified, results are inserted into the database using the ConnectionPool.
The class process.mtrlogobserver.EqConnectionPool subclass of ConnectionPool can be useful to pass as value for dbpool, to avoid having config reloads think the Buildstep is changed just because it gets a new ConnectionPool instance (even though connection parameters are unchanged).
autoCreateTables Boolean, defaults to False. If True (and dbpool is specified), the necessary database tables will be created automatically if they do not exist already. Alternatively, the tables can be created manually from the SQL statements found in the mtrlogobserver.py source file.
test_type test_info Two descriptive strings that will be inserted in the database tables if dbpool is specified. The test_type string, if specified, will also appear on the waterfall page."""
description=None, descriptionDone=None, autoCreateTables=False, textLimit=5, testNameLimit=16, parallel=4, logfiles = {}, lazylogfiles = True, warningPattern="MTR's internal check of the test case '.*' failed", mtr_subdir="mysql-test", **kwargs):
if description is None: description = ["testing"] if test_type: description.append(test_type) if descriptionDone is None: descriptionDone = ["test"] if test_type: descriptionDone.append(test_type) Test.__init__(self, logfiles=logfiles, lazylogfiles=lazylogfiles, description=description, descriptionDone=descriptionDone, warningPattern=warningPattern, **kwargs) self.dbpool = dbpool self.test_type = test_type self.test_info = test_info self.autoCreateTables = autoCreateTables self.textLimit = textLimit self.testNameLimit = testNameLimit self.parallel = parallel self.mtr_subdir = mtr_subdir self.progressMetrics += ('tests',)
# Add mysql server logfiles. for mtr in range(0, self.parallel+1): for mysqld in range(1, 4+1): if mtr == 0: logname = "mysqld.%d.err" % mysqld filename = "var/log/mysqld.%d.err" % mysqld else: logname = "mysqld.%d.err.%d" % (mysqld, mtr) filename = "var/%d/log/mysqld.%d.err" % (mtr, mysqld) self.addLogFile(logname, self.mtr_subdir + "/" + filename)
self.myMtr = self.MyMtrLogObserver(textLimit=self.textLimit, testNameLimit=self.testNameLimit, testType=self.test_type) self.addLogObserver("stdio", self.myMtr) # Insert a row for this test run into the database and set up # build properties, then start the command proper. d = self.registerInDB() d.addCallback(self.afterRegisterInDB) d.addErrback(self.failed)
return self.myMtr.makeText(True)
""" Run a database transaction with dbpool.runInteraction, but retry the transaction in case of a temporary error (like connection lost).
This is needed to be robust against things like database connection idle timeouts.
The passed callable that implements the transaction must be retryable, ie. it must not have any destructive side effects in the case where an exception is thrown and/or rollback occurs that would prevent it from functioning correctly when called again."""
def runWithRetry(txn, *args, **kw): retryCount = 0 while(True): try: return actionFn(txn, *args, **kw) except txn.OperationalError: retryCount += 1 if retryCount >= 5: raise excType, excValue, excTraceback = sys.exc_info() log.msg("Database transaction failed (caught exception %s(%s)), retrying ..." % (excType, excValue)) txn.close() txn.reconnect() txn.reopen()
return self.dbpool.runInteraction(runWithRetry, *args, **kw)
""" Run a database query, like with dbpool.runQuery, but retry the query in case of a temporary error (like connection lost).
This is needed to be robust against things like database connection idle timeouts."""
def runQuery(txn, *args, **kw): txn.execute(*args, **kw) return txn.fetchall()
return self.runInteractionWithRetry(runQuery, *args, **kw)
if self.dbpool: return self.runInteractionWithRetry(self.doRegisterInDB) else: return defer.succeed(0)
# The real database work is done in a thread in a synchronous way. # Auto create tables. # This is off by default, as it gives warnings in log file # about tables already existing (and I did not find the issue # important enough to find a better fix). if self.autoCreateTables: txn.execute(""" CREATE TABLE IF NOT EXISTS test_run( id INT PRIMARY KEY AUTO_INCREMENT, branch VARCHAR(100), revision VARCHAR(32) NOT NULL, platform VARCHAR(100) NOT NULL, dt TIMESTAMP NOT NULL, bbnum INT NOT NULL, typ VARCHAR(32) NOT NULL, info VARCHAR(255), KEY (branch, revision), KEY (dt), KEY (platform, bbnum) ) ENGINE=innodb """) txn.execute(""" CREATE TABLE IF NOT EXISTS test_failure( test_run_id INT NOT NULL, test_name VARCHAR(100) NOT NULL, test_variant VARCHAR(16) NOT NULL, info_text VARCHAR(255), failure_text TEXT, PRIMARY KEY (test_run_id, test_name, test_variant) ) ENGINE=innodb """) txn.execute(""" CREATE TABLE IF NOT EXISTS test_warnings( test_run_id INT NOT NULL, list_id INT NOT NULL, list_idx INT NOT NULL, test_name VARCHAR(100) NOT NULL, PRIMARY KEY (test_run_id, list_id, list_idx) ) ENGINE=innodb """)
revision = self.getProperty("got_revision") if revision is None: revision = self.getProperty("revision") typ = "mtr" if self.test_type: typ = self.test_type txn.execute(""" INSERT INTO test_run(branch, revision, platform, dt, bbnum, typ, info) VALUES (%s, %s, %s, CURRENT_TIMESTAMP(), %s, %s, %s) """, (self.getProperty("branch"), revision, self.getProperty("buildername"), self.getProperty("buildnumber"), typ, self.test_info))
return txn.lastrowid
self.setProperty("mtr_id", insert_id) self.setProperty("mtr_warn_id", 0)
Test.start(self)
log.msg("Error in async insert into database: %s" % err)
# Insert asynchronously into database. dbpool = self.step.dbpool run_id = self.step.getProperty("mtr_id") if dbpool == None: return defer.succeed(None) if variant == None: variant = "" d = self.step.runQueryWithRetry(""" INSERT INTO test_failure(test_run_id, test_name, test_variant, info_text, failure_text) VALUES (%s, %s, %s, %s, %s) """, (run_id, testname, variant, info, text))
d.addErrback(self.step.reportError) return d
# Insert asynchronously into database. dbpool = self.step.dbpool if dbpool == None: return defer.succeed(None) run_id = self.step.getProperty("mtr_id") warn_id = self.step.getProperty("mtr_warn_id") self.step.setProperty("mtr_warn_id", warn_id + 1) q = ("INSERT INTO test_warnings(test_run_id, list_id, list_idx, test_name) " + "VALUES " + ", ".join(map(lambda x: "(%s, %s, %s, %s)", testList))) v = [] idx = 0 for t in testList: v.extend([run_id, warn_id, idx, t]) idx = idx + 1 d = self.step.runQueryWithRetry(q, tuple(v)) d.addErrback(self.step.reportError) return d |