From patchwork Wed Jul 20 15:10:54 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Larson X-Patchwork-Id: 2874 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id D996123F42 for ; Wed, 20 Jul 2011 15:10:57 +0000 (UTC) Received: from mail-qy0-f180.google.com (mail-qy0-f180.google.com [209.85.216.180]) by fiordland.canonical.com (Postfix) with ESMTP id 8EC60A1876F for ; Wed, 20 Jul 2011 15:10:57 +0000 (UTC) Received: by qyk30 with SMTP id 30so227954qyk.11 for ; Wed, 20 Jul 2011 08:10:57 -0700 (PDT) Received: by 10.229.25.212 with SMTP id a20mr7204864qcc.148.1311174656721; Wed, 20 Jul 2011 08:10:56 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.229.217.78 with SMTP id hl14cs116521qcb; Wed, 20 Jul 2011 08:10:56 -0700 (PDT) Received: by 10.216.79.5 with SMTP id h5mr7438010wee.110.1311174655501; Wed, 20 Jul 2011 08:10:55 -0700 (PDT) Received: from adelie.canonical.com (adelie.canonical.com [91.189.90.139]) by mx.google.com with ESMTP id v34si653947weq.114.2011.07.20.08.10.55; Wed, 20 Jul 2011 08:10:55 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) client-ip=91.189.90.139; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) smtp.mail=bounces@canonical.com Received: from loganberry.canonical.com ([91.189.90.37]) by adelie.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1QjYQI-0004iF-OA for ; Wed, 20 Jul 2011 15:10:54 +0000 Received: from loganberry.canonical.com (localhost [127.0.0.1]) by loganberry.canonical.com (Postfix) with ESMTP id 94ED22EA024 for ; Wed, 20 Jul 2011 15:10:54 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-test X-Launchpad-Branch: ~linaro-validation/lava-test/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 76 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-test/trunk] Rev 76: Stream output from tests while they are running, a side effect is that Message-Id: <20110720151054.23145.51411.launchpad@loganberry.canonical.com> Date: Wed, 20 Jul 2011 15:10:54 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="13468"; Instance="initZopeless config overlay" X-Launchpad-Hash: d14279a3bcc13d818447638ae0c43a5f6c6ab860 Merge authors: Paul Larson (pwlars) Related merge proposals: https://code.launchpad.net/~pwlars/lava-test/stream-output/+merge/68419 proposed by: Paul Larson (pwlars) ------------------------------------------------------------ revno: 76 [merge] committer: Paul Larson branch nick: lava-test timestamp: Wed 2011-07-20 10:05:24 -0500 message: Stream output from tests while they are running, a side effect is that this will not allow the test to run things that expect input from the user though (such as sudo), so tests need to be run with root if the require it. Typically these days though, we are only running fully automated tests with this, so I don't think this is a bad assumption. modified: abrek/builtins.py abrek/command.py abrek/testdef.py abrek/utils.py --- lp:lava-test https://code.launchpad.net/~linaro-validation/lava-test/trunk You are subscribed to branch lp:lava-test. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-test/trunk/+edit-subscription === modified file 'abrek/builtins.py' --- abrek/builtins.py 2011-03-09 14:41:27 +0000 +++ abrek/builtins.py 2011-07-11 18:24:25 +0000 @@ -70,6 +70,7 @@ arglist = ['*testname'] def run(self): + self.checkroot() if len(self.args) != 1: print "please specify the name of the test to install" sys.exit(1) @@ -93,6 +94,7 @@ help="Store processed test output to FILE")] def run(self): + self.checkroot() if len(self.args) != 1: print "please specify the name of the test to run" sys.exit(1) === modified file 'abrek/command.py' --- abrek/command.py 2010-10-12 02:23:36 +0000 +++ abrek/command.py 2011-07-11 18:24:25 +0000 @@ -14,6 +14,8 @@ # along with this program. If not, see . from optparse import OptionParser +import os +import sys class _AbrekOptionParser(OptionParser): @@ -97,6 +99,11 @@ def get_subcommand(self, name): return None + def checkroot(self): + if os.getuid() != 0: + print >> sys.stderr, ("**** WARNING: ROOT PERMISSIONS ARE OFTEN" + "REQUIRED FOR THIS OPERATION ****") + class AbrekCmdWithSubcommands(AbrekCmd): === modified file 'abrek/testdef.py' --- abrek/testdef.py 2011-06-14 18:43:45 +0000 +++ abrek/testdef.py 2011-07-19 16:00:42 +0000 @@ -27,7 +27,7 @@ from abrek.api import ITest from abrek.bundle import DocumentIO from abrek.config import get_config -from abrek.utils import Tee, geturl, run_and_log, write_file +from abrek.utils import geturl, run_and_log, write_file class AbrekTest(ITest): @@ -222,9 +222,9 @@ def _runsteps(self, resultsdir, quiet=False): outputlog = os.path.join(resultsdir, 'testoutput.log') - with Tee(outputlog, 'a', quiet=quiet) as fd: + with open(outputlog, 'a') as fd: for cmd in self.steps: - run_and_log(cmd, fd) + run_and_log(cmd, fd, quiet) def run(self, resultsdir, quiet=False): self.starttime = datetime.utcnow() @@ -245,10 +245,10 @@ For example: If your testoutput had lines that look like: "test01: PASS", then you could use a pattern like this: "^(?P\w+):\W+(?P\w+)" - This would result in identifying "test01" as testid and "PASS" - as result. Once parse() has been called, self.results.test_results[] - contains a list of dicts of all the key,value pairs found for - each test result + This would result in identifying "test01" as testid and + "PASS" as result. Once parse() has been called, + self.results.test_results[] contains a list of dicts of all the + key,value pairs found for each test result fixupdict - dict of strings to convert test results to standard strings For example: if you want to standardize on having pass/fail results in lower case, but your test outputs them in upper case, you could @@ -280,7 +280,9 @@ try: pat = re.compile(self.pattern) except Exception as strerror: - raise RuntimeError("AbrekTestParser - Invalid regular expression '%s' - %s" %(self.pattern,strerror)) + raise RuntimeError( + "AbrekTestParser - Invalid regular expression '%s' - %s" % ( + self.pattern,strerror)) with open(filename, 'r') as stream: for lineno, line in enumerate(stream, 1): @@ -310,7 +312,8 @@ def appendtoall(self, entry): """Append entry to each item in the test_results. - entry - dict of key,value pairs to add to each item in the test_results + entry - dict of key,value pairs to add to each item in the + test_results """ for t in self.results['test_results']: t.update(entry) === modified file 'abrek/utils.py' --- abrek/utils.py 2011-04-20 08:39:25 +0000 +++ abrek/utils.py 2011-07-19 16:00:42 +0000 @@ -15,15 +15,16 @@ import os import shutil +import subprocess import sys import urllib2 import urlparse -from subprocess import Popen, PIPE _fake_files = None _fake_paths = None _fake_machine = None + class Tee(file): """ A file-like object that optionally mimics tee functionality. @@ -42,26 +43,29 @@ if self.quiet is False: sys.stdout.write(data) + def geturl(url, path=""): urlpath = urlparse.urlsplit(url).path filename = os.path.basename(urlpath) if path: - filename = os.path.join(path,filename) + filename = os.path.join(path, filename) fd = open(filename, "w") try: response = urllib2.urlopen(urllib2.quote(url, safe=":/")) fd = open(filename, 'wb') - shutil.copyfileobj(response,fd,0x10000) + shutil.copyfileobj(response, fd, 0x10000) fd.close() response.close() except: raise RuntimeError("Could not retrieve %s" % url) return filename + def write_file(data, path): with open(path, "w") as fd: fd.write(data) + def read_file(path): global _fake_files global _fake_paths @@ -75,6 +79,7 @@ data = fd.read() return data + def fake_file(path, data=None, newpath=None): """ Set up a fake file to be read with read_file() in testing @@ -93,6 +98,7 @@ _fake_paths = {} _fake_paths[path] = newpath + def fake_machine(type): """ Set up a fake machine type for testing @@ -100,27 +106,34 @@ global _fake_machine _fake_machine = type + def clear_fakes(): global _fake_files global _fake_paths _fake_files = {} _fake_paths = {} + def clear_fake_machine(): global _fake_machine _fake_machine = None -def run_and_log(cmd, fd): + +def run_and_log(cmd, fd, quiet=False): """ Run a command and log the output to fd """ - proc = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) - output, err = proc.communicate() - if output is not None: - fd.write(output) - + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, shell=True) + while proc.returncode == None: + proc.poll() + data = proc.stdout.readline() + fd.write(data) + if quiet is False: + sys.stdout.write(data) return proc.returncode + def get_machine_type(): """ Return the machine type @@ -129,4 +142,3 @@ if _fake_machine is None: return os.uname()[-1] return _fake_machine -