From patchwork Thu Sep 15 08:44:13 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Spring Zhang X-Patchwork-Id: 4083 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 6F41423EF8 for ; Thu, 15 Sep 2011 08:44:27 +0000 (UTC) Received: from mail-fx0-f52.google.com (mail-fx0-f52.google.com [209.85.161.52]) by fiordland.canonical.com (Postfix) with ESMTP id 615DFA1883A for ; Thu, 15 Sep 2011 08:44:27 +0000 (UTC) Received: by mail-fx0-f52.google.com with SMTP id 23so564959fxe.11 for ; Thu, 15 Sep 2011 01:44:27 -0700 (PDT) Received: by 10.223.94.134 with SMTP id z6mr586738fam.8.1316076265255; Thu, 15 Sep 2011 01:44:25 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.152.11.8 with SMTP id m8cs84126lab; Thu, 15 Sep 2011 01:44:25 -0700 (PDT) Received: by 10.216.72.144 with SMTP id t16mr875207wed.13.1316076264756; Thu, 15 Sep 2011 01:44:24 -0700 (PDT) Received: from indium.canonical.com (indium.canonical.com [91.189.90.7]) by mx.google.com with ESMTPS id r47si7008950wec.116.2011.09.15.01.44.24 (version=TLSv1/SSLv3 cipher=OTHER); Thu, 15 Sep 2011 01:44:24 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1R47YV-0007Gh-Sw for ; Thu, 15 Sep 2011 08:44:23 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id D0BF5E0444 for ; Thu, 15 Sep 2011 08:44:13 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 110 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 110: merge kernel inject branch Message-Id: <20110915084413.13279.45889.launchpad@ackee.canonical.com> Date: Thu, 15 Sep 2011 08:44:13 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="13921"; Instance="initZopeless config overlay" X-Launchpad-Hash: 9a1eea3b37eb8e5266861f93bee0efeadf4e9c97 Merge authors: Spring Zhang (qzhang) Related merge proposals: https://code.launchpad.net/~qzhang/lava-dispatcher/another-inject-kernel/+merge/72927 proposed by: Spring Zhang (qzhang) https://code.launchpad.net/~qzhang/lava-dispatcher/inject-kernel/+merge/69975 proposed by: Spring Zhang (qzhang) review: Resubmit - Spring Zhang (qzhang) review: Needs Information - Paul Larson (pwlars) ------------------------------------------------------------ revno: 110 [merge] committer: Spring Zhang branch nick: another-inject-kernel timestamp: Thu 2011-09-15 15:46:03 +0800 message: merge kernel inject branch added: doc/android-new-kernel.json doc/lava-new-kernel.json renamed: doc/lava-ltp-job => doc/lava-ltp-job.json modified: lava_dispatcher/actions/android_deploy.py lava_dispatcher/actions/deploy.py lava_dispatcher/client.py lava_dispatcher/utils.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === added file 'doc/android-new-kernel.json' --- doc/android-new-kernel.json 1970-01-01 00:00:00 +0000 +++ doc/android-new-kernel.json 2011-08-16 06:15:23 +0000 @@ -0,0 +1,37 @@ +{ + "job_name": "android_new_kernel", + "image_type": "android", + "target": "panda01", + "timeout": 18000, + "actions": [ + { + "command": "deploy_linaro_android_image", + "parameters": + { + "boot": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/artifact/build/out/target/product/pandaboard/boot.tar.bz2", + "system": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/artifact/build/out/target/product/pandaboard/system.tar.bz2", + "data": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/61/artifact/build/out/target/product/pandaboard/userdata.tar.bz2", + "pkg": "https://android-build.linaro.org/jenkins/job/linaro-android_leb-panda/171/artifact/build/out/target/product/pandaboard/boot.tar.bz2" + }, + "metadata": + { + "rootfs.type": "android", + "rootfs.build": "61" + } + }, + { + "command": "boot_linaro_android_image" + }, + { + "command": "test_android_basic" + }, + { + "command": "submit_results_on_host", + "parameters": + { + "server": "http://validation.linaro.org/launch-control", + "stream": "/anonymous/android-panda01-basic/" + } + } + ] +} === renamed file 'doc/lava-ltp-job' => 'doc/lava-ltp-job.json' === added file 'doc/lava-new-kernel.json' --- doc/lava-new-kernel.json 1970-01-01 00:00:00 +0000 +++ doc/lava-new-kernel.json 2011-08-18 10:40:22 +0000 @@ -0,0 +1,30 @@ +{ + "job_name": "new-kernel", + "target": "mx53loco01", + "timeout": 6000, + "actions": [ + { + "command": "deploy_linaro_image", + "parameters": + { + "rootfs": +"http://snapshots.linaro.org/11.05-daily/linaro-nano/20110612/0/images/tar/nano-n-tar-20110612-0.tar.gz", + "hwpack": +"http://snapshots.linaro.org/11.05-daily/linaro-hwpacks/lt-mx53loco/20110609/0/images/hwpack/hwpack_linaro-lt-mx53loco_20110609-0_armel_supported.tar.gz", + "kernel_matrix":[ +"http://pkgserver/original/linux-image-2.6.38-1000-linaro-lt-mx5_2.6.38-1000.7_armel.deb", "linux-image-2.6.38"] + } + }, + { + "command": "boot_linaro_image" + }, + { + "command": "submit_results", + "parameters": + { + "server": "http://validation.linaro.org/launch-control", + "stream": "/anonymous/testresult/" + } + } + ] +} === modified file 'lava_dispatcher/actions/android_deploy.py' --- lava_dispatcher/actions/android_deploy.py 2011-09-05 22:39:47 +0000 +++ lava_dispatcher/actions/android_deploy.py 2011-09-14 04:18:40 +0000 @@ -29,7 +29,7 @@ from lava_dispatcher.client import CriticalError class cmd_deploy_linaro_android_image(BaseAction): - def run(self, boot, system, data, use_cache=True): + def run(self, boot, system, data, pkg=None, use_cache=True): LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir LAVA_IMAGE_URL = self.context.lava_image_url client = self.client @@ -49,8 +49,8 @@ raise CriticalError("Unable to reach LAVA server, check network") try: - boot_tbz2, system_tbz2, data_tbz2 = self.download_tarballs(boot, - system, data, use_cache) + boot_tbz2, system_tbz2, data_tbz2, pkg_tbz2 = \ + self.download_tarballs(boot, system, data, pkg, use_cache) except: tb = traceback.format_exc() client.sio.write(tb) @@ -66,9 +66,16 @@ LAVA_IMAGE_URL, system_tarball]) data_url = '/'.join(u.strip('/') for u in [ LAVA_IMAGE_URL, data_tarball]) + if pkg_tbz2: + pkg_tarball = pkg_tbz2.replace(LAVA_IMAGE_TMPDIR, '') + pkg_url = '/'.join(u.strip('/') for u in [ + LAVA_IMAGE_URL, pkg_tarball]) try: - self.deploy_linaro_android_testboot(boot_url) + if pkg_tbz2: + self.deploy_linaro_android_testboot(boot_url, pkg_url) + else: + self.deploy_linaro_android_testboot(boot_url) self.deploy_linaro_android_testrootfs(system_url) self.purge_linaro_android_sdcard() except: @@ -78,12 +85,14 @@ finally: shutil.rmtree(self.tarball_dir) - def download_tarballs(self, boot_url, system_url, data_url, use_cache=True): + def download_tarballs(self, boot_url, system_url, data_url, pkg_url=None, + use_cache=True): """Download tarballs from a boot, system and data tarball url :param boot_url: url of the Linaro Android boot tarball to download :param system_url: url of the Linaro Android system tarball to download :param data_url: url of the Linaro Android data tarball to download + :param pkg_url: url of the custom kernel tarball to download :param use_cache: whether or not to use the cached copy (if it exists) """ lava_cachedir = self.context.lava_cachedir @@ -96,13 +105,21 @@ boot_path = download_with_cache(boot_url, tarball_dir, lava_cachedir) system_path = download_with_cache(system_url, tarball_dir, lava_cachedir) data_path = download_with_cache(data_url, tarball_dir, lava_cachedir) + if pkg_url: + pkg_path = download_with_cache(pkg_url, tarball_dir) + else: + pkg_path = None else: boot_path = download(boot_url, tarball_dir) system_path = download(system_url, tarball_dir) data_path = download(data_url, tarball_dir) - return boot_path, system_path, data_path + if pkg_url: + pkg_path = download(pkg_url, tarball_dir) + else: + pkg_path = None + return boot_path, system_path, data_path, pkg_path - def deploy_linaro_android_testboot(self, boottbz2): + def deploy_linaro_android_testboot(self, boottbz2, pkgbz2=None): client = self.client client.run_cmd_master('mkfs.vfat /dev/disk/by-label/testboot ' '-n testboot') @@ -111,6 +128,10 @@ client.run_cmd_master('mount /dev/disk/by-label/testboot ' '/mnt/lava/boot') client.run_cmd_master('wget -qO- %s |tar --numeric-owner -C /mnt/lava -xjf -' % boottbz2) + if pkgbz2: + client.run_shell_command( + 'wget -qO- %s |tar --numeric-owner -C /mnt/lava -xjf -' + % pkgbz2, response = MASTER_STR) self.recreate_uInitrd() === modified file 'lava_dispatcher/actions/deploy.py' --- lava_dispatcher/actions/deploy.py 2011-09-08 23:35:05 +0000 +++ lava_dispatcher/actions/deploy.py 2011-09-15 07:46:03 +0000 @@ -15,11 +15,11 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along -# with this program; if not, see . +# along with this program; if not, see . from commands import getoutput, getstatusoutput import os +import sys import re import shutil import traceback @@ -31,13 +31,15 @@ class cmd_deploy_linaro_image(BaseAction): - def run(self, hwpack, rootfs, use_cache=True): + def run(self, hwpack, rootfs, kernel_matrix=None, use_cache=True): LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir LAVA_IMAGE_URL = self.context.lava_image_url client = self.client print "deploying on %s" % client.hostname print " hwpack: %s" % hwpack print " rootfs: %s" % rootfs + if kernel_matrix: + print " package: %s" % kernel_matrix[0] print "Booting master image" client.boot_master_image() @@ -49,9 +51,17 @@ client.sio.write(tb) raise CriticalError("Unable to reach LAVA server, check network") + if kernel_matrix: + hwpack = self.refresh_hwpack(kernel_matrix, hwpack, use_cache) + #make new hwpack downloadable + hwpack = hwpack.replace(LAVA_IMAGE_TMPDIR, '') + hwpack = '/'.join(u.strip('/') for u in [ + LAVA_IMAGE_URL, hwpack]) + print " hwpack with new kernel: %s" % hwpack + try: - boot_tgz, root_tgz = self.generate_tarballs( - hwpack, rootfs, use_cache) + boot_tgz, root_tgz = self.generate_tarballs(hwpack, rootfs, + use_cache) except: tb = traceback.format_exc() client.sio.write(tb) @@ -119,6 +129,7 @@ self.tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR) tarball_dir = self.tarball_dir os.chmod(tarball_dir, 0755) + #fix me: if url is not http-prefix, copy it to tarball_dir if use_cache: hwpack_path = download_with_cache(hwpack_url, tarball_dir, lava_cachedir) rootfs_path = download_with_cache(rootfs_url, tarball_dir, lava_cachedir) @@ -184,3 +195,45 @@ 'wget -qO- %s |tar --numeric-owner -C /mnt/boot -xzf -' % bootfs) client.run_cmd_master('umount /mnt/boot') + def refresh_hwpack(self, kernel_matrix, hwpack, use_cache=True): + client = self.client + LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir + print "Deploying new kernel" + new_kernel = kernel_matrix[0] + deb_prefix = kernel_matrix[1] + filesuffix = new_kernel.split(".")[-1] + + if filesuffix != "deb": + raise CriticalError("New kernel only support deb kernel package!") + + # download package to local + tarball_dir = mkdtemp(dir=LAVA_IMAGE_TMPDIR) + os.chmod(tarball_dir, 0755) + if use_cache: + kernel_path = download_with_cache(new_kernel, tarball_dir) + hwpack_path = download_with_cache(hwpack, tarball_dir) + else: + kernel_path = download(new_kernel, tarball_dir) + hwpack_path = download(hwpack, tarball_dir) + + cmd = ("sudo linaro-hwpack-replace -t %s -p %s -r %s" + % (hwpack_path, kernel_path, deb_prefix)) + + rc, output = getstatusoutput(cmd) + if rc: + shutil.rmtree(tarball_dir) + tb = traceback.format_exc() + client.sio.write(tb) + raise RuntimeError("linaro-hwpack-replace failed: %s" % output) + + #fix it:l-h-r doesn't make a output option to specify the output hwpack, + #so it needs to do manually here + + #remove old hwpack and leave only new hwpack in tarball_dir + os.remove(hwpack_path) + hwpack_list = os.listdir(tarball_dir) + for hp in hwpack_list: + if hp.split(".")[-1] == "gz": + new_hwpack_path = os.path.join(tarball_dir, hp) + return new_hwpack_path + === modified file 'lava_dispatcher/client.py' --- lava_dispatcher/client.py 2011-09-14 16:05:43 +0000 +++ lava_dispatcher/client.py 2011-09-15 07:02:08 +0000 @@ -130,7 +130,7 @@ # Details: system PS1 is set in /etc/bash.bashrc and user PS1 is set in # /root/.bashrc, it is # "${debian_chroot:+($debian_chroot)}\u@\h:\w\$ " - self.proc.sendline('export PS1="$PS1 rc=$(echo \$?) "') + self.proc.sendline('export PS1="$PS1 [rc=$(echo \$?)]: "') self.proc.expect(self.tester_str) def enter_uboot(self): === modified file 'lava_dispatcher/utils.py' --- lava_dispatcher/utils.py 2011-09-01 03:03:09 +0000 +++ lava_dispatcher/utils.py 2011-09-15 07:02:08 +0000 @@ -45,7 +45,11 @@ if os.path.exists(cache_loc): filename = os.path.basename(cache_loc) file_location = os.path.join(path, filename) - os.link(cache_loc, file_location) + try: + os.link(cache_loc, file_location) + except OSError, err: + if err.errno == 18: + shutil.copy(cache_loc, file_location) else: file_location = download(url, path) try: