From patchwork Tue Jun 19 04:22:11 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Andy Doan X-Patchwork-Id: 9424 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 778D723EE3 for ; Tue, 19 Jun 2012 04:22:15 +0000 (UTC) Received: from mail-yx0-f180.google.com (mail-yx0-f180.google.com [209.85.213.180]) by fiordland.canonical.com (Postfix) with ESMTP id 245BCA18593 for ; Tue, 19 Jun 2012 04:22:15 +0000 (UTC) Received: by yenq6 with SMTP id q6so4813281yen.11 for ; Mon, 18 Jun 2012 21:22:14 -0700 (PDT) X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=20120113; h=x-forwarded-to:x-forwarded-for:delivered-to:received-spf :content-type:mime-version:x-launchpad-project:x-launchpad-branch :x-launchpad-message-rationale:x-launchpad-branch-revision-number :x-launchpad-notification-type:to:from:subject:message-id:date :reply-to:sender:errors-to:precedence:x-generated-by :x-launchpad-hash:x-gm-message-state; bh=iuB8T9JKgLkxWtyj/hdxWdjE0qkyBKlxDnQa2iJBGZo=; b=V4WL+ncEjxGb9ELSU2SzxXrk19nLZiTgw9PJ2va9JUBBwf3R+F441oIEdcQ543oAnv 3StMAooMMPDp4zbS+MvitVJTbEgO8auZj4iqKI75PfpVi/C+U66Vu6WUaz4DOq3t2tXY lcOnetSpZY3cJ7IbL4MYnAEc4LcPQdTQ1aKX54znP8XDWD4v7Mk0E3nsPyz2+6tlSQZZ T4LLueM/6Q86Ewntc5Y1RBrXhtbFk3AhffP+0xDvZ5YDU2lRhcnzi9P8IaANaBz934JZ 363VMTyTf8zvuuthLiQGeel58iAdHV9ERJXPP8aqv7XEEODZT+uxDz2d3Ng8zLkgSS8j s2CQ== Received: by 10.42.155.73 with SMTP id t9mr6680828icw.48.1340079734456; Mon, 18 Jun 2012 21:22:14 -0700 (PDT) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.231.24.148 with SMTP id v20csp105063ibb; Mon, 18 Jun 2012 21:22:13 -0700 (PDT) Received: by 10.216.227.101 with SMTP id c79mr9545543weq.188.1340079732523; Mon, 18 Jun 2012 21:22:12 -0700 (PDT) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id t18si24682630wiv.32.2012.06.18.21.22.12 (version=TLSv1/SSLv3 cipher=OTHER); Mon, 18 Jun 2012 21:22:12 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1SgpxD-0002Ea-Ub for ; Tue, 19 Jun 2012 04:22:11 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id DBDB1E016F for ; Tue, 19 Jun 2012 04:22:11 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dispatcher X-Launchpad-Branch: ~linaro-validation/lava-dispatcher/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 329 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dispatcher/trunk] Rev 329: updates for new downloader code and qemu Message-Id: <20120619042211.16245.65402.launchpad@ackee.canonical.com> Date: Tue, 19 Jun 2012 04:22:11 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="15435"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: 664b73d536751938001482c832993b3798eb3b5c X-Gm-Message-State: ALoCoQnsJSspZJ3hFtn8I9o4CvcG+KHFdy01bG1s9uHIFcCs3lA7pkhMTwqD6mkCzAxuBs/wckSN Merge authors: Andy Doan (doanac) Related merge proposals: https://code.launchpad.net/~doanac/lava-dispatcher/download-qemu-updates/+merge/110936 proposed by: Andy Doan (doanac) ------------------------------------------------------------ revno: 329 [merge] committer: Andy Doan branch nick: lava-dispatcher timestamp: Mon 2012-06-18 23:18:31 -0500 message: updates for new downloader code and qemu added: lava_dispatcher/default-config/lava-dispatcher/device-types/qemu.conf modified: lava_dispatcher/client/master.py lava_dispatcher/client/qemu.py lava_dispatcher/utils.py --- lp:lava-dispatcher https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk You are subscribed to branch lp:lava-dispatcher. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription === modified file 'lava_dispatcher/client/master.py' --- lava_dispatcher/client/master.py 2012-06-13 07:35:23 +0000 +++ lava_dispatcher/client/master.py 2012-06-19 04:18:31 +0000 @@ -34,7 +34,7 @@ import errno from lava_dispatcher.utils import ( - download, + download_image, logging_spawn, logging_system, string_to_list, @@ -354,18 +354,6 @@ def _close_logging_spawn(self): self.proc.close(True) - def decompress(self, image_file): - for suffix, command in [('.gz', 'gunzip'), - ('.xz', 'unxz'), - ('.bz2', 'bunzip2')]: - if image_file.endswith(suffix): - logging.info("Uncompressing %s with %s", image_file, command) - uncompressed_name = image_file[:-len(suffix)] - subprocess.check_call( - [command, '-c', image_file], stdout=open(uncompressed_name, 'w')) - return uncompressed_name - return image_file - def _tarball_url_to_cache(self, url, cachedir): cache_loc = url_to_cache(url, cachedir) # can't have a folder name same as file name. replacing '.' with '.' @@ -426,6 +414,14 @@ raise return True + def _remove_cache_lock(self, image, lava_cachedir, cache_loc=None): + if not cache_loc: + cache_loc = self._tarball_url_to_cache(image, lava_cachedir) + path = os.path.join(cache_loc, "tarballs-cache-ongoing") + if os.path.exists(path): + logging.debug("Removing cache lock for %s" % path) + shutil.rmtree(path) + def _cache_tarballs(self, image, boot_tgz, root_tgz, lava_cachedir): cache_loc = self._tarball_url_to_cache(image, lava_cachedir) if not os.path.exists(cache_loc): @@ -434,9 +430,6 @@ c_root_tgz = os.path.join(cache_loc, "root.tgz") shutil.copy(boot_tgz, c_boot_tgz) shutil.copy(root_tgz, c_root_tgz) - path = os.path.join(cache_loc, "tarballs-cache-ongoing") - if os.path.exists(path): - shutil.rmtree(path) def _download(self, url, directory): lava_proxy = self.context.lava_proxy @@ -476,15 +469,16 @@ # caching of same tarballs exact at the same time. One of them will successfully # get the lock directory. The rest will skip the caching if _about_to_cache_tarballs # return false. - should_cache = self._about_to_cache_tarballs(image, lava_cachedir) - image_file = self._download(image, tarball_dir) - image_file = self.decompress(image_file) - boot_tgz, root_tgz = self._generate_tarballs(image_file) - if should_cache: - self._cache_tarballs(image, boot_tgz, root_tgz, lava_cachedir) + try: + should_cache = self._about_to_cache_tarballs(image, lava_cachedir) + image_file = download_image(image, self.context, tarball_dir) + boot_tgz, root_tgz = self._generate_tarballs(image_file) + if should_cache: + self._cache_tarballs(image, boot_tgz, root_tgz, lava_cachedir) + finally: + self._remove_cache_lock(image, lava_cachedir) else: - image_file = self._download(image, tarball_dir) - image_file = self.decompress(image_file) + image_file = download_image(image, self.context, tarball_dir) boot_tgz, root_tgz = self._generate_tarballs(image_file) # remove the cached tarballs cache_loc = self._tarball_url_to_cache(image, lava_cachedir) === modified file 'lava_dispatcher/client/qemu.py' --- lava_dispatcher/client/qemu.py 2012-05-08 14:01:53 +0000 +++ lava_dispatcher/client/qemu.py 2012-06-13 20:46:12 +0000 @@ -33,6 +33,7 @@ image_partition_mounted, ) from lava_dispatcher.utils import ( + download_image, logging_spawn, logging_system, ) @@ -49,7 +50,7 @@ if image is None: image_file = generate_image(self, hwpack, rootfs, kernel_matrix, use_cache, rootfstype) else: - image_file = image + image_file = download_image(image, self.context) self._lava_image = image_file with image_partition_mounted(self._lava_image, self.root_part) as mntdir: logging_system('echo linaro > %s/etc/hostname' % mntdir) === added file 'lava_dispatcher/default-config/lava-dispatcher/device-types/qemu.conf' --- lava_dispatcher/default-config/lava-dispatcher/device-types/qemu.conf 1970-01-01 00:00:00 +0000 +++ lava_dispatcher/default-config/lava-dispatcher/device-types/qemu.conf 2012-06-13 20:47:32 +0000 @@ -0,0 +1,6 @@ +# NOTE: you also need to set something like: +# DEFAULT_QEMU_BINARY = qemu-system-arm +# in your lava-dispatcher.conf file +qemu_machine_type = beaglexm +qemu_drive_interface = sd +client_type=qemu === modified file 'lava_dispatcher/utils.py' --- lava_dispatcher/utils.py 2012-06-07 23:38:28 +0000 +++ lava_dispatcher/utils.py 2012-06-13 20:20:03 +0000 @@ -23,9 +23,11 @@ import logging import os import shutil +import subprocess import urllib2 import urlparse from shlex import shlex +from tempfile import mkdtemp import pexpect @@ -54,6 +56,28 @@ raise RuntimeError("Could not retrieve %s" % url) return filename +def decompress(image_file): + for suffix, command in [('.gz', 'gunzip'), + ('.xz', 'unxz'), + ('.bz2', 'bunzip2')]: + if image_file.endswith(suffix): + logging.info("Uncompressing %s with %s", image_file, command) + uncompressed_name = image_file[:-len(suffix)] + subprocess.check_call( + [command, '-c', image_file], stdout=open(uncompressed_name, 'w')) + return uncompressed_name + return image_file + +def download_image(url, context, imgdir=None): + ''' common download function to be used by clients. This will download + and decompress the image using LMC_COOKIES and/or LMC_PROXY settings + ''' + logging.info("Downloading image: %s" % url) + if not imgdir: + imgdir = mkdtemp(dir=context.lava_image_tmpdir) + img = download(url, imgdir, context.lava_proxy, context.lava_cookies) + return decompress(img) + def link_or_copy_file(src, dest): try: dir = os.path.dirname(dest)