=== modified file 'lava_dispatcher/actions/android_deploy.py'
@@ -20,8 +20,6 @@
# along with this program; if not, see <http://www.gnu.org/licenses>.
from lava_dispatcher.actions import BaseAction
-from lava_dispatcher.client.fastmodel import LavaFastModelClient
-from lava_dispatcher.client.master import LavaMasterImageClient
class cmd_deploy_linaro_android_image(BaseAction):
@@ -38,7 +36,4 @@
}
def run(self, boot, system, data, rootfstype='ext4'):
- if not isinstance(self.client, LavaMasterImageClient) and \
- not isinstance(self.client, LavaFastModelClient):
- raise RuntimeError("Invalid LavaClient for this action")
self.client.deploy_linaro_android(boot, system, data, rootfstype)
=== modified file 'lava_dispatcher/actions/android_install_binaries.py'
@@ -19,7 +19,6 @@
import logging
from lava_dispatcher.actions import BaseAction, null_or_empty_schema
-from lava_dispatcher.client.master import _deploy_tarball_to_board
class cmd_android_install_binaries(BaseAction):
@@ -32,9 +31,9 @@
logging.error("android_binary_drivers not defined in any config")
return
- with self.client._master_session() as session:
+ with self.client.target_device._as_master() as session:
session.run(
'mount /dev/disk/by-label/testrootfs /mnt/lava/system')
- _deploy_tarball_to_board(
+ self.client.target_device.target_extract(
session, driver_tarball, '/mnt/lava/system', timeout=600)
session.run('umount /mnt/lava/system')
=== modified file 'lava_dispatcher/actions/deploy.py'
@@ -18,9 +18,6 @@
# along with this program; if not, see <http://www.gnu.org/licenses>.
from lava_dispatcher.actions import BaseAction
-from lava_dispatcher.client.fastmodel import LavaFastModelClient
-from lava_dispatcher.client.master import LavaMasterImageClient
-from lava_dispatcher.client.qemu import LavaQEMUClient
class cmd_deploy_linaro_image(BaseAction):
@@ -73,10 +70,5 @@
raise ValueError('must specify image if not specifying a hwpack')
def run(self, hwpack=None, rootfs=None, image=None, rootfstype='ext3'):
- if not isinstance(self.client, LavaMasterImageClient) and \
- not isinstance(self.client, LavaQEMUClient) and \
- not isinstance(self.client, LavaFastModelClient):
- raise RuntimeError("Invalid LavaClient for this action")
-
self.client.deploy_linaro(
hwpack=hwpack, rootfs=rootfs, image=image, rootfstype=rootfstype)
=== removed file 'lava_dispatcher/actions/fastmodel_deploy.py'
@@ -1,41 +0,0 @@
-# Copyright (C) 2012 Linaro Limited
-#
-# Author: Andy Doan <andy.doan@linaro.org>
-#
-# This file is part of LAVA Dispatcher.
-#
-# LAVA Dispatcher is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# LAVA Dispatcher is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, see <http://www.gnu.org/licenses>.
-
-from lava_dispatcher.actions import BaseAction
-from lava_dispatcher.client.fastmodel import LavaFastModelClient
-
-
-class cmd_deploy_fastmodel_image(BaseAction):
-
- parameters_schema = {
- 'type': 'object',
- 'properties': {
- 'image': {'type': 'string', 'optional': False},
- 'axf': {'type': 'string', 'optional': False},
- 'image_type': {
- 'type': 'string', 'optional': True, 'default': 'ubuntu',
- 'enum': ['android', 'ubuntu']},
- },
- 'additionalProperties': False,
- }
-
- def run(self, image, axf, image_type='ubuntu'):
- if not isinstance(self.client, LavaFastModelClient):
- raise RuntimeError("Invalid LavaClient for this action")
- self.client.deploy_image(image, axf, image_type=='android')
=== modified file 'lava_dispatcher/client/base.py'
@@ -19,19 +19,17 @@
# along
# with this program; if not, see <http://www.gnu.org/licenses>.
-import atexit
import commands
import contextlib
import logging
-import os
import pexpect
-import shutil
import sys
import time
import traceback
+import lava_dispatcher.utils as utils
+
from cStringIO import StringIO
-from tempfile import mkdtemp
from lava_dispatcher.test_data import create_attachment
@@ -58,15 +56,6 @@
self.match_id = None
self.match = None
- def _empty_pexpect_buffer(self):
- """Make sure there is nothing in the pexpect buffer."""
- # Do we really need this? It wastes at least 1 second per command
- # invocation, if nothing else.
- index = 0
- while index == 0:
- index = self._connection.expect(
- ['.+', pexpect.EOF, pexpect.TIMEOUT], timeout=1, lava_no_logging=1)
-
def run(self, cmd, response=None, timeout=-1, failok=False):
"""Run `cmd` and wait for a shell response.
@@ -80,7 +69,7 @@
:return: The exit value of the command, if wait_for_rc not explicitly
set to False during construction.
"""
- self._empty_pexpect_buffer()
+ self._connection.empty_buffer()
self._connection.sendline(cmd)
start = time.time()
if response is not None:
@@ -386,13 +375,9 @@
logging.info("System is in test image now")
def get_www_scratch_dir(self):
- """returns a temporary directory available for downloads that's gets
- deleted when the process exits"""
-
- d = mkdtemp(dir=self.context.config.lava_image_tmpdir)
- atexit.register(shutil.rmtree, d)
- os.chmod(d, 0755)
- return d
+ """ returns a temporary directory available for downloads that gets
+ deleted when the process exits """
+ return utils.mkdtemp(self.context.config.lava_image_tmpdir)
def get_test_data_attachments(self):
'''returns attachments to go in the "lava_results" test run'''
=== removed file 'lava_dispatcher/client/master.py'
@@ -1,874 +0,0 @@
-# Copyright (C) 2011 Linaro Limited
-#
-# Author: Michael Hudson-Doyle <michael.hudson@linaro.org>
-# Author: Paul Larson <paul.larson@linaro.org>
-#
-# This file is part of LAVA Dispatcher.
-#
-# LAVA Dispatcher is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# LAVA Dispatcher is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along
-# with this program; if not, see <http://www.gnu.org/licenses>.
-
-import contextlib
-import logging
-import os
-import re
-import shutil
-import time
-import traceback
-import atexit
-
-import pexpect
-import errno
-
-from lava_dispatcher.downloader import (
- download_image,
- )
-from lava_dispatcher.utils import (
- logging_spawn,
- logging_system,
- string_to_list,
- url_to_cache, link_or_copy_file)
-from lava_dispatcher.client.base import (
- CommandRunner,
- CriticalError,
- LavaClient,
- NetworkCommandRunner,
- OperationFailed,
- )
-from lava_dispatcher.client.lmc_utils import (
- generate_image,
- image_partition_mounted,
- )
-
-
-def _extract_partition(image, partno, tarfile):
- """Mount a partition and produce a tarball of it
-
- :param image: The image to mount
- :param partno: The index of the partition in the image
- :param tarfile: path and filename of the tgz to output
- """
-
- with image_partition_mounted(image, partno) as mntdir:
- cmd = "sudo tar -C %s -czf %s ." % (mntdir, tarfile)
- rc = logging_system(cmd)
- if rc:
- raise RuntimeError("Failed to create tarball: %s" % tarfile)
-
-WGET_DEBUGGING_OPTIONS = '-S --progress=dot -e dotbytes=2M'
-
-def _deploy_tarball_to_board(session, tarball_url, dest, timeout=-1, num_retry=5):
- decompression_char = ''
- if tarball_url.endswith('.gz') or tarball_url.endswith('.tgz'):
- decompression_char = 'z'
- elif tarball_url.endswith('.bz2'):
- decompression_char = 'j'
-
- deploy_ok = False
-
- while num_retry > 0:
- try:
- session.run(
- 'wget --no-check-certificate --no-proxy --connect-timeout=30 %s -O- %s |'
- 'tar --warning=no-timestamp --numeric-owner -C %s -x%sf -'
- % (WGET_DEBUGGING_OPTIONS, tarball_url, dest, decompression_char),
- timeout=timeout)
- except (OperationFailed, pexpect.TIMEOUT):
- logging.warning("Deploy %s failed. %d retry left." % (tarball_url, num_retry - 1))
- else:
- deploy_ok = True
- break
-
- if num_retry > 1:
- # send CTRL C in case wget still hasn't exited.
- session._client.proc.sendcontrol("c")
- session._client.proc.sendline("echo 'retry left %s time(s)'" % (num_retry - 1))
- # And wait a little while.
- sleep_time = 60
- logging.info("Wait %d second before retry" % sleep_time)
- time.sleep(sleep_time)
- num_retry = num_retry - 1
-
- if not deploy_ok:
- raise Exception("Deploy tarball (%s) to board failed" % tarball_url);
-
-def _deploy_linaro_rootfs(session, rootfs):
- logging.info("Deploying linaro image")
- session.run('udevadm trigger')
- session.run('mkdir -p /mnt/root')
- session.run('mount /dev/disk/by-label/testrootfs /mnt/root')
- # The timeout has to be this long for vexpress. For a full desktop it
- # takes 214 minutes, plus about 25 minutes for the mkfs ext3, add
- # another hour to err on the side of caution.
- _deploy_tarball_to_board(session, rootfs, '/mnt/root', timeout=18000)
-
- session.run('echo %s > /mnt/root/etc/hostname'
- % session._client.config.tester_hostname)
- #DO NOT REMOVE - diverting flash-kernel and linking it to /bin/true
- #prevents a serious problem where packages getting installed that
- #call flash-kernel can update the kernel on the master image
- if session.run('chroot /mnt/root which dpkg-divert', failok=True) == 0:
- session.run(
- 'chroot /mnt/root dpkg-divert --local /usr/sbin/flash-kernel')
- session.run(
- 'chroot /mnt/root ln -sf /bin/true /usr/sbin/flash-kernel')
- session.run('umount /mnt/root')
-
-def _deploy_linaro_bootfs(session, bootfs):
- logging.info("Deploying linaro bootfs")
- session.run('udevadm trigger')
- session.run('mkdir -p /mnt/boot')
- session.run('mount /dev/disk/by-label/testboot /mnt/boot')
- _deploy_tarball_to_board(session, bootfs, '/mnt/boot')
- session.run('umount /mnt/boot')
-
-def _deploy_linaro_android_testboot(session, boottbz2):
- logging.info("Deploying test boot filesystem")
- session.run('umount /dev/disk/by-label/testboot', failok=True)
- session.run('mkfs.vfat /dev/disk/by-label/testboot '
- '-n testboot')
- session.run('udevadm trigger')
- session.run('mkdir -p /mnt/lava/boot')
- session.run('mount /dev/disk/by-label/testboot '
- '/mnt/lava/boot')
- _deploy_tarball_to_board(session, boottbz2, '/mnt/lava')
-
- _recreate_uInitrd(session)
-
-def _update_uInitrd_partitions(session, rc_filename):
- # Original android sdcard partition layout by l-a-m-c
- sys_part_org = session._client.config.sys_part_android_org
- cache_part_org = session._client.config.cache_part_android_org
- data_part_org = session._client.config.data_part_android_org
- # Sdcard layout in Lava image
- sys_part_lava = session._client.config.sys_part_android
- data_part_lava = session._client.config.data_part_android
-
- session.run(
- 'sed -i "/mount ext4 \/dev\/block\/mmcblk0p%s/d" %s'
- % (cache_part_org, rc_filename), failok=True)
-
- session.run('sed -i "s/mmcblk0p%s/mmcblk0p%s/g" %s'
- % (data_part_org, data_part_lava, rc_filename), failok=True)
- session.run('sed -i "s/mmcblk0p%s/mmcblk0p%s/g" %s'
- % (sys_part_org, sys_part_lava, rc_filename), failok=True)
- session.run('sed -i "s/mmcblk1p%s/mmcblk1p%s/g" %s'
- % (data_part_org, data_part_lava, rc_filename), failok=True)
- session.run('sed -i "s/mmcblk1p%s/mmcblk1p%s/g" %s'
- % (sys_part_org, sys_part_lava, rc_filename), failok=True)
-
-def _recreate_uInitrd(session):
- logging.debug("Recreate uInitrd")
-
- session.run('mkdir -p ~/tmp/')
- session.run('mv /mnt/lava/boot/uInitrd ~/tmp')
- session.run('cd ~/tmp/')
-
- session.run('dd if=uInitrd of=uInitrd.data ibs=64 skip=1')
- session.run('mv uInitrd.data ramdisk.cpio.gz')
- session.run(
- 'gzip -d -f ramdisk.cpio.gz; cpio -i -F ramdisk.cpio')
-
- # The mount partitions have moved from init.rc to init.partitions.rc
- # For backward compatible with early android build, we updatep both rc files
- _update_uInitrd_partitions(session, 'init.rc')
- _update_uInitrd_partitions(session, 'init.partitions.rc')
-
- session.run(
- 'sed -i "/export PATH/a \ \ \ \ export PS1 root@linaro: " init.rc')
-
- session.run("cat init.rc")
- session.run("cat init.partitions.rc", failok=True)
-
- session.run(
- 'cpio -i -t -F ramdisk.cpio | cpio -o -H newc | \
- gzip > ramdisk_new.cpio.gz')
-
- session.run(
- 'mkimage -A arm -O linux -T ramdisk -n "Android Ramdisk Image" \
- -d ramdisk_new.cpio.gz uInitrd')
-
- session.run('cd -')
- session.run('mv ~/tmp/uInitrd /mnt/lava/boot/uInitrd')
- session.run('rm -rf ~/tmp')
-
-def _deploy_linaro_android_testrootfs(session, systemtbz2, rootfstype):
- logging.info("Deploying the test root filesystem")
-
- session.run('umount /dev/disk/by-label/testrootfs', failok=True)
- session.run(
- 'mkfs -t %s -q /dev/disk/by-label/testrootfs -L testrootfs' % rootfstype, timeout=1800)
- session.run('udevadm trigger')
- session.run('mkdir -p /mnt/lava/system')
- session.run(
- 'mount /dev/disk/by-label/testrootfs /mnt/lava/system')
- _deploy_tarball_to_board(session, systemtbz2, '/mnt/lava', timeout=600)
-
- if session.has_partition_with_label('userdata') and \
- session.has_partition_with_label('sdcard') and \
- session.is_file_exist('/mnt/lava/system/etc/vold.fstab'):
- # If there is no userdata partition on the sdcard(like iMX and Origen),
- # then the sdcard partition will be used as the userdata partition as
- # before, and so cannot be used here as the sdcard on android
- sdcard_part_lava = session._client.config.sdcard_part_android
- sdcard_part_org = session._client.config.sdcard_part_android_org
- original = 'dev_mount sdcard /mnt/sdcard %s ' % sdcard_part_org
- replacement = 'dev_mount sdcard /mnt/sdcard %s ' % sdcard_part_lava
- sed_cmd = "s@{original}@{replacement}@".format(original=original,
- replacement=replacement)
- session.run(
- 'sed -i "%s" /mnt/lava/system/etc/vold.fstab' % sed_cmd,
- failok=True)
- session.run("cat /mnt/lava/system/etc/vold.fstab", failok=True)
-
- script_path = '%s/%s' % ('/mnt/lava', '/system/bin/disablesuspend.sh')
- if not session.is_file_exist(script_path):
- git_url = session._client.config.git_url_disablesuspend_sh
- lava_proxy = session._client.context.config.lava_proxy
- session.run("sh -c 'export http_proxy=%s'" % lava_proxy)
- session.run('wget --no-check-certificate %s -O %s' % (git_url, script_path))
- session.run('chmod +x %s' % script_path)
- session.run('chown :2000 %s' % script_path)
-
- session.run(
- 'sed -i "s/^PS1=.*$/PS1=\'root@linaro: \'/" /mnt/lava/system/etc/mkshrc',
- failok=True)
-
- session.run('umount /mnt/lava/system')
-
-
-def _purge_linaro_android_sdcard(session):
- logging.info("Reformatting Linaro Android sdcard filesystem")
- session.run('mkfs.vfat /dev/disk/by-label/sdcard -n sdcard')
- session.run('udevadm trigger')
-
-def _deploy_linaro_android_data(session, datatbz2):
-
- data_label = 'userdata'
- if not session.has_partition_with_label(data_label):
- ##consider the compatiblity, here use the existed sdcard partition
- data_label = 'sdcard'
-
- session.run('umount /dev/disk/by-label/%s' % data_label, failok=True)
- session.run('mkfs.ext4 -q /dev/disk/by-label/%s -L %s' % (data_label, data_label))
- session.run('udevadm trigger')
- session.run('mkdir -p /mnt/lava/data')
- session.run('mount /dev/disk/by-label/%s /mnt/lava/data' % (data_label))
- _deploy_tarball_to_board(session, datatbz2, '/mnt/lava', timeout=600)
- session.run('umount /mnt/lava/data')
-
-class PrefixCommandRunner(CommandRunner):
- """A CommandRunner that prefixes every command run with a given string.
-
- The motivating use case is to prefix every command with 'chroot
- $LOCATION'.
- """
-
- def __init__(self, prefix, connection, prompt_str):
- super(PrefixCommandRunner, self).__init__(connection, prompt_str)
- if not prefix.endswith(' '):
- prefix += ' '
- self._prefix = prefix
-
- def run(self, cmd, response=None, timeout=-1, failok=False):
- return super(PrefixCommandRunner, self).run(
- self._prefix + cmd, response, timeout, failok)
-
-
-class MasterCommandRunner(NetworkCommandRunner):
- """A CommandRunner to use when the board is booted into the master image.
-
- See `LavaClient.master_session`.
- """
-
- def __init__(self, client):
- super(MasterCommandRunner, self).__init__(client, client.config.master_str)
-
- def get_master_ip(self):
- #get master image ip address
- try:
- self.wait_network_up()
- except:
- logging.warning(traceback.format_exc())
- return None
- #tty device uses minimal match, see pexpect wiki
- pattern1 = "<(\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?)>"
- cmd = ("ifconfig %s | grep 'inet addr' | awk -F: '{print $2}' |"
- "awk '{print \"<\" $1 \">\"}'" % self._client.config.default_network_interface)
- self.run(
- cmd, [pattern1, pexpect.EOF, pexpect.TIMEOUT], timeout=5)
- if self.match_id == 0:
- ip = self.match.group(1)
- logging.debug("Master image IP is %s" % ip)
- return ip
- return None
-
- def has_partition_with_label(self, label):
- if not label:
- return False
-
- path = '/dev/disk/by-label/%s' % label
- return self.is_file_exist(path)
-
- def is_file_exist(self, path):
- cmd = 'ls %s' % path
- rc = self.run(cmd, failok=True)
- if rc == 0:
- return True
- return False
-
-
-class LavaMasterImageClient(LavaClient):
-
- def __init__(self, context, config):
- super(LavaMasterImageClient, self).__init__(context, config)
- pre_connect = self.config.pre_connect_command
- if pre_connect:
- logging_system(pre_connect)
- self.proc = self._connect_carefully()
- atexit.register(self._close_logging_spawn)
-
- def _connect_carefully(self):
- cmd = self.config.connection_command
-
- retry_count = 0
- retry_limit = 3
-
- port_stuck_message = 'Data Buffering Suspended\.'
- conn_closed_message = 'Connection closed by foreign host\.'
-
- expectations = {
- port_stuck_message: 'reset-port',
- 'Connected\.\r': 'all-good',
- conn_closed_message: 'retry',
- pexpect.TIMEOUT: 'all-good',
- }
- patterns = []
- results = []
- for pattern, result in expectations.items():
- patterns.append(pattern)
- results.append(result)
-
- while retry_count < retry_limit:
- proc = logging_spawn(cmd, timeout=1200)
- proc.logfile_read = self.sio
- #serial can be slow, races do funny things if you don't increase delay
- proc.delaybeforesend = 1
- logging.info('Attempting to connect to device')
- match = proc.expect(patterns, timeout=10)
- result = results[match]
- logging.info('Matched %r which means %s', patterns[match], result)
- if result == 'retry':
- proc.close(True)
- retry_count += 1
- time.sleep(5)
- continue
- elif result == 'all-good':
- return proc
- elif result == 'reset-port':
- reset_port = self.config.reset_port_command
- if reset_port:
- logging_system(reset_port)
- else:
- raise OperationFailed("no reset_port command configured")
- proc.close(True)
- retry_count += 1
- time.sleep(5)
- raise OperationFailed("could execute connection_command successfully")
-
- def _close_logging_spawn(self):
- self.proc.close(True)
-
- def _tarball_url_to_cache(self, url, cachedir):
- cache_loc = url_to_cache(url, cachedir)
- # can't have a folder name same as file name. replacing '.' with '.'
- return os.path.join(cache_loc.replace('.', '-'), "tarballs")
-
- def _are_tarballs_cached(self, image, lava_cachedir):
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
- cached = os.path.exists(os.path.join(cache_loc, "boot.tgz")) and \
- os.path.exists(os.path.join(cache_loc, "root.tgz"))
-
- if cached:
- return True;
-
- # Check if there is an other lava-dispatch instance have start to cache the same image
- # see the _about_to_cache_tarballs
- if not os.path.exists(os.path.join(cache_loc, "tarballs-cache-ongoing")):
- return False
-
- # wait x minute for caching is done.
- waittime = 20
-
- logging.info("Waiting for the other instance of lava-dispatcher to finish the caching of %s", image)
- while waittime > 0:
- if not os.path.exists(os.path.join(cache_loc, "tarballs-cache-ongoing")):
- waittime = 0
- else:
- time.sleep(60)
- waittime = waittime - 1
- if (waittime % 5) == 0:
- logging.info("%d minute left..." % waittime)
-
- return os.path.exists(os.path.join(cache_loc, "boot.tgz")) and \
- os.path.exists(os.path.join(cache_loc, "root.tgz"))
-
- def _get_cached_tarballs(self, image, tarball_dir, lava_cachedir):
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
-
- boot_tgz = os.path.join(tarball_dir, "boot.tgz")
- root_tgz = os.path.join(tarball_dir, "root.tgz")
- link_or_copy_file(os.path.join(cache_loc, "root.tgz"), root_tgz)
- link_or_copy_file(os.path.join(cache_loc, "boot.tgz"), boot_tgz)
-
- return (boot_tgz, root_tgz)
-
- def _about_to_cache_tarballs(self, image, lava_cachedir):
- # create this folder to indicate this instance of lava-dispatcher is caching this image.
- # see _are_tarballs_cached
- # return false if unable to create the directory. The caller should not cache the tarballs
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
- path = os.path.join(cache_loc, "tarballs-cache-ongoing")
- try:
- os.makedirs(path)
- except OSError as exc: # Python >2.5
- if exc.errno == errno.EEXIST:
- # other dispatcher process already caching - concurrency issue
- return False
- else:
- raise
- return True
-
- def _remove_cache_lock(self, image, lava_cachedir, cache_loc=None):
- if not cache_loc:
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
- path = os.path.join(cache_loc, "tarballs-cache-ongoing")
- if os.path.exists(path):
- logging.debug("Removing cache lock for %s" % path)
- shutil.rmtree(path)
-
- def _cache_tarballs(self, image, boot_tgz, root_tgz, lava_cachedir):
- cache_loc = self._tarball_url_to_cache(image, lava_cachedir)
- if not os.path.exists(cache_loc):
- os.makedirs(cache_loc)
- c_boot_tgz = os.path.join(cache_loc, "boot.tgz")
- c_root_tgz = os.path.join(cache_loc, "root.tgz")
- shutil.copy(boot_tgz, c_boot_tgz)
- shutil.copy(root_tgz, c_root_tgz)
-
- def deploy_linaro(self, hwpack=None, rootfs=None, image=None, rootfstype='ext3'):
- LAVA_IMAGE_TMPDIR = self.context.config.lava_image_tmpdir
- LAVA_IMAGE_URL = self.context.config.lava_image_url
- # validate in parameters
- if image is None:
- if hwpack is None or rootfs is None:
- raise CriticalError(
- "must specify both hwpack and rootfs when not specifying image")
- else:
- if hwpack is not None or rootfs is not None:
- raise CriticalError(
- "cannot specify hwpack or rootfs when specifying image")
-
- # generate image if needed
- try:
- tarball_dir = self.get_www_scratch_dir()
- if image is None:
- image_file = generate_image(self, hwpack, rootfs, tarball_dir)
- boot_tgz, root_tgz = self._generate_tarballs(image_file)
- else:
- os.chmod(tarball_dir, 0755)
- lava_cachedir = self.context.config.lava_cachedir
- if self.context.job_data.get('health_check', False):
- if self._are_tarballs_cached(image, lava_cachedir):
- logging.info("Reusing cached tarballs")
- boot_tgz, root_tgz = self._get_cached_tarballs(image, tarball_dir, lava_cachedir)
- else:
- logging.info("Downloading and caching the tarballs")
- # in some corner case, there can be more than one lava-dispatchers execute
- # caching of same tarballs exact at the same time. One of them will successfully
- # get the lock directory. The rest will skip the caching if _about_to_cache_tarballs
- # return false.
- try:
- should_cache = self._about_to_cache_tarballs(image, lava_cachedir)
- image_file = download_image(image, self.context, tarball_dir)
- boot_tgz, root_tgz = self._generate_tarballs(image_file)
- if should_cache:
- self._cache_tarballs(image, boot_tgz, root_tgz, lava_cachedir)
- finally:
- self._remove_cache_lock(image, lava_cachedir)
- else:
- image_file = download_image(image, self.context, tarball_dir)
- boot_tgz, root_tgz = self._generate_tarballs(image_file)
-
- except CriticalError:
- raise
- except:
- logging.error("Deployment tarballs preparation failed")
- tb = traceback.format_exc()
- self.sio.write(tb)
- raise CriticalError("Deployment tarballs preparation failed")
-
- # deploy the boot image and rootfs to target
- logging.info("Booting master image")
- self.boot_master_image()
- boot_tarball = boot_tgz.replace(LAVA_IMAGE_TMPDIR, '')
- root_tarball = root_tgz.replace(LAVA_IMAGE_TMPDIR, '')
- boot_url = '/'.join(u.strip('/') for u in [
- LAVA_IMAGE_URL, boot_tarball])
- root_url = '/'.join(u.strip('/') for u in [
- LAVA_IMAGE_URL, root_tarball])
- with self._master_session() as session:
- self._format_testpartition(session, rootfstype)
-
- logging.info("Waiting for network to come up")
- try:
- session.wait_network_up()
- except:
- logging.error("Unable to reach LAVA server, check network")
- tb = traceback.format_exc()
- self.sio.write(tb)
- raise CriticalError("Unable to reach LAVA server, check network")
-
- try:
- _deploy_linaro_rootfs(session, root_url)
- _deploy_linaro_bootfs(session, boot_url)
- except:
- logging.error("Deployment failed")
- tb = traceback.format_exc()
- self.sio.write(tb)
- raise CriticalError("Deployment failed")
-
- def deploy_linaro_android(self, boot, system, data, rootfstype='ext4'):
- LAVA_IMAGE_TMPDIR = self.context.lava_image_tmpdir
- LAVA_IMAGE_URL = self.context.lava_image_url
- logging.info("Deploying Android on %s" % self.hostname)
- logging.info(" boot: %s" % boot)
- logging.info(" system: %s" % system)
- logging.info(" data: %s" % data)
- logging.info("Boot master image")
- try:
- self.boot_master_image()
- with self._master_session() as session:
- logging.info("Waiting for network to come up...")
- try:
- session.wait_network_up()
- except:
- logging.error("Unable to reach LAVA server, check network")
- tb = traceback.format_exc()
- self.sio.write(tb)
- raise CriticalError("Unable to reach LAVA server, check network")
-
- try:
- boot_tbz2, system_tbz2, data_tbz2 = \
- self._download_tarballs(boot, system, data)
- except:
- logging.error("Unable to download artifacts for deployment")
- tb = traceback.format_exc()
- self.sio.write(tb)
- raise CriticalError("Unable to download artifacts for deployment")
-
- boot_tarball = boot_tbz2.replace(LAVA_IMAGE_TMPDIR, '')
- system_tarball = system_tbz2.replace(LAVA_IMAGE_TMPDIR, '')
- data_tarball = data_tbz2.replace(LAVA_IMAGE_TMPDIR, '')
-
- boot_url = '/'.join(u.strip('/') for u in [
- LAVA_IMAGE_URL, boot_tarball])
- system_url = '/'.join(u.strip('/') for u in [
- LAVA_IMAGE_URL, system_tarball])
- data_url = '/'.join(u.strip('/') for u in [
- LAVA_IMAGE_URL, data_tarball])
-
- try:
- _deploy_linaro_android_testboot(session, boot_url)
- _deploy_linaro_android_testrootfs(session, system_url, rootfstype)
- _deploy_linaro_android_data(session, data_url)
- if session.has_partition_with_label('userdata') and \
- session.has_partition_with_label('sdcard'):
- _purge_linaro_android_sdcard(session)
- except:
- logging.error("Android deployment failed")
- tb = traceback.format_exc()
- self.sio.write(tb)
- raise CriticalError("Android deployment failed")
- finally:
- logging.info("Android image deployment exiting")
-
- def _download_tarballs(self, boot_url, system_url, data_url):
- """Download tarballs from a boot, system and data tarball url
-
- :param boot_url: url of the Linaro Android boot tarball to download
- :param system_url: url of the Linaro Android system tarball to download
- :param data_url: url of the Linaro Android data tarball to download
- :param pkg_url: url of the custom kernel tarball to download
- """
- tarball_dir = self.get_www_scratch_dir()
- logging.info("Downloading the image files")
-
- boot_path = download_image(boot_url, self.context, tarball_dir, decompress=False)
- system_path = download_image(system_url, self.context, tarball_dir, decompress=False)
- data_path = download_image(data_url, self.context, tarball_dir, decompress=False)
- logging.info("Downloaded the image files")
- return boot_path, system_path, data_path
-
- def boot_master_image(self):
- """
- reboot the system, and check that we are in a master shell
- """
- logging.info("Boot the system master image")
- try:
- self.soft_reboot()
- image_boot_msg = self.config.image_boot_msg
- self.proc.expect(image_boot_msg, timeout=300)
- self._in_master_shell(300)
- except:
- logging.exception("in_master_shell failed")
- self.hard_reboot()
- image_boot_msg = self.config.image_boot_msg
- self.proc.expect(image_boot_msg, timeout=300)
- self._in_master_shell(300)
- self.proc.sendline('export PS1="$PS1 [rc=$(echo \$?)]: "')
- self.proc.expect(self.config.master_str, timeout=120, lava_no_logging=1)
- self.setup_proxy(self.config.master_str)
- logging.info("System is in master image now")
-
- def _format_testpartition(self, session, fstype):
- logging.info("Format testboot and testrootfs partitions")
- session.run('umount /dev/disk/by-label/testrootfs', failok=True)
- session.run(
- 'mkfs -t %s -q /dev/disk/by-label/testrootfs -L testrootfs'
- % fstype, timeout=1800)
- session.run('umount /dev/disk/by-label/testboot', failok=True)
- session.run('mkfs.vfat /dev/disk/by-label/testboot -n testboot')
-
- def _generate_tarballs(self, image_file):
- """Generate tarballs from a hwpack and rootfs url
-
- :param hwpack_url: url of the Linaro hwpack to download
- :param rootfs_url: url of the Linaro image to download
- """
- tarball_dir = os.path.dirname(image_file)
- boot_tgz = os.path.join(tarball_dir, "boot.tgz")
- root_tgz = os.path.join(tarball_dir, "root.tgz")
- try:
- _extract_partition(image_file, self.config.boot_part, boot_tgz)
- _extract_partition(image_file, self.config.root_part, root_tgz)
- except:
- logging.error("Failed to generate tarballs")
- shutil.rmtree(tarball_dir)
- tb = traceback.format_exc()
- self.sio.write(tb)
- raise
- return boot_tgz, root_tgz
-
- def reliable_session(self):
- return self._partition_session('testrootfs')
-
- def retrieve_results(self, result_disk):
- with self._master_session() as session:
-
- session.run('mkdir -p /mnt/root')
- session.run(
- 'mount /dev/disk/by-label/%s /mnt/root' % result_disk)
- lava_result_dir = self.context.config.lava_result_dir
- # Clean results directory on master image
- session.run(
- 'rm -rf /tmp/lava_results.tgz /tmp/%s' % lava_result_dir)
- session.run('mkdir -p /tmp/%s' % lava_result_dir)
- session.run(
- 'cp /mnt/root/%s/*.bundle /tmp/%s' % (lava_result_dir, lava_result_dir))
- # Clean result bundle on test image
- session.run(
- 'rm -f /mnt/root/%s/*.bundle' % (lava_result_dir))
- session.run('umount /mnt/root')
-
- # Create tarball of all results
- logging.info("Creating lava results tarball")
- session.run('cd /tmp')
- session.run(
- 'tar czf /tmp/lava_results.tgz -C /tmp/%s .' % lava_result_dir)
-
- # start gather_result job, status
- err_msg = ''
- master_ip = session.get_master_ip()
- if not master_ip:
- err_msg = (err_msg + "Getting master image IP address failed, "
- "no test case result retrieved.")
- logging.warning(err_msg)
- return 'fail', err_msg, None
- # Set 80 as server port
- session.run('python -m SimpleHTTPServer 80 &> /dev/null &')
- try:
- time.sleep(3)
-
- result_tarball = "http://%s/lava_results.tgz" % master_ip
- tarball_dir = self.get_www_scratch_dir()
-
- # download test result with a retry mechanism
- # set retry timeout to 5 mins
- logging.info("About to download the result tarball to host")
- now = time.time()
- timeout = 300
- tries = 0
-
- while True:
- try:
- result_path = download_image(result_tarball,
- self.context, tarball_dir, decompress=False)
- return 'pass', '', result_path
- except RuntimeError:
- tries += 1
- if time.time() >= now + timeout:
- logging.error(
- "download '%s' failed. Nr tries = %s" % (
- result_tarball, tries))
- return 'fail', err_msg, None
- else:
- logging.info(
- "Sleep one minute and retry (%d)" % tries)
- time.sleep(60)
- finally:
- session.run('kill %1')
- session.run('')
-
-
- @contextlib.contextmanager
- def _partition_session(self, partition):
- """A session that can be used to run commands in a given test
- partition.
-
- Anything that uses this will have to be done differently for images
- that are not deployed via a master image (e.g. using a JTAG to blow
- the image onto the card or testing under QEMU).
- """
- with self._master_session() as master_session:
- directory = '/mnt/' + partition
- master_session.run('mkdir -p %s' % directory)
- master_session.run('mount /dev/disk/by-label/%s %s' % (
- partition, directory))
- master_session.run(
- '[ -e %s/etc/resolv.conf ] && cp -f %s/etc/resolv.conf %s/etc/resolv.conf.bak' % (
- directory, directory, directory))
- master_session.run('cp -L /etc/resolv.conf %s/etc' % directory)
- #eliminate warning: Can not write log, openpty() failed
- # (/dev/pts not mounted?), does not work
- master_session.run('mount --rbind /dev %s/dev' % directory)
- try:
- yield PrefixCommandRunner(
- 'chroot ' + directory, self.proc, self.config.master_str)
- finally:
- master_session.run(
- '[ -e %s/etc/resolv.conf.bak ] && cp -f %s/etc/resolv.conf.bak %s/etc/resolv.conf || rm %s/etc/resolv.conf' % (
- directory, directory, directory, directory))
- cmd = ('cat /proc/mounts | awk \'{print $2}\' | grep "^%s/dev"'
- '| sort -r | xargs umount' % directory)
- master_session.run(cmd)
- master_session.run('umount ' + directory)
-
- def _in_master_shell(self, timeout=10):
- """
- Check that we are in a shell on the master image
- """
- self.proc.sendline("")
- match_id = self.proc.expect(
- [self.config.master_str, pexpect.TIMEOUT], timeout=timeout, lava_no_logging=1)
- if match_id == 1:
- raise OperationFailed
-
- @contextlib.contextmanager
- def _master_session(self):
- """A session that can be used to run commands in the master image.
-
- Anything that uses this will have to be done differently for images
- that are not deployed via a master image (e.g. using a JTAG to blow
- the image onto the card or testing under QEMU).
- """
- try:
- self._in_master_shell()
- except OperationFailed:
- self.boot_master_image()
- yield MasterCommandRunner(self)
-
- def soft_reboot(self):
- logging.info("Perform soft reboot the system")
- cmd = self.config.soft_boot_cmd
- # make sure in the shell (sometime the earlier command has not exit) by sending CTRL + C
- self.proc.sendline("\003")
- if cmd != "":
- self.proc.sendline(cmd)
- else:
- self.proc.sendline("reboot")
- # Looking for reboot messages or if they are missing, the U-Boot message will also indicate the
- # reboot is done.
- match_id = self.proc.expect(
- ['Restarting system.', 'The system is going down for reboot NOW',
- 'Will now restart', 'U-Boot', pexpect.TIMEOUT], timeout=120)
- if match_id not in [0, 1, 2, 3]:
- raise Exception("Soft reboot failed")
-
- def hard_reboot(self):
- logging.info("Perform hard reset on the system")
- cmd = self.config.hard_reset_command
- if cmd != "":
- logging_system(cmd)
- else:
- self.proc.send("~$")
- self.proc.sendline("hardreset")
- # after hardreset empty the pexpect buffer
- self._empty_pexpect_buffer()
-
- def _empty_pexpect_buffer(self):
- """Make sure there is nothing in the pexpect buffer."""
- index = 0
- while index == 0:
- index = self.proc.expect(
- ['.+', pexpect.EOF, pexpect.TIMEOUT], timeout=1, lava_no_logging=1)
-
- def _enter_uboot(self):
- interrupt_boot_prompt = self.config.interrupt_boot_prompt
- if self.proc.expect(interrupt_boot_prompt) != 0:
- raise Exception("Faile to enter uboot")
-
- interrupt_boot_command = self.config.interrupt_boot_command
- self.proc.sendline(interrupt_boot_command)
-
- def _boot_linaro_image(self):
- boot_cmds = 'boot_cmds' #default commands to boot ubuntu image
- for option in self.boot_options:
- keyval = option.split('=')
- if len(keyval) != 2:
- logging.warn("Invalid boot option format: %s" % option)
- elif keyval[0] != 'boot_cmds':
- logging.warn("Invalid boot option: %s" % keyval[0])
- else:
- boot_cmds = keyval[1].strip()
-
- self._boot(string_to_list(getattr(self.config, boot_cmds)))
-
- def _boot_linaro_android_image(self):
- self._boot(string_to_list(self.config.boot_cmds_android))
-
- def _boot(self, boot_cmds):
- try:
- self.soft_reboot()
- self._enter_uboot()
- except:
- logging.exception("_enter_uboot failed")
- self.hard_reboot()
- self._enter_uboot()
- self.proc.sendline(boot_cmds[0])
- bootloader_prompt = re.escape(self.config.bootloader_prompt)
- for line in range(1, len(boot_cmds)):
- self.proc.expect(bootloader_prompt, timeout=300)
- self.proc.sendline(boot_cmds[line])
-
=== removed file 'lava_dispatcher/client/qemu.py'
@@ -1,135 +0,0 @@
-# Copyright (C) 2011 Linaro Limited
-#
-# Author: Michael Hudson-Doyle <michael.hudson@linaro.org>
-#
-# This file is part of LAVA Dispatcher.
-#
-# LAVA Dispatcher is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# LAVA Dispatcher is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along
-# with this program; if not, see <http://www.gnu.org/licenses>.
-
-import contextlib
-import logging
-import os
-import pexpect
-from tempfile import mkdtemp
-
-from lava_dispatcher.client.base import (
- CommandRunner,
- LavaClient,
- )
-from lava_dispatcher.client.lmc_utils import (
- generate_image,
- image_partition_mounted,
- )
-from lava_dispatcher.downloader import (
- download_image,
- )
-from lava_dispatcher.utils import (
- logging_spawn,
- logging_system,
- )
-
-
-
-class LavaQEMUClient(LavaClient):
-
- def __init__(self, context, config):
- super(LavaQEMUClient, self).__init__(context, config)
- self._lava_image = None
-
- def deploy_linaro(self, hwpack=None, rootfs=None, image=None, rootfstype='ext3'):
- if image is None:
- odir = self.get_www_scratch_dir()
- image_file = generate_image(self, hwpack, rootfs, odir, rootfstype)
- else:
- image_file = download_image(image, self.context)
- self._lava_image = image_file
- with image_partition_mounted(self._lava_image, self.config.root_part) as mntdir:
- logging_system('echo %s > %s/etc/hostname' % (self.config.tester_hostname,
- mntdir))
-
- @contextlib.contextmanager
- def _mnt_prepared_for_qemu(self, mntdir):
- logging_system('sudo cp %s/etc/resolv.conf %s/etc/resolv.conf.bak' % (mntdir, mntdir))
- logging_system('sudo cp %s/etc/hosts %s/etc/hosts.bak' % (mntdir, mntdir))
- logging_system('sudo cp /etc/hosts %s/etc/hosts' % (mntdir,))
- logging_system('sudo cp /etc/resolv.conf %s/etc/resolv.conf' % (mntdir,))
- logging_system('sudo cp /usr/bin/qemu-arm-static %s/usr/bin/' % (mntdir,))
- try:
- yield
- finally:
- logging_system('sudo mv %s/etc/resolv.conf.bak %s/etc/resolv.conf' % (mntdir, mntdir))
- logging_system('sudo mv %s/etc/hosts.bak %s/etc/hosts' % (mntdir, mntdir))
- logging_system('sudo rm %s/usr/bin/qemu-arm-static' % (mntdir,))
-
- @contextlib.contextmanager
- def _chroot_into_rootfs_session(self):
- with image_partition_mounted(self._lava_image, self.config.root_part) as mntdir:
- with self._mnt_prepared_for_qemu(mntdir):
- cmd = pexpect.spawn('chroot ' + mntdir, logfile=self.sio, timeout=None)
- try:
- cmd.sendline('export PS1="root@host-mount:# [rc=$(echo \$?)] "')
- cmd.expect('root@host-mount:#')
- yield CommandRunner(cmd, 'root@host-mount:#')
- finally:
- cmd.sendline('exit')
- cmd.close()
-
- def reliable_session(self):
- # We could use _chroot_into_rootfs_session instead, but in my testing
- # as of 2011-11-30, the network works better in a tested image than
- # qemu-arm-static works to run the complicated commands of test
- # installation.
- return self.tester_session()
-
- def boot_master_image(self):
- raise RuntimeError("QEMU devices do not have a master image to boot.")
-
- def boot_linaro_image(self):
- """
- Boot the system to the test image
- """
- if self.proc is not None:
- self.proc.sendline('sync')
- self.proc.expect([self.config.tester_str, pexpect.TIMEOUT], timeout=10)
- self.proc.close()
- qemu_cmd = ('%s -M %s -drive if=%s,cache=writeback,file=%s '
- '-clock unix -device usb-kbd -device usb-mouse -usb '
- '-device usb-net,netdev=mynet -netdev user,id=mynet '
- '-net nic -net user -nographic') % (
- self.context.config.default_qemu_binary,
- self.config.qemu_machine_type,
- self.config.qemu_drive_interface,
- self._lava_image)
- logging.info('launching qemu with command %r' % qemu_cmd)
- self.proc = logging_spawn(
- qemu_cmd, logfile=self.sio, timeout=None)
- self.proc.expect(self.config.tester_str, timeout=300)
- # set PS1 to include return value of last command
- self.proc.sendline('export PS1="$PS1 [rc=$(echo \$?)]: "')
- self.proc.expect(self.config.tester_str, timeout=10)
-
- def retrieve_results(self, result_disk):
- if self.proc is not None:
- self.proc.sendline('sync')
- self.proc.expect([self.config.tester_str, pexpect.TIMEOUT], timeout=10)
- self.proc.close()
- tardir = mkdtemp()
- tarfile = os.path.join(tardir, "lava_results.tgz")
- with image_partition_mounted(self._lava_image, self.config.root_part) as mntdir:
- logging_system(
- 'tar czf %s -C %s%s .' % (
- tarfile, mntdir, self.context.lava_result_dir))
- logging_system('rm %s%s/*.bundle' % (mntdir, self.context.lava_result_dir))
- return 'pass', '', tarfile
=== renamed file 'lava_dispatcher/client/fastmodel.py' => 'lava_dispatcher/client/targetdevice.py'
@@ -18,128 +18,36 @@
# along
# with this program; if not, see <http://www.gnu.org/licenses>.
-import atexit
-import codecs
-import cStringIO
import logging
import os
import shutil
-import stat
-import threading
import time
from lava_dispatcher.client.base import (
+ CommandRunner,
CriticalError,
- TesterCommandRunner,
LavaClient,
)
-from lava_dispatcher.client.lmc_utils import (
- image_partition_mounted,
- generate_android_image,
- generate_fastmodel_image,
- )
-from lava_dispatcher.downloader import (
- download_image,
- )
-from lava_dispatcher.test_data import (
- create_attachment,
+from lava_dispatcher.device.target import (
+ get_target,
)
from lava_dispatcher.utils import (
- logging_spawn,
logging_system,
)
-class LavaFastModelClient(LavaClient):
-
- PORT_PATTERN = 'terminal_0: Listening for serial connection on port (\d+)'
- ANDROID_WALLPAPER = 'system/wallpaper_info.xml'
- SYS_PARTITION = 2
- DATA_PARTITION = 5
-
- BOOT_OPTIONS = {
- 'motherboard.smsc_91c111.enabled': '1',
- 'motherboard.hostbridge.userNetworking': '1',
- 'coretile.cache_state_modelled': '0',
- 'coretile.cluster0.cpu0.semihosting-enable': '1',
- }
-
- # a list of allowable values for BOOT_OPTIONS
- BOOT_VALS = [ '0', '1' ]
+class TargetBasedClient(LavaClient):
+ '''This is a wrapper around the lava_dispatcher.device.target class that
+ provides the additional functionality that's needed by lava-dispatcher
+ actions that depend on a LavaClient
+ '''
def __init__(self, context, config):
- super(LavaFastModelClient, self).__init__(context, config)
- self._sim_binary = config.simulator_binary
- lic_server = config.license_server
- if not self._sim_binary or not lic_server:
- raise RuntimeError("The device type config for this device "
- "requires settings for 'simulator_binary' and 'license_server'")
-
- os.putenv('ARMLMD_LICENSE_FILE', lic_server)
- self._sim_proc = None
-
- def get_android_adb_interface(self):
- return 'lo'
-
- def _customize_android(self):
- with image_partition_mounted(self._sd_image, self.DATA_PARTITION) as d:
- wallpaper = '%s/%s' % (d, self.ANDROID_WALLPAPER)
- # delete the android active wallpaper as slows things down
- logging_system('sudo rm -f %s' % wallpaper)
-
- with image_partition_mounted(self._sd_image, self.SYS_PARTITION) as d:
- script_path = '%s/%s' % (d, 'bin/disablesuspend.sh')
- if self.config.git_url_disablesuspend_sh:
- logging_system('sudo wget %s -O %s' % (
- self.config.git_url_disablesuspend_sh,
- script_path))
- logging_system('sudo chmod +x %s' % script_path)
- logging_system('sudo chown :2000 %s' % script_path)
-
- #make sure PS1 is what we expect it to be
- logging_system(
- 'sudo sh -c \'echo "PS1=%s: ">> %s/etc/mkshrc\'' % (self.config.tester_str, d))
- # fast model usermode networking does not support ping
- logging_system(
- 'sudo sh -c \'echo "alias ping=\\\"echo LAVA-ping override 1 received\\\"">> %s/etc/mkshrc\'' % d)
-
- def _customize_ubuntu(self):
- with image_partition_mounted(self._sd_image, self.config.root_part) as mntdir:
- logging_system('sudo echo %s > %s/etc/hostname'
- % (self.config.tester_hostname, mntdir))
-
- def deploy_image(self, image, axf, is_android=False):
- self._axf = download_image(axf, self.context)
- self._sd_image = download_image(image, self.context)
-
- logging.debug("image file is: %s" % self._sd_image)
- if is_android:
- self._customize_android()
- else:
- self._customize_ubuntu()
-
- def _copy_axf(self, partno, fname):
- with image_partition_mounted(self._sd_image, partno) as mntdir:
- src = '%s/%s' % (mntdir,fname)
- odir = os.path.dirname(self._sd_image)
- self._axf = '%s/%s' % (odir, os.path.split(src)[1])
- shutil.copyfile(src, self._axf)
+ super(TargetBasedClient, self).__init__(context, config)
+ self.target_device = get_target(context, config)
def deploy_linaro_android(self, boot, system, data, rootfstype='ext4'):
- logging.info("Deploying Android on %s" % self.hostname)
-
- self._boot = download_image(boot, self.context, decompress=False)
- self._data = download_image(data, self.context, decompress=False)
- self._system = download_image(system, self.context, decompress=False)
-
- self._sd_image = '%s/android.img' % os.path.dirname(self._system)
-
- generate_android_image(
- 'vexpress-a9', self._boot, self._data, self._system, self._sd_image)
-
- self._copy_axf(self.config.boot_part, 'linux-system-ISW.axf')
-
- self._customize_android()
+ self.target_device.deploy_android(boot, system, data)
def deploy_linaro(self, hwpack=None, rootfs=None, image=None,
rootfstype='ext3'):
@@ -152,121 +60,19 @@
"cannot specify hwpack or rootfs when specifying image")
if image is None:
- hwpack = download_image(hwpack, self.context, decompress=False)
- rootfs = download_image(rootfs, self.context, decompress=False)
- odir = os.path.dirname(rootfs)
-
- generate_fastmodel_image(hwpack, rootfs, odir)
- self._sd_image = '%s/sd.img' % odir
- self._axf = '%s/img.axf' % odir
+ self.target_device.deploy_linaro(hwpack, rootfs)
else:
- self._sd_image = download_image(image, self.context)
- self._copy_axf(self.config.root_part, 'boot/img.axf')
-
- self._customize_ubuntu()
-
- def _fix_perms(self):
- ''' The directory created for the image download/creation gets created
- with tempfile.mkdtemp which grants permission only to the creator of
- the directory. We need group access because the dispatcher may run
- the simulator as a different user
- '''
- d = os.path.dirname(self._sd_image)
- os.chmod(d, stat.S_IRWXG|stat.S_IRWXU)
- os.chmod(self._sd_image, stat.S_IRWXG|stat.S_IRWXU)
- os.chmod(self._axf, stat.S_IRWXG|stat.S_IRWXU)
-
- #lmc ignores the parent directories group owner
- st = os.stat(d)
- os.chown(self._axf, st.st_uid, st.st_gid)
- os.chown(self._sd_image, st.st_uid, st.st_gid)
-
- def _boot_options(self):
- options = dict(self.BOOT_OPTIONS)
- for option in self.boot_options:
- keyval = option.split('=')
- if len(keyval) != 2:
- logging.warn("Invalid boot option format: %s" % option)
- elif keyval[0] not in self.BOOT_OPTIONS:
- logging.warn("Invalid boot option: %s" % keyval[0])
- elif keyval[1] not in self.BOOT_VALS:
- logging.warn("Invalid boot option value: %s" % option)
- else:
- options[keyval[0]] = keyval[1]
-
- return ' '.join(['-C %s=%s' %(k,v) for k,v in options.iteritems()])
-
- def _get_sim_cmd(self):
- options = self._boot_options()
- return ("%s -a coretile.cluster0.*=%s "
- "-C motherboard.mmc.p_mmc_file=%s "
- "-C motherboard.hostbridge.userNetPorts='5555=5555' %s") % (
- self._sim_binary, self._axf, self._sd_image, options)
-
- def _stop(self):
- if self.proc is not None:
- logging.info("performing sync on target filesystem")
- r = TesterCommandRunner(self)
- r.run("sync", timeout=10, failok=True)
- self.proc.close()
- self.proc = None
- if self._sim_proc is not None:
- self._sim_proc.close()
- self._sim_proc = None
-
- def _create_rtsm_ostream(self, ofile):
- '''the RTSM binary uses the windows code page(cp1252), but the
- dashboard and celery needs data with a utf-8 encoding'''
- return codecs.EncodedFile(ofile, 'cp1252', 'utf-8')
-
-
- def _drain_sim_proc(self):
- '''pexpect will continue to get data for the simproc process. We need
- to keep this pipe drained so that it won't get full and then stop block
- the process from continuing to execute'''
-
- f = cStringIO.StringIO()
- self._sim_proc.logfile = self._create_rtsm_ostream(f)
- _pexpect_drain(self._sim_proc).start()
+ self.target_device.deploy_linaro_prebuilt(image)
def _boot_linaro_image(self):
- self._stop()
-
- self._fix_perms()
- sim_cmd = self._get_sim_cmd()
-
- # the simulator proc only has stdout/stderr about the simulator
- # we hook up into a telnet port which emulates a serial console
- logging.info('launching fastmodel with command %r' % sim_cmd)
- self._sim_proc = logging_spawn(
- sim_cmd,
- logfile=self.sio,
- timeout=1200)
- atexit.register(self._stop)
- self._sim_proc.expect(self.PORT_PATTERN, timeout=300)
- self._serial_port = self._sim_proc.match.groups()[0]
- logging.info('serial console port on: %s' % self._serial_port)
-
- match = self._sim_proc.expect(["ERROR: License check failed!",
- "Simulation is started"])
- if match == 0:
- raise RuntimeError("fast model license check failed")
-
- self._drain_sim_proc()
-
- logging.info('simulator is started connecting to serial port')
- self.proc = logging_spawn(
- 'telnet localhost %s' % self._serial_port,
- logfile=self._create_rtsm_ostream(self.sio),
- timeout=90)
- atexit.register(self._stop)
+ self.proc = self.target_device.power_on()
def _boot_linaro_android_image(self):
''' booting android or ubuntu style images don't differ much'''
logging.info('ensuring ADB port is ready')
while logging_system("sh -c 'netstat -an | grep 5555.*TIME_WAIT'") == 0:
- logging.info ("waiting for TIME_WAIT 5555 socket to finish")
+ logging.info("waiting for TIME_WAIT 5555 socket to finish")
time.sleep(3)
self._boot_linaro_image()
@@ -275,34 +81,17 @@
return self.tester_session()
def retrieve_results(self, result_disk):
- self._stop()
+ td = self.target_device
+ td.power_off(self.proc)
- tardir = os.path.dirname(self._sd_image)
- tarfile = os.path.join(tardir, 'lava_results.tgz')
- with image_partition_mounted(self._sd_image, self.config.root_part) as mnt:
- logging_system(
- 'tar czf %s -C %s%s .' % (
- tarfile, mnt, self.context.lava_result_dir))
- return 'pass', '', tarfile
+ tarbase = os.path.join(td.scratch_dir, 'lava_results')
+ result_dir = self.context.config.lava_result_dir
+ with td.file_system(td.config.root_part, result_dir) as mnt:
+ tarbase = shutil.make_archive(tarbase, 'gztar', mnt)
+ return 'pass', '', tarbase
def get_test_data_attachments(self):
'''returns attachments to go in the "lava_results" test run'''
- a = super(LavaFastModelClient, self).get_test_data_attachments()
-
- # if the simulator never got started we won't even get to a logfile
- if getattr(self._sim_proc, 'logfile', None) is not None:
- content = self._sim_proc.logfile.getvalue()
- a.append( create_attachment('rtsm.log', content) )
+ a = super(TargetBasedClient, self).get_test_data_attachments()
+ a.extend(self.target_device.get_test_data_attachments())
return a
-
-class _pexpect_drain(threading.Thread):
- ''' The simulator process can dump a lot of information to its console. If
- don't actively read from it, the pipe will get full and the process will
- be blocked. This allows us to keep the pipe empty so the process can run
- '''
- def __init__(self, proc):
- threading.Thread.__init__(self)
- self.proc = proc
- self.daemon = True #allows thread to die when main main proc exits
- def run(self):
- self.proc.drain()
=== modified file 'lava_dispatcher/context.py'
@@ -24,9 +24,7 @@
import tempfile
from lava_dispatcher.config import get_device_config
-from lava_dispatcher.client.fastmodel import LavaFastModelClient
-from lava_dispatcher.client.master import LavaMasterImageClient
-from lava_dispatcher.client.qemu import LavaQEMUClient
+from lava_dispatcher.client.targetdevice import TargetBasedClient
from lava_dispatcher.test_data import LavaTestData
@@ -36,17 +34,7 @@
self.job_data = job_data
device_config = get_device_config(
target, dispatcher_config.config_dir)
- client_type = device_config.client_type
- if client_type == 'master' or client_type == 'conmux':
- self._client = LavaMasterImageClient(self, device_config)
- elif client_type == 'qemu':
- self._client = LavaQEMUClient(self, device_config)
- elif client_type == 'fastmodel':
- self._client = LavaFastModelClient(self, device_config)
- else:
- raise RuntimeError(
- "this version of lava-dispatcher only supports master, qemu, "
- "and fastmodel clients, not %r" % client_type)
+ self._client = TargetBasedClient(self, device_config)
self.test_data = LavaTestData()
self.oob_file = oob_file
self._host_result_dir = None
=== modified file 'lava_dispatcher/default-config/lava-dispatcher/device-defaults.conf'
@@ -110,9 +110,6 @@
# QEMU drive interface.
qemu_drive_interface = sd
-# This is used for snowball soft reset fix, since the reboot command is hanging.
-soft_boot_cmd = reboot
-
# This is for android build where the network is not up by default. 1 or 0
enable_network_after_boot_android = 1
=== added directory 'lava_dispatcher/device'
=== added file 'lava_dispatcher/device/__init__.py'
@@ -0,0 +1,19 @@
+# Copyright (C) 2011 Linaro Limited
+#
+# Author: Michael Hudson-Doyle <michael.hudson@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
=== added file 'lava_dispatcher/device/fastmodel.py'
@@ -0,0 +1,258 @@
+# Copyright (C) 2012 Linaro Limited
+#
+# Author: Andy Doan <andy.doan@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+import codecs
+import contextlib
+import cStringIO
+import logging
+import os
+import shutil
+import stat
+import threading
+
+from lava_dispatcher.device.target import (
+ Target
+)
+from lava_dispatcher.client.lmc_utils import (
+ image_partition_mounted,
+ generate_android_image,
+ generate_fastmodel_image,
+ )
+from lava_dispatcher.downloader import (
+ download_image,
+ )
+from lava_dispatcher.test_data import (
+ create_attachment,
+ )
+from lava_dispatcher.utils import (
+ ensure_directory,
+ logging_spawn,
+ logging_system,
+ )
+
+
+class FastModelTarget(Target):
+
+ PORT_PATTERN = 'terminal_0: Listening for serial connection on port (\d+)'
+ ANDROID_WALLPAPER = 'system/wallpaper_info.xml'
+ SYS_PARTITION = 2
+ DATA_PARTITION = 5
+
+ BOOT_OPTIONS = {
+ 'motherboard.smsc_91c111.enabled': '1',
+ 'motherboard.hostbridge.userNetworking': '1',
+ 'coretile.cache_state_modelled': '0',
+ 'coretile.cluster0.cpu0.semihosting-enable': '1',
+ }
+
+ # a list of allowable values for BOOT_OPTIONS
+ BOOT_VALS = ['0', '1']
+
+ def __init__(self, context, config):
+ super(FastModelTarget, self).__init__(context, config)
+ self._sim_binary = config.simulator_binary
+ lic_server = config.license_server
+ if not self._sim_binary or not lic_server:
+ raise RuntimeError("The device type config for this device "
+ "requires settings for 'simulator_binary' and 'license_server'"
+ )
+
+ os.putenv('ARMLMD_LICENSE_FILE', lic_server)
+ self._sim_proc = None
+
+ def _customize_android(self):
+ with image_partition_mounted(self._sd_image, self.DATA_PARTITION) as d:
+ wallpaper = '%s/%s' % (d, self.ANDROID_WALLPAPER)
+ # delete the android active wallpaper as slows things down
+ logging_system('sudo rm -f %s' % wallpaper)
+
+ with image_partition_mounted(self._sd_image, self.SYS_PARTITION) as d:
+ #make sure PS1 is what we expect it to be
+ logging_system(
+ 'sudo sh -c \'echo "PS1=%s: ">> %s/etc/mkshrc\'' %
+ (self.config.tester_str, d))
+ self.deployment_data = Target.android_deployment_data
+
+ def _customize_ubuntu(self):
+ rootpart = self.config.root_part
+ with image_partition_mounted(self._sd_image, rootpart) as d:
+ logging_system('sudo echo %s > %s/etc/hostname'
+ % (self.config.tester_hostname, d))
+ self.deployment_data = Target.ubuntu_deployment_data
+
+ def _copy_axf(self, partno, fname):
+ with image_partition_mounted(self._sd_image, partno) as mntdir:
+ src = '%s/%s' % (mntdir, fname)
+ odir = os.path.dirname(self._sd_image)
+ self._axf = '%s/%s' % (odir, os.path.split(src)[1])
+ shutil.copyfile(src, self._axf)
+
+ def deploy_android(self, boot, system, data):
+ logging.info("Deploying Android on %s" % self.config.hostname)
+
+ self._boot = download_image(boot, self.context, decompress=False)
+ self._data = download_image(data, self.context, decompress=False)
+ self._system = download_image(system, self.context, decompress=False)
+
+ self._sd_image = '%s/android.img' % os.path.dirname(self._system)
+
+ generate_android_image(
+ 'vexpress-a9', self._boot, self._data, self._system, self._sd_image
+ )
+
+ self._copy_axf(self.config.boot_part, 'linux-system-ISW.axf')
+
+ self._customize_android()
+
+ def deploy_linaro(self, hwpack=None, rootfs=None):
+ hwpack = download_image(hwpack, self.context, decompress=False)
+ rootfs = download_image(rootfs, self.context, decompress=False)
+ odir = os.path.dirname(rootfs)
+
+ generate_fastmodel_image(hwpack, rootfs, odir)
+ self._sd_image = '%s/sd.img' % odir
+ self._axf = '%s/img.axf' % odir
+
+ self._customize_ubuntu()
+
+ def deploy_linaro_prebuilt(self, image):
+ self._sd_image = download_image(image, self.context)
+ self._copy_axf(self.config.root_part, 'boot/img.axf')
+
+ self._customize_ubuntu()
+
+ @contextlib.contextmanager
+ def file_system(self, partition, directory):
+ with image_partition_mounted(self._sd_image, partition) as mntdir:
+ path = '%s/%s' % (mntdir, directory)
+ ensure_directory(path)
+ yield path
+
+ def _fix_perms(self):
+ ''' The directory created for the image download/creation gets created
+ with tempfile.mkdtemp which grants permission only to the creator of
+ the directory. We need group access because the dispatcher may run
+ the simulator as a different user
+ '''
+ d = os.path.dirname(self._sd_image)
+ os.chmod(d, stat.S_IRWXG | stat.S_IRWXU)
+ os.chmod(self._sd_image, stat.S_IRWXG | stat.S_IRWXU)
+ os.chmod(self._axf, stat.S_IRWXG | stat.S_IRWXU)
+
+ #lmc ignores the parent directories group owner
+ st = os.stat(d)
+ os.chown(self._axf, st.st_uid, st.st_gid)
+ os.chown(self._sd_image, st.st_uid, st.st_gid)
+
+ def _boot_options(self):
+ options = dict(self.BOOT_OPTIONS)
+ for option in self.boot_options:
+ keyval = option.split('=')
+ if len(keyval) != 2:
+ logging.warn("Invalid boot option format: %s" % option)
+ elif keyval[0] not in self.BOOT_OPTIONS:
+ logging.warn("Invalid boot option: %s" % keyval[0])
+ elif keyval[1] not in self.BOOT_VALS:
+ logging.warn("Invalid boot option value: %s" % option)
+ else:
+ options[keyval[0]] = keyval[1]
+
+ return ' '.join(['-C %s=%s' % (k, v) for k, v in options.iteritems()])
+
+ def _get_sim_cmd(self):
+ options = self._boot_options()
+ return ("%s -a coretile.cluster0.*=%s "
+ "-C motherboard.mmc.p_mmc_file=%s "
+ "-C motherboard.hostbridge.userNetPorts='5555=5555' %s") % (
+ self._sim_binary, self._axf, self._sd_image, options)
+
+ def power_off(self, proc):
+ if proc is not None:
+ proc.close()
+ if self._sim_proc is not None:
+ self._sim_proc.close()
+
+ def _create_rtsm_ostream(self, ofile):
+ '''the RTSM binary uses the windows code page(cp1252), but the
+ dashboard and celery needs data with a utf-8 encoding'''
+ return codecs.EncodedFile(ofile, 'cp1252', 'utf-8')
+
+ def _drain_sim_proc(self):
+ '''pexpect will continue to get data for the simproc process. We need
+ to keep this pipe drained so that it won't get full and then stop block
+ the process from continuing to execute'''
+
+ f = cStringIO.StringIO()
+ self._sim_proc.logfile = self._create_rtsm_ostream(f)
+ _pexpect_drain(self._sim_proc).start()
+
+ def power_on(self):
+ self._fix_perms()
+ sim_cmd = self._get_sim_cmd()
+
+ # the simulator proc only has stdout/stderr about the simulator
+ # we hook up into a telnet port which emulates a serial console
+ logging.info('launching fastmodel with command %r' % sim_cmd)
+ self._sim_proc = logging_spawn(
+ sim_cmd,
+ logfile=self.sio,
+ timeout=1200)
+ self._sim_proc.expect(self.PORT_PATTERN, timeout=300)
+ self._serial_port = self._sim_proc.match.groups()[0]
+ logging.info('serial console port on: %s' % self._serial_port)
+
+ match = self._sim_proc.expect(["ERROR: License check failed!",
+ "Simulation is started"])
+ if match == 0:
+ raise RuntimeError("fast model license check failed")
+
+ self._drain_sim_proc()
+
+ logging.info('simulator is started connecting to serial port')
+ self.proc = logging_spawn(
+ 'telnet localhost %s' % self._serial_port,
+ logfile=self._create_rtsm_ostream(self.sio),
+ timeout=90)
+ return self.proc
+
+ def get_test_data_attachments(self):
+ '''returns attachments to go in the "lava_results" test run'''
+ # if the simulator never got started we won't even get to a logfile
+ if getattr(self._sim_proc, 'logfile', None) is not None:
+ content = self._sim_proc.logfile.getvalue()
+ return [create_attachment('rtsm.log', content)]
+ return []
+
+
+class _pexpect_drain(threading.Thread):
+ ''' The simulator process can dump a lot of information to its console. If
+ don't actively read from it, the pipe will get full and the process will
+ be blocked. This allows us to keep the pipe empty so the process can run
+ '''
+ def __init__(self, proc):
+ threading.Thread.__init__(self)
+ self.proc = proc
+
+ self.daemon = True # allow thread to die when main main proc exits
+
+ def run(self):
+ self.proc.drain()
+
+target_class = FastModelTarget
=== added file 'lava_dispatcher/device/master.py'
@@ -0,0 +1,661 @@
+# Copyright (C) 2011 Linaro Limited
+#
+# Author: Michael Hudson-Doyle <michael.hudson@linaro.org>
+# Author: Paul Larson <paul.larson@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+import atexit
+import contextlib
+import logging
+import os
+import shutil
+import tarfile
+import time
+import traceback
+
+import pexpect
+
+import lava_dispatcher.tarballcache as tarballcache
+
+from lava_dispatcher.device.target import (
+ Target
+ )
+from lava_dispatcher.downloader import (
+ download_image,
+ download_with_retry,
+ )
+from lava_dispatcher.utils import (
+ logging_spawn,
+ logging_system,
+ string_to_list,
+ )
+from lava_dispatcher.client.base import (
+ CriticalError,
+ NetworkCommandRunner,
+ OperationFailed,
+ )
+from lava_dispatcher.client.lmc_utils import (
+ generate_image,
+ image_partition_mounted,
+ )
+
+
+class MasterImageTarget(Target):
+
+ def __init__(self, context, config):
+ super(MasterImageTarget, self).__init__(context, config)
+
+ Target.android_deployment_data['boot_cmds'] = 'boot_cmds_android'
+ Target.ubuntu_deployment_data['boot_cmds'] = 'boot_cmds'
+
+ self.master_ip = None
+
+ if config.pre_connect_command:
+ logging_system(config.pre_connect_command)
+
+ self.proc = self._connect_carefully(config.connection_command)
+ atexit.register(self._close_logging_spawn)
+
+ def power_on(self):
+ self._boot_linaro_image()
+ return self.proc
+
+ def power_off(self, proc):
+ # we always leave master image devices powered on
+ pass
+
+ def _customize_ubuntu(self, image):
+ with image_partition_mounted(image, self.config.root_part) as d:
+ logging_system('sudo echo %s > %s/etc/hostname'
+ % (self.config.tester_hostname, d))
+
+ def deploy_linaro(self, hwpack, rfs):
+ image_file = generate_image(self, hwpack, rfs, self.scratch_dir)
+ boot_tgz, root_tgz = self._generate_tarballs(image_file)
+
+ self._deploy_tarballs(boot_tgz, root_tgz)
+ self.deployment_data = Target.ubuntu_deployment_data
+
+ def deploy_android(self, boot, system, userdata):
+ sdir = self.scratch_dir
+ boot = download_image(boot, self.context, sdir, decompress=False)
+ system = download_image(system, self.context, sdir, decompress=False)
+ data = download_image(userdata, self.context, sdir, decompress=False)
+
+ tmpdir = self.context.config.lava_image_tmpdir
+ url = self.context.config.lava_image_url
+
+ boot = boot.replace(tmpdir, '')
+ system = system.replace(tmpdir, '')
+ data = data.replace(tmpdir, '')
+
+ boot_url = '/'.join(u.strip('/') for u in [url, boot])
+ system_url = '/'.join(u.strip('/') for u in [url, system])
+ data_url = '/'.join(u.strip('/') for u in [url, data])
+
+ with self._as_master() as master:
+ self._format_testpartition(master, 'ext4')
+ _deploy_linaro_android_boot(master, boot_url)
+ _deploy_linaro_android_system(master, system_url)
+ _deploy_linaro_android_data(master, data_url)
+
+ if master.has_partition_with_label('userdata') and \
+ master.has_partition_with_label('sdcard'):
+ _purge_linaro_android_sdcard(master)
+
+ self.deployment_data = Target.android_deployment_data
+
+ def deploy_linaro_prebuilt(self, image):
+ if self.context.job_data.get('health_check', False):
+ (boot_tgz, root_tgz) = tarballcache.get_tarballs(
+ self.context, image, self.scratch_dir, self._generate_tarballs)
+ else:
+ image_file = download_image(image, self.context, self.scratch_dir)
+ boot_tgz, root_tgz = self._generate_tarballs(image_file)
+
+ self._deploy_tarballs(boot_tgz, root_tgz)
+ self.deployment_data = Target.ubuntu_deployment_data
+
+ def _deploy_tarballs(self, boot_tgz, root_tgz):
+ logging.info("Booting master image")
+ self.boot_master_image()
+
+ tmpdir = self.context.config.lava_image_tmpdir
+ url = self.context.config.lava_image_url
+
+ boot_tarball = boot_tgz.replace(tmpdir, '')
+ root_tarball = root_tgz.replace(tmpdir, '')
+ boot_url = '/'.join(u.strip('/') for u in [url, boot_tarball])
+ root_url = '/'.join(u.strip('/') for u in [url, root_tarball])
+ with self._as_master() as master:
+ self._format_testpartition(master, 'ext4')
+ try:
+ _deploy_linaro_rootfs(master, root_url)
+ _deploy_linaro_bootfs(master, boot_url)
+ except:
+ logging.error("Deployment failed")
+ tb = traceback.format_exc()
+ self.sio.write(tb)
+ raise CriticalError("Deployment failed")
+
+ def _format_testpartition(self, runner, fstype):
+ logging.info("Format testboot and testrootfs partitions")
+ runner.run('umount /dev/disk/by-label/testrootfs', failok=True)
+ runner.run('mkfs -t %s -q /dev/disk/by-label/testrootfs -L testrootfs'
+ % fstype, timeout=1800)
+ runner.run('umount /dev/disk/by-label/testboot', failok=True)
+ runner.run('mkfs.vfat /dev/disk/by-label/testboot -n testboot')
+
+ def _generate_tarballs(self, image_file):
+ self._customize_ubuntu(image_file)
+ boot_tgz = os.path.join(self.scratch_dir, "boot.tgz")
+ root_tgz = os.path.join(self.scratch_dir, "root.tgz")
+ try:
+ _extract_partition(image_file, self.config.boot_part, boot_tgz)
+ _extract_partition(image_file, self.config.root_part, root_tgz)
+ except:
+ logging.error("Failed to generate tarballs")
+ tb = traceback.format_exc()
+ self.sio.write(tb)
+ raise
+ return boot_tgz, root_tgz
+
+ def target_extract(self, runner, tar_url, dest, timeout=-1, num_retry=5):
+ decompression_char = ''
+ if tar_url.endswith('.gz') or tar_url.endswith('.tgz'):
+ decompression_char = 'z'
+ elif tar_url.endswith('.bz2'):
+ decompression_char = 'j'
+ else:
+ raise RuntimeError('bad file extension: %s' % tar_url)
+
+ while num_retry > 0:
+ try:
+ runner.run(
+ 'wget --no-check-certificate --no-proxy '
+ '--connect-timeout=30 -S --progress=dot -e dotbytes=2M '
+ '-O- %s | '
+ 'tar --warning=no-timestamp --numeric-owner -C %s -x%sf -'
+ % (tar_url, dest, decompression_char),
+ timeout=timeout)
+ return
+ except (OperationFailed, pexpect.TIMEOUT):
+ logging.warning(("transfering %s failed. %d retry left."
+ % (tar_url, num_retry - 1)))
+
+ if num_retry > 1:
+ # send CTRL C in case wget still hasn't exited.
+ self.proc.sendcontrol("c")
+ self.proc.sendline(
+ "echo 'retry left %s time(s)'" % (num_retry - 1))
+ # And wait a little while.
+ sleep_time = 60
+ logging.info("Wait %d second before retry" % sleep_time)
+ time.sleep(sleep_time)
+ num_retry = num_retry - 1
+
+ raise RuntimeError('extracting %s on target failed' % tar_url)
+
+ @contextlib.contextmanager
+ def file_system(self, partition, directory):
+ logging.info('attempting to access master filesystem %r:%s' %
+ (partition, directory))
+
+ if partition == self.config.boot_part:
+ partition = '/dev/disk/by-label/testboot'
+ elif partition == self.config.root_part:
+ partition = '/dev/disk/by-label/testrootfs'
+ elif partition != self.config.data_part_android_org:
+ raise RuntimeError(
+ 'unknown master image partition(%d)' % partition)
+
+ with self._as_master() as runner:
+ if partition == self.config.data_part_android_org:
+ lbl = _android_data_label(runner)
+ partition = '/dev/disk/by-label/%s' % lbl
+
+ runner.run('mount %s /mnt' % partition)
+ try:
+ targetdir = os.path.join('/mnt/%s' % directory)
+ if not runner.is_file_exist(targetdir):
+ runner.run('mkdir %s' % targetdir)
+
+ runner.run('tar -czf /tmp/fs.tgz -C %s ./' % targetdir)
+ runner.run('cd /tmp') # need to be in same dir as fs.tgz
+ self.proc.sendline('python -m SimpleHTTPServer 0 2>/dev/null')
+ match_id = self.proc.expect([
+ 'Serving HTTP on 0.0.0.0 port (\d+) \.\.',
+ pexpect.EOF, pexpect.TIMEOUT])
+ if match_id != 0:
+ msg = "Unable to start HTTP server on master"
+ logging.error(msg)
+ raise CriticalError(msg)
+ port = self.proc.match.groups()[match_id]
+
+ url = "http://%s:%s/fs.tgz" % (self.master_ip, port)
+ tf = download_with_retry(
+ self.context, self.scratch_dir, url, False)
+
+ tfdir = os.path.join(self.scratch_dir, str(time.time()))
+ try:
+ os.mkdir(tfdir)
+ tar = tarfile.open(tf, 'r:gz')
+ tar.extractall(tfdir)
+ yield tfdir
+
+ finally:
+ tf = os.path.join(self.scratch_dir, 'fs')
+ tf = shutil.make_archive(tf, 'gztar', tfdir)
+ shutil.rmtree(tfdir)
+
+ self.proc.sendcontrol('c') # kill SimpleHTTPServer
+
+ # get the last 2 parts of tf, ie "scratchdir/tf.tgz"
+ tf = '/'.join(tf.split('/')[-2:])
+ url = '%s/%s' % (self.context.config.lava_image_url, tf)
+ self.target_extract(runner, url, targetdir)
+
+ finally:
+ self.proc.sendcontrol('c') # kill SimpleHTTPServer
+ runner.run('umount /mnt')
+
+ def _connect_carefully(self, cmd):
+ retry_count = 0
+ retry_limit = 3
+
+ port_stuck_message = 'Data Buffering Suspended\.'
+ conn_closed_message = 'Connection closed by foreign host\.'
+
+ expectations = {
+ port_stuck_message: 'reset-port',
+ 'Connected\.\r': 'all-good',
+ conn_closed_message: 'retry',
+ pexpect.TIMEOUT: 'all-good',
+ }
+ patterns = []
+ results = []
+ for pattern, result in expectations.items():
+ patterns.append(pattern)
+ results.append(result)
+
+ while retry_count < retry_limit:
+ proc = logging_spawn(cmd, timeout=1200)
+ proc.logfile_read = self.sio
+ #serial can be slow, races do funny things, so increase delay
+ proc.delaybeforesend = 1
+ logging.info('Attempting to connect to device')
+ match = proc.expect(patterns, timeout=10)
+ result = results[match]
+ logging.info('Matched %r which means %s', patterns[match], result)
+ if result == 'retry':
+ proc.close(True)
+ retry_count += 1
+ time.sleep(5)
+ continue
+ elif result == 'all-good':
+ return proc
+ elif result == 'reset-port':
+ reset_port = self.config.reset_port_command
+ if reset_port:
+ logging_system(reset_port)
+ else:
+ raise OperationFailed("no reset_port command configured")
+ proc.close(True)
+ retry_count += 1
+ time.sleep(5)
+ raise OperationFailed("could execute connection_command successfully")
+
+ def _close_logging_spawn(self):
+ self.proc.close(True)
+
+ def boot_master_image(self):
+ """
+ reboot the system, and check that we are in a master shell
+ """
+ logging.info("Boot the system master image")
+ try:
+ self._soft_reboot()
+ self.proc.expect(self.config.image_boot_msg, timeout=300)
+ self._in_master_shell(300)
+ except:
+ logging.exception("in_master_shell failed")
+ self._hard_reboot()
+ self.proc.expect(self.config.image_boot_msg, timeout=300)
+ self._in_master_shell(300)
+ self.proc.sendline('export PS1="$PS1 [rc=$(echo \$?)]: "')
+ self.proc.expect(
+ self.config.master_str, timeout=120, lava_no_logging=1)
+
+ lava_proxy = self.context.config.lava_proxy
+ if lava_proxy:
+ logging.info("Setting up http proxy")
+ self.proc.sendline("export http_proxy=%s" % lava_proxy)
+ self.proc.expect(self.config.master_str, timeout=30)
+ logging.info("System is in master image now")
+
+ def _in_master_shell(self, timeout=10):
+ self.proc.sendline("")
+ match_id = self.proc.expect(
+ [self.config.master_str, pexpect.TIMEOUT],
+ timeout=timeout, lava_no_logging=1)
+ if match_id == 1:
+ raise OperationFailed
+
+ if not self.master_ip:
+ runner = MasterCommandRunner(self)
+ self.master_ip = runner.get_master_ip()
+
+ @contextlib.contextmanager
+ def _as_master(self):
+ """A session that can be used to run commands in the master image.
+
+ Anything that uses this will have to be done differently for images
+ that are not deployed via a master image (e.g. using a JTAG to blow
+ the image onto the card or testing under QEMU).
+ """
+ try:
+ self._in_master_shell()
+ yield MasterCommandRunner(self)
+ except OperationFailed:
+ self.boot_master_image()
+ yield MasterCommandRunner(self)
+
+ def _soft_reboot(self):
+ logging.info("Perform soft reboot the system")
+ self.master_ip = None
+ # make sure in the shell (sometime the earlier command has not exit)
+ self.proc.sendcontrol('c')
+ self.proc.sendline(self.config.soft_boot_cmd)
+ # Looking for reboot messages or if they are missing, the U-Boot
+ # message will also indicate the reboot is done.
+ match_id = self.proc.expect(
+ ['Restarting system.', 'The system is going down for reboot NOW',
+ 'Will now restart', 'U-Boot', pexpect.TIMEOUT], timeout=120)
+ if match_id not in [0, 1, 2, 3]:
+ raise Exception("Soft reboot failed")
+
+ def _hard_reboot(self):
+ logging.info("Perform hard reset on the system")
+ self.master_ip = None
+ if self.config.hard_reset_command != "":
+ logging_system(self.config.hard_reset_command)
+ else:
+ self.proc.send("~$")
+ self.proc.sendline("hardreset")
+ self.proc.empty_buffer()
+
+ def _enter_uboot(self):
+ if self.proc.expect(self.config.interrupt_boot_prompt) != 0:
+ raise Exception("Faile to enter uboot")
+ self.proc.sendline(self.config.interrupt_boot_command)
+
+ def _boot_linaro_image(self):
+ boot_cmds = self.deployment_data['boot_cmds']
+ for option in self.boot_options:
+ keyval = option.split('=')
+ if len(keyval) != 2:
+ logging.warn("Invalid boot option format: %s" % option)
+ elif keyval[0] != 'boot_cmds':
+ logging.warn("Invalid boot option: %s" % keyval[0])
+ else:
+ boot_cmds = keyval[1].strip()
+
+ boot_cmds = getattr(self.config, boot_cmds)
+ self._boot(string_to_list(boot_cmds))
+
+ def _boot(self, boot_cmds):
+ try:
+ self._soft_reboot()
+ self._enter_uboot()
+ except:
+ logging.exception("_enter_uboot failed")
+ self.hard_reboot()
+ self._enter_uboot()
+ self.proc.sendline(boot_cmds[0])
+ for line in range(1, len(boot_cmds)):
+ self.proc.expect(self.config.bootloader_prompt, timeout=300)
+ self.proc.sendline(boot_cmds[line])
+
+
+target_class = MasterImageTarget
+
+
+class MasterCommandRunner(NetworkCommandRunner):
+ """A CommandRunner to use when the board is booted into the master image.
+ """
+
+ def __init__(self, target):
+ super(MasterCommandRunner, self).__init__(
+ target, target.config.master_str)
+
+ def get_master_ip(self):
+ logging.info("Waiting for network to come up")
+ try:
+ self.wait_network_up()
+ except:
+ msg = "Unable to reach LAVA server, check network"
+ logging.error(msg)
+ self._client.sio.write(traceback.format_exc())
+ raise CriticalError(msg)
+
+ pattern1 = "<(\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?\.\d?\d?\d?)>"
+ cmd = ("ifconfig %s | grep 'inet addr' | awk -F: '{print $2}' |"
+ "awk '{print \"<\" $1 \">\"}'" %
+ self._client.config.default_network_interface)
+ self.run(
+ cmd, [pattern1, pexpect.EOF, pexpect.TIMEOUT], timeout=5)
+ if self.match_id != 0:
+ msg = "Unable to determine master image IP address"
+ logging.error(msg)
+ raise CriticalError(msg)
+
+ ip = self.match.group(1)
+ logging.debug("Master image IP is %s" % ip)
+ return ip
+
+ def has_partition_with_label(self, label):
+ if not label:
+ return False
+
+ path = '/dev/disk/by-label/%s' % label
+ return self.is_file_exist(path)
+
+ def is_file_exist(self, path):
+ cmd = 'ls %s' % path
+ rc = self.run(cmd, failok=True)
+ if rc == 0:
+ return True
+ return False
+
+
+def _extract_partition(image, partno, tarfile):
+ """Mount a partition and produce a tarball of it
+
+ :param image: The image to mount
+ :param partno: The index of the partition in the image
+ :param tarfile: path and filename of the tgz to output
+ """
+ with image_partition_mounted(image, partno) as mntdir:
+ cmd = "sudo tar -C %s -czf %s ." % (mntdir, tarfile)
+ rc = logging_system(cmd)
+ if rc:
+ raise RuntimeError("Failed to create tarball: %s" % tarfile)
+
+
+def _deploy_linaro_rootfs(session, rootfs):
+ logging.info("Deploying linaro image")
+ session.run('udevadm trigger')
+ session.run('mkdir -p /mnt/root')
+ session.run('mount /dev/disk/by-label/testrootfs /mnt/root')
+ # The timeout has to be this long for vexpress. For a full desktop it
+ # takes 214 minutes, plus about 25 minutes for the mkfs ext3, add
+ # another hour to err on the side of caution.
+ session._client.target_extract(session, rootfs, '/mnt/root', timeout=18000)
+
+ #DO NOT REMOVE - diverting flash-kernel and linking it to /bin/true
+ #prevents a serious problem where packages getting installed that
+ #call flash-kernel can update the kernel on the master image
+ if session.run('chroot /mnt/root which dpkg-divert', failok=True) == 0:
+ session.run(
+ 'chroot /mnt/root dpkg-divert --local /usr/sbin/flash-kernel')
+ session.run(
+ 'chroot /mnt/root ln -sf /bin/true /usr/sbin/flash-kernel')
+ session.run('umount /mnt/root')
+
+
+def _deploy_linaro_bootfs(session, bootfs):
+ logging.info("Deploying linaro bootfs")
+ session.run('udevadm trigger')
+ session.run('mkdir -p /mnt/boot')
+ session.run('mount /dev/disk/by-label/testboot /mnt/boot')
+ session._client.target_extract(session, bootfs, '/mnt/boot')
+ session.run('umount /mnt/boot')
+
+
+def _deploy_linaro_android_boot(session, boottbz2):
+ logging.info("Deploying test boot filesystem")
+ session.run('mount /dev/disk/by-label/testboot /mnt/lava/boot')
+ session._client.target_extract(session, boottbz2, '/mnt/lava')
+ _recreate_uInitrd(session)
+
+
+def _update_uInitrd_partitions(session, rc_filename):
+ # Original android sdcard partition layout by l-a-m-c
+ sys_part_org = session._client.config.sys_part_android_org
+ cache_part_org = session._client.config.cache_part_android_org
+ data_part_org = session._client.config.data_part_android_org
+ # Sdcard layout in Lava image
+ sys_part_lava = session._client.config.sys_part_android
+ data_part_lava = session._client.config.data_part_android
+
+ session.run(
+ 'sed -i "/mount ext4 \/dev\/block\/mmcblk0p%s/d" %s'
+ % (cache_part_org, rc_filename), failok=True)
+
+ session.run('sed -i "s/mmcblk0p%s/mmcblk0p%s/g" %s'
+ % (data_part_org, data_part_lava, rc_filename), failok=True)
+ session.run('sed -i "s/mmcblk0p%s/mmcblk0p%s/g" %s'
+ % (sys_part_org, sys_part_lava, rc_filename), failok=True)
+ # for snowball the mcvblk1 is used instead of mmcblk0.
+ session.run('sed -i "s/mmcblk1p%s/mmcblk1p%s/g" %s'
+ % (data_part_org, data_part_lava, rc_filename), failok=True)
+ session.run('sed -i "s/mmcblk1p%s/mmcblk1p%s/g" %s'
+ % (sys_part_org, sys_part_lava, rc_filename), failok=True)
+
+
+def _recreate_uInitrd(session):
+ logging.debug("Recreate uInitrd")
+
+ session.run('mkdir -p ~/tmp/')
+ session.run('mv /mnt/lava/boot/uInitrd ~/tmp')
+ session.run('cd ~/tmp/')
+
+ session.run('dd if=uInitrd of=uInitrd.data ibs=64 skip=1')
+ session.run('mv uInitrd.data ramdisk.cpio.gz')
+ session.run('gzip -d -f ramdisk.cpio.gz; cpio -i -F ramdisk.cpio')
+
+ # The mount partitions have moved from init.rc to init.partitions.rc
+ # For backward compatible with early android build, we update both rc files
+ _update_uInitrd_partitions(session, 'init.rc')
+ _update_uInitrd_partitions(session, 'init.partitions.rc')
+
+ session.run(
+ 'sed -i "/export PATH/a \ \ \ \ export PS1 root@linaro: " init.rc')
+
+ session.run("cat init.rc")
+ session.run("cat init.partitions.rc", failok=True)
+
+ session.run('cpio -i -t -F ramdisk.cpio | cpio -o -H newc | \
+ gzip > ramdisk_new.cpio.gz')
+
+ session.run(
+ 'mkimage -A arm -O linux -T ramdisk -n "Android Ramdisk Image" \
+ -d ramdisk_new.cpio.gz uInitrd')
+
+ session.run('cd -')
+ session.run('mv ~/tmp/uInitrd /mnt/lava/boot/uInitrd')
+ session.run('rm -rf ~/tmp')
+
+
+def _deploy_linaro_android_system(session, systemtbz2):
+ logging.info("Deploying the system filesystem")
+ target = session._client
+
+ session.run('mkdir -p /mnt/lava/system')
+ session.run('mount /dev/disk/by-label/testrootfs /mnt/lava/system')
+ session._client.target_extract(
+ session, systemtbz2, '/mnt/lava', timeout=600)
+
+ if session.has_partition_with_label('userdata') and \
+ session.has_partition_with_label('sdcard') and \
+ session.is_file_exist('/mnt/lava/system/etc/vold.fstab'):
+ # If there is no userdata partition on the sdcard(like iMX and Origen),
+ # then the sdcard partition will be used as the userdata partition as
+ # before, and so cannot be used here as the sdcard on android
+ original = 'dev_mount sdcard /mnt/sdcard %s ' % (
+ target.config.sdcard_part_android_org)
+ replacement = 'dev_mount sdcard /mnt/sdcard %s ' % (
+ target.sdcard_part_lava)
+ sed_cmd = "s@{original}@{replacement}@".format(original=original,
+ replacement=replacement)
+ session.run(
+ 'sed -i "%s" /mnt/lava/system/etc/vold.fstab' % sed_cmd,
+ failok=True)
+ session.run("cat /mnt/lava/system/etc/vold.fstab", failok=True)
+
+ script_path = '%s/%s' % ('/mnt/lava', '/system/bin/disablesuspend.sh')
+ if not session.is_file_exist(script_path):
+ session.run("sh -c 'export http_proxy=%s'" %
+ target.context.config.lava_proxy)
+ session.run('wget --no-check-certificate %s -O %s' %
+ (target.config.git_url_disablesuspend_sh, script_path))
+ session.run('chmod +x %s' % script_path)
+ session.run('chown :2000 %s' % script_path)
+
+ session.run(
+ 'sed -i "s/^PS1=.*$/PS1=\'root@linaro: \'/" '
+ '/mnt/lava/system/etc/mkshrc',
+ failok=True)
+
+ session.run('umount /mnt/lava/system')
+
+
+def _purge_linaro_android_sdcard(session):
+ logging.info("Reformatting Linaro Android sdcard filesystem")
+ session.run('mkfs.vfat /dev/disk/by-label/sdcard -n sdcard')
+ session.run('udevadm trigger')
+
+
+def _android_data_label(session):
+ data_label = 'userdata'
+ if not session.has_partition_with_label(data_label):
+ #consider the compatiblity, here use the existed sdcard partition
+ data_label = 'sdcard'
+ return data_label
+
+
+def _deploy_linaro_android_data(session, datatbz2):
+ data_label = _android_data_label(session)
+ session.run('umount /dev/disk/by-label/%s' % data_label, failok=True)
+ session.run('mkfs.ext4 -q /dev/disk/by-label/%s -L %s' %
+ (data_label, data_label))
+ session.run('udevadm trigger')
+ session.run('mkdir -p /mnt/lava/data')
+ session.run('mount /dev/disk/by-label/%s /mnt/lava/data' % (data_label))
+ session._client.target_extract(session, datatbz2, '/mnt/lava', timeout=600)
+ session.run('umount /mnt/lava/data')
=== added file 'lava_dispatcher/device/qemu.py'
@@ -0,0 +1,86 @@
+# Copyright (C) 2011 Linaro Limited
+#
+# Author: Michael Hudson-Doyle <michael.hudson@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+import contextlib
+import logging
+
+from lava_dispatcher.device.target import (
+ Target
+)
+from lava_dispatcher.client.lmc_utils import (
+ generate_image,
+ image_partition_mounted,
+ )
+from lava_dispatcher.downloader import (
+ download_image,
+ )
+from lava_dispatcher.utils import (
+ ensure_directory,
+ logging_spawn,
+ )
+
+
+class QEMUTarget(Target):
+
+ def __init__(self, context, config):
+ super(QEMUTarget, self).__init__(context, config)
+ self._sd_image = None
+
+ def _customize_ubuntu(self):
+ root_part = self.config.root_part
+ with image_partition_mounted(self._sd_image, root_part) as mnt:
+ with open('%s/etc/hostname' % mnt, 'w') as f:
+ f.write('%s\n' % self.config.tester_hostname)
+ self.deployment_data = Target.ubuntu_deployment_data
+
+ def deploy_linaro(self, hwpack=None, rootfs=None):
+ odir = self.scratch_dir
+ self._sd_image = generate_image(self, hwpack, rootfs, odir)
+ self._customize_ubuntu()
+
+ def deploy_linaro_prebuilt(self, image):
+ self._sd_image = download_image(image, self.context)
+ self._customize_ubuntu()
+
+ @contextlib.contextmanager
+ def file_system(self, partition, directory):
+ with image_partition_mounted(self._sd_image, partition) as mntdir:
+ path = '%s/%s' % (mntdir, directory)
+ ensure_directory(path)
+ yield path
+
+ def power_off(self, proc):
+ if proc is not None:
+ proc.close()
+
+ def power_on(self):
+ qemu_cmd = ('%s -M %s -drive if=%s,cache=writeback,file=%s '
+ '-clock unix -device usb-kbd -device usb-mouse -usb '
+ '-device usb-net,netdev=mynet -netdev user,id=mynet '
+ '-net nic -net user -nographic') % (
+ self.context.config.default_qemu_binary,
+ self.config.qemu_machine_type,
+ self.config.qemu_drive_interface,
+ self._sd_image)
+ logging.info('launching qemu with command %r' % qemu_cmd)
+ proc = logging_spawn(qemu_cmd, logfile=self.sio, timeout=None)
+ return proc
+
+target_class = QEMUTarget
=== added file 'lava_dispatcher/device/target.py'
@@ -0,0 +1,133 @@
+# Copyright (C) 2012 Linaro Limited
+#
+# Author: Andy Doan <andy.doan@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+import contextlib
+import logging
+import sys
+
+import lava_dispatcher.utils as utils
+
+from cStringIO import StringIO
+
+
+def get_target(context, device_config):
+ ipath = 'lava_dispatcher.device.%s' % device_config.client_type
+ m = __import__(ipath, fromlist=[ipath])
+ return m.target_class(context, device_config)
+
+
+class Target(object):
+ """ Defines the contract needed by the dispatcher for dealing with a
+ target device
+ """
+
+ # The target deployment functions will point self.deployment_data to
+ # the appropriate dictionary below. Code such as actions can contribute
+ # to these structures with special handling logic
+ android_deployment_data = {}
+ ubuntu_deployment_data = {}
+
+ def __init__(self, context, device_config):
+ self.context = context
+ self.config = device_config
+ self.deployment_data = None
+ self.sio = SerialIO(sys.stdout)
+
+ self.boot_options = []
+ self.scratch_dir = utils.mkdtemp(context.config.lava_image_tmpdir)
+ self.deployment_data = {}
+
+ def power_on(self):
+ """ responsible for powering on the target device and returning an
+ instance of a pexpect session
+ """
+ raise NotImplementedError('power_on')
+
+ def power_off(self, proc):
+ """ responsible for powering off the target device
+ """
+ raise NotImplementedError('power_off')
+
+ def deploy_linaro(self, hwpack, rfs):
+ raise NotImplementedError('deploy_image')
+
+ def deploy_android(self, boot, system, userdata):
+ raise NotImplementedError('deploy_android_image')
+
+ def deploy_linaro_prebuilt(self, image):
+ raise NotImplementedError('deploy_linaro_prebuilt')
+
+ @contextlib.contextmanager
+ def file_system(self, partition, directory):
+ """ Allows the caller to interact directly with a directory on
+ the target. This method yields a directory where the caller can
+ interact from. Upon the exit of this context, the changes will be
+ applied to the target.
+
+ The partition parameter refers to partition number the directory
+ would reside in as created by linaro-media-create. ie - the boot
+ partition would be 1. In the case of something like the master
+ image, the target implementation must map this number to the actual
+ partition its using.
+
+ NOTE: due to difference in target implementations, the caller should
+ try and interact with the smallest directory locations possible.
+ """
+ raise NotImplementedError('file_system')
+
+ @contextlib.contextmanager
+ def runner(self):
+ """ Powers on the target, returning a CommandRunner object and will
+ power off the target when the context is exited
+ """
+ proc = runner = None
+ try:
+ proc = self.power_on()
+ from lava_dispatcher.client.base import CommandRunner
+ runner = CommandRunner(proc, self.config.tester_str)
+ yield runner
+ finally:
+ if proc:
+ logging.info('attempting a filesystem sync before power_off')
+ runner.run('sync', timeout=20)
+ self.power_off(proc)
+
+ def get_test_data_attachments(self):
+ return []
+
+
+class SerialIO(file):
+ def __init__(self, logfile):
+ self.serialio = StringIO()
+ self.logfile = logfile
+
+ def write(self, text):
+ self.serialio.write(text)
+ self.logfile.write(text)
+
+ def close(self):
+ self.serialio.close()
+ self.logfile.close()
+
+ def flush(self):
+ self.logfile.flush()
+
+ def getvalue(self):
+ return self.serialio.getvalue()
=== modified file 'lava_dispatcher/downloader.py'
@@ -26,12 +26,15 @@
import re
import shutil
import subprocess
+import time
+import traceback
import urllib2
import urlparse
import zlib
from tempfile import mkdtemp
+
@contextlib.contextmanager
def _scp_stream(url, proxy=None, cookies=None):
process = None
@@ -46,6 +49,7 @@
if process:
process.kill()
+
@contextlib.contextmanager
def _http_stream(url, proxy=None, cookies=None):
resp = None
@@ -65,6 +69,7 @@
if resp:
resp.close()
+
@contextlib.contextmanager
def _file_stream(url, proxy=None, cookies=None):
fd = None
@@ -75,6 +80,7 @@
if fd:
fd.close()
+
@contextlib.contextmanager
def _decompressor_stream(url, imgdir, decompress):
fd = None
@@ -101,6 +107,7 @@
if fd:
fd.close
+
def _url_to_fname_suffix(url, path='/tmp'):
filename = os.path.basename(url.path)
parts = filename.split('.')
@@ -108,6 +115,7 @@
filename = os.path.join(path, '.'.join(parts[:-1]))
return (filename, suffix)
+
def _url_mapping(url, context):
'''allows the downloader to override a URL so that something like:
http://blah/ becomes file://localhost/blah
@@ -126,6 +134,7 @@
logging.info('url mapped to: %s', url)
return url
+
def download_image(url, context, imgdir=None,
delete_on_exit=True, decompress=True):
'''downloads a image that's been compressed as .bz2 or .gz and
@@ -140,7 +149,6 @@
url = _url_mapping(url, context)
url = urlparse.urlparse(url)
- stream = None
if url.scheme == 'scp':
reader = _scp_stream
elif url.scheme == 'http' or url.scheme == 'https':
@@ -160,3 +168,24 @@
buff = r.read(bsize)
return fname
+
+def download_with_retry(context, imgdir, url, decompress=True, timeout=300):
+ '''
+ download test result with a retry mechanism and 5 minute default timeout
+ '''
+ logging.info("About to download %s to the host" % url)
+ now = time.time()
+ tries = 0
+
+ while True:
+ try:
+ return download_image(url, context, imgdir, decompress)
+ except:
+ logging.warn("unable to download: %r" % traceback.format_exc())
+ tries += 1
+ if time.time() >= now + timeout:
+ raise RuntimeError(
+ 'downloading %s failed after %d tries' % (url, tries))
+ else:
+ logging.info('Sleep one minute and retry (%d)' % tries)
+ time.sleep(60)
=== added file 'lava_dispatcher/tarballcache.py'
@@ -0,0 +1,88 @@
+# Copyright (C) 2012 Linaro Limited
+#
+# Author: Andy Doan <andy.doan@linaro.org>
+#
+# This file is part of LAVA Dispatcher.
+#
+# LAVA Dispatcher is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# LAVA Dispatcher is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along
+# with this program; if not, see <http://www.gnu.org/licenses>.
+
+import contextlib
+import errno
+import fcntl
+import logging
+import os
+
+import lava_dispatcher.utils as utils
+
+from lava_dispatcher.downloader import (
+ download_image,
+ )
+
+
+def get_tarballs(context, image_url, scratch_dir, generator):
+ '''
+ Tries to return a cached copy array of (boot_tgz, root_tgz). If no cache
+ exists for this image_url, then it:
+ * places a global lock for the image_url to prevent other dispatchers
+ from concurrently building tarballs for the same image
+ * downloads the image
+ * calls the generator function to build the tarballs
+
+ generator - a callback to a function that can generate the tarballs given
+ a local copy .img file
+ '''
+ logging.info('try to find cached tarballs for %s' % image_url)
+ with _cache_locked(image_url, context.config.lava_cachedir) as cachedir:
+ boot_tgz = os.path.join(cachedir, 'boot.tgz')
+ root_tgz = os.path.join(cachedir, 'root.tgz')
+
+ if os.path.exists(boot_tgz) and os.path.exists(root_tgz):
+ logging.info('returning cached copies')
+ (boot_tgz, root_tgz) = _link(boot_tgz, root_tgz, scratch_dir)
+ return (boot_tgz, root_tgz)
+
+ logging.info('no cache found for %s' % image_url)
+ image = download_image(image_url, context, cachedir)
+ (boot_tgz, root_tgz) = generator(image)
+ _link(boot_tgz, root_tgz, cachedir)
+ os.unlink(image)
+ return (boot_tgz, root_tgz)
+
+
+def _link(boot_tgz, root_tgz, destdir):
+ dboot_tgz = os.path.join(destdir, 'boot.tgz')
+ droot_tgz = os.path.join(destdir, 'root.tgz')
+ os.link(boot_tgz, dboot_tgz)
+ os.link(root_tgz, droot_tgz)
+ return (dboot_tgz, droot_tgz)
+
+
+@contextlib.contextmanager
+def _cache_locked(image_url, cachedir):
+ cachedir = utils.url_to_cache(image_url, cachedir).replace('.', '-')
+ try:
+ os.makedirs(cachedir)
+ except OSError as e:
+ if e.errno != errno.EEXIST: # directory may already exist and is okay
+ raise
+
+ lockfile = os.path.join(cachedir, 'lockfile')
+ with open(lockfile, 'w') as f:
+ logging.info('aquiring lock for %s' % lockfile)
+ try:
+ fcntl.lockf(f, fcntl.LOCK_EX)
+ yield cachedir
+ finally:
+ fcntl.lockf(f, fcntl.LOCK_UN)
=== modified file 'lava_dispatcher/utils.py'
@@ -18,6 +18,7 @@
# along
# with this program; if not, see <http://www.gnu.org/licenses>.
+import atexit
import datetime
import errno
import logging
@@ -25,11 +26,13 @@
import select
import sys
import shutil
+import tempfile
import urlparse
from shlex import shlex
import pexpect
+
def link_or_copy_file(src, dest):
try:
dir = os.path.dirname(dest)
@@ -44,6 +47,7 @@
else:
logging.exception("os.link '%s' with '%s' failed" % (src, dest))
+
def copy_file(src, dest):
dir = os.path.dirname(dest)
if not os.path.exists(dir):
@@ -51,6 +55,23 @@
shutil.copy(src, dest)
+def mkdtemp(basedir='/tmp'):
+ ''' returns a temporary directory that's deleted when the process exits
+ '''
+
+ d = tempfile.mkdtemp(dir=basedir)
+ atexit.register(shutil.rmtree, d)
+ os.chmod(d, 0755)
+ return d
+
+
+def ensure_directory(path):
+ ''' ensures the path exists, if it doesn't it will be created
+ '''
+ if not os.path.exists(path):
+ os.mkdir(path)
+
+
def url_to_cache(url, cachedir):
url_parts = urlparse.urlsplit(url)
path = os.path.join(cachedir, url_parts.netloc,
@@ -100,6 +121,13 @@
return super(logging_spawn, self).expect(*args, **kw)
+ def empty_buffer(self):
+ """Make sure there is nothing in the pexpect buffer."""
+ index = 0
+ while index == 0:
+ index = self.expect(
+ ['.+', pexpect.EOF, pexpect.TIMEOUT], timeout=1, lava_no_logging=1)
+
def drain(self):
"""this is a one-off of the pexect __interact that ignores STDIN and
handles an error that happens when we call read just after the process exits
@@ -115,6 +143,7 @@
logging.warn("error while draining pexpect buffers: %r", einfo)
pass
+
# XXX Duplication: we should reuse lava-test TestArtifacts
def generate_bundle_file_name(test_name):
return ("{test_id}.{time.tm_year:04}-{time.tm_mon:02}-{time.tm_mday:02}T"