From patchwork Mon Apr 4 21:26:14 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Lo=C3=AFc_Minier?= X-Patchwork-Id: 900 Return-Path: Delivered-To: unknown Received: from imap.gmail.com (74.125.159.109) by localhost6.localdomain6 with IMAP4-SSL; 08 Jun 2011 14:47:12 -0000 Delivered-To: patches@linaro.org Received: by 10.68.42.132 with SMTP id o4cs77556pbl; Mon, 4 Apr 2011 14:26:19 -0700 (PDT) Received: by 10.227.159.204 with SMTP id k12mr3053895wbx.162.1301952376533; Mon, 04 Apr 2011 14:26:16 -0700 (PDT) Received: from adelie.canonical.com (adelie.canonical.com [91.189.90.139]) by mx.google.com with ESMTP id b13si10794829wbi.108.2011.04.04.14.26.15; Mon, 04 Apr 2011 14:26:16 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) client-ip=91.189.90.139; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.139 as permitted sender) smtp.mail=bounces@canonical.com Received: from loganberry.canonical.com ([91.189.90.37]) by adelie.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1Q6rHq-0003GI-Bz for ; Mon, 04 Apr 2011 21:26:14 +0000 Received: from loganberry.canonical.com (localhost [127.0.0.1]) by loganberry.canonical.com (Postfix) with ESMTP id 56BA22E84FB for ; Mon, 4 Apr 2011 21:26:14 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: linaro-image-tools X-Launchpad-Branch: ~linaro-maintainers/linaro-image-tools/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 310 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-maintainers/linaro-image-tools/trunk] Rev 310: Merge lp:~lool/linaro-image-tools/btrfs-support; various improvements to the Message-Id: <20110404212614.13543.91336.launchpad@loganberry.canonical.com> Date: Mon, 04 Apr 2011 21:26:14 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="12734"; Instance="initZopeless config overlay" X-Launchpad-Hash: 02efdb1bb2371db58f86e594d97700c92ed9be80 Merge authors: Loïc Minier (lool) Related merge proposals: https://code.launchpad.net/~lool/linaro-image-tools/btrfs-support/+merge/56165 proposed by: Loïc Minier (lool) review: Approve - Guilherme Salgado (salgado) ------------------------------------------------------------ revno: 310 [merge] fixes bug(s): https://launchpad.net/bugs/715932 https://launchpad.net/bugs/745728 https://launchpad.net/bugs/745771 committer: Loïc Minier branch nick: linaro-image-tools timestamp: Mon 2011-04-04 23:23:58 +0200 message: Merge lp:~lool/linaro-image-tools/btrfs-support; various improvements to the btrfs support and related fixes: - set proper fstab mount options; LP: #745771 - install btrfs-tools if missing; LP #715932 - fix missing final newline in fstab; LP: #745728 renamed: linaro_image_tools/media_create/hwpack.py => linaro_image_tools/media_create/chroot_utils.py modified: linaro-media-create linaro_image_tools/cmd_runner.py linaro_image_tools/media_create/rootfs.py linaro_image_tools/media_create/tests/test_media_create.py linaro_image_tools/tests/fixtures.py linaro_image_tools/tests/test_cmd_runner.py linaro_image_tools/media_create/chroot_utils.py --- lp:linaro-image-tools https://code.launchpad.net/~linaro-maintainers/linaro-image-tools/trunk You are subscribed to branch lp:linaro-image-tools. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-maintainers/linaro-image-tools/trunk/+edit-subscription === modified file 'linaro-media-create' --- linaro-media-create 2011-03-24 18:41:59 +0000 +++ linaro-media-create 2011-04-04 13:17:56 +0000 @@ -28,7 +28,10 @@ from linaro_image_tools.media_create.boards import board_configs from linaro_image_tools.media_create.check_device import ( confirm_device_selection_and_ensure_it_is_ready) -from linaro_image_tools.media_create.hwpack import install_hwpacks +from linaro_image_tools.media_create.chroot_utils import ( + install_hwpacks, + install_packages, + ) from linaro_image_tools.media_create.partitions import ( Media, setup_partitions, @@ -78,10 +81,10 @@ if not is_arm_host(): required_commands.append('qemu-arm-static') required_commands.append('qemu-img') - if args.rootfs in ['ext2', 'ext3', 'ext4']: + if args.rootfs in ['btrfs', 'ext2', 'ext3', 'ext4']: required_commands.append('mkfs.%s' % args.rootfs) else: - required_commands.append('mkfs.btrfs') + raise AssertionError('Unsupported rootfs type %s' % args.rootfs) for command in required_commands: ensure_command(command) @@ -122,6 +125,9 @@ install_hwpacks( ROOTFS_DIR, TMP_DIR, lmc_dir, args.hwpack_force_yes, *hwpacks) + if args.rootfs == 'btrfs': + install_packages(ROOTFS_DIR, TMP_DIR, "btrfs-tools") + boot_partition, root_partition = setup_partitions( board_config, media, args.image_size, args.boot_label, args.rfs_label, args.rootfs, args.should_create_partitions, args.should_format_bootfs, === modified file 'linaro_image_tools/cmd_runner.py' --- linaro_image_tools/cmd_runner.py 2011-03-24 22:12:56 +0000 +++ linaro_image_tools/cmd_runner.py 2011-04-04 10:38:07 +0000 @@ -22,6 +22,7 @@ DEFAULT_PATH = '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' +CHROOT_ARGS = ['chroot'] SUDO_ARGS = ['sudo', '-E'] @@ -34,7 +35,8 @@ env['PATH'] = os.pathsep.join(dirs) -def run(args, as_root=False, stdin=None, stdout=None, stderr=None): +def run(args, as_root=False, chroot=None, stdin=None, stdout=None, + stderr=None): """Run the given command as a sub process. Return a Popen instance. @@ -44,6 +46,7 @@ :param command: A list or tuple containing the command to run and the arguments that should be passed to it. :param as_root: Should the given command be run as root (with sudo)? + :param chroot: A directory to chroot into (implies as_root). :param stdin: Same as in subprocess.Popen(). :param stdout: Same as in subprocess.Popen(). :param stderr: Same as in subprocess.Popen(). @@ -52,6 +55,9 @@ "The command to run must be a list or tuple, found: %s" % type(args)) if isinstance(args, tuple): args = list(args) + if chroot is not None: + args = CHROOT_ARGS + [chroot] + args + as_root = True if as_root and os.getuid() != 0: args = SUDO_ARGS + args return Popen(args, stdin=stdin, stdout=stdout, stderr=stderr) === renamed file 'linaro_image_tools/media_create/hwpack.py' => 'linaro_image_tools/media_create/chroot_utils.py' --- linaro_image_tools/media_create/hwpack.py 2011-03-24 10:10:38 +0000 +++ linaro_image_tools/media_create/chroot_utils.py 2011-04-04 21:20:09 +0000 @@ -32,10 +32,8 @@ # functions would only be called after l-m-c.py exits. local_atexit = [] -def install_hwpacks( - chroot_dir, tmp_dir, tools_dir, hwpack_force_yes, *hwpack_files): - """Install the given hwpacks onto the given chroot.""" - +def prepare_chroot(chroot_dir, tmp_dir): + """Prepares a chroot to run commands in it (networking and QEMU setup).""" chroot_etc = os.path.join(chroot_dir, 'etc') temporarily_overwrite_file_on_dir('/etc/resolv.conf', chroot_etc, tmp_dir) temporarily_overwrite_file_on_dir('/etc/hosts', chroot_etc, tmp_dir) @@ -44,6 +42,11 @@ copy_file('/usr/bin/qemu-arm-static', os.path.join(chroot_dir, 'usr', 'bin')) +def install_hwpacks( + chroot_dir, tmp_dir, tools_dir, hwpack_force_yes, *hwpack_files): + """Install the given hwpacks onto the given chroot.""" + prepare_chroot(chroot_dir, tmp_dir) + linaro_hwpack_install_path = find_command( 'linaro-hwpack-install', prefer_dir=tools_dir) # FIXME: shouldn't use chroot/usr/bin as this might conflict with installed @@ -70,15 +73,36 @@ hwpack_basename = os.path.basename(hwpack_file) copy_file(hwpack_file, chroot_dir) print "-" * 60 - print "Installing (apt-get) %s in target rootfs." % hwpack_basename - args = ['chroot', chroot_dir, 'linaro-hwpack-install'] + print "Installing (linaro-hwpack-install) %s in target rootfs." % ( + hwpack_basename) + args = ['linaro-hwpack-install'] if hwpack_force_yes: args.append('--force-yes') args.append('/%s' % hwpack_basename) - cmd_runner.run(args, as_root=True).wait() + cmd_runner.run(args, as_root=True, chroot=chroot_dir).wait() print "-" * 60 +def install_packages(chroot_dir, tmp_dir, *packages): + """Install packages in the given chroot. + + This does not run apt-get update before hand.""" + prepare_chroot(chroot_dir, tmp_dir) + + try: + mount_chroot_proc(chroot_dir) + print "-" * 60 + print "Installing (apt-get) %s in target rootfs." % " ".join(packages) + args = ("apt-get", "--yes", "install") + packages + cmd_runner.run(args, as_root=True, chroot=chroot_dir).wait() + print "Cleaning up downloaded packages." + args = ("apt-get", "clean") + cmd_runner.run(args, as_root=True, chroot=chroot_dir).wait() + print "-" * 60 + finally: + run_local_atexit_funcs() + + def mount_chroot_proc(chroot_dir): """Mount a /proc filesystem on the given chroot. === modified file 'linaro_image_tools/media_create/rootfs.py' --- linaro_image_tools/media_create/rootfs.py 2011-03-24 10:10:38 +0000 +++ linaro_image_tools/media_create/rootfs.py 2011-04-04 14:00:26 +0000 @@ -24,6 +24,15 @@ from linaro_image_tools import cmd_runner +def rootfs_mount_options(rootfs_type): + """Return mount options for the specific rootfs type.""" + if rootfs_type == "btrfs": + return "defaults" + if rootfs_type in ('ext2', 'ext3', 'ext4'): + return "errors=remount-ro" + raise ValueError('Unsupported rootfs type') + + def populate_rootfs(content_dir, root_disk, partition, rootfs_type, rootfs_uuid, should_create_swap, swap_size, partition_offset): @@ -47,8 +56,9 @@ move_contents(content_dir, root_disk) - fstab_additions = ["UUID=%s / %s errors=remount-ro 0 1 " % ( - rootfs_uuid, rootfs_type)] + mount_options = rootfs_mount_options(rootfs_type) + fstab_additions = ["UUID=%s / %s %s 0 1" % ( + rootfs_uuid, rootfs_type, mount_options)] if should_create_swap: print "\nCreating SWAP File\n" if has_space_left_for_swap(root_disk, swap_size): @@ -115,7 +125,7 @@ def append_to_fstab(root_disk, fstab_additions): fstab = os.path.join(root_disk, 'etc', 'fstab') - data = open(fstab).read() + '\n' + '\n'.join(fstab_additions) + data = open(fstab).read() + '\n' + '\n'.join(fstab_additions) + '\n' write_data_to_protected_file(fstab, data) === modified file 'linaro_image_tools/media_create/tests/test_media_create.py' --- linaro_image_tools/media_create/tests/test_media_create.py 2011-04-04 20:13:55 +0000 +++ linaro_image_tools/media_create/tests/test_media_create.py 2011-04-04 21:23:58 +0000 @@ -54,11 +54,13 @@ _get_mlo_file, _run_mkimage, ) -from linaro_image_tools.media_create.hwpack import ( +from linaro_image_tools.media_create.chroot_utils import ( copy_file, install_hwpack, install_hwpacks, + install_packages, mount_chroot_proc, + prepare_chroot, run_local_atexit_funcs, temporarily_overwrite_file_on_dir, ) @@ -78,10 +80,12 @@ _parse_blkid_output, ) from linaro_image_tools.media_create.rootfs import ( + append_to_fstab, create_flash_kernel_config, has_space_left_for_swap, move_contents, populate_rootfs, + rootfs_mount_options, write_data_to_protected_file, ) from linaro_image_tools.media_create.tests.fixtures import ( @@ -100,6 +104,7 @@ from linaro_image_tools.utils import find_command, preferred_tools_dir +chroot_args = " ".join(cmd_runner.CHROOT_ARGS) sudo_args = " ".join(cmd_runner.SUDO_ARGS) @@ -1035,7 +1040,7 @@ swap_size=100, partition_offset=0) self.assertEqual( - ['UUID=uuid / ext3 errors=remount-ro 0 1 ', + ['UUID=uuid / ext3 errors=remount-ro 0 1', '/SWAP.swap none swap sw 0 0'], self.lines_added_to_fstab) self.assertEqual(True, self.create_flash_kernel_config_called) @@ -1111,6 +1116,27 @@ fixture.mock.commands_executed) self.assertEqual(data, open(tmpfile).read()) + def test_rootfs_mount_options_for_btrfs(self): + self.assertEqual("defaults", rootfs_mount_options('btrfs')) + + def test_rootfs_mount_options_for_ext4(self): + self.assertEqual("errors=remount-ro", rootfs_mount_options('ext4')) + + def test_rootfs_mount_options_for_unknown(self): + self.assertRaises(ValueError, rootfs_mount_options, 'unknown') + + def test_append_to_fstab(self): + tempdir = self.useFixture(CreateTempDirFixture()).get_temp_dir() + etc = os.path.join(tempdir, 'etc') + os.mkdir(etc) + fstab = os.path.join(etc, 'fstab') + open(fstab, "w").close() + append_to_fstab(tempdir, ['foo', 'bar']) + f = open(fstab) + contents = f.read() + f.close() + self.assertEquals("\nfoo\nbar\n", contents) + class TestCheckDevice(TestCaseWithFixtures): @@ -1191,6 +1217,13 @@ class TestInstallHWPack(TestCaseWithFixtures): + def mock_prepare_chroot(self, chroot_dir, tmp_dir): + def fake_prepare_chroot(chroot_dir, tmp_dir): + cmd_runner.run(['prepare_chroot %s %s' % (chroot_dir, tmp_dir)], + as_root=True).wait() + self.useFixture(MockSomethingFixture( + linaro_image_tools.media_create.chroot_utils, 'prepare_chroot', + fake_prepare_chroot)) def test_temporarily_overwrite_file_on_dir(self): fixture = self.useFixture(MockCmdRunnerPopenFixture()) @@ -1236,49 +1269,92 @@ self.useFixture(MockSomethingFixture( sys, 'stdout', open('/dev/null', 'w'))) fixture = self.useFixture(MockCmdRunnerPopenFixture()) + chroot_dir = 'chroot_dir' force_yes = False - install_hwpack('chroot', 'hwpack.tgz', force_yes) + install_hwpack(chroot_dir, 'hwpack.tgz', force_yes) self.assertEquals( - ['%s cp hwpack.tgz chroot' % sudo_args, - '%s chroot chroot linaro-hwpack-install /hwpack.tgz' - % sudo_args], + ['%s cp hwpack.tgz %s' % (sudo_args, chroot_dir), + '%s %s %s linaro-hwpack-install /hwpack.tgz' + % (sudo_args, chroot_args, chroot_dir)], fixture.mock.commands_executed) fixture.mock.calls = [] run_local_atexit_funcs() self.assertEquals( - ['%s rm -f chroot/hwpack.tgz' % sudo_args], + ['%s rm -f %s/hwpack.tgz' % (sudo_args, chroot_dir)], fixture.mock.commands_executed) def test_install_hwpacks(self): self.useFixture(MockSomethingFixture( sys, 'stdout', open('/dev/null', 'w'))) fixture = self.useFixture(MockCmdRunnerPopenFixture()) + chroot_dir = 'chroot_dir' + tmp_dir = 'tmp_dir' + self.mock_prepare_chroot(chroot_dir, tmp_dir) force_yes = True prefer_dir = preferred_tools_dir() install_hwpacks( - 'chroot', '/tmp/dir', prefer_dir, force_yes, 'hwpack1.tgz', + chroot_dir, tmp_dir, prefer_dir, force_yes, 'hwpack1.tgz', 'hwpack2.tgz') linaro_hwpack_install = find_command( 'linaro-hwpack-install', prefer_dir=prefer_dir) expected = [ + 'prepare_chroot %(chroot_dir)s %(tmp_dir)s', + 'cp %(linaro_hwpack_install)s %(chroot_dir)s/usr/bin', + 'mount proc %(chroot_dir)s/proc -t proc', + 'cp hwpack1.tgz %(chroot_dir)s', + ('%(chroot_args)s %(chroot_dir)s linaro-hwpack-install ' + '--force-yes /hwpack1.tgz'), + 'cp hwpack2.tgz %(chroot_dir)s', + ('%(chroot_args)s %(chroot_dir)s linaro-hwpack-install ' + '--force-yes /hwpack2.tgz'), + 'rm -f %(chroot_dir)s/hwpack2.tgz', + 'rm -f %(chroot_dir)s/hwpack1.tgz', + 'umount -v %(chroot_dir)s/proc', + 'rm -f %(chroot_dir)s/usr/bin/linaro-hwpack-install'] + keywords = dict( + chroot_dir=chroot_dir, tmp_dir=tmp_dir, chroot_args=chroot_args, + linaro_hwpack_install=linaro_hwpack_install) + expected = [ + "%s %s" % (sudo_args, line % keywords) for line in expected] + self.assertEquals(expected, fixture.mock.commands_executed) + + def test_install_packages(self): + self.useFixture(MockSomethingFixture( + sys, 'stdout', open('/dev/null', 'w'))) + fixture = self.useFixture(MockCmdRunnerPopenFixture()) + chroot_dir = 'chroot_dir' + tmp_dir = 'tmp_dir' + self.mock_prepare_chroot(chroot_dir, tmp_dir) + + install_packages(chroot_dir, tmp_dir, 'pkg1', 'pkg2') + expected = [ + 'prepare_chroot %(chroot_dir)s %(tmp_dir)s', + 'mount proc %(chroot_dir)s/proc -t proc', + '%(chroot_args)s %(chroot_dir)s apt-get --yes install pkg1 pkg2', + '%(chroot_args)s %(chroot_dir)s apt-get clean', + 'umount -v %(chroot_dir)s/proc'] + keywords = dict( + chroot_dir=chroot_dir, tmp_dir=tmp_dir, chroot_args=chroot_args) + expected = [ + "%s %s" % (sudo_args, line % keywords) for line in expected] + self.assertEquals(expected, fixture.mock.commands_executed) + + def test_prepare_chroot(self): + self.useFixture(MockSomethingFixture( + sys, 'stdout', open('/dev/null', 'w'))) + fixture = self.useFixture(MockCmdRunnerPopenFixture()) + + prepare_chroot('chroot', '/tmp/dir') + run_local_atexit_funcs() + expected = [ 'mv -f chroot/etc/resolv.conf /tmp/dir/resolv.conf', 'cp /etc/resolv.conf chroot/etc', 'mv -f chroot/etc/hosts /tmp/dir/hosts', 'cp /etc/hosts chroot/etc', 'cp /usr/bin/qemu-arm-static chroot/usr/bin', - 'cp %s chroot/usr/bin' % linaro_hwpack_install, - 'mount proc chroot/proc -t proc', - 'cp hwpack1.tgz chroot', - 'chroot chroot linaro-hwpack-install --force-yes /hwpack1.tgz', - 'cp hwpack2.tgz chroot', - 'chroot chroot linaro-hwpack-install --force-yes /hwpack2.tgz', - 'rm -f chroot/hwpack2.tgz', - 'rm -f chroot/hwpack1.tgz', - 'umount -v chroot/proc', - 'rm -f chroot/usr/bin/linaro-hwpack-install', 'rm -f chroot/usr/bin/qemu-arm-static', 'mv -f /tmp/dir/hosts chroot/etc', 'mv -f /tmp/dir/resolv.conf chroot/etc'] @@ -1301,13 +1377,12 @@ # run_local_atexit_funcs() runs the atexit handlers in LIFO order, but # even though the first function called (raising_func) will raise # an exception, the second one will still be called after it. - linaro_image_tools.media_create.hwpack.local_atexit = [ + linaro_image_tools.media_create.chroot_utils.local_atexit = [ behaving_func, raising_func] # run_local_atexit_funcs() also propagates the last exception raised # by one of the functions. - self.assertRaises( - TestException, - linaro_image_tools.media_create.hwpack.run_local_atexit_funcs) + chroot_utils = linaro_image_tools.media_create.chroot_utils + self.assertRaises(TestException, chroot_utils.run_local_atexit_funcs) self.assertEquals( ['raising_func', 'behaving_func'], self.call_order) @@ -1324,10 +1399,11 @@ sys, 'stdout', open('/dev/null', 'w'))) self.useFixture(MockCmdRunnerPopenFixture()) self.useFixture(MockSomethingFixture( - linaro_image_tools.media_create.hwpack, 'install_hwpack', + linaro_image_tools.media_create.chroot_utils, 'install_hwpack', mock_install_hwpack)) self.useFixture(MockSomethingFixture( - linaro_image_tools.media_create.hwpack, 'run_local_atexit_funcs', + linaro_image_tools.media_create.chroot_utils, + 'run_local_atexit_funcs', mock_run_local_atexit_functions)) force_yes = True @@ -1346,5 +1422,5 @@ # Ensure the list of cleanup functions gets cleared to make sure tests # don't interfere with one another. def clear_atexits(): - linaro_image_tools.media_create.hwpack.local_atexit = [] + linaro_image_tools.media_create.chroot_utils.local_atexit = [] self.addCleanup(clear_atexits) === modified file 'linaro_image_tools/tests/fixtures.py' --- linaro_image_tools/tests/fixtures.py 2011-03-24 11:05:42 +0000 +++ linaro_image_tools/tests/fixtures.py 2011-04-04 11:29:01 +0000 @@ -69,6 +69,10 @@ child_finished = True def __call__(self, cmd, *args, **kwargs): + if not self.child_finished: + raise AssertionError( + "You should call wait() or communicate() to ensure " + "the subprocess is finished before proceeding.") self.child_finished = False if self.calls is None: self.calls = [] === modified file 'linaro_image_tools/tests/test_cmd_runner.py' --- linaro_image_tools/tests/test_cmd_runner.py 2011-03-24 22:12:56 +0000 +++ linaro_image_tools/tests/test_cmd_runner.py 2011-04-04 12:57:33 +0000 @@ -28,6 +28,7 @@ sudo_args = " ".join(cmd_runner.SUDO_ARGS) +chroot_args = " ".join(cmd_runner.CHROOT_ARGS) class TestSanitizePath(TestCaseWithFixtures): @@ -82,6 +83,13 @@ self.assertEqual( ['%s foo bar' % sudo_args], fixture.mock.commands_executed) + def test_chrooted(self): + fixture = self.useFixture(MockCmdRunnerPopenFixture()) + cmd_runner.run(['foo', 'bar'], chroot='chroot_dir').wait() + self.assertEqual( + ['%s %s chroot_dir foo bar' % (sudo_args, chroot_args)], + fixture.mock.commands_executed) + def test_run_succeeds_on_zero_return_code(self): proc = cmd_runner.run(['true']) # Need to wait() here as we're using the real Popen.