From patchwork Fri Dec 16 04:27:15 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Paul Larson X-Patchwork-Id: 5806 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id 89DD823E03 for ; Fri, 16 Dec 2011 04:27:17 +0000 (UTC) Received: from mail-ee0-f52.google.com (mail-ee0-f52.google.com [74.125.83.52]) by fiordland.canonical.com (Postfix) with ESMTP id 75001A182AE for ; Fri, 16 Dec 2011 04:27:17 +0000 (UTC) Received: by eeke52 with SMTP id e52so3216065eek.11 for ; Thu, 15 Dec 2011 20:27:17 -0800 (PST) Received: by 10.204.131.74 with SMTP id w10mr2662989bks.36.1324009637094; Thu, 15 Dec 2011 20:27:17 -0800 (PST) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.205.129.2 with SMTP id hg2cs60886bkc; Thu, 15 Dec 2011 20:27:16 -0800 (PST) Received: by 10.180.107.97 with SMTP id hb1mr10104049wib.18.1324009635599; Thu, 15 Dec 2011 20:27:15 -0800 (PST) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id a21si5240598wed.67.2011.12.15.20.27.15 (version=TLSv1/SSLv3 cipher=OTHER); Thu, 15 Dec 2011 20:27:15 -0800 (PST) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1RbPO7-0006ys-Co for ; Fri, 16 Dec 2011 04:27:15 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id 56EB6E01D4 for ; Fri, 16 Dec 2011 04:27:15 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-scheduler X-Launchpad-Branch: ~linaro-validation/lava-scheduler/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 109 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-scheduler/trunk] Rev 109: Add support for device tags in the scheduler Message-Id: <20111216042715.26921.98906.launchpad@ackee.canonical.com> Date: Fri, 16 Dec 2011 04:27:15 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="14523"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: a534a76b70df3505581a0693c1256bf8980369e8 Merge authors: Michael Hudson-Doyle (mwhudson) Related merge proposals: https://code.launchpad.net/~mwhudson/lava-scheduler/device-tags/+merge/85788 proposed by: Michael Hudson-Doyle (mwhudson) ------------------------------------------------------------ revno: 109 [merge] committer: Paul Larson branch nick: lava-scheduler timestamp: Thu 2011-12-15 22:06:06 -0600 message: Add support for device tags in the scheduler added: lava_scheduler_app/migrations/0011_auto__add_tag.py modified: lava_scheduler_app/admin.py lava_scheduler_app/models.py lava_scheduler_app/templates/lava_scheduler_app/device.html lava_scheduler_app/templates/lava_scheduler_app/job_sidebar.html lava_scheduler_app/tests.py lava_scheduler_daemon/dbjobsource.py --- lp:lava-scheduler https://code.launchpad.net/~linaro-validation/lava-scheduler/trunk You are subscribed to branch lp:lava-scheduler. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-scheduler/trunk/+edit-subscription === modified file 'lava_scheduler_app/admin.py' --- lava_scheduler_app/admin.py 2011-06-17 03:55:23 +0000 +++ lava_scheduler_app/admin.py 2011-12-14 01:07:33 +0000 @@ -1,6 +1,7 @@ from django.contrib import admin -from lava_scheduler_app.models import Device, DeviceType, TestJob +from lava_scheduler_app.models import Device, DeviceType, TestJob, Tag admin.site.register(Device) admin.site.register(DeviceType) admin.site.register(TestJob) +admin.site.register(Tag) === added file 'lava_scheduler_app/migrations/0011_auto__add_tag.py' --- lava_scheduler_app/migrations/0011_auto__add_tag.py 1970-01-01 00:00:00 +0000 +++ lava_scheduler_app/migrations/0011_auto__add_tag.py 2011-12-15 03:16:40 +0000 @@ -0,0 +1,122 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding model 'Tag' + db.create_table('lava_scheduler_app_tag', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.SlugField')(unique=True, max_length=50, db_index=True)), + ('description', self.gf('django.db.models.fields.TextField')(null=True, blank=True)), + )) + db.send_create_signal('lava_scheduler_app', ['Tag']) + + # Adding M2M table for field tags on 'Device' + db.create_table('lava_scheduler_app_device_tags', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('device', models.ForeignKey(orm['lava_scheduler_app.device'], null=False)), + ('tag', models.ForeignKey(orm['lava_scheduler_app.tag'], null=False)) + )) + db.create_unique('lava_scheduler_app_device_tags', ['device_id', 'tag_id']) + + # Adding M2M table for field tags on 'TestJob' + db.create_table('lava_scheduler_app_testjob_tags', ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('testjob', models.ForeignKey(orm['lava_scheduler_app.testjob'], null=False)), + ('tag', models.ForeignKey(orm['lava_scheduler_app.tag'], null=False)) + )) + db.create_unique('lava_scheduler_app_testjob_tags', ['testjob_id', 'tag_id']) + + + def backwards(self, orm): + + # Deleting model 'Tag' + db.delete_table('lava_scheduler_app_tag') + + # Removing M2M table for field tags on 'Device' + db.delete_table('lava_scheduler_app_device_tags') + + # Removing M2M table for field tags on 'TestJob' + db.delete_table('lava_scheduler_app_testjob_tags') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'lava_scheduler_app.device': { + 'Meta': {'object_name': 'Device'}, + 'current_job': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.TestJob']", 'unique': 'True', 'null': 'True', 'blank': 'True'}), + 'device_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.DeviceType']"}), + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '200', 'primary_key': 'True'}), + 'status': ('django.db.models.fields.IntegerField', [], {'default': '1'}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['lava_scheduler_app.Tag']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'lava_scheduler_app.devicetype': { + 'Meta': {'object_name': 'DeviceType'}, + 'name': ('django.db.models.fields.SlugField', [], {'max_length': '50', 'primary_key': 'True', 'db_index': 'True'}) + }, + 'lava_scheduler_app.tag': { + 'Meta': {'object_name': 'Tag'}, + 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'}) + }, + 'lava_scheduler_app.testjob': { + 'Meta': {'object_name': 'TestJob'}, + 'actual_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.Device']"}), + 'definition': ('django.db.models.fields.TextField', [], {}), + 'description': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'end_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'log_file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'requested_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.Device']"}), + 'requested_device_type': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.DeviceType']"}), + 'results_link': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '400', 'null': 'True', 'blank': 'True'}), + 'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'status': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'submit_time': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'submitter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['lava_scheduler_app.Tag']", 'symmetrical': 'False', 'blank': 'True'}) + } + } + + complete_apps = ['lava_scheduler_app'] === modified file 'lava_scheduler_app/models.py' --- lava_scheduler_app/models.py 2011-10-28 00:24:13 +0000 +++ lava_scheduler_app/models.py 2011-12-15 03:16:40 +0000 @@ -9,6 +9,16 @@ """Error raised when JSON is syntactically valid but ill-formed.""" +class Tag(models.Model): + + name = models.SlugField(unique=True) + + description = models.TextField(null=True, blank=True) + + def __unicode__(self): + return self.name + + class DeviceType(models.Model): """ A class of device, for example a pandaboard or a snowball. @@ -51,6 +61,8 @@ current_job = models.ForeignKey( "TestJob", blank=True, unique=True, null=True) + tags = models.ManyToManyField(Tag, blank=True) + status = models.IntegerField( choices = STATUS_CHOICES, default = IDLE, @@ -137,6 +149,8 @@ requested_device_type = models.ForeignKey( DeviceType, null=True, default=None, related_name='+', blank=True) + tags = models.ManyToManyField(Tag, blank=True) + # This is set once the job starts. actual_device = models.ForeignKey( Device, null=True, default=None, related_name='+', blank=True) @@ -206,6 +220,8 @@ definition=json_data, submitter=user, requested_device=target, requested_device_type=device_type, description=job_name) job.save() + for tag_name in job_data.get('device_tags', []): + job.tags.add(Tag.objects.get_or_create(name=tag_name)[0]) return job def can_cancel(self, user): === modified file 'lava_scheduler_app/templates/lava_scheduler_app/device.html' --- lava_scheduler_app/templates/lava_scheduler_app/device.html 2011-10-28 00:24:13 +0000 +++ lava_scheduler_app/templates/lava_scheduler_app/device.html 2011-12-15 03:49:36 +0000 @@ -37,6 +37,13 @@
Device type:
{{ device.device_type }}
+ +
Device Tags
+ {% for tag in device.tags.all %} +
{{ tag.name }}
+ {% empty %} +
None
+ {% endfor %}
Status:
=== modified file 'lava_scheduler_app/templates/lava_scheduler_app/job_sidebar.html' --- lava_scheduler_app/templates/lava_scheduler_app/job_sidebar.html 2011-12-09 03:55:33 +0000 +++ lava_scheduler_app/templates/lava_scheduler_app/job_sidebar.html 2011-12-15 20:55:43 +0000 @@ -25,6 +25,17 @@
{{ job.requested_device_type }}
{% endif %} + {% for tag in job.tags.all %} + {% if forloop.first %} + {% if forloop.revcounter > 1 %} +
Required Device Tags
+ {% else %} +
Required Device Tag
+ {% endif %} + {% endif %} +
{{ tag.name }}
+ {% endfor %} + {% if job.description %}
Description:
{{ job.description }}
=== modified file 'lava_scheduler_app/tests.py' --- lava_scheduler_app/tests.py 2011-12-14 02:10:04 +0000 +++ lava_scheduler_app/tests.py 2011-12-14 05:02:36 +0000 @@ -9,7 +9,7 @@ from django_testscenarios.ubertest import TestCase -from lava_scheduler_app.models import Device, DeviceType, TestJob +from lava_scheduler_app.models import Device, DeviceType, Tag, TestJob @@ -135,6 +135,41 @@ json.dumps({'device_type':'panda'}), self.factory.make_user()) self.assertEqual(job.status, TestJob.SUBMITTED) + def test_from_json_and_user_sets_no_tags_if_no_tags(self): + self.factory.ensure_device_type(name='panda') + job = TestJob.from_json_and_user( + json.dumps({'device_type':'panda', 'device_tags':[]}), + self.factory.make_user()) + self.assertEqual(set(job.tags.all()), set([])) + + def test_from_json_and_user_sets_tag_from_device_tags(self): + self.factory.ensure_device_type(name='panda') + job = TestJob.from_json_and_user( + json.dumps({'device_type':'panda', 'device_tags':['tag']}), + self.factory.make_user()) + self.assertEqual( + set(tag.name for tag in job.tags.all()), set(['tag'])) + + def test_from_json_and_user_sets_multiple_tag_from_device_tags(self): + self.factory.ensure_device_type(name='panda') + job = TestJob.from_json_and_user( + json.dumps({'device_type':'panda', 'device_tags':['tag1', 'tag2']}), + self.factory.make_user()) + self.assertEqual( + set(tag.name for tag in job.tags.all()), set(['tag1', 'tag2'])) + + def test_from_json_and_user_reuses_tag_objects(self): + self.factory.ensure_device_type(name='panda') + job1 = TestJob.from_json_and_user( + json.dumps({'device_type':'panda', 'device_tags':['tag']}), + self.factory.make_user()) + job2 = TestJob.from_json_and_user( + json.dumps({'device_type':'panda', 'device_tags':['tag']}), + self.factory.make_user()) + self.assertEqual( + set(tag.pk for tag in job1.tags.all()), + set(tag.pk for tag in job2.tags.all())) + class TestSchedulerAPI(TestCaseWithFactory): @@ -298,6 +333,53 @@ None, DatabaseJobSource().getJobForBoard_impl('panda02')) + def _makeBoardWithTags(self, tags): + board = self.factory.make_device() + for tag_name in tags: + board.tags.add(Tag.objects.get_or_create(name=tag_name)[0]) + return board + + def _makeJobWithTagsForBoard(self, tags, board): + job = self.factory.make_testjob(requested_device=board) + for tag_name in tags: + job.tags.add(Tag.objects.get_or_create(name=tag_name)[0]) + return job + + def assertBoardWithTagsGetsJobWithTags(self, board_tags, job_tags): + board = self._makeBoardWithTags(board_tags) + self._makeJobWithTagsForBoard(job_tags, board) + self.assertEqual( + board.hostname, + DatabaseJobSource().getJobForBoard_impl(board.hostname)['target']) + + def assertBoardWithTagsDoesNotGetJobWithTags(self, board_tags, job_tags): + board = self._makeBoardWithTags(board_tags) + self._makeJobWithTagsForBoard(job_tags, board) + self.assertEqual( + None, + DatabaseJobSource().getJobForBoard_impl(board.hostname)) + + def test_getJobForBoard_does_not_return_job_if_board_lacks_tag(self): + self.assertBoardWithTagsDoesNotGetJobWithTags([], ['tag']) + + def test_getJobForBoard_returns_job_if_board_has_tag(self): + self.assertBoardWithTagsGetsJobWithTags(['tag'], ['tag']) + + def test_getJobForBoard_returns_job_if_board_has_both_tags(self): + self.assertBoardWithTagsGetsJobWithTags(['tag1', 'tag2'], ['tag1', 'tag2']) + + def test_getJobForBoard_returns_job_if_board_has_extra_tags(self): + self.assertBoardWithTagsGetsJobWithTags(['tag1', 'tag2'], ['tag1']) + + def test_getJobForBoard_does_not_return_job_if_board_has_only_one_tag(self): + self.assertBoardWithTagsDoesNotGetJobWithTags(['tag1'], ['tag1', 'tag2']) + + def test_getJobForBoard_does_not_return_job_if_board_has_unrelated_tag(self): + self.assertBoardWithTagsDoesNotGetJobWithTags(['tag1'], ['tag2']) + + def test_getJobForBoard_does_not_return_job_if_only_one_tag_matches(self): + self.assertBoardWithTagsDoesNotGetJobWithTags(['tag1', 'tag2'], ['tag1', 'tag3']) + def test_getJobForBoard_sets_start_time(self): device = self.factory.make_device(hostname='panda01') job = self.factory.make_testjob(requested_device=device) === modified file 'lava_scheduler_daemon/dbjobsource.py' --- lava_scheduler_daemon/dbjobsource.py 2011-12-01 03:58:08 +0000 +++ lava_scheduler_daemon/dbjobsource.py 2011-12-15 03:06:49 +0000 @@ -87,7 +87,17 @@ | Q(requested_device_type=device.device_type), status=TestJob.SUBMITTED) jobs_for_device = jobs_for_device.extra( - select={'is_targeted': 'requested_device_id is not NULL'}, + select={ + 'is_targeted': 'requested_device_id is not NULL', + 'missing_tags': ''' + select count(*) from lava_scheduler_app_testjob_tags + where testjob_id = lava_scheduler_app_testjob.id + and tag_id not in (select tag_id + from lava_scheduler_app_device_tags + where device_id = '%s') + ''' % device.hostname, + }, + where=['missing_tags = 0'], order_by=['-is_targeted', 'submit_time']) jobs = jobs_for_device[:1] if jobs: