diff mbox

[Branch,~linaro-validation/lava-scheduler/trunk] Rev 67: Job cancellation

Message ID 20110819040713.13401.82408.launchpad@ackee.canonical.com
State Accepted
Headers show

Commit Message

Michael-Doyle Hudson Aug. 19, 2011, 4:07 a.m. UTC 
Merge authors:
  Michael Hudson-Doyle (mwhudson)
------------------------------------------------------------
revno: 67 [merge]
committer: Michael-Doyle Hudson <michael.hudson@linaro.org>
branch nick: trunk
timestamp: Fri 2011-08-19 16:03:01 +1200
message:
  Job cancellation
   * the process running the dispatcher checks with the job source every 10
     seconds to see if the dispatcher should be killed (i.e. if the status is
     no longer RUNNING)
   * add model apis to check if a user can cancel a job and to cancel a job
   * add RPC to cancel a job
   * add a button in the ui to cancel a job
modified:
  lava_scheduler_app/api.py
  lava_scheduler_app/models.py
  lava_scheduler_app/templates/lava_scheduler_app/job.html
  lava_scheduler_app/tests.py
  lava_scheduler_app/urls.py
  lava_scheduler_app/views.py
  lava_scheduler_daemon/board.py
  lava_scheduler_daemon/dbjobsource.py


--
lp:lava-scheduler
https://code.launchpad.net/~linaro-validation/lava-scheduler/trunk

You are subscribed to branch lp:lava-scheduler.
To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-scheduler/trunk/+edit-subscription
diff mbox

Patch

=== modified file 'lava_scheduler_app/api.py'
--- lava_scheduler_app/api.py	2011-06-13 23:37:25 +0000
+++ lava_scheduler_app/api.py	2011-08-19 03:40:59 +0000
@@ -13,3 +13,12 @@ 
         if not self.user.has_perm('lava_scheduler_app.add_testjob'):
             raise xmlrpclib.Fault(403, "Permission denied.")
         return TestJob.from_json_and_user(job_data, self.user).id
+
+    def cancel_job(self, job_id):
+        if not self.user:
+            raise xmlrpclib.Fault(401, "Authentication required.")
+        job = TestJob.objects.get(pk=job_id)
+        if not job.can_cancel(self.user):
+            raise xmlrpclib.Fault(403, "Permission denied.")
+        job.cancel()
+        return True

=== modified file 'lava_scheduler_app/models.py'
--- lava_scheduler_app/models.py	2011-08-17 03:02:01 +0000
+++ lava_scheduler_app/models.py	2011-08-19 03:24:11 +0000
@@ -69,6 +69,7 @@ 
     COMPLETE = 2
     INCOMPLETE = 3
     CANCELED = 4
+    CANCELING = 5
 
     STATUS_CHOICES = (
         (SUBMITTED, 'Submitted'),
@@ -76,6 +77,7 @@ 
         (COMPLETE, 'Complete'),
         (INCOMPLETE, 'Incomplete'),
         (CANCELED, 'Canceled'),
+        (CANCELING, 'Canceling'),
     )
 
     id = models.AutoField(primary_key=True)
@@ -92,13 +94,13 @@ 
 
     # Only one of these two should be non-null.
     requested_device = models.ForeignKey(
-        Device, null=True, default=None, related_name='+')
+        Device, null=True, default=None, related_name='+', blank=True)
     requested_device_type = models.ForeignKey(
-        DeviceType, null=True, default=None, related_name='+')
+        DeviceType, null=True, default=None, related_name='+', blank=True)
 
     # This is set once the job starts.
     actual_device = models.ForeignKey(
-        Device, null=True, default=None, related_name='+')
+        Device, null=True, default=None, related_name='+', blank=True)
 
     #priority = models.IntegerField(
     #    verbose_name = _(u"Priority"),
@@ -133,7 +135,7 @@ 
         editable = False,
     )
     log_file = models.FileField(
-        upload_to='lava-logs', default=None, null=True)
+        upload_to='lava-logs', default=None, null=True, blank=True)
 
     results_link = models.CharField(
         max_length=400, default=None, null=True, blank=True)
@@ -158,3 +160,13 @@ 
             requested_device_type=device_type)
         job.save()
         return job
+
+    def can_cancel(self, user):
+        return user.is_superuser or user == self.submitter
+
+    def cancel(self):
+        if self.status == TestJob.RUNNING:
+            self.status = TestJob.CANCELING
+        else:
+            self.status = TestJob.CANCELED
+        self.save()

=== modified file 'lava_scheduler_app/templates/lava_scheduler_app/job.html'
--- lava_scheduler_app/templates/lava_scheduler_app/job.html	2011-08-17 03:30:25 +0000
+++ lava_scheduler_app/templates/lava_scheduler_app/job.html	2011-08-19 03:24:11 +0000
@@ -21,6 +21,14 @@ 
 {% block content %}
 <h2>Job {{ job.pk }}</h2>
 
+{% if show_cancel %}
+<form style="display:inline; float:right" method="POST" 
+      action="{% url lava_scheduler_app.views.job_cancel job.pk %}">
+  {% csrf_token %}
+  <button id="cancel-button">Cancel</button>
+</form>
+{% endif %}
+
 <div id="columns">
   <div class="column">
     <dt>Submitted by:</dt>
@@ -155,6 +163,7 @@ 
         }
       }
     );
+    $("#cancel-button").button();
   }
 );
 </script>

=== modified file 'lava_scheduler_app/tests.py'
--- lava_scheduler_app/tests.py	2011-08-18 03:24:09 +0000
+++ lava_scheduler_app/tests.py	2011-08-19 03:38:44 +0000
@@ -66,10 +66,11 @@ 
         device.save()
         return device
 
-    def make_testjob(self, definition=None, **kwargs):
+    def make_testjob(self, definition=None, submitter=None, **kwargs):
         if definition is None:
             definition = json.dumps({})
-        submitter = self.make_user()
+        if submitter is None:
+            submitter = self.make_user()
         testjob = TestJob(
             definition=definition, submitter=submitter, **kwargs)
         testjob.save()
@@ -139,7 +140,7 @@ 
             'http://localhost/RPC2/',
             transport=TestTransport(user=user, password=password))
 
-    def test_api_rejects_anonymous(self):
+    def test_submit_job_rejects_anonymous(self):
         server = self.server_proxy()
         try:
             server.scheduler.submit_job("{}")
@@ -148,7 +149,7 @@ 
         else:
             self.fail("fault not raised")
 
-    def test_api_rejects_unpriv_user(self):
+    def test_submit_job_rejects_unpriv_user(self):
         User.objects.create_user('test', 'e@mail.invalid', 'test').save()
         server = self.server_proxy('test', 'test')
         try:
@@ -158,7 +159,7 @@ 
         else:
             self.fail("fault not raised")
 
-    def test_sets_definition(self):
+    def test_submit_job_sets_definition(self):
         user = User.objects.create_user('test', 'e@mail.invalid', 'test')
         user.user_permissions.add(
             Permission.objects.get(codename='add_testjob'))
@@ -170,6 +171,36 @@ 
         job = TestJob.objects.get(id=job_id)
         self.assertEqual(definition, job.definition)
 
+    def test_cancel_job_rejects_anonymous(self):
+        job = self.factory.make_testjob()
+        server = self.server_proxy()
+        try:
+            server.scheduler.cancel_job(job.id)
+        except xmlrpclib.Fault as f:
+            self.assertEqual(401, f.faultCode)
+        else:
+            self.fail("fault not raised")
+
+    def test_cancel_job_rejects_unpriv_user(self):
+        job = self.factory.make_testjob()
+        User.objects.create_user('test', 'e@mail.invalid', 'test').save()
+        server = self.server_proxy('test', 'test')
+        try:
+            server.scheduler.cancel_job(job.id)
+        except xmlrpclib.Fault as f:
+            self.assertEqual(403, f.faultCode)
+        else:
+            self.fail("fault not raised")
+
+    def test_cancel_job_cancels_job(self):
+        user = User.objects.create_user('test', 'e@mail.invalid', 'test')
+        user.save()
+        job = self.factory.make_testjob(submitter=user)
+        server = self.server_proxy('test', 'test')
+        server.scheduler.cancel_job(job.id)
+        job = TestJob.objects.get(pk=job.pk)
+        self.assertEqual(TestJob.CANCELED, job.status)
+
 
 from django.test import TransactionTestCase
 

=== modified file 'lava_scheduler_app/urls.py'
--- lava_scheduler_app/urls.py	2011-07-25 03:09:15 +0000
+++ lava_scheduler_app/urls.py	2011-08-19 03:24:11 +0000
@@ -6,4 +6,5 @@ 
     url(r'^alljobs$', 'alljobs'),
     url(r'^job/(?P<pk>[0-9]+)$', 'job'),
     url(r'^job/(?P<pk>[0-9]+)/output$', 'job_output'),
+    url(r'^job/(?P<pk>[0-9]+)/cancel$', 'job_cancel'),
     )

=== modified file 'lava_scheduler_app/views.py'
--- lava_scheduler_app/views.py	2011-07-26 05:06:11 +0000
+++ lava_scheduler_app/views.py	2011-08-19 03:32:49 +0000
@@ -1,8 +1,8 @@ 
 import os
 
-from django.http import HttpResponse
+from django.http import HttpResponse, HttpResponseForbidden
 from django.template import RequestContext
-from django.shortcuts import render_to_response
+from django.shortcuts import redirect, render_to_response
 
 from lava_scheduler_app.models import Device, TestJob
 
@@ -33,6 +33,7 @@ 
         {
             'log_file_present': bool(job.log_file),
             'job': TestJob.objects.get(pk=pk),
+            'show_cancel': job.status <= TestJob.RUNNING and job.can_cancel(request.user),
         },
         RequestContext(request))
 
@@ -69,3 +70,13 @@ 
     if job.status != TestJob.RUNNING:
         response['X-Is-Finished'] = '1'
     return response
+
+
+def job_cancel(request, pk):
+    job = TestJob.objects.get(pk=pk)
+    if job.can_cancel(request.user):
+        job.cancel()
+        return redirect('lava_scheduler_app.views.job', pk=job.pk)
+    else:
+        return HttpResponseForbidden(
+            "you cannot cancel this job", content_type="text/plain")

=== modified file 'lava_scheduler_daemon/board.py'
--- lava_scheduler_daemon/board.py	2011-08-18 04:02:33 +0000
+++ lava_scheduler_daemon/board.py	2011-08-19 03:59:00 +0000
@@ -1,10 +1,11 @@ 
 import json
 import os
+import signal
 import tempfile
 import logging
 
 from twisted.internet.protocol import ProcessProtocol
-from twisted.internet import defer
+from twisted.internet import defer, task
 from twisted.protocols.basic import LineReceiver
 
 
@@ -69,6 +70,17 @@ 
         self.board_name = board_name
         self.reactor = reactor
         self._json_file = None
+        self._source_lock = defer.DeferredLock()
+        self._checkCancel_call = task.LoopingCall(self._checkCancel)
+
+    def _checkCancel(self):
+        return self._source_lock.run(
+            self.source.jobCheckForCancellation, self.board_name).addCallback(
+            self._maybeCancel)
+
+    def _maybeCancel(self, cancel):
+        if cancel:
+            self._protocol.transport.signalProcess(signal.SIGINT)
 
     def run(self):
         d = self.source.getLogFileForJobOnBoard(self.board_name)
@@ -81,12 +93,13 @@ 
         fd, self._json_file = tempfile.mkstemp()
         with os.fdopen(fd, 'wb') as f:
             json.dump(json_data, f)
+        self._protocol = DispatcherProcessProtocol(
+            d, log_file, self.source, self.board_name)
         self.reactor.spawnProcess(
-            DispatcherProcessProtocol(
-                d, log_file, self.source, self.board_name),
-            self.dispatcher, args=[
+            self._protocol, self.dispatcher, args=[
                 self.dispatcher, self._json_file, '--oob-fd', '3'],
             childFDs={0:0, 1:'r', 2:'r', 3:'r'}, env=None)
+        self._checkCancel_call.start(10)
         d.addBoth(self._exited)
         return d
 
@@ -95,7 +108,9 @@ 
         if self._json_file is not None:
             os.unlink(self._json_file)
         self.logger.info("reporting job completed")
-        return self.source.jobCompleted(self.board_name).addCallback(
+        self._source_lock.run(self._checkCancel_call.stop)
+        return self._source_lock.run(
+            self.source.jobCompleted, self.board_name).addCallback(
             lambda r:result)
 
 
@@ -128,7 +143,7 @@ 
             SimplePP(d), 'lava-scheduler-monitor', childFDs={0:0, 1:1, 2:2},
             env=None, args=[
                 'lava-scheduler-monitor', self.dispatcher,
-                self.board_name, self._json_file])
+                str(self.board_name), self._json_file])
         d.addBoth(self._exited)
         return d
 

=== modified file 'lava_scheduler_daemon/dbjobsource.py'
--- lava_scheduler_daemon/dbjobsource.py	2011-08-18 03:24:09 +0000
+++ lava_scheduler_daemon/dbjobsource.py	2011-08-19 02:38:22 +0000
@@ -127,7 +127,14 @@ 
         device.status = Device.IDLE
         job = device.current_job
         device.current_job = None
-        job.status = TestJob.COMPLETE
+        if job.status == TestJob.RUNNING:
+            job.status = TestJob.COMPLETE
+        elif job.status == TestJob.CANCELING:
+            job.status = TestJob.CANCELED
+        else:
+            self.logger.error(
+                "Unexpected job state in jobCompleted: %s" % job.status)
+            job.status = TestJob.COMPLETE
         job.end_time = datetime.datetime.utcnow()
         device.save()
         job.save()
@@ -146,3 +153,12 @@ 
 
     def jobOobData(self, board_name, key, value):
         return self.deferForDB(self.jobOobData_impl, board_name, key, value)
+
+    def jobCheckForCancellation_impl(self, board_name):
+        device = Device.objects.get(hostname=board_name)
+        device.status = Device.IDLE
+        job = device.current_job
+        return job.status != TestJob.RUNNING
+
+    def jobCheckForCancellation(self, board_name):
+        return self.deferForDB(self.jobCheckForCancellation_impl, board_name)