diff mbox

[Branch,~linaro-validation/lava-dispatcher/trunk] Rev 300: Added retry on image deployment

Message ID 20120524230111.29963.2422.launchpad@ackee.canonical.com
State Accepted
Headers show

Commit Message

Le Chi Thu May 24, 2012, 11:01 p.m. UTC
Merge authors:
  Le Chi Thu le.chi.thu@linaro.org <le.chi.thu@linaro.org>
Related merge proposals:
  https://code.launchpad.net/~le-chi-thu/lava-dispatcher/add-deploy-retry/+merge/107293
  proposed by: Le Chi Thu (le-chi-thu)
  review: Approve - Michael Hudson-Doyle (mwhudson)
------------------------------------------------------------
revno: 300 [merge]
committer: Le Chi Thu le.chi.thu@linaro.org <le.chi.thu@linaro.org>
branch nick: trunk
timestamp: Fri 2012-05-25 00:59:46 +0200
message:
  Added retry on image deployment
modified:
  doc/changes.rst
  lava_dispatcher/client/master.py


--
lp:lava-dispatcher
https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk

You are subscribed to branch lp:lava-dispatcher.
To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dispatcher/trunk/+edit-subscription
diff mbox

Patch

=== modified file 'doc/changes.rst'
--- doc/changes.rst	2012-05-23 21:49:59 +0000
+++ doc/changes.rst	2012-05-24 17:32:08 +0000
@@ -3,6 +3,7 @@ 
 
 * Fixed reboot issues
 * Skip raising exception on the home screen has not displayed for health check jobs
+* Retry deployment if failed.
 
 .. _version_0_7_1:
 

=== modified file 'lava_dispatcher/client/master.py'
--- lava_dispatcher/client/master.py	2012-05-22 13:03:31 +0000
+++ lava_dispatcher/client/master.py	2012-05-24 21:12:04 +0000
@@ -67,16 +67,43 @@ 
         if rc:
             raise RuntimeError("Failed to create tarball: %s" % tarfile)
 
-def _deploy_tarball_to_board(session, tarball_url, dest, timeout=-1):
+def _deploy_tarball_to_board(session, tarball_url, dest, timeout=-1, num_retry=2):
     decompression_char = ''
     if tarball_url.endswith('.gz') or tarball_url.endswith('.tgz'):
         decompression_char = 'z'
     elif tarball_url.endswith('.bz2'):
         decompression_char = 'j'
-    session.run(
-        'wget --no-proxy --connect-timeout=30 -S -O- %s --progress=dot -e dotbytes=2M | tar --numeric-owner -C %s -x%sf -' % (
-            tarball_url, dest, decompression_char),
-        timeout=timeout)
+
+    deploy_ok=False
+
+    while (num_retry > 0 and not deploy_ok):
+        try:
+            session._empty_pexpect_buffer()
+            session._client.proc.sendline(
+                'wget --no-proxy --connect-timeout=30 -S -O- %s --progress=dot -e dotbytes=2M |'
+                'tar --numeric-owner -C %s -x%sf -'
+                % (tarball_url, dest, decompression_char))
+
+            match_id = session._client.proc.expect(["tar: Error", "unexpected end of file",
+                                                    "tar: Unexpected EOF in archive",session._prompt_str], timeout=timeout)
+            if match_id != 3:
+                logging.warning("Deploy %s failed. %d retry left." %(tarball_url, num_retry-1))
+            else:
+                deploy_ok=True
+        except:
+            logging.warning("Deploy %s failed. %d retry left." %(tarball_url, num_retry-1))
+        # do some delay before retry
+        if (num_retry > 1 and not deploy_ok):
+            sleep_time=5*60
+            logging.info("Wait %d second before retry" % sleep_time)
+            time.sleep(sleep_time)
+            # send CTRL C to exit last command.
+            session._client.proc.sendline("\003")
+            session._client.proc.sendline("echo 'retry left %s'" % num_retry-1)
+        num_retry = num_retry - 1
+
+    if not deploy_ok:
+        raise Exception("Deploy tarball (%s) to board failed" % tarball_url);
 
 def _deploy_linaro_rootfs(session, rootfs):
     logging.info("Deploying linaro image")
@@ -754,6 +781,7 @@ 
             self.proc.sendline("hardreset")
         # after hardreset empty the pexpect buffer
         self._empty_pexpect_buffer()
+
     def _empty_pexpect_buffer(self):
         """Make sure there is nothing in the pexpect buffer."""
         index = 0