[Branch,~linaro-validation/lava-dashboard/trunk] Rev 291: add a compressed representation for bundles on disk.

=== modified file 'dashboard_app/admin.py'
--- dashboard_app/admin.py	2011-09-27 13:06:33 +0000
+++ dashboard_app/admin.py	2011-12-20 04:09:12 +0000
@@ -56,7 +56,7 @@ 
     date_hierarchy = 'uploaded_on'
     fieldsets = (
         ('Document', {
-            'fields': ('content', 'content_filename')}),
+            'fields': ('_raw_content', '_gz_content', 'content_filename')}),
         ('Upload Details', {
             'fields': ('bundle_stream', 'uploaded_by')}),
         ('Deserialization', {

=== added file 'dashboard_app/migrations/0012_auto__del_field_bundle_content__add_field_bundle__raw_content__add_fie.py'
--- dashboard_app/migrations/0012_auto__del_field_bundle_content__add_field_bundle__raw_content__add_fie.py	1970-01-01 00:00:00 +0000
+++ dashboard_app/migrations/0012_auto__del_field_bundle_content__add_field_bundle__raw_content__add_fie.py	2011-12-19 23:03:42 +0000
@@ -0,0 +1,212 @@ 
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding field 'Bundle._gz_content'
+        db.add_column('dashboard_app_bundle', '_gz_content', self.gf('dashboard_app.models.GzFileField')(max_length=100, null=True, db_column='gz_content'), keep_default=False)
+
+
+    def backwards(self, orm):
+
+        # Deleting field 'Bundle._gz_content'
+        db.delete_column('dashboard_app_bundle', 'gz_content')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'dashboard_app.attachment': {
+            'Meta': {'object_name': 'Attachment'},
+            'content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}),
+            'content_filename': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '64'}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'public_url': ('django.db.models.fields.URLField', [], {'max_length': '512', 'blank': 'True'})
+        },
+        'dashboard_app.bundle': {
+            'Meta': {'ordering': "['-uploaded_on']", 'object_name': 'Bundle'},
+            '_gz_content': ('dashboard_app.models.GzFileField', [], {'max_length': '100', 'null': 'True', 'db_column': "'gz_content'"}),
+            '_raw_content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'db_column': "'content'"}),
+            'bundle_stream': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bundles'", 'to': "orm['dashboard_app.BundleStream']"}),
+            'content_filename': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'content_sha1': ('django.db.models.fields.CharField', [], {'max_length': '40', 'unique': 'True', 'null': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_deserialized': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'uploaded_by': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'uploaded_bundles'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'uploaded_on': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'})
+        },
+        'dashboard_app.bundledeserializationerror': {
+            'Meta': {'object_name': 'BundleDeserializationError'},
+            'bundle': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'deserialization_error'", 'unique': 'True', 'primary_key': 'True', 'to': "orm['dashboard_app.Bundle']"}),
+            'error_message': ('django.db.models.fields.CharField', [], {'max_length': '1024'}),
+            'traceback': ('django.db.models.fields.TextField', [], {'max_length': '32768'})
+        },
+        'dashboard_app.bundlestream': {
+            'Meta': {'object_name': 'BundleStream'},
+            'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_anonymous': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'pathname': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}),
+            'slug': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'})
+        },
+        'dashboard_app.hardwaredevice': {
+            'Meta': {'object_name': 'HardwareDevice'},
+            'description': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'device_type': ('django.db.models.fields.CharField', [], {'max_length': '32'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'})
+        },
+        'dashboard_app.namedattribute': {
+            'Meta': {'unique_together': "(('object_id', 'name'),)", 'object_name': 'NamedAttribute'},
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.TextField', [], {}),
+            'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'value': ('django.db.models.fields.TextField', [], {})
+        },
+        'dashboard_app.softwarepackage': {
+            'Meta': {'unique_together': "(('name', 'version'),)", 'object_name': 'SoftwarePackage'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'dashboard_app.softwarepackagescratch': {
+            'Meta': {'object_name': 'SoftwarePackageScratch'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'version': ('django.db.models.fields.CharField', [], {'max_length': '128'})
+        },
+        'dashboard_app.softwaresource': {
+            'Meta': {'object_name': 'SoftwareSource'},
+            'branch_revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'branch_url': ('django.db.models.fields.CharField', [], {'max_length': '256'}),
+            'branch_vcs': ('django.db.models.fields.CharField', [], {'max_length': '10'}),
+            'commit_timestamp': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project_name': ('django.db.models.fields.CharField', [], {'max_length': '32'})
+        },
+        'dashboard_app.tag': {
+            'Meta': {'object_name': 'Tag'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '256', 'db_index': 'True'})
+        },
+        'dashboard_app.test': {
+            'Meta': {'object_name': 'Test'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}),
+            'test_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'})
+        },
+        'dashboard_app.testcase': {
+            'Meta': {'unique_together': "(('test', 'test_case_id'),)", 'object_name': 'TestCase'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'test': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_cases'", 'to': "orm['dashboard_app.Test']"}),
+            'test_case_id': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'units': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'})
+        },
+        'dashboard_app.testingeffort': {
+            'Meta': {'object_name': 'TestingEffort'},
+            'description': ('django.db.models.fields.TextField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'testing_efforts'", 'to': "orm['lava_projects.Project']"}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'testing_efforts'", 'symmetrical': 'False', 'to': "orm['dashboard_app.Tag']"})
+        },
+        'dashboard_app.testresult': {
+            'Meta': {'ordering': "('_order',)", 'object_name': 'TestResult'},
+            '_order': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'filename': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'lineno': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'measurement': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '10', 'blank': 'True'}),
+            'message': ('django.db.models.fields.TextField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}),
+            'microseconds': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}),
+            'relative_index': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'result': ('django.db.models.fields.PositiveSmallIntegerField', [], {}),
+            'test_case': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'test_results'", 'null': 'True', 'to': "orm['dashboard_app.TestCase']"}),
+            'test_run': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_results'", 'to': "orm['dashboard_app.TestRun']"}),
+            'timestamp': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'dashboard_app.testrun': {
+            'Meta': {'ordering': "['-import_assigned_date']", 'object_name': 'TestRun'},
+            'analyzer_assigned_date': ('django.db.models.fields.DateTimeField', [], {}),
+            'analyzer_assigned_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'}),
+            'bundle': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_runs'", 'to': "orm['dashboard_app.Bundle']"}),
+            'devices': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.HardwareDevice']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'import_assigned_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'packages': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.SoftwarePackage']"}),
+            'sources': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.SoftwareSource']"}),
+            'sw_image_desc': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.Tag']"}),
+            'test': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_runs'", 'to': "orm['dashboard_app.Test']"}),
+            'time_check_performed': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        'dashboard_app.testrundenormalization': {
+            'Meta': {'object_name': 'TestRunDenormalization'},
+            'count_fail': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'count_pass': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'count_skip': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'count_unknown': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'test_run': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'denormalization'", 'unique': 'True', 'primary_key': 'True', 'to': "orm['dashboard_app.TestRun']"})
+        },
+        'lava_projects.project': {
+            'Meta': {'object_name': 'Project'},
+            'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'identifier': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100', 'db_index': 'True'}),
+            'is_aggregate': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'registered_by': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'projects'", 'to': "orm['auth.User']"}),
+            'registered_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['dashboard_app']

=== modified file 'dashboard_app/models.py'
--- dashboard_app/models.py	2011-11-25 01:48:45 +0000
+++ dashboard_app/models.py	2011-12-20 04:10:57 +0000
@@ -21,6 +21,8 @@ 
 """
 
 import datetime
+import errno
+import gzip
 import hashlib
 import logging
 import os
@@ -28,10 +30,13 @@ 
 import traceback
 import contextlib
 
+from django.conf import settings
 from django.contrib.auth.models import User
 from django.contrib.contenttypes import generic
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
+from django.core.files import locks, File
+from django.core.files.storage import FileSystemStorage
 from django.core.urlresolvers import reverse
 from django.db import models
 from django.template import Template, Context
@@ -311,6 +316,62 @@ 
         return user, group, slug, is_public, is_anonymous
 
 
+class GzipFileSystemStorage(FileSystemStorage):
+
+    def _open(self, name, mode='rb'):
+        full_path = self.path(name)
+        gzip_file = gzip.GzipFile(full_path, mode)
+        gzip_file.name = full_path
+        return File(gzip_file)
+
+    # This is a copy-paste-hack of FileSystemStorage._save
+    def _save(self, name, content):
+        full_path = self.path(name)
+
+        directory = os.path.dirname(full_path)
+        if not os.path.exists(directory):
+            os.makedirs(directory)
+        elif not os.path.isdir(directory):
+            raise IOError("%s exists and is not a directory." % directory)
+
+        # There's a potential race condition between get_available_name and
+        # saving the file; it's possible that two threads might return the
+        # same name, at which point all sorts of fun happens. So we need to
+        # try to create the file, but if it already exists we have to go back
+        # to get_available_name() and try again.
+
+        while True:
+            try:
+                # This fun binary flag incantation makes os.open throw an
+                # OSError if the file already exists before we open it.
+                fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
+                # This line, and the use of gz_file.write below, are the
+                # changes from the original version of this.
+                gz_file = gzip.GzipFile(fileobj=os.fdopen(fd, 'wb'))
+                try:
+                    locks.lock(fd, locks.LOCK_EX)
+                    for chunk in content.chunks():
+                        gz_file.write(chunk)
+                finally:
+                    locks.unlock(fd)
+                    gz_file.close()
+            except OSError, e:
+                if e.errno == errno.EEXIST:
+                    # Ooops, the file exists. We need a new file name.
+                    name = self.get_available_name(name)
+                    full_path = self.path(name)
+                else:
+                    raise
+            else:
+                # OK, the file save worked. Break out of the loop.
+                break
+
+        if settings.FILE_UPLOAD_PERMISSIONS is not None:
+            os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS)
+
+        return name
+
+
 class Bundle(models.Model):
     """
     Model for "Dashboard Bundles"
@@ -337,11 +398,34 @@ 
                 " into the database"),
             editable = False)
 
-    content = models.FileField(
+    _raw_content = models.FileField(
             verbose_name = _(u"Content"),
             help_text = _(u"Document in Dashboard Bundle Format 1.0"),
             upload_to = 'bundles',
-            null = True)
+            null = True,
+            db_column = 'content')
+
+    _gz_content = models.FileField(
+            verbose_name = _(u"Compressed content"),
+            help_text = _(u"Compressed document in Dashboard Bundle Format 1.0"),
+            upload_to = 'compressed-bundles',
+            null = True,
+            db_column = 'gz_content',
+            storage = GzipFileSystemStorage())
+
+    def _get_content(self):
+        r = self._gz_content
+        if not r:
+            return self._raw_content
+        else:
+            return r
+
+    content = property(_get_content)
+
+    def compress(self):
+        c = self._raw_content
+        self._gz_content.save(c.name, c)
+        c.delete()
 
     content_sha1 = models.CharField(
             editable = False,