From patchwork Wed Jan 18 20:27:13 2012 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Michael-Doyle Hudson X-Patchwork-Id: 6282 Return-Path: X-Original-To: patchwork@peony.canonical.com Delivered-To: patchwork@peony.canonical.com Received: from fiordland.canonical.com (fiordland.canonical.com [91.189.94.145]) by peony.canonical.com (Postfix) with ESMTP id D022B23E0C for ; Wed, 18 Jan 2012 20:27:15 +0000 (UTC) Received: from mail-bk0-f52.google.com (mail-bk0-f52.google.com [209.85.214.52]) by fiordland.canonical.com (Postfix) with ESMTP id AD29EA182EE for ; Wed, 18 Jan 2012 20:27:15 +0000 (UTC) Received: by bkbzt4 with SMTP id zt4so3513548bkb.11 for ; Wed, 18 Jan 2012 12:27:15 -0800 (PST) Received: by 10.205.139.66 with SMTP id iv2mr9158967bkc.27.1326918435370; Wed, 18 Jan 2012 12:27:15 -0800 (PST) X-Forwarded-To: linaro-patchwork@canonical.com X-Forwarded-For: patch@linaro.org linaro-patchwork@canonical.com Delivered-To: patches@linaro.org Received: by 10.205.82.144 with SMTP id ac16cs158819bkc; Wed, 18 Jan 2012 12:27:14 -0800 (PST) Received: by 10.216.137.148 with SMTP id y20mr44485wei.32.1326918433546; Wed, 18 Jan 2012 12:27:13 -0800 (PST) Received: from indium.canonical.com (indium.canonical.com. [91.189.90.7]) by mx.google.com with ESMTPS id s55si16943180wec.11.2012.01.18.12.27.13 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 18 Jan 2012 12:27:13 -0800 (PST) Received-SPF: pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) client-ip=91.189.90.7; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of bounces@canonical.com designates 91.189.90.7 as permitted sender) smtp.mail=bounces@canonical.com Received: from ackee.canonical.com ([91.189.89.26]) by indium.canonical.com with esmtp (Exim 4.71 #1 (Debian)) id 1Rnc6D-0001oh-9d for ; Wed, 18 Jan 2012 20:27:13 +0000 Received: from ackee.canonical.com (localhost [127.0.0.1]) by ackee.canonical.com (Postfix) with ESMTP id 37154E01EB for ; Wed, 18 Jan 2012 20:27:13 +0000 (UTC) MIME-Version: 1.0 X-Launchpad-Project: lava-dashboard X-Launchpad-Branch: ~linaro-validation/lava-dashboard/trunk X-Launchpad-Message-Rationale: Subscriber X-Launchpad-Branch-Revision-Number: 291 X-Launchpad-Notification-Type: branch-revision To: Linaro Patch Tracker From: noreply@launchpad.net Subject: [Branch ~linaro-validation/lava-dashboard/trunk] Rev 291: add a compressed representation for bundles on disk. Message-Id: <20120118202713.23804.98510.launchpad@ackee.canonical.com> Date: Wed, 18 Jan 2012 20:27:13 -0000 Reply-To: noreply@launchpad.net Sender: bounces@canonical.com Errors-To: bounces@canonical.com Precedence: bulk X-Generated-By: Launchpad (canonical.com); Revision="14681"; Instance="launchpad-lazr.conf" X-Launchpad-Hash: ea844718b1003afcda4a7a4b7c54621b622a6093 Merge authors: Michael Hudson-Doyle (mwhudson) Related merge proposals: https://code.launchpad.net/~mwhudson/lava-dashboard/compress-bundles/+merge/86331 proposed by: Michael Hudson-Doyle (mwhudson) review: Approve - Zygmunt Krynicki (zkrynicki) ------------------------------------------------------------ revno: 291 [merge] committer: Michael Hudson-Doyle branch nick: trunk timestamp: Thu 2012-01-19 09:25:33 +1300 message: add a compressed representation for bundles on disk. bundles will not be automatically compressed yet, however. added: dashboard_app/migrations/0012_auto__del_field_bundle_content__add_field_bundle__raw_content__add_fie.py modified: dashboard_app/admin.py dashboard_app/models.py --- lp:lava-dashboard https://code.launchpad.net/~linaro-validation/lava-dashboard/trunk You are subscribed to branch lp:lava-dashboard. To unsubscribe from this branch go to https://code.launchpad.net/~linaro-validation/lava-dashboard/trunk/+edit-subscription === modified file 'dashboard_app/admin.py' --- dashboard_app/admin.py 2011-09-27 13:06:33 +0000 +++ dashboard_app/admin.py 2011-12-20 04:09:12 +0000 @@ -56,7 +56,7 @@ date_hierarchy = 'uploaded_on' fieldsets = ( ('Document', { - 'fields': ('content', 'content_filename')}), + 'fields': ('_raw_content', '_gz_content', 'content_filename')}), ('Upload Details', { 'fields': ('bundle_stream', 'uploaded_by')}), ('Deserialization', { === added file 'dashboard_app/migrations/0012_auto__del_field_bundle_content__add_field_bundle__raw_content__add_fie.py' --- dashboard_app/migrations/0012_auto__del_field_bundle_content__add_field_bundle__raw_content__add_fie.py 1970-01-01 00:00:00 +0000 +++ dashboard_app/migrations/0012_auto__del_field_bundle_content__add_field_bundle__raw_content__add_fie.py 2011-12-19 23:03:42 +0000 @@ -0,0 +1,212 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + + # Adding field 'Bundle._gz_content' + db.add_column('dashboard_app_bundle', '_gz_content', self.gf('dashboard_app.models.GzFileField')(max_length=100, null=True, db_column='gz_content'), keep_default=False) + + + def backwards(self, orm): + + # Deleting field 'Bundle._gz_content' + db.delete_column('dashboard_app_bundle', 'gz_content') + + + models = { + 'auth.group': { + 'Meta': {'object_name': 'Group'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'Meta': {'object_name': 'User'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'dashboard_app.attachment': { + 'Meta': {'object_name': 'Attachment'}, + 'content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True'}), + 'content_filename': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mime_type': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'public_url': ('django.db.models.fields.URLField', [], {'max_length': '512', 'blank': 'True'}) + }, + 'dashboard_app.bundle': { + 'Meta': {'ordering': "['-uploaded_on']", 'object_name': 'Bundle'}, + '_gz_content': ('dashboard_app.models.GzFileField', [], {'max_length': '100', 'null': 'True', 'db_column': "'gz_content'"}), + '_raw_content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'db_column': "'content'"}), + 'bundle_stream': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bundles'", 'to': "orm['dashboard_app.BundleStream']"}), + 'content_filename': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'content_sha1': ('django.db.models.fields.CharField', [], {'max_length': '40', 'unique': 'True', 'null': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_deserialized': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'uploaded_by': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'uploaded_bundles'", 'null': 'True', 'to': "orm['auth.User']"}), + 'uploaded_on': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}) + }, + 'dashboard_app.bundledeserializationerror': { + 'Meta': {'object_name': 'BundleDeserializationError'}, + 'bundle': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'deserialization_error'", 'unique': 'True', 'primary_key': 'True', 'to': "orm['dashboard_app.Bundle']"}), + 'error_message': ('django.db.models.fields.CharField', [], {'max_length': '1024'}), + 'traceback': ('django.db.models.fields.TextField', [], {'max_length': '32768'}) + }, + 'dashboard_app.bundlestream': { + 'Meta': {'object_name': 'BundleStream'}, + 'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_anonymous': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'pathname': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), + 'slug': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}) + }, + 'dashboard_app.hardwaredevice': { + 'Meta': {'object_name': 'HardwareDevice'}, + 'description': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'device_type': ('django.db.models.fields.CharField', [], {'max_length': '32'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}) + }, + 'dashboard_app.namedattribute': { + 'Meta': {'unique_together': "(('object_id', 'name'),)", 'object_name': 'NamedAttribute'}, + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.TextField', [], {}), + 'object_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'value': ('django.db.models.fields.TextField', [], {}) + }, + 'dashboard_app.softwarepackage': { + 'Meta': {'unique_together': "(('name', 'version'),)", 'object_name': 'SoftwarePackage'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'dashboard_app.softwarepackagescratch': { + 'Meta': {'object_name': 'SoftwarePackageScratch'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'version': ('django.db.models.fields.CharField', [], {'max_length': '128'}) + }, + 'dashboard_app.softwaresource': { + 'Meta': {'object_name': 'SoftwareSource'}, + 'branch_revision': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'branch_url': ('django.db.models.fields.CharField', [], {'max_length': '256'}), + 'branch_vcs': ('django.db.models.fields.CharField', [], {'max_length': '10'}), + 'commit_timestamp': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'project_name': ('django.db.models.fields.CharField', [], {'max_length': '32'}) + }, + 'dashboard_app.tag': { + 'Meta': {'object_name': 'Tag'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '256', 'db_index': 'True'}) + }, + 'dashboard_app.test': { + 'Meta': {'object_name': 'Test'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), + 'test_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'}) + }, + 'dashboard_app.testcase': { + 'Meta': {'unique_together': "(('test', 'test_case_id'),)", 'object_name': 'TestCase'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'test': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_cases'", 'to': "orm['dashboard_app.Test']"}), + 'test_case_id': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'units': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}) + }, + 'dashboard_app.testingeffort': { + 'Meta': {'object_name': 'TestingEffort'}, + 'description': ('django.db.models.fields.TextField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'project': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'testing_efforts'", 'to': "orm['lava_projects.Project']"}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'testing_efforts'", 'symmetrical': 'False', 'to': "orm['dashboard_app.Tag']"}) + }, + 'dashboard_app.testresult': { + 'Meta': {'ordering': "('_order',)", 'object_name': 'TestResult'}, + '_order': ('django.db.models.fields.IntegerField', [], {'default': '0'}), + 'filename': ('django.db.models.fields.CharField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'lineno': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'measurement': ('django.db.models.fields.DecimalField', [], {'null': 'True', 'max_digits': '20', 'decimal_places': '10', 'blank': 'True'}), + 'message': ('django.db.models.fields.TextField', [], {'max_length': '1024', 'null': 'True', 'blank': 'True'}), + 'microseconds': ('django.db.models.fields.BigIntegerField', [], {'null': 'True', 'blank': 'True'}), + 'relative_index': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'result': ('django.db.models.fields.PositiveSmallIntegerField', [], {}), + 'test_case': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'test_results'", 'null': 'True', 'to': "orm['dashboard_app.TestCase']"}), + 'test_run': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_results'", 'to': "orm['dashboard_app.TestRun']"}), + 'timestamp': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}) + }, + 'dashboard_app.testrun': { + 'Meta': {'ordering': "['-import_assigned_date']", 'object_name': 'TestRun'}, + 'analyzer_assigned_date': ('django.db.models.fields.DateTimeField', [], {}), + 'analyzer_assigned_uuid': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'}), + 'bundle': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_runs'", 'to': "orm['dashboard_app.Bundle']"}), + 'devices': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.HardwareDevice']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'import_assigned_date': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'packages': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.SoftwarePackage']"}), + 'sources': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.SoftwareSource']"}), + 'sw_image_desc': ('django.db.models.fields.CharField', [], {'max_length': '100', 'blank': 'True'}), + 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'test_runs'", 'blank': 'True', 'to': "orm['dashboard_app.Tag']"}), + 'test': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'test_runs'", 'to': "orm['dashboard_app.Test']"}), + 'time_check_performed': ('django.db.models.fields.BooleanField', [], {'default': 'False'}) + }, + 'dashboard_app.testrundenormalization': { + 'Meta': {'object_name': 'TestRunDenormalization'}, + 'count_fail': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'count_pass': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'count_skip': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'count_unknown': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'test_run': ('django.db.models.fields.related.OneToOneField', [], {'related_name': "'denormalization'", 'unique': 'True', 'primary_key': 'True', 'to': "orm['dashboard_app.TestRun']"}) + }, + 'lava_projects.project': { + 'Meta': {'object_name': 'Project'}, + 'description': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'identifier': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '100', 'db_index': 'True'}), + 'is_aggregate': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'registered_by': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'projects'", 'to': "orm['auth.User']"}), + 'registered_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['dashboard_app'] === modified file 'dashboard_app/models.py' --- dashboard_app/models.py 2011-11-25 01:48:45 +0000 +++ dashboard_app/models.py 2011-12-20 04:10:57 +0000 @@ -21,6 +21,8 @@ """ import datetime +import errno +import gzip import hashlib import logging import os @@ -28,10 +30,13 @@ import traceback import contextlib +from django.conf import settings from django.contrib.auth.models import User from django.contrib.contenttypes import generic from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ValidationError +from django.core.files import locks, File +from django.core.files.storage import FileSystemStorage from django.core.urlresolvers import reverse from django.db import models from django.template import Template, Context @@ -311,6 +316,62 @@ return user, group, slug, is_public, is_anonymous +class GzipFileSystemStorage(FileSystemStorage): + + def _open(self, name, mode='rb'): + full_path = self.path(name) + gzip_file = gzip.GzipFile(full_path, mode) + gzip_file.name = full_path + return File(gzip_file) + + # This is a copy-paste-hack of FileSystemStorage._save + def _save(self, name, content): + full_path = self.path(name) + + directory = os.path.dirname(full_path) + if not os.path.exists(directory): + os.makedirs(directory) + elif not os.path.isdir(directory): + raise IOError("%s exists and is not a directory." % directory) + + # There's a potential race condition between get_available_name and + # saving the file; it's possible that two threads might return the + # same name, at which point all sorts of fun happens. So we need to + # try to create the file, but if it already exists we have to go back + # to get_available_name() and try again. + + while True: + try: + # This fun binary flag incantation makes os.open throw an + # OSError if the file already exists before we open it. + fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0)) + # This line, and the use of gz_file.write below, are the + # changes from the original version of this. + gz_file = gzip.GzipFile(fileobj=os.fdopen(fd, 'wb')) + try: + locks.lock(fd, locks.LOCK_EX) + for chunk in content.chunks(): + gz_file.write(chunk) + finally: + locks.unlock(fd) + gz_file.close() + except OSError, e: + if e.errno == errno.EEXIST: + # Ooops, the file exists. We need a new file name. + name = self.get_available_name(name) + full_path = self.path(name) + else: + raise + else: + # OK, the file save worked. Break out of the loop. + break + + if settings.FILE_UPLOAD_PERMISSIONS is not None: + os.chmod(full_path, settings.FILE_UPLOAD_PERMISSIONS) + + return name + + class Bundle(models.Model): """ Model for "Dashboard Bundles" @@ -337,11 +398,34 @@ " into the database"), editable = False) - content = models.FileField( + _raw_content = models.FileField( verbose_name = _(u"Content"), help_text = _(u"Document in Dashboard Bundle Format 1.0"), upload_to = 'bundles', - null = True) + null = True, + db_column = 'content') + + _gz_content = models.FileField( + verbose_name = _(u"Compressed content"), + help_text = _(u"Compressed document in Dashboard Bundle Format 1.0"), + upload_to = 'compressed-bundles', + null = True, + db_column = 'gz_content', + storage = GzipFileSystemStorage()) + + def _get_content(self): + r = self._gz_content + if not r: + return self._raw_content + else: + return r + + content = property(_get_content) + + def compress(self): + c = self._raw_content + self._gz_content.save(c.name, c) + c.delete() content_sha1 = models.CharField( editable = False,