From b06a6b79389cbc2b03f41b3432055a6c8a17d59d Mon Sep 17 00:00:00 2001 From: Joseph Atkins-Turkish Date: Mon, 11 Apr 2016 14:25:40 -0700 Subject: [PATCH] Remove support for AWS_ENABLED==False --- app.json | 1 - cloudpebble/settings.py | 1 - ide/api/project.py | 2 -- ide/api/resource.py | 16 ++++-------- ide/models/build.py | 45 ++++++---------------------------- ide/models/files.py | 18 +++----------- ide/models/meta.py | 1 + ide/models/s3file.py | 43 +++----------------------------- ide/models/scriptfile.py | 1 - ide/tasks/archive.py | 20 +++++---------- utils/s3.py | 53 ++++++++++++++-------------------------- 11 files changed, 45 insertions(+), 156 deletions(-) diff --git a/app.json b/app.json index ccb92b0d..ed33ef07 100644 --- a/app.json +++ b/app.json @@ -7,7 +7,6 @@ "AWS_ACCESS_KEY_ID": { "required": true }, - "AWS_ENABLED": "yes", "AWS_S3_BUILDS_BUCKET": "builds-staging.cloudpebble.net", "AWS_S3_EXPORT_BUCKET": "export-staging.cloudpebble.net", "AWS_S3_SOURCE_BUCKET": "source-staging.cloudpebble.net", diff --git a/cloudpebble/settings.py b/cloudpebble/settings.py index 2384b21a..91de943b 100644 --- a/cloudpebble/settings.py +++ b/cloudpebble/settings.py @@ -302,7 +302,6 @@ MAILCHIMP_API_KEY = _environ.get('MAILCHIMP_API_KEY', None) MAILCHIMP_LIST_ID = _environ.get('MAILCHIMP_LIST_ID', None) -AWS_ENABLED = 'AWS_ENABLED' in _environ AWS_ACCESS_KEY_ID = _environ.get('AWS_ACCESS_KEY_ID', None) AWS_SECRET_ACCESS_KEY = _environ.get('AWS_SECRET_ACCESS_KEY', None) diff --git a/ide/api/project.py b/ide/api/project.py index 89c2121f..2fe77d88 100644 --- a/ide/api/project.py +++ b/ide/api/project.py @@ -1,6 +1,4 @@ -import os import re -import tempfile import time import json from django.conf import settings diff --git a/ide/api/resource.py b/ide/api/resource.py index cfd53b22..aeafce0b 100644 --- a/ide/api/resource.py +++ b/ide/api/resource.py @@ -1,5 +1,4 @@ import json -from django.conf import settings from django.contrib.auth.decorators import login_required from django.db import transaction from django.http import HttpResponse, HttpResponseRedirect @@ -241,13 +240,8 @@ def show_resource(request, project_id, resource_id, variant): } content_disposition = "attachment; filename=\"%s\"" % resource.file_name content_type = content_types[resource.kind] - if settings.AWS_ENABLED: - headers = { - 'response-content-disposition': content_disposition, - 'Content-Type': content_type - } - return HttpResponseRedirect(s3.get_signed_url('source', variant.s3_path, headers=headers)) - else: - response = HttpResponse(open(variant.local_filename), content_type=content_type) - response['Content-Disposition'] = content_disposition - return response + headers = { + 'response-content-disposition': content_disposition, + 'Content-Type': content_type + } + return HttpResponseRedirect(s3.get_signed_url('source', variant.s3_path, headers=headers)) \ No newline at end of file diff --git a/ide/models/build.py b/ide/models/build.py index 11ba73b1..2fe82899 100644 --- a/ide/models/build.py +++ b/ide/models/build.py @@ -1,8 +1,5 @@ import uuid import json -import shutil -import os -import os.path from django.conf import settings from django.db import models from ide.models.project import Project @@ -40,19 +37,10 @@ class BuildResult(IdeModel): finished = models.DateTimeField(blank=True, null=True) def _get_dir(self): - if settings.AWS_ENABLED: - return '%s/' % self.uuid - else: - path = '%s%s/%s/%s/' % (settings.MEDIA_ROOT, self.uuid[0], self.uuid[1], self.uuid) - if not os.path.exists(path): - os.makedirs(path) - return path + return '%s/' % self.uuid def get_url(self): - if settings.AWS_ENABLED: - return "%s%s/" % (settings.MEDIA_URL, self.uuid) - else: - return '%s%s/%s/%s/' % (settings.MEDIA_URL, self.uuid[0], self.uuid[1], self.uuid) + return "%s%s/" % (settings.MEDIA_URL, self.uuid) def get_pbw_filename(self): return '%swatchface.pbw' % self._get_dir() @@ -79,39 +67,20 @@ def get_simplyjs_url(self): return '%ssimply.js' % self.get_url() def save_build_log(self, text): - if not settings.AWS_ENABLED: - with open(self.build_log, 'w') as f: - f.write(text) - else: - s3.save_file('builds', self.build_log, text, public=True, content_type='text/plain') + s3.save_file('builds', self.build_log, text, public=True, content_type='text/plain') def read_build_log(self): - if not settings.AWS_ENABLED: - with open(self.build_log, 'r') as f: - return f.read() - else: - return s3.read_file('builds', self.build_log) + return s3.read_file('builds', self.build_log) def save_debug_info(self, json_info, platform, kind): text = json.dumps(json_info) - if not settings.AWS_ENABLED: - with open(self.get_debug_info_filename(platform, kind), 'w') as f: - f.write(text) - else: - s3.save_file('builds', self.get_debug_info_filename(platform, kind), text, public=True, content_type='application/json') + s3.save_file('builds', self.get_debug_info_filename(platform, kind), text, public=True, content_type='application/json') def save_pbw(self, pbw_path): - if not settings.AWS_ENABLED: - shutil.move(pbw_path, self.pbw) - else: - s3.upload_file('builds', self.pbw, pbw_path, public=True, download_filename='%s.pbw' % self.project.app_short_name.replace('/','-')) + s3.upload_file('builds', self.pbw, pbw_path, public=True, download_filename='%s.pbw' % self.project.app_short_name.replace('/','-')) def save_simplyjs(self, javascript): - if not settings.AWS_ENABLED: - with open(self.simplyjs, 'w') as f: - f.write(javascript) - else: - s3.save_file('builds', self.simplyjs, javascript, public=True, content_type='text/javascript') + s3.save_file('builds', self.simplyjs, javascript, public=True, content_type='text/javascript') pbw = property(get_pbw_filename) build_log = property(get_build_log) diff --git a/ide/models/files.py b/ide/models/files.py index 4ca9b1cf..99a619de 100644 --- a/ide/models/files.py +++ b/ide/models/files.py @@ -1,10 +1,6 @@ import os -import shutil import traceback -import datetime import json -from django.conf import settings -from django.core.validators import RegexValidator from django.db import models from django.db.models.signals import post_delete from django.dispatch import receiver @@ -251,13 +247,7 @@ class Meta(IdeModel.Meta): @receiver(post_delete) def delete_file(sender, instance, **kwargs): if issubclass(sender, S3File): - if settings.AWS_ENABLED: - try: - s3.delete_file(sender.bucket_name, instance.s3_path) - except: - traceback.print_exc() - else: - try: - os.unlink(instance.local_filename) - except OSError: - pass + try: + s3.delete_file(sender.bucket_name, instance.s3_path) + except: + traceback.print_exc() diff --git a/ide/models/meta.py b/ide/models/meta.py index 4c24c907..a0373c31 100644 --- a/ide/models/meta.py +++ b/ide/models/meta.py @@ -1,5 +1,6 @@ from django.db import models + class IdeModel(models.Model): class Meta: abstract = True diff --git a/ide/models/s3file.py b/ide/models/s3file.py index a86be5ac..5856d838 100644 --- a/ide/models/s3file.py +++ b/ide/models/s3file.py @@ -1,6 +1,3 @@ -import shutil -import os - from django.utils.translation import ugettext as _ from django.conf import settings from django.utils.timezone import now @@ -13,7 +10,6 @@ class S3File(IdeModel): bucket_name = 'source' folder = None project = None - _create_local_if_not_exists = False @property def padded_id(self): @@ -32,41 +28,11 @@ def s3_id(self): def s3_path(self): return '%s/%s' % (self.folder, self.s3_id) - def _get_contents_local(self): - try: - return open(self.local_filename).read() - except IOError: - if self._create_local_if_not_exists: - return '' - else: - raise - - def _save_string_local(self, string): - if not os.path.exists(os.path.dirname(self.local_filename)): - os.makedirs(os.path.dirname(self.local_filename)) - with open(self.local_filename, 'wb') as out: - out.write(string) - - def _copy_to_path_local(self, path): - try: - shutil.copy(self.local_filename, path) - except IOError as err: - if err.errno == 2 and self._crete_local_if_not_exists: - open(path, 'w').close() # create the file if it's missing. - else: - raise - def get_contents(self): - if not settings.AWS_ENABLED: - return self._get_contents_local() - else: - return s3.read_file(self.bucket_name, self.s3_path) + return s3.read_file(self.bucket_name, self.s3_path) def save_string(self, string): - if not settings.AWS_ENABLED: - self._save_string_local(string) - else: - s3.save_file(self.bucket_name, self.s3_path, string) + s3.save_file(self.bucket_name, self.s3_path, string) if self.project: self.project.last_modified = now() self.project.save() @@ -80,10 +46,7 @@ def save_text(self, content): self.save_string(content.encode('utf-8')) def copy_to_path(self, path): - if not settings.AWS_ENABLED: - self._copy_to_path_local(path) - else: - s3.read_file_to_filesystem(self.bucket_name, self.s3_path, path) + s3.read_file_to_filesystem(self.bucket_name, self.s3_path, path) class Meta(IdeModel.Meta): abstract = True diff --git a/ide/models/scriptfile.py b/ide/models/scriptfile.py index 25b5ff3e..cccdfd0c 100644 --- a/ide/models/scriptfile.py +++ b/ide/models/scriptfile.py @@ -10,7 +10,6 @@ class ScriptFile(S3File): """ ScriptFiles add support to TextFiles for last-modified timestamps and code folding """ last_modified = models.DateTimeField(blank=True, null=True, auto_now=True) folded_lines = models.TextField(default="[]") - _create_local_if_not_exists = True def was_modified_since(self, expected_modification_time): if isinstance(expected_modification_time, int): diff --git a/ide/tasks/archive.py b/ide/tasks/archive.py index faa01a2e..341a31b8 100644 --- a/ide/tasks/archive.py +++ b/ide/tasks/archive.py @@ -63,16 +63,9 @@ def create_archive(project_id): send_td_event('cloudpebble_export_project', project=project) - if not settings.AWS_ENABLED: - outfile = '%s%s/%s.zip' % (settings.EXPORT_DIRECTORY, u, prefix) - os.makedirs(os.path.dirname(outfile), 0755) - shutil.copy(filename, outfile) - os.chmod(outfile, 0644) - return '%s%s/%s.zip' % (settings.EXPORT_ROOT, u, prefix) - else: - outfile = '%s/%s.zip' % (u, prefix) - s3.upload_file('export', outfile, filename, public=True, content_type='application/zip') - return '%s%s' % (settings.EXPORT_ROOT, outfile) + outfile = '%s/%s.zip' % (u, prefix) + s3.upload_file('export', outfile, filename, public=True, content_type='application/zip') + return '%s%s' % (settings.EXPORT_ROOT, outfile) @task(acks_late=True) @@ -105,7 +98,7 @@ def get_filename_variant(file_name, resource_suffix_map): split = file_name_parts[0].split("~") tags = split[1:] try: - ids = [resource_suffix_map['~'+tag] for tag in tags] + ids = [resource_suffix_map['~' + tag] for tag in tags] except KeyError as key: raise ValueError('Unrecognised tag %s' % key) root_file_name = split[0] + file_name_parts[1] @@ -146,7 +139,6 @@ def do_import_archive(project_id, archive, delete_project=False): raise Exception("Too many files in zip file.") file_list = [x.filename for x in contents] - base_dir = find_project_root(file_list) dir_end = len(base_dir) @@ -223,9 +215,9 @@ def make_valid_filename(zip_entry): filename = make_valid_filename(zipitem) if filename is False or not filename.startswith(RES_PATH): continue - filename = filename[len(RES_PATH)+1:] + filename = filename[len(RES_PATH) + 1:] try: - extracted = z.open("%s%s/%s"%(base_dir, RES_PATH, filename)) + extracted = z.open("%s%s/%s" % (base_dir, RES_PATH, filename)) except KeyError: print "Failed to open %s" % filename continue diff --git a/utils/s3.py b/utils/s3.py index 4294326d..1c8b031b 100644 --- a/utils/s3.py +++ b/utils/s3.py @@ -4,6 +4,7 @@ from django.conf import settings import urllib + def _ensure_bucket_exists(s3, bucket): try: s3.create_bucket(bucket) @@ -12,57 +13,43 @@ def _ensure_bucket_exists(s3, bucket): else: print "Created bucket %s" % bucket -if settings.AWS_ENABLED: - if settings.AWS_S3_FAKE_S3 is None: - _s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) - else: - host, port = (settings.AWS_S3_FAKE_S3.split(':', 2) + [80])[:2] - port = int(port) - _s3 = boto.connect_s3("key_id", "secret_key", is_secure=False, port=port, - host=host, calling_format=OrdinaryCallingFormat()) - _ensure_bucket_exists(_s3, settings.AWS_S3_SOURCE_BUCKET) - _ensure_bucket_exists(_s3, settings.AWS_S3_EXPORT_BUCKET) - _ensure_bucket_exists(_s3, settings.AWS_S3_BUILDS_BUCKET) - - _buckets = { - 'source': _s3.get_bucket(settings.AWS_S3_SOURCE_BUCKET), - 'export': _s3.get_bucket(settings.AWS_S3_EXPORT_BUCKET), - 'builds': _s3.get_bucket(settings.AWS_S3_BUILDS_BUCKET), - } -else: - _s3 = None - _buckets = None +if settings.AWS_S3_FAKE_S3 is None: + _s3 = boto.connect_s3(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY) +else: + host, port = (settings.AWS_S3_FAKE_S3.split(':', 2) + [80])[:2] + port = int(port) + _s3 = boto.connect_s3("key_id", "secret_key", is_secure=False, port=port, + host=host, calling_format=OrdinaryCallingFormat()) + _ensure_bucket_exists(_s3, settings.AWS_S3_SOURCE_BUCKET) + _ensure_bucket_exists(_s3, settings.AWS_S3_EXPORT_BUCKET) + _ensure_bucket_exists(_s3, settings.AWS_S3_BUILDS_BUCKET) -def _requires_aws(fn): - if settings.AWS_ENABLED: - return fn - else: - def complain(*args, **kwargs): - raise Exception("AWS_ENABLED must be True to call %s" % fn.__name__) - return complain +_buckets = { + 'source': _s3.get_bucket(settings.AWS_S3_SOURCE_BUCKET), + 'export': _s3.get_bucket(settings.AWS_S3_EXPORT_BUCKET), + 'builds': _s3.get_bucket(settings.AWS_S3_BUILDS_BUCKET), +} -@_requires_aws def read_file(bucket_name, path): bucket = _buckets[bucket_name] key = bucket.get_key(path) return key.get_contents_as_string() -@_requires_aws def read_file_to_filesystem(bucket_name, path, destination): bucket = _buckets[bucket_name] key = bucket.get_key(path) key.get_contents_to_filename(destination) -@_requires_aws + def delete_file(bucket_name, path): bucket = _buckets[bucket_name] key = bucket.get_key(path) key.delete() -@_requires_aws + def save_file(bucket_name, path, value, public=False, content_type='application/octet-stream'): bucket = _buckets[bucket_name] key = Key(bucket) @@ -76,7 +63,6 @@ def save_file(bucket_name, path, value, public=False, content_type='application/ key.set_contents_from_string(value, policy=policy, headers={'Content-Type': content_type}) -@_requires_aws def upload_file(bucket_name, dest_path, src_path, public=False, content_type='application/octet-stream', download_filename=None): bucket = _buckets[bucket_name] key = Key(bucket) @@ -92,12 +78,11 @@ def upload_file(bucket_name, dest_path, src_path, public=False, content_type='ap } if download_filename is not None: - headers['Content-Disposition'] = 'attachment;filename="%s"' % download_filename.replace(' ','_') + headers['Content-Disposition'] = 'attachment;filename="%s"' % download_filename.replace(' ', '_') key.set_contents_from_filename(src_path, policy=policy, headers=headers) -@_requires_aws def get_signed_url(bucket_name, path, headers=None): bucket = _buckets[bucket_name] key = bucket.get_key(path)