Update sanic-rest references

flamingo/main.py

@@ -1,14 +1,16 @@
 # pylint: disable=wrong-import-position
 import os
 import sys
+
+from sanic_rest import exceptions
+
 sys.path.append(os.path.dirname(__file__))
 
 from sanic import Sanic
 from sanic_openapi import swagger_blueprint
 
 import settings
 import views
-import exceptions
 
 app = Sanic(name='flamingo', error_handler=exceptions.rest_error_handler)
 app.update_config(settings)

flamingo/models/build.py

@@ -2,8 +2,8 @@
 from typing import List
 
 from gcp_pilot.datastore import EmbeddedDocument
+from sanic_rest import exceptions
 
-import exceptions
 import settings
 from models.buildpack import BuildPack
 from models.label import Label

flamingo/models/repository.py

@@ -3,8 +3,8 @@
 from gcp_pilot.build import AnyEventType, CloudBuild
 from gcp_pilot.datastore import EmbeddedDocument
 from github import Github
+from sanic_rest import exceptions
 
-import exceptions
 import settings
 from models.project import Project
 

flamingo/views/app_views.py

@@ -2,8 +2,8 @@
 
 from sanic import Blueprint
 from sanic.request import Request
+from sanic_rest import exceptions
 
-import exceptions
 from models.app import App
 from models.database import Database
 from models.env_var import EnvVar

tsuru.py

@@ -0,0 +1,229 @@
+import json
+import os
+from urllib.parse import urlparse
+
+import requests
+
+
+class Tsuru:
+    API_URL = 'http://10.2.0.2:8080'  # This URL is in your `tsuru target list`
+    TOKEN = None
+
+    def __init__(self, email, password):
+        self.email = email
+        self.password = password
+
+    @property
+    def token(self):
+        if not self.TOKEN:
+            url = f"{self.API_URL}/1.0/users/{self.email}/tokens"
+            self.TOKEN = requests.post(url=url, data={'password': self.password}).json()['token']
+        return self.TOKEN
+
+    def _get(self, endpoint, params=None):
+        url = f"{self.API_URL}{endpoint}"
+        headers = {"Authorization": f"Bearer {self.token}"}
+        response = requests.get(url=url, params=params, headers=headers)
+        response.raise_for_status()
+        return response.json()
+
+    def env_get(self, name):
+        endpoint = f'/1.0/apps/{name}/env'
+        return self._get(endpoint=endpoint)
+
+    def app_list(self):
+        endpoint = f'/1.0/apps'
+        return self._get(endpoint=endpoint)
+
+    def dump(self, filename='./tsuru.json'):
+        apps = []
+        for app in self.app_list():
+            apps.append(dict(**app, env=self.env_get(name=app['name'])))
+        json.dump(apps, open(str(filename), 'w'))
+
+
+class Flamingo:
+    API_URL = 'https://flamingo.eduk.dev'
+    SKIP_ENV_VARS = [
+        'DATABASE_URL',
+        'RABBITMQ_URL',
+        'REDIS_URL',
+        'DEBUG',
+        'GCP_B64',
+        'GCP_JSON',
+        'PYPI_PASSWORD',
+        'GOOGLE_CHAT_HOOK_URL',
+    ]
+
+    def __init__(self, token=None):
+        self.token = token or self._get_token()
+
+    def _get_token(self):
+        stream = os.popen('gcloud auth print-identity-token')
+        return stream.read().strip()
+
+    def restore(self, filename='./tsuru.json', apps=None):
+        tsuru_data = json.load(open(str(filename)))
+        for app_data in tsuru_data:
+            if apps and app_data['name'] not in apps:
+                continue
+            flamingo = self._build_flamingo_data(data=app_data)
+            self.create_or_update(flamingo)
+
+    def _call_flamingo(self, method, endpoint, **kwargs):
+        url = f"{self.API_URL}{endpoint}"
+        headers = {"Authorization": f"Bearer {self.token}"}
+        response = getattr(requests, method)(url=url, headers=headers, **kwargs)
+        return response.json(), response.status_code
+
+    def _platform_to_buildpack(self, name, platform):
+        if platform == 'python' and 'api' in name:
+            return 'django'
+
+    def _detect_size(self, units):
+        return max(2 * len(units), 10)
+
+    def _parse_envs(self, name, env_name, platform, envs):
+        all_envs = []
+        for e in envs:
+            key, value, is_secret = e['name'], e['value'], bool(not e['public'])
+
+            if key.startswith('TSURU_') or key in self.SKIP_ENV_VARS:
+                continue
+
+            if key.endswith('_API_URL'):
+                value = value.replace('-stg.eduktsuru.vpc', '.stg.eduk.dev').replace('-prd.eduktsuru.vpc', '.eduk.dev')
+
+            if '.vpc' in value:
+                print(f"{value} wont be reachable from outside the VPC")
+
+            if 'TOKEN' in name or 'KEY' in name:
+                is_secret = True
+
+            all_envs.append({
+                "key": key,
+                "value": value,
+                "is_secret": is_secret,
+            })
+        return all_envs
+
+    def _get_env_var(self, env_vars, key):
+        for e in env_vars:
+            if e['name'] == key:
+                return e['value']
+        return None
+
+    def _get_db(self, name, env_name, platform, url):
+        if url:
+            if env_name == 'stg':
+                instance = "shared-pg-stg"
+                tier = "db-f1-micro"
+            else:
+                instance = name
+                tier = "db-n1-standard-1"
+            parts = urlparse(url)
+            return {
+                "database": {
+                    "instance": instance,
+                    "name": parts.path.replace('/', ''),
+                    "user": parts.username,
+                    "password": parts.password,
+                    "version": "POSTGRES_12" if parts.scheme == 'postgres' else 'MYSQL_5_6',
+                    "tier": tier,
+                    "region": "us-east1",
+                    "project": {
+                        "id": f"eduk-{env_name}",
+                    },
+                    "env_var": "DATABASE_URL" if platform == 'python' else 'DB_*',
+                    "high_availability": False
+                }
+            }
+        return {}
+
+    def _build_flamingo_data(self, data):
+        name, env_name = data['name'].rsplit('-', 1)
+        cpu = data['plan']['cpushare']  # Cant use this because it's a relative number among all units in the same pool
+        ram = data['plan']['memory'] / 1024 / 1024
+        platform = data['platform']
+
+        env_vars = self._parse_envs(name=name, env_name=env_name, platform=platform, envs=data['env'])
+        db_data = self._get_db(
+            name=name,
+            env_name=env_name,
+            platform=platform,
+            url=self._get_env_var(data['env'], 'DATABASE_URL')
+        )
+
+        return {
+            "name": name,
+            "environment_name": env_name,
+            "build_setup": {
+                "build_pack_name": self._platform_to_buildpack(name=name, platform=platform),
+                "deploy_branch": None,
+                "deploy_tag": f"^{env_name}.*",
+                "post_build_commands": [],
+                "os_dependencies": [],
+                "labels": [],
+                "memory": int(ram),
+                # "cpu": 1,
+                # "min_instances": 0,
+                "max_instances": self._detect_size(data['units']),
+                # "timeout": 900,
+                # "concurrency": 80,
+                "is_authenticated": True
+            },
+            "repository": {
+                "name": f"edukorg/{name}",
+                "mirrored": True,
+            },
+            "domains": [
+                f"{name}.{env_name}.eduk.dev" if env_name == 'stg' else f"{name}.eduk.dev"
+            ],
+            "vars": env_vars,
+            "bucket": {
+                "name": identifier,
+                "env_var": "GCS_BUCKET_NAME",
+                "region": "us-east1",
+                "project": {
+                    "id": f"eduk-{env_name}",
+                }
+            },
+            "region": "us-east1",
+            "service_account": {
+                "name": f"{name}",
+                "description": f"{name} Service Account",
+                "display_name": f"{name}",
+                "roles": [
+                    "organizations/970900718166/roles/ServerlessCloudApp"
+                ],
+                "project": {
+                    "id": f"eduk-{env_name}",
+                }
+            },
+            **db_data
+        }
+
+    def create_or_update(self, data):
+        name = f'{data["name"]}-{data["environment_name"]}'
+        r, status = self._call_flamingo('get', endpoint=f'/apps/{name}')
+        if status == 404:
+            r, status = self._call_flamingo('post', endpoint=f'/apps', json=data)
+        elif status == 200:
+            r, status = self._call_flamingo('patch', endpoint=f'/apps/{name}', json=data)
+        else:
+            raise Exception(f"failed getting app: {r}")
+
+        if status not in [201, 200]:
+            raise Exception(f'failed creating app: {r}')
+
+        r, status = self._call_flamingo('post', endpoint=f'/apps/{name}/check', json=data)
+        if status != 200:
+            raise Exception(f'failed checking app: {r}')
+
+        r, status = self._call_flamingo('post', endpoint=f'/apps/{name}/apply', json=data)
+        if status != 200:
+            raise Exception(f'failed applying app: {r}')
+
+
+# Tsuru(email='[email protected]', password='352515').dump()
+Flamingo().restore(apps=['boss-api-stg'])