Merge pull request #15 from katulu-io/GerardoGR/edge-identity

Edge identity service foundation

.gitmodules

@@ -1,3 +1,9 @@
+[submodule "components/vendor/kubeflow-manifests"]
+	path = components/vendor/kubeflow-manifests
+	url = https://github.com/kubeflow/manifests.git
 [submodule "components/vendor/kubeflow"]
 	path = components/vendor/kubeflow
-	url = [email protected]:kubeflow/manifests.git
+	url = https://github.com/kubeflow/kubeflow.git
+[submodule "components/vendor/spire-api-sdk"]
+	path = components/vendor/spire-api-sdk
+	url = https://github.com/spiffe/spire-api-sdk.git

Makefile

@@ -22,7 +22,7 @@ commitlint:
 .PHONY: commitlint
 
 dockerlint:
-	@find . -maxdepth 5 -type f -name Dockerfile | xargs -I {} hadolint {}
+	@find . -maxdepth 5 -type f -name Dockerfile ! -path "*/vendor/*" | xargs -I {} hadolint {}
 .PHONY: dockerlint
 
 LINTTARGETS = $(SUBDIRS:%=lint--%)

components/centraldashboard/src/app/config.service.ts

@@ -2,6 +2,12 @@ import { Injectable } from '@angular/core';
 
 const APPS = [
   { path: '_/jupyter', location: '/jupyter/', label: 'Notebooks', icon: 'book' },
+  {
+    path: '_/edges',
+    location: '/edges/',
+    label: 'Edges',
+    icon: 'developer_board',
+  },
   {
     path: '_/volumes',
     location: '/volumes/',

components/edge-identity/.gitignore

@@ -0,0 +1,5 @@
+/.poetry.stamp
+
+/backend/edge-identity.db
+/kustomize/base/kustomization.yaml
+/backend/apps/default/static/

components/edge-identity/Dockerfile

@@ -0,0 +1,68 @@
+# --- Build the frontend kubeflow library ---
+FROM node:16.13.1 as frontend-kubeflow-lib
+
+WORKDIR /src
+
+ENV NG_CLI_ANALYTICS "ci"
+COPY ./vendor/kubeflow/components/crud-web-apps/common/frontend/kubeflow-common-lib/package.json ./
+COPY ./vendor/kubeflow/components/crud-web-apps/common/frontend/kubeflow-common-lib/package-lock.json ./
+RUN npm ci
+
+COPY ./vendor/kubeflow/components/crud-web-apps/common/frontend/kubeflow-common-lib/projects ./projects
+COPY ./vendor/kubeflow/components/crud-web-apps/common/frontend/kubeflow-common-lib/angular.json .
+COPY ./vendor/kubeflow/components/crud-web-apps/common/frontend/kubeflow-common-lib/tsconfig.json .
+RUN npm run build
+
+# --- Build the frontend ---
+FROM node:16.13.1-buster-slim as frontend
+
+WORKDIR /src
+
+COPY ./edge-identity/frontend/package.json ./
+COPY ./edge-identity/frontend/package-lock.json ./
+COPY ./edge-identity/frontend/tsconfig.json ./
+COPY ./edge-identity/frontend/tsconfig.app.json ./
+COPY ./edge-identity/frontend/tsconfig.spec.json ./
+COPY ./edge-identity/frontend/angular.json ./
+COPY ./edge-identity/frontend/src ./src
+
+ENV NG_CLI_ANALYTICS "ci"
+RUN npm ci
+COPY --from=frontend-kubeflow-lib /src/dist/kubeflow/ ./node_modules/kubeflow/
+
+RUN npm run build -- --output-path=./dist/default --configuration=production
+
+FROM python:3.9-slim-buster as webapp
+
+SHELL ["/bin/bash", "-euxo", "pipefail", "-c"]
+
+# Install poetry
+ENV POETRY_HOME "/opt/poetry"
+ENV PATH $POETRY_HOME/bin:$PATH
+ENV POETRY_VIRTUALENVS_CREATE "false"
+# hadolint ignore=DL3008
+RUN url="https://install.python-poetry.org"; \
+    apt-get update -qq; \
+    apt-get install -qq -y --no-install-recommends wget; \
+    \
+    wget -q -O- "$url" | POETRY_VERSION=1.1.13 python3 -; \
+    poetry --version; \
+    rm -rf /var/lib/apt/lists/*;
+
+WORKDIR /app/src
+
+COPY ./edge-identity/backend/pyproject.toml .
+COPY ./edge-identity/backend/poetry.lock .
+
+RUN mkdir /vendor
+COPY ./vendor/kubeflow /vendor/kubeflow
+
+RUN poetry install
+
+COPY ./edge-identity/backend/spire/ ./spire
+COPY ./edge-identity/backend/apps/ ./apps
+COPY ./edge-identity/backend/entrypoint.py .
+
+COPY --from=frontend /src/dist/default/ /app/src/apps/default/static/
+
+ENTRYPOINT ["/bin/bash", "-c", "FLASK_APP=apps.default flask init-db && gunicorn -w 3 --bind 0.0.0.0:5000 --access-logfile - entrypoint:app"]

components/edge-identity/Makefile

@@ -0,0 +1,75 @@
+-include ../../.devcontainer/targets.mk
+
+MAKEVAR_REGISTRY?=localhost:5000
+
+MAKEVAR_VERSION?=0.0.0-dev-$(shell git log -n 1 --pretty=format:%h)
+CONTAINER_TAG = $(shell echo "${MAKEVAR_VERSION}" | sed "s/\+[0-9a-zA-Z-]*//")
+IMAGE_NAME = edge-identity
+IMAGE_URL = ${MAKEVAR_REGISTRY}/${IMAGE_NAME}:${CONTAINER_TAG}
+# We're changing the version a bit to make it work with pip, as PEP 440 isn't semver compatible.
+PIP_VERSION = $(shell echo "${MAKEVAR_VERSION}" | sed -e "s/-/+/" -e "s/-/./g")
+POETRY_STAMP := .poetry.stamp
+
+# Build is not needed because the dist target builds the application via docker multi-stage
+lint test build:
+	@echo "$@ not implemented"
+.PHONY: lint test build
+
+${POETRY_STAMP}: backend/poetry.lock
+	@cd backend/ && \
+		poetry install
+	touch ${POETRY_STAMP}
+
+dependencies: ${POETRY_STAMP}
+
+backend/edge-identity.db: ${POETRY_STAMP} backend/apps/default/schema.sql
+	@cd backend/ && FLASK_APP=apps.default poetry run flask init-db
+
+dist: manifests
+	@cd backend/ && \
+		poetry version ${PIP_VERSION}
+
+	# Using ../ as docker context to be able to retrieve the components/vendor/kubeflow for the common backend/frontend code
+	DOCKER_BUILDKIT=1 docker build -t ${IMAGE_URL} -f Dockerfile ..
+.PHONY: dist
+
+manifests:
+	@cd kustomize/base/ && \
+		cp kustomization.tpl.yaml kustomization.yaml && \
+		kustomize edit set image ${IMAGE_NAME}=${IMAGE_URL}
+.PHONY: manifests
+
+
+run: ${POETRY_STAMP} backend/edge-identity.db
+	@cd backend/ && \
+		poetry run gunicorn -w 3 --bind 0.0.0.0:8080 --access-logfile - entrypoint:app
+run: export BACKEND_MODE = dev
+run: export APP_SECURE_COOKIES = False
+run: export APP_PREFIX = /
+run: export UI_FLAVOR = default
+.PHONY: run
+
+spire-sdk:
+	@cd backend/ && \
+		poetry run python -m grpc_tools.protoc \
+			--python_out=. --grpc_python_out=. \
+			-I ../../vendor/spire-api-sdk/proto/ \
+			../../vendor/spire-api-sdk/proto/spire/api/server/agent/v1/agent.proto \
+			../../vendor/spire-api-sdk/proto/spire/api/types/*.proto
+.PHONY: spire-sdk
+
+push:
+	docker push ${IMAGE_URL}
+.PHONY: push
+
+install:
+	kustomize build kustomize/overlays/istio | kubectl apply -f -
+.PHONY: install
+
+uninstall:
+	kustomize build kustomize/overlays/istio | kubectl delete -f -
+.PHONY: uninstall
+
+maintainer-clean:
+	rm -fr ../vendor/kubeflow/components/crud-web-apps/common/frontend/kubeflow-common-lib/dist/kubeflow frontend/node_modules frontend/dist/
+.PHONY: maintainer-clean

components/edge-identity/README.md

@@ -0,0 +1,31 @@
+# Edge identity service
+
+## Local development
+
+### Build the common (kubeflow) frontend library
+```shell
+# build the common library
+cd $FL_SUITE_ROOT_DIR/components/vendor/kubeflow/components/crud-web-apps/common/frontend/kubeflow-common-lib
+npm i
+npm run build
+cd dist/kubeflow
+npm link
+```
+
+### Build the frontend
+
+```shell
+cd $FL_SUITE_ROOT_DIR/components/edge-identity/frontend
+npm i
+npm link kubeflow
+npm run build:watch
+```
+
+### Run the backend service
+
+The backend service serves the frontend (static files).
+
+```shell
+cd $FL_SUITE_ROOT_DIR/components/edge-identity
+make run
+```

components/edge-identity/backend/apps/default/__init__.py

@@ -0,0 +1,29 @@
+import os
+
+import kubeflow.kubeflow.crud_backend as base
+from kubeflow.kubeflow.crud_backend import config, logging
+
+from . import db
+from .routes import bp as routes_bp
+
+log = logging.getLogger(__name__)
+
+
+def create_app(name=__name__, cfg: config.Config = None):
+    cfg = config.Config() if cfg is None else cfg
+
+    # Properly set the static serving directory
+    static_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "static")
+
+    cfg = config.Config() if cfg is None else cfg
+
+    app = base.create_app(name, static_dir, cfg)
+
+    log.info("Setting STATIC_DIR to: " + static_dir)
+    app.config["STATIC_DIR"] = static_dir
+
+    # Register the app's blueprints
+    app.register_blueprint(routes_bp)
+    db.init_app(app)
+
+    return app

components/edge-identity/backend/apps/default/db.py

@@ -0,0 +1,52 @@
+import sqlite3
+from typing import Any, List, Union
+
+import click
+from flask import current_app, g
+from flask.cli import with_appcontext
+
+DATABASE = "edge-identity.db"
+
+
+def get_db():
+    if "db" not in g:
+        g.db = sqlite3.connect(DATABASE, detect_types=sqlite3.PARSE_DECLTYPES)
+        g.db.row_factory = sqlite3.Row
+
+    return g.db
+
+
+def query_db(query, args=(), one=False) -> Union[Any, List[Any], None]:
+    cur = get_db().execute(query, args)
+    rv = cur.fetchall()
+    cur.close()
+    return (rv[0] if rv else None) if one else rv
+
+
+def close_db(exception=None):
+    db = g.pop("db", None)
+
+    if db is not None:
+        db.close()
+
+
+def init_db():
+    db = get_db()
+
+    with current_app.open_resource("schema.sql") as f:
+        db.executescript(f.read().decode("utf8"))
+
+    db.commit()
+
+
+@click.command("init-db")
+@with_appcontext
+def init_db_command():
+    """Clear the existing data and create new tables."""
+    init_db()
+    click.echo("Initialized the database.")
+
+
+def init_app(app):
+    app.teardown_appcontext(close_db)
+    app.cli.add_command(init_db_command)

components/edge-identity/backend/apps/default/routes/__init__.py

@@ -0,0 +1,5 @@
+from flask import Blueprint
+
+bp = Blueprint("default_routes", __name__)
+
+from . import delete, get, post  # noqa: F401, E402

components/edge-identity/backend/apps/default/routes/delete.py

@@ -0,0 +1,28 @@
+from kubeflow.kubeflow.crud_backend import api, logging
+
+from ..db import get_db
+from . import bp
+
+log = logging.getLogger(__name__)
+
+
+@bp.route("/api/namespaces/<namespace>/edges/<edge_name>", methods=["DELETE"])
+def delete_edge(edge_name, namespace):
+    """
+    Deregister an edge
+    """
+    log.info("Deregistering Edge %s/%s...", namespace, edge_name)
+
+    # TODO: Refactor into model
+    db_connection = get_db()
+    cur = db_connection.cursor()
+    cur.execute("PRAGMA foreign_keys = ON")
+    sql = "DELETE FROM Edge WHERE name = ? and namespace = ?"
+    cur.execute(sql, (edge_name, namespace))
+    db_connection.commit()
+
+    log.info("Successfully deregsitered Edge %s/%s", namespace, edge_name)
+
+    return api.success_response(
+        "message", "Edge %s successfully deregistered." % edge_name
+    )

components/edge-identity/backend/apps/default/routes/get.py

@@ -0,0 +1,34 @@
+from kubeflow.kubeflow.crud_backend import api, helpers, logging, status
+
+from ..db import query_db
+from . import bp
+
+log = logging.getLogger(__name__)
+
+
+@bp.route("/api/namespaces/<namespace>/edges")
+def get_edges(namespace):
+    edge_list = []
+
+    # TODO: Refactor into model
+    results = query_db(
+        "SELECT "
+        "e.edge_id, e.name, e.namespace, e.created_at, a.join_token "
+        "FROM Edge AS e join JoinTokenAuth as a ON a.edge_id = e.edge_id "
+        "WHERE e.namespace = ?;",
+        (namespace,),
+    )
+    if results is not None:
+        for edge in results:
+            edge_list.append(
+                {
+                    "name": edge["name"],
+                    "namespace": edge["namespace"],
+                    # The status is hardcoded because it is only used for presentation purposes
+                    "status": status.create_status(status.STATUS_PHASE.READY, "Ready"),
+                    "age": helpers.get_uptime(edge["created_at"]),
+                    "join_token": edge["join_token"],
+                }
+            )
+
+    return api.success_response("edges", edge_list)