Add k8s installation

k8s-installation/Makefile

@@ -0,0 +1,29 @@
+PLATFORM ?= linux/amd64
+IMAGE_NAME ?= airflow
+TAG ?= latest
+REGISTRY ?= "localhost:5001"
+RELEASE_NAME ?= "airflow"
+NAMESPACE ?= "airflow"
+SSH_KEY ?= "~/.ssh/dags_ssh"
+
+.PHONY: build
+build:
+	-cd .. && docker build -f docker-installation/Dockerfile --platform $(PLATFORM) -t $(REGISTRY)/$(IMAGE_NAME):$(TAG) .	
+
+.PHONY: push
+push: 
+	-docker push $(REGISTRY)/$(IMAGE_NAME):$(TAG)
+
+.PHONY: deploy
+deploy:
+	-helm install $(RELEASE_NAME) apache-airflow/airflow --namespace $(NAMESPACE) --debug --timeout 10m0s -f values.yaml
+	-create-connections
+	-git-sync-credentials
+
+.PHONY: create-connections
+create_connections:
+	-kubectl create secret generic airflow-connections --from-file=airflow-connections.yaml -n $(NAMESPACE)
+
+.PHONY: git-sync-credentials
+git-sync-credentials:
+	-kubectl create secret generic airflow-ssh-git-secret --from-file=gitSshKey=$(SSH_KEY) -n $(NAMESPACE)

k8s-installation/kind-cluster.yaml

@@ -0,0 +1,34 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes: 
+- role: control-plane
+- role: worker
+  kubeadmConfigPatches:
+  - |
+    kind: JoinConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "node=worker_1"
+  extraMounts:
+    - hostPath: ./data
+      containerPath: /tmp/data
+- role: worker
+  kubeadmConfigPatches:
+  - |
+    kind: JoinConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "node=worker_2"
+  extraMounts:
+    - hostPath: ./data
+      containerPath: /tmp/data
+- role: worker
+  kubeadmConfigPatches:
+  - |
+    kind: JoinConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "node=worker_3"
+  extraMounts:
+    - hostPath: ./data
+      containerPath: /tmp/data

requirements.txt

@@ -0,0 +1,3 @@
+kafka-python-ng
+apache-airflow-providers-mongo
+apache-airflow-providers-apache-kafka