diff --git a/k8s-installation/Makefile b/k8s-installation/Makefile new file mode 100644 index 0000000..a08366e --- /dev/null +++ b/k8s-installation/Makefile @@ -0,0 +1,29 @@ +PLATFORM ?= linux/amd64 +IMAGE_NAME ?= airflow +TAG ?= latest +REGISTRY ?= "localhost:5001" +RELEASE_NAME ?= "airflow" +NAMESPACE ?= "airflow" +SSH_KEY ?= "~/.ssh/dags_ssh" + +.PHONY: build +build: + -cd .. && docker build -f docker-installation/Dockerfile --platform $(PLATFORM) -t $(REGISTRY)/$(IMAGE_NAME):$(TAG) . + +.PHONY: push +push: + -docker push $(REGISTRY)/$(IMAGE_NAME):$(TAG) + +.PHONY: deploy +deploy: + -helm install $(RELEASE_NAME) apache-airflow/airflow --namespace $(NAMESPACE) --debug --timeout 10m0s -f values.yaml + -create-connections + -git-sync-credentials + +.PHONY: create-connections +create_connections: + -kubectl create secret generic airflow-connections --from-file=airflow-connections.yaml -n $(NAMESPACE) + +.PHONY: git-sync-credentials +git-sync-credentials: + -kubectl create secret generic airflow-ssh-git-secret --from-file=gitSshKey=$(SSH_KEY) -n $(NAMESPACE) diff --git a/k8s-installation/kind-cluster.yaml b/k8s-installation/kind-cluster.yaml new file mode 100644 index 0000000..47a54ee --- /dev/null +++ b/k8s-installation/kind-cluster.yaml @@ -0,0 +1,34 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "node=worker_1" + extraMounts: + - hostPath: ./data + containerPath: /tmp/data +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "node=worker_2" + extraMounts: + - hostPath: ./data + containerPath: /tmp/data +- role: worker + kubeadmConfigPatches: + - | + kind: JoinConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "node=worker_3" + extraMounts: + - hostPath: ./data + containerPath: /tmp/data diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..790b625 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +kafka-python-ng +apache-airflow-providers-mongo +apache-airflow-providers-apache-kafka