Fix Spark image

dynatrace-research · Feb 1, 2024 · 84cd2f6 · 84cd2f6
1 parent 1d5b5c9
commit 84cd2f6
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 4 deletions.
diff --git a/shuffle-spark/Dockerfile b/shuffle-spark/Dockerfile
@@ -24,9 +24,9 @@ RUN mkdir -p /opt && \
 
 COPY build/libs/shuffle-spark-1.0-SNAPSHOT.jar .
 
-#ADD common.sh spark-master spark-worker /
+ADD common.sh spark-master spark-worker /
 RUN cd /
-#RUN chmod +x spark-master
-#RUN chmod +x spark-worker
-#ADD spark-defaults.conf /opt/spark/conf/spark-defaults.conf
+RUN chmod +x spark-master
+RUN chmod +x spark-worker
+ADD spark-defaults.conf /opt/spark/conf/spark-defaults.conf
 ENV PATH $PATH:/opt/spark/bin
diff --git a/shuffle-spark/common.sh b/shuffle-spark/common.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# unset variable set by kubernetes
+unset SPARK_MASTER_PORT
diff --git a/shuffle-spark/spark-defaults.conf b/shuffle-spark/spark-defaults.conf
@@ -0,0 +1,2 @@
+spark.master spark://spark-master:7077
+spark.driver.extraLibraryPath /opt/hadoop/lib/native
diff --git a/shuffle-spark/spark-master b/shuffle-spark/spark-master
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. /common.sh
+
+echo "$(hostname -i) spark-master" >> /etc/hosts
+
+/opt/spark/bin/spark-class org.apache.spark.deploy.master.Master --ip spark-master --port 7077 --webui-port 8080
diff --git a/shuffle-spark/spark-worker b/shuffle-spark/spark-worker
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+. /common.sh
+
+if ! getent hosts spark-master; then
+  sleep 5
+  exit 0
+fi
+
+/opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077 --webui-port 8081
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		spark.master spark://spark-master:7077
		spark.driver.extraLibraryPath /opt/hadoop/lib/native