diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 745ef857..cf330593 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -109,17 +109,3 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha,src=docker/env-cuda-11-8
           cache-to: type=gha,mode=max
-
-      # Build and push Docker image with Buildx (don't push on PR)
-      # https://github.com/docker/build-push-action
-      - name: Build and push Docker image (github-worker-cuda-11-8)
-        id: build-and-push
-        uses: docker/build-push-action@v4
-        with:
-          context: "{{defaultContext}}:docker/github-worker-cuda-11-8"
-          push: ${{ github.event_name != 'pull_request' }} # Don't push on PR
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:github-worker-cuda-11-8
-          # tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha,src=docker/github-worker-cuda-11-8
-          cache-to: type=gha,mode=max
diff --git a/.gitignore b/.gitignore
index 5ec6c499..2e2fbd86 100644
--- a/.gitignore
+++ b/.gitignore
@@ -165,6 +165,9 @@ notebook/scratch/*
 */lightning_logs/
 */config.yaml
 
+# Add back script files used in docker images
+!docker/**/*.sh
+
 # Add .github items
 !.github/
 
diff --git a/docker/env-cuda-11-8/Dockerfile b/docker/env-cuda-11-8/Dockerfile
index 2a6c38ea..720d961b 100644
--- a/docker/env-cuda-11-8/Dockerfile
+++ b/docker/env-cuda-11-8/Dockerfile
@@ -19,4 +19,7 @@ RUN pip3 install lightning==2.0.5 deepspeed==0.10.0 \
 
 # Install the misc packages we might need for the various experiments
 RUN pip3 install \
-    papermill aiocsv aiofiles
\ No newline at end of file
+    papermill aiocsv aiofiles
+
+# Configure default dir, to the home directory
+WORKDIR /root