From 2e0def2c3c8f4023aeaf58e363a284a132311e78 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Tue, 22 Aug 2023 02:28:26 +0000
Subject: [PATCH 01/16] Update to world tokenizer count

---
 RWKV-v4neo/src/model.py | 2 +-
 RWKV-v5/src/model.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/RWKV-v4neo/src/model.py b/RWKV-v4neo/src/model.py
index 7f9bcb3b..c92554ac 100644
--- a/RWKV-v4neo/src/model.py
+++ b/RWKV-v4neo/src/model.py
@@ -1256,7 +1256,7 @@ def __init__(
             tokenizer_file = os.path.join(SCRIPT_DIR,"./dataflow/20B_tokenizer.json")
             tokenizer = PreTrainedTokenizerFast(tokenizer_file=tokenizer_file)
             self.fastTokenizer = tokenizer
-        elif vocab_size == 65529 or vocab_size == 65536:
+        elif vocab_size == 65536:
             # Use the world tokenizer
             from .dataflow.trie_tokenizer import MT_TRIE_TOKENIZER
             world_tokenizer = MT_TRIE_TOKENIZER(os.path.join(SCRIPT_DIR, "./dataflow/rwkv_vocab_v20230424.txt"))
diff --git a/RWKV-v5/src/model.py b/RWKV-v5/src/model.py
index 8ef2c1c7..ce404795 100644
--- a/RWKV-v5/src/model.py
+++ b/RWKV-v5/src/model.py
@@ -1395,7 +1395,7 @@ def __init__(
             tokenizer_file = os.path.join(SCRIPT_DIR,"./dataflow/20B_tokenizer.json")
             tokenizer = PreTrainedTokenizerFast(tokenizer_file=tokenizer_file)
             self.fastTokenizer = tokenizer
-        elif vocab_size == 65529 or vocab_size == 65536:
+        elif vocab_size == 65536:
             # Use the world tokenizer
             from .dataflow.trie_tokenizer import MT_TRIE_TOKENIZER
             world_tokenizer = MT_TRIE_TOKENIZER(os.path.join(SCRIPT_DIR, "./dataflow/rwkv_vocab_v20230424.txt"))

From e636e7ddd54ba77548bedd4d0d6935576e916332 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Tue, 22 Aug 2023 07:44:07 +0000
Subject: [PATCH 02/16] WIP github worker pipeline

---
 .github/workflows/docker-build.yml        | 16 +++++++++++++++-
 docker/github-worker-cuda-11-8/Dockerfile |  8 ++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 docker/github-worker-cuda-11-8/Dockerfile

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index f91a3eba..395e5eef 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -98,7 +98,7 @@ jobs:
       
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
-      - name: Build and push Docker image
+      - name: Build and push Docker image (env-cuda-11-8)
         id: build-and-push
         uses: docker/build-push-action@v4
         with:
@@ -109,3 +109,17 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha,src=docker/env-cuda-11-8
           cache-to: type=gha,mode=max
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image (github-worker-cuda-11-8)
+        id: build-and-push
+        uses: docker/build-push-action@v4
+        with:
+          context: "{{defaultContext}}:docker/github-worker-cuda-11-8"
+          push: ${{ github.event_name != 'pull_request' }} # Don't push on PR
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:github-worker-cuda-11-8
+          # tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,src=docker/github-worker-cuda-11-8
+          cache-to: type=gha,mode=max
diff --git a/docker/github-worker-cuda-11-8/Dockerfile b/docker/github-worker-cuda-11-8/Dockerfile
new file mode 100644
index 00000000..93951375
--- /dev/null
+++ b/docker/github-worker-cuda-11-8/Dockerfile
@@ -0,0 +1,8 @@
+# Temporary, until the rwkv package is public
+FROM ghcr.io/picocreator/rwkv-lm-lora:env-cuda-11-8
+# FROM ghcr.io/rwkv/rwkv-infctx-trainer:env-cuda-11-8
+
+RUN mkdir actions-runner && cd actions-runner && \
+    curl -o actions-runner-linux-x64-2.308.0.tar.gz \
+    -L https://github.com/actions/runner/releases/download/v2.308.0/actions-runner-linux-x64-2.308.0.tar.gz && \
+    tar xzf ./actions-runner-linux-x64-2.308.0.tar.gz

From 51af911b539740b79f7dd419307c5b44a684a655 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Tue, 22 Aug 2023 07:47:12 +0000
Subject: [PATCH 03/16] enable back , "main-dev-infctx"

---
 .github/workflows/docker-build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 395e5eef..745ef857 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -2,11 +2,11 @@ name: Docker Env Image (cuda-11-8)
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ "main", "main-dev-infctx" ]
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
   pull_request:
-    branches: [ "main" ]
+    branches: [ "main", "main-dev-infctx" ]
 
 env:
   # Use docker.io for Docker Hub if empty

From c9bbc1a79a0b8af89da3c585818bb86548e5464e Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Tue, 22 Aug 2023 09:29:45 +0000
Subject: [PATCH 04/16] build-and-push is limtied to once per pipeline workflow

---
 .github/workflows/docker-build.yml | 14 --------------
 .gitignore                         |  3 +++
 docker/env-cuda-11-8/Dockerfile    |  5 ++++-
 3 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index 745ef857..cf330593 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -109,17 +109,3 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha,src=docker/env-cuda-11-8
           cache-to: type=gha,mode=max
-
-      # Build and push Docker image with Buildx (don't push on PR)
-      # https://github.com/docker/build-push-action
-      - name: Build and push Docker image (github-worker-cuda-11-8)
-        id: build-and-push
-        uses: docker/build-push-action@v4
-        with:
-          context: "{{defaultContext}}:docker/github-worker-cuda-11-8"
-          push: ${{ github.event_name != 'pull_request' }} # Don't push on PR
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:github-worker-cuda-11-8
-          # tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha,src=docker/github-worker-cuda-11-8
-          cache-to: type=gha,mode=max
diff --git a/.gitignore b/.gitignore
index 5ec6c499..2e2fbd86 100644
--- a/.gitignore
+++ b/.gitignore
@@ -165,6 +165,9 @@ notebook/scratch/*
 */lightning_logs/
 */config.yaml
 
+# Add back script files used in docker images
+!docker/**/*.sh
+
 # Add .github items
 !.github/
 
diff --git a/docker/env-cuda-11-8/Dockerfile b/docker/env-cuda-11-8/Dockerfile
index 2a6c38ea..720d961b 100644
--- a/docker/env-cuda-11-8/Dockerfile
+++ b/docker/env-cuda-11-8/Dockerfile
@@ -19,4 +19,7 @@ RUN pip3 install lightning==2.0.5 deepspeed==0.10.0 \
 
 # Install the misc packages we might need for the various experiments
 RUN pip3 install \
-    papermill aiocsv aiofiles
\ No newline at end of file
+    papermill aiocsv aiofiles
+
+# Configure default dir, to the home directory
+WORKDIR /root

From ea5baa363ecd8ce88eb25a29aa1780106eb4c683 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Tue, 22 Aug 2023 09:46:53 +0000
Subject: [PATCH 05/16] WIP runner setup

---
 docker/github-worker-cuda-11-8/Dockerfile    | 20 +++++++++--
 docker/github-worker-cuda-11-8/entrypoint.sh | 35 ++++++++++++++++++++
 2 files changed, 53 insertions(+), 2 deletions(-)
 create mode 100644 docker/github-worker-cuda-11-8/entrypoint.sh

diff --git a/docker/github-worker-cuda-11-8/Dockerfile b/docker/github-worker-cuda-11-8/Dockerfile
index 93951375..52cb6ebc 100644
--- a/docker/github-worker-cuda-11-8/Dockerfile
+++ b/docker/github-worker-cuda-11-8/Dockerfile
@@ -2,7 +2,23 @@
 FROM ghcr.io/picocreator/rwkv-lm-lora:env-cuda-11-8
 # FROM ghcr.io/rwkv/rwkv-infctx-trainer:env-cuda-11-8
 
-RUN mkdir actions-runner && cd actions-runner && \
+# Install the github runner
+RUN cd / && mkdir actions-runner && cd actions-runner && \
     curl -o actions-runner-linux-x64-2.308.0.tar.gz \
     -L https://github.com/actions/runner/releases/download/v2.308.0/actions-runner-linux-x64-2.308.0.tar.gz && \
-    tar xzf ./actions-runner-linux-x64-2.308.0.tar.gz
+    tar xzf ./actions-runner-linux-x64-2.308.0.tar.gz && \
+    rm ./actions-runner-linux-x64-2.308.0.tar.gz
+
+# Install dependencies
+RUN cd actions-runner && ./bin/installdependencies.sh 
+
+# Copy the entrypoint script, and set it up
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]
+
+# Configure default env variables
+ENV RUNNER_LABELS="cuda-11-8"
+ENV RUNNER_NAME=""
+ENV RUNNER_TOKEN=""
+ENV RUNNER_REPO_URL="https://github.com/RWKV/RWKV-infctx-trainer"
\ No newline at end of file
diff --git a/docker/github-worker-cuda-11-8/entrypoint.sh b/docker/github-worker-cuda-11-8/entrypoint.sh
new file mode 100644
index 00000000..362a1bfe
--- /dev/null
+++ b/docker/github-worker-cuda-11-8/entrypoint.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+export RUNNER_ALLOW_RUNASROOT="1"
+cd /actions-runner
+
+# Check the URL, token, and name of the runner from the container ENV vars
+# and if they are not set, provide default values
+if [[ -z "${RUNNER_NAME}" ]]; then
+    export RUNNER_NAME=$(hostname)
+fi
+if [[ -z "${RUNNER_TOKEN}" ]]; then
+    echo "# [WARNING] RUNNER_TOKEN is missing, skipping github runner setup"
+else
+    # Configure unattended
+    ./config.sh \
+        --unattended \
+        --url "${RUNNER_REPO_URL}" \
+        --token "${RUNNER_TOKEN}" \
+        --name "${RUNNER_NAME}" \
+        --replace \
+        --labels "${RUNNER_LABELS}"
+
+    # Run it in background, and get the PID
+    ./run.sh &
+    RUNNER_PID=$!
+fi
+
+# Follow up on any forwarded command args
+if [[ $# -gt 0 ]]; then
+    exec "$@"
+fi
+
+# Wait for everything to exit
+# wait $RUNNER_PID
+wait 
\ No newline at end of file

From e8e23c494c1f1a600643b04dc1dff8e6588e6554 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Tue, 22 Aug 2023 09:58:15 +0000
Subject: [PATCH 06/16] setup github worker build

---
 .github/workflows/docker-build.yml | 98 +++++++++++++++++++++++++++++-
 1 file changed, 97 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index cf330593..bf9274f1 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -15,7 +15,7 @@ env:
   IMAGE_NAME: ${{ github.repository }}
 
 jobs:
-  build:
+  build_env:
     name: Docker Env Image (cuda-11-8)
   
     runs-on: ubuntu-latest
@@ -109,3 +109,99 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha,src=docker/env-cuda-11-8
           cache-to: type=gha,mode=max
+
+  build_runner:
+    name: Docker Env Image (github-worker-11-8)
+  
+    needs: build_env
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      # Get and log the free space
+      - name: Get system free space (Before reclaim)
+        run: |
+          echo "Free space:"
+          df -h
+      
+      # Due to the docker image being > available space on the runner
+      # we need to do some optimization, to create more space.
+      # https://github.com/marketplace/actions/disk-space-reclaimer
+      # https://stackoverflow.com/questions/76294509/github-actions-docker-service-container-25gb-cannot-be-loaded
+      - name: Maximize build space
+        uses: insightsengineering/disk-space-reclaimer@v1
+        with:
+          # this might remove tools that are actually needed,
+          # if set to "true" but frees about 6 GB
+          tools-cache: true
+
+          # all of these default to true, but feel free to set to
+          # "false" if necessary for your workflow
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+          docker-images: true
+      
+      # Get and log the free space
+      - name: Get system free space (After reclaim)
+        run: |
+          echo "Free space:"
+          df -h
+
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0
+        with:
+          cosign-release: 'v1.11.0'
+
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v2
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      
+      - name: downcase IMAGE_NAME
+        run: |
+          echo "IMAGE_NAME_LC=${IMAGE_NAME,,}" >>${GITHUB_ENV}
+      
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image (env-cuda-11-8)
+        id: build-and-push
+        uses: docker/build-push-action@v4
+        with:
+          context: "{{defaultContext}}:docker/github-worker-11-8"
+          push: ${{ github.event_name != 'pull_request' }} # Don't push on PR
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:github-worker-11-8
+          # tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,src=docker/github-worker-11-8
+          cache-to: type=gha,mode=max

From b64f2bfb1bdb0ad4040c6d869dd9ac46b51d6c07 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Tue, 22 Aug 2023 10:08:18 +0000
Subject: [PATCH 07/16] prototype github worker runner, with lane2 support

---
 docker/github-worker-cuda-11-8/Dockerfile    | 18 ++++++++++++++++-
 docker/github-worker-cuda-11-8/entrypoint.sh | 21 +++++++++++++++++++-
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/docker/github-worker-cuda-11-8/Dockerfile b/docker/github-worker-cuda-11-8/Dockerfile
index 52cb6ebc..a3db79f1 100644
--- a/docker/github-worker-cuda-11-8/Dockerfile
+++ b/docker/github-worker-cuda-11-8/Dockerfile
@@ -9,6 +9,9 @@ RUN cd / && mkdir actions-runner && cd actions-runner && \
     tar xzf ./actions-runner-linux-x64-2.308.0.tar.gz && \
     rm ./actions-runner-linux-x64-2.308.0.tar.gz
 
+# Clone the runner, for lane2 track
+RUN cd / && cp -r /actions-runner /actions-runner-lane2
+
 # Install dependencies
 RUN cd actions-runner && ./bin/installdependencies.sh 
 
@@ -21,4 +24,17 @@ ENTRYPOINT ["/entrypoint.sh"]
 ENV RUNNER_LABELS="cuda-11-8"
 ENV RUNNER_NAME=""
 ENV RUNNER_TOKEN=""
-ENV RUNNER_REPO_URL="https://github.com/RWKV/RWKV-infctx-trainer"
\ No newline at end of file
+ENV RUNNER_REPO_URL="https://github.com/RWKV/RWKV-infctx-trainer"
+
+# Runner with lane2 track
+# ---
+# this helps setup dual runs on the same machine
+# to help ensure better utilization of GPUs.
+#
+# In general DS2/3_offload should be used.
+#
+# Tags should be adjusted to be half their original spec
+# to account for the fact that we are running two runners
+#
+# This is only useful for high GPU, and high ram count machines
+ENV RUNNER_LANE2="false"
\ No newline at end of file
diff --git a/docker/github-worker-cuda-11-8/entrypoint.sh b/docker/github-worker-cuda-11-8/entrypoint.sh
index 362a1bfe..eb713bfd 100644
--- a/docker/github-worker-cuda-11-8/entrypoint.sh
+++ b/docker/github-worker-cuda-11-8/entrypoint.sh
@@ -22,11 +22,30 @@ else
 
     # Run it in background, and get the PID
     ./run.sh &
-    RUNNER_PID=$!
+
+    # If lane2 runner is enabled, start it
+    # this is enabled with RUNNER_LANE2=true
+    if [[ -z "${RUNNER_LANE2}" ]]; then
+        echo "# [INFO] lane2 runner is disabled"
+    else
+        echo "# [INFO] lane2 runner is enabled"
+        cd /actions-runner-lane2
+        ./config.sh \
+            --unattended \
+            --url "${RUNNER_REPO_URL}" \
+            --token "${RUNNER_TOKEN}" \
+            --name "${RUNNER_NAME}-lane2" \
+            --replace \
+            --labels "${RUNNER_LABELS},lane2"
+
+        # Run it in background, and get the PID
+        ./run.sh &
+    fi
 fi
 
 # Follow up on any forwarded command args
 if [[ $# -gt 0 ]]; then
+    cd /root
     exec "$@"
 fi
 

From 97e2432705f6db75b1c1cc24a873fede3766ea08 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Tue, 22 Aug 2023 10:26:26 +0000
Subject: [PATCH 08/16] Add git attribute

---
 .gitattributes | 2 ++
 .gitignore     | 1 +
 2 files changed, 3 insertions(+)
 create mode 100644 .gitattributes

diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 00000000..665955b4
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,2 @@
+# Disable code % attribution of the project from notebooks
+notebook/* linguist-vendored
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 2e2fbd86..1adc7889 100644
--- a/.gitignore
+++ b/.gitignore
@@ -170,6 +170,7 @@ notebook/scratch/*
 
 # Add .github items
 !.github/
+!.git*
 
 # Add back requirements.txt
 !requirements.txt

From a2c8376416e766e664b8720e0fd1ca287b3c4df0 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Tue, 22 Aug 2023 10:30:39 +0000
Subject: [PATCH 09/16] fixing build workflow

---
 .github/workflows/docker-build.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index bf9274f1..c10eeb1d 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -194,14 +194,14 @@ jobs:
       
       # Build and push Docker image with Buildx (don't push on PR)
       # https://github.com/docker/build-push-action
-      - name: Build and push Docker image (env-cuda-11-8)
+      - name: Build and push Docker image (github-worker-cuda-11-8)
         id: build-and-push
         uses: docker/build-push-action@v4
         with:
-          context: "{{defaultContext}}:docker/github-worker-11-8"
+          context: "{{defaultContext}}:docker/github-worker-cuda-11-8"
           push: ${{ github.event_name != 'pull_request' }} # Don't push on PR
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:github-worker-11-8
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME_LC }}:github-worker-cuda-11-8
           # tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha,src=docker/github-worker-11-8
+          cache-from: type=gha,src=docker/github-worker-cuda-11-8
           cache-to: type=gha,mode=max

From 56f6aab2a13055e8712ec9c5c58c77314c1e11e7 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Wed, 23 Aug 2023 02:06:07 +0000
Subject: [PATCH 10/16] dependency setup for runners

---
 docker/github-worker-cuda-11-8/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/github-worker-cuda-11-8/Dockerfile b/docker/github-worker-cuda-11-8/Dockerfile
index a3db79f1..d833a6cf 100644
--- a/docker/github-worker-cuda-11-8/Dockerfile
+++ b/docker/github-worker-cuda-11-8/Dockerfile
@@ -13,7 +13,8 @@ RUN cd / && mkdir actions-runner && cd actions-runner && \
 RUN cd / && cp -r /actions-runner /actions-runner-lane2
 
 # Install dependencies
-RUN cd actions-runner && ./bin/installdependencies.sh 
+RUN cd /actions-runner && ./bin/installdependencies.sh && \
+    cd /actions-runner-lane2 && ./bin/installdependencies.sh
 
 # Copy the entrypoint script, and set it up
 COPY entrypoint.sh /entrypoint.sh

From 33a7d5f3ad523a90c2b5b6a76deba39eeb7ff61c Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Wed, 23 Aug 2023 02:26:26 +0000
Subject: [PATCH 11/16] setup cuda versiononig in labels

---
 docker/github-worker-cuda-11-8/entrypoint.sh | 45 ++++++++++++++------
 1 file changed, 32 insertions(+), 13 deletions(-)

diff --git a/docker/github-worker-cuda-11-8/entrypoint.sh b/docker/github-worker-cuda-11-8/entrypoint.sh
index eb713bfd..76ae482f 100644
--- a/docker/github-worker-cuda-11-8/entrypoint.sh
+++ b/docker/github-worker-cuda-11-8/entrypoint.sh
@@ -3,6 +3,9 @@
 export RUNNER_ALLOW_RUNASROOT="1"
 cd /actions-runner
 
+# CUDA version for label
+CUDA_VER="cuda-11-8"
+
 # Check the URL, token, and name of the runner from the container ENV vars
 # and if they are not set, provide default values
 if [[ -z "${RUNNER_NAME}" ]]; then
@@ -11,24 +14,40 @@ fi
 if [[ -z "${RUNNER_TOKEN}" ]]; then
     echo "# [WARNING] RUNNER_TOKEN is missing, skipping github runner setup"
 else
-    # Configure unattended
-    ./config.sh \
-        --unattended \
-        --url "${RUNNER_REPO_URL}" \
-        --token "${RUNNER_TOKEN}" \
-        --name "${RUNNER_NAME}" \
-        --replace \
-        --labels "${RUNNER_LABELS}"
-
-    # Run it in background, and get the PID
-    ./run.sh &
+    echo "# [INFO] lane1 starting up ... "
 
     # If lane2 runner is enabled, start it
     # this is enabled with RUNNER_LANE2=true
     if [[ -z "${RUNNER_LANE2}" ]]; then
+
+        # Configure unattended
+        ./config.sh \
+            --unattended \
+            --url "${RUNNER_REPO_URL}" \
+            --token "${RUNNER_TOKEN}" \
+            --name "${RUNNER_NAME}" \
+            --replace \
+            --labels "${RUNNER_LABELS},nolane,${CUDA_VER}"
+
+        # Run it in background, and get the PID
+        ./run.sh &
+
         echo "# [INFO] lane2 runner is disabled"
     else
-        echo "# [INFO] lane2 runner is enabled"
+        # Configure unattended
+        ./config.sh \
+            --unattended \
+            --url "${RUNNER_REPO_URL}" \
+            --token "${RUNNER_TOKEN}" \
+            --name "${RUNNER_NAME}-lane1" \
+            --replace \
+            --labels "${RUNNER_LABELS},lane1,${CUDA_VER}"
+
+        # Run it in background, and get the PID
+        ./run.sh &
+
+        echo "# [INFO] lane2 starting up ... "
+
         cd /actions-runner-lane2
         ./config.sh \
             --unattended \
@@ -36,7 +55,7 @@ else
             --token "${RUNNER_TOKEN}" \
             --name "${RUNNER_NAME}-lane2" \
             --replace \
-            --labels "${RUNNER_LABELS},lane2"
+            --labels "${RUNNER_LABELS},lane2,${CUDA_VER}"
 
         # Run it in background, and get the PID
         ./run.sh &

From c52dde676b7f1541634a0a4dc0bc1378c432d3a0 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Wed, 23 Aug 2023 02:59:07 +0000
Subject: [PATCH 12/16] remove cuda form default label

---
 docker/github-worker-cuda-11-8/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/github-worker-cuda-11-8/Dockerfile b/docker/github-worker-cuda-11-8/Dockerfile
index d833a6cf..6c0a92f3 100644
--- a/docker/github-worker-cuda-11-8/Dockerfile
+++ b/docker/github-worker-cuda-11-8/Dockerfile
@@ -22,7 +22,7 @@ RUN chmod +x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]
 
 # Configure default env variables
-ENV RUNNER_LABELS="cuda-11-8"
+ENV RUNNER_LABELS=""
 ENV RUNNER_NAME=""
 ENV RUNNER_TOKEN=""
 ENV RUNNER_REPO_URL="https://github.com/RWKV/RWKV-infctx-trainer"

From 1aa95f4d4eab25381d4fd9e4c58cee58e4f6d216 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Wed, 23 Aug 2023 02:59:44 +0000
Subject: [PATCH 13/16] reorder the labels

---
 docker/github-worker-cuda-11-8/entrypoint.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/github-worker-cuda-11-8/entrypoint.sh b/docker/github-worker-cuda-11-8/entrypoint.sh
index 76ae482f..b410bf4b 100644
--- a/docker/github-worker-cuda-11-8/entrypoint.sh
+++ b/docker/github-worker-cuda-11-8/entrypoint.sh
@@ -27,7 +27,7 @@ else
             --token "${RUNNER_TOKEN}" \
             --name "${RUNNER_NAME}" \
             --replace \
-            --labels "${RUNNER_LABELS},nolane,${CUDA_VER}"
+            --labels "nolane,${CUDA_VER},${RUNNER_LABELS}"
 
         # Run it in background, and get the PID
         ./run.sh &
@@ -41,7 +41,7 @@ else
             --token "${RUNNER_TOKEN}" \
             --name "${RUNNER_NAME}-lane1" \
             --replace \
-            --labels "${RUNNER_LABELS},lane1,${CUDA_VER}"
+            --labels "lane1,${CUDA_VER},${RUNNER_LABELS}"
 
         # Run it in background, and get the PID
         ./run.sh &
@@ -55,7 +55,7 @@ else
             --token "${RUNNER_TOKEN}" \
             --name "${RUNNER_NAME}-lane2" \
             --replace \
-            --labels "${RUNNER_LABELS},lane2,${CUDA_VER}"
+            --labels "lane2,${CUDA_VER},${RUNNER_LABELS}"
 
         # Run it in background, and get the PID
         ./run.sh &

From 9cf281994ee686027d1c0c69be4c47bdd4c9688a Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Wed, 23 Aug 2023 03:35:25 +0000
Subject: [PATCH 14/16] multilane logic tweak

---
 docker/github-worker-cuda-11-8/entrypoint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/github-worker-cuda-11-8/entrypoint.sh b/docker/github-worker-cuda-11-8/entrypoint.sh
index b410bf4b..662d4515 100644
--- a/docker/github-worker-cuda-11-8/entrypoint.sh
+++ b/docker/github-worker-cuda-11-8/entrypoint.sh
@@ -18,7 +18,7 @@ else
 
     # If lane2 runner is enabled, start it
     # this is enabled with RUNNER_LANE2=true
-    if [[ -z "${RUNNER_LANE2}" ]]; then
+    if [ "$RUNNER_LANE2" != true ]; then
 
         # Configure unattended
         ./config.sh \

From e991ff47ae02037ee46c4bfff0ac65e220e32a37 Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Wed, 23 Aug 2023 03:39:55 +0000
Subject: [PATCH 15/16] notebook run controls

---
 .github/workflows/notebook-run.yml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 .github/workflows/notebook-run.yml

diff --git a/.github/workflows/notebook-run.yml b/.github/workflows/notebook-run.yml
new file mode 100644
index 00000000..5c66171e
--- /dev/null
+++ b/.github/workflows/notebook-run.yml
@@ -0,0 +1,19 @@
+name: Notebook-Run
+
+on:
+  workflow_dispatch:
+    inputs:
+      cudaVersion:
+        description: 'cuda version'     
+        required: true
+        default: '11-8'
+
+env:
+
+jobs:
+  build_env:
+    name: Notebook-Run
+    runs-on: cuda-${{github.event.inputs.cudaVersion}}
+    steps:
+    - run: |
+        echo "Cuda Version: ${{github.event.inputs.cudaVersion}}"
\ No newline at end of file

From 2e4c6180ee6e152bf0bd69f99be2448940a76fcc Mon Sep 17 00:00:00 2001
From: "@picocreator (Eugene Cheah)" <eugeneqin@gmail.com>
Date: Wed, 23 Aug 2023 03:48:55 +0000
Subject: [PATCH 16/16] Disable main-dev

---
 .github/workflows/docker-build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index c10eeb1d..4649350c 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -2,11 +2,11 @@ name: Docker Env Image (cuda-11-8)
 
 on:
   push:
-    branches: [ "main", "main-dev-infctx" ]
+    branches: [ "main" ]
     # Publish semver tags as releases.
     tags: [ 'v*.*.*' ]
   pull_request:
-    branches: [ "main", "main-dev-infctx" ]
+    branches: [ "main" ]
 
 env:
   # Use docker.io for Docker Hub if empty