diff --git a/.github/RELEASE_PROCESS.md b/.github/RELEASE_PROCESS.md
index d5295d38..529ea64f 100644
--- a/.github/RELEASE_PROCESS.md
+++ b/.github/RELEASE_PROCESS.md
@@ -7,7 +7,7 @@
 3. Run the release script:
 
     ```bash
-    ./src/scripts/release.sh
+    ./src/scripts/release/release.sh
     ```
 
     This will commit the changes to the CHANGELOG and `version.py` files and then create a new tag in git
@@ -15,10 +15,4 @@
 
 ## Fixing a failed release
 
-If for some reason the GitHub Actions release workflow failed with an error that needs to be fixed, you'll have to delete both the tag and corresponding release from GitHub. After you've pushed a fix, delete the tag from your local clone with
-
-```bash
-git tag -l | xargs git tag -d && git fetch -t
-```
-
-Then repeat the steps above.
+If for some reason the GitHub Actions release workflow failed with an error that needs to be fixed, you'll have to delete the tag on GitHub. Once you've pushed a fix you can simply repeat the steps above.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a7f89480..4d7758df 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -239,7 +239,7 @@ jobs:
       - name: Generate release notes
         run: |
           . .venv/bin/activate
-          python src/scripts/release_notes.py > ${{ github.workspace }}-RELEASE_NOTES.md
+          python src/scripts/release/release_notes.py > ${{ github.workspace }}-RELEASE_NOTES.md
 
       - name: Publish package to PyPI
         run: |
@@ -260,4 +260,4 @@ jobs:
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
-          ./scripts/add_pr_comments_on_release.sh
+          ./src/scripts/release/add_pr_comments_on_release.sh
diff --git a/src/scripts/add_pr_comments_on_release.sh b/src/scripts/release/add_pr_comments_on_release.sh
similarity index 99%
rename from src/scripts/add_pr_comments_on_release.sh
rename to src/scripts/release/add_pr_comments_on_release.sh
index f1f383ef..4e4f9e49 100755
--- a/src/scripts/add_pr_comments_on_release.sh
+++ b/src/scripts/release/add_pr_comments_on_release.sh
@@ -3,6 +3,7 @@
 set -e
 
 repo_url=https://github.com/allenai/OLMo-core
+
 tags=$(git tag -l --sort=-version:refname 'v*' | head -n 2)
 current_tag=$(echo "$tags" | head -n 1)
 last_tag=$(echo "$tags" | tail -n 1)
diff --git a/src/scripts/prepare_changelog.py b/src/scripts/release/prepare_changelog.py
similarity index 100%
rename from src/scripts/prepare_changelog.py
rename to src/scripts/release/prepare_changelog.py
diff --git a/src/scripts/release.sh b/src/scripts/release/release.sh
similarity index 81%
rename from src/scripts/release.sh
rename to src/scripts/release/release.sh
index 5b9df6ed..d9e9cc0a 100755
--- a/src/scripts/release.sh
+++ b/src/scripts/release/release.sh
@@ -2,9 +2,12 @@
 
 set -e
 
-TAG=$(python -c 'from olmo_core.version import VERSION; print("v" + VERSION)')
+# Make sure clone is up-to-date with remote.
+git pull > /dev/null
+git tag -l | xargs git tag -d > /dev/null
+git fetch -t > /dev/null
 
-git pull
+TAG=$(python -c 'from olmo_core.version import VERSION; print("v" + VERSION)')
 
 # Make sure tag/release doesn't already exist.
 STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" "https://github.com/allenai/OLMo-core/releases/tag/${TAG}")
@@ -13,7 +16,7 @@ if [[ $STATUS_CODE == "200" ]]; then
     exit 1
 fi
 
-python src/scripts/prepare_changelog.py
+python src/scripts/release/prepare_changelog.py
 
 read -rp "Creating new release for $TAG. Do you want to continue? [Y/n] " prompt
 
diff --git a/src/scripts/release_notes.py b/src/scripts/release/release_notes.py
similarity index 100%
rename from src/scripts/release_notes.py
rename to src/scripts/release/release_notes.py