ci: update structure of wheel storage

.github/ci-scripts/upload_wheel_to_s3.py

@@ -2,43 +2,44 @@
 from pathlib import Path
 
 import boto3
-import constants
 import wheellib
 
+BUCKET_NAME = "github-actions-artifacts-bucket"
+WHEEL_SUFFIX = ".whl"
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument("--commit-hash", required=True)
-    parser.add_argument("--platform-substring", required=True, choices=["x86", "aarch", "arm"])
-    parser.add_argument("--path-to-wheel-dir", required=True)
+    parser.add_argument("--commit-hash", type=str, required=True)
+    parser.add_argument("--subdir", type=str, required=True)
+    parser.add_argument("--platform-substring", type=str, required=True, choices=["x86", "aarch", "arm"])
+    parser.add_argument("--path-to-wheel-dir", type=Path, required=True)
     args = parser.parse_args()
 
-    commit_hash = args.commit_hash
-    platform_substring = args.platform_substring
-    path_to_wheel_dir = Path(args.path_to_wheel_dir)
-
-    assert path_to_wheel_dir.exists(), f"Path to wheel directory does not exist: {path_to_wheel_dir}"
-    wheelpaths = iter(filepath for filepath in path_to_wheel_dir.iterdir() if filepath.suffix == constants.WHEEL_SUFFIX)
+    assert args.path_to_wheel_dir.exists(), f"Path to wheel directory does not exist: {args.path_to_wheel_dir}"
+    wheelpaths = iter(filepath for filepath in args.path_to_wheel_dir.iterdir() if filepath.suffix == WHEEL_SUFFIX)
 
     def f(wheelpath: Path) -> bool:
         platform_tag = wheellib.get_platform_tag(wheelpath.name)
-        return platform_substring in platform_tag
+        return args.platform_substring in platform_tag
 
     filtered_wheelpaths: list[Path] = list(filter(f, wheelpaths))
 
     length = len(filtered_wheelpaths)
     if length == 0:
-        raise RuntimeError(f"No wheels found that match the given platform substring: {platform_substring}; expected 1")
+        raise RuntimeError(
+            f"No wheels found that match the given platform substring: {args.platform_substring}; expected 1"
+        )
     elif length > 1:
         raise RuntimeError(
-            f"""Multiple wheels found that match the given platform substring: {platform_substring}; expected just 1
+            f"""Multiple wheels found that match the given platform substring: {args.platform_substring}; expected just 1
 Wheels available: {wheelpaths}"""
         )
     [wheelpath] = filtered_wheelpaths
     s3 = boto3.client("s3")
-    destination = Path("builds") / commit_hash / wheelpath.name
+    destination = Path("builds") / args.commit_hash / args.subdir / wheelpath.name
     s3.upload_file(
         Filename=wheelpath,
-        Bucket=constants.BUCKET_NAME,
+        Bucket=BUCKET_NAME,
         Key=str(destination),
         ExtraArgs={"ACL": "public-read"},
     )

.github/workflows/build-commit.yaml

@@ -34,6 +34,11 @@ jobs:
         fi
         echo "platform_substring=$platform_substring" >> $GITHUB_ENV
         echo "Running on $platform_substring build machines"
+    - name: Set unique run id
+      run: |
+        run_id="${{ github.run_id }}_${{ github.run_attempt }}"
+        echo "run_id=$run_id" >> $GITHUB_ENV
+        echo "Unique run id: $run_id"
     - name: Checkout repo
       uses: actions/checkout@v4
       with:
@@ -59,46 +64,26 @@ jobs:
         uv v
         source .venv/bin/activate
         uv pip install boto3 packaging
-    - name: Check if build already exists in AWS S3
-      run: |
-        source .venv/bin/activate
-        wheel_name=$(python .github/ci-scripts/get_wheel_name_from_s3.py \
-          --commit-hash "${{ github.sha }}" \
-          --platform-substring "$platform_substring" \
-        )
-        if [ "$wheel_name" ]; then
-          echo "Python wheel for this commit already built and uploaded"
-        else
-          echo "No python wheel for this commit found; proceeding with build"
-        fi
-        echo "wheel_name=$wheel_name" >> $GITHUB_ENV
     - name: Build release wheel
       run: |
-        if [ "$wheel_name" ]; then
-          echo "Python wheel for this commit already built and uploaded"
-          exit 0
-        fi
-        export CARGO_TARGET_DIR=~/target
         source .venv/bin/activate
+        export CARGO_TARGET_DIR=~/target
         uv pip install pip maturin boto3
         maturin build --release
     - name: Upload wheel to AWS S3
       run: |
-        if [ "$wheel_name" ]; then
-          echo "Python wheel for this commit already built and uploaded"
-          exit 0
-        fi
         source .venv/bin/activate
         wheel_name=$(python .github/ci-scripts/upload_wheel_to_s3.py \
-          --commit-hash "${{ github.sha }}" \
-          --platform-substring "$platform_substring" \
+          --commit-hash="${{ github.sha }}" \
+          --subdir="$run_id" \
+          --platform-substring="$platform_substring" \
           --path-to-wheel-dir ~/target/wheels \
         )
         echo "wheel_name=$wheel_name" >> $GITHUB_ENV
     - name: Print url of the built wheel to GitHub Actions Summary Page
       run: |
-        console_url="https://us-west-2.console.aws.amazon.com/s3/object/github-actions-artifacts-bucket?prefix=builds/${{ github.sha }}/$wheel_name"
-        download_url="https://github-actions-artifacts-bucket.s3.us-west-2.amazonaws.com/builds/${{ github.sha }}/$wheel_name"
+        console_url="https://us-west-2.console.aws.amazon.com/s3/object/github-actions-artifacts-bucket?prefix=builds/${{ github.sha }}/$run_id/$wheel_name"
+        download_url="https://github-actions-artifacts-bucket.s3.us-west-2.amazonaws.com/builds/${{ github.sha }}/$run_id/$wheel_name"
 
         echo "View the location of the built wheel in the AWS console here:" >> $GITHUB_STEP_SUMMARY
         echo "$console_url" >> $GITHUB_STEP_SUMMARY