Bump s3 client lib and parameterize region in s3 tests + devcontainer (…

…#147)

.devcontainer/Dockerfile

@@ -0,0 +1,50 @@
+FROM mcr.microsoft.com/devcontainers/python:3.11-bullseye
+
+# Copy environment.yml (if found) to a temp location so we update the environment. Also
+# copy "noop.txt" so the COPY instruction does not fail if no environment.yml exists.
+COPY environment.yml* .devcontainer/noop.txt /tmp/conda-tmp/
+RUN if [ -f "/tmp/conda-tmp/environment.yml" ]; then umask 0002 && /opt/conda/bin/conda env update -n base -f /tmp/conda-tmp/environment.yml; fi \
+    && rm -rf /tmp/conda-tmp
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends \
+    vim \
+    build-essential \
+    libtool \
+    autoconf \
+    unzip \
+    wget \
+    pkg-config \
+    libssl-dev
+
+RUN if [ $(uname -m) = "aarch64" ] ; then \
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o "awscliv2.zip"; \
+    else \
+    curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"; \
+    fi
+
+RUN unzip awscliv2.zip && ./aws/install
+
+USER vscode
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+
+WORKDIR /workspaces
+
+RUN wget "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+ENV DEBIAN_FRONTEND noninteractive
+RUN bash Miniforge3-$(uname)-$(uname -m).sh -b
+
+ENV PATH="/home/vscode/miniforge3/bin:${PATH}"
+
+RUN mkdir -p /workspaces/dolma
+COPY ../python /workspaces/dolma/python
+COPY ../src /workspaces/dolma/src
+COPY ../Cargo.toml /workspaces/dolma/Cargo.toml
+COPY ../Cargo.lock /workspaces/dolma/Cargo.lock
+COPY ../pyproject.toml /workspaces/dolma/pyproject.toml
+
+COPY .devcontainer/postInstall.sh .
+RUN ./postInstall.sh
+
+RUN echo "source activate base" >> ~/.bashrc
+RUN echo "echo 'Happy hacking!\n'" >> ~/.bashrc

.devcontainer/README.md

@@ -0,0 +1,36 @@
+# VSCode Devcontainer Setup Guide
+
+This guide will walk you through the steps to set up and use the [devcontainer](https://code.visualstudio.com/docs/devcontainers/containers) configuration in this directory to start a VSCode devcontainer.
+
+Note: This is confirmed working on Apple M series chips. Need to confirm it works on Intel but it should...
+
+## Prerequisites
+
+Before you begin, make sure you have the following installed/configured on your machine:
+
+- [Docker](https://www.docker.com/)
+- [AWS Credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) (Make sure it is in the standard location)
+
+## Getting Started
+
+To get started with the devcontainer, follow these steps:
+
+1. Open Visual Studio Code.
+
+2. If prompted, install the recommended extensions for the devcontainer.
+
+3. Press `Ctrl+Shift+P` (or `Cmd+Shift+P` on macOS) to open the command palette.
+
+4. Type `Dev Containers: Reopen in Container` and select `dolma-container` from the list.
+
+5. Wait for the devcontainer to build and start. This may take a few minutes the first time you build the image.
+
+6. Once the devcontainer is up and running, you can start working on your project inside the devcontainer.
+
+7. Open a terminal within VSCode for all cli related tasks. (ie: `make test`) Simplest option is `Ctrl+Shift+P` (or `Cmd+Shift+P` on macOS) and select `Terminal: Create New Terminal`
+
+## Additional Configuration
+
+If you need to customize the devcontainer configuration, you can modify the `.devcontainer/devcontainer.json` file in this directory. Refer to the [VSCode Remote - Containers documentation](https://code.visualstudio.com/docs/remote/containers) for more information on configuring devcontainers. Just be selective about merging changes that might impact correctness across platforms/OS.
+
+Happy hacking!

.devcontainer/devcontainer.json

@@ -0,0 +1,10 @@
+{
+	"name": "dolma-container",
+	"build": {
+		"context": "..",
+		"dockerfile": "Dockerfile"
+	},
+	"mounts": [
+		"source=${localEnv:HOME}/.aws,target=/home/vscode/.aws,type=bind,consistency=cached"
+	]
+}

.devcontainer/noop.txt

@@ -0,0 +1,3 @@
+This file is copied into the container along with environment.yml* from the
+parent folder. This is done to prevent the Dockerfile COPY instruction from 
+failing if no environment.yml is found.

.devcontainer/postInstall.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+PATH=/home/vscode/.cargo/bin:$PATH
+cd dolma
+source /home/vscode/miniforge3/bin/activate && pip install cmake "maturin[patchelf]>=1.1,<2.0"

.gitignore

@@ -17,7 +17,6 @@ pip-wheel-metadata/
 .vscode/
 /*.iml
 
-
 # jupyter notebooks
 
 .ipynb_checkpoints