Update to the new distro mechanism (#23)

* Update to the new distro mechanism

* Revert tip

* Update configs/nomad.yaml

Co-authored-by: Hampus Näsström <[email protected]>

* Add generate dep script

* Add volume and git instruction

* Delete nomad.yaml

* Add nomad.yaml to Dockerfile

* Update docker image

* Add run scripts

* Add all examples

* Update lockfile

* Example upload scripts

* Improve docker build step

* Delete version number

* Update readme

* Copy jupyterhub config

* Separate build step and lock step

* Use python3.12

* Constraint hyperspy

* Avoid fail fast

* Change image url

* Update permissions

* Remove example uploads

* Use uv project interface

* Move permissions in GA

* Use static versioning

* Include plugins extra

* Delete extra scripts

* Install plugins in jupyterhub

* Use repo name

* Optimize dockerfile

* Fix warning

---------

Co-authored-by: Hampus Näsström <[email protected]>
Co-authored-by: github-actions <[email protected]>

.github/workflows/build-app.yml

@@ -0,0 +1,94 @@
+# Build and publish a Docker image.
+name: Build and publish docker images
+
+on:
+  push:
+    branches: ["main"]
+    # Publish semver tags as releases.
+    tags: ["v*.*.*"]
+  pull_request:
+    branches: ["main"]
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+
+permissions:
+  contents: write
+  packages: write
+  attestations: write
+  id-token: write
+
+jobs:
+  # Job 1: Update Lock File
+  update-lockfile:
+    name: Update Python Lock File
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+          submodules: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v2
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Update lock file
+        run: uv lock --upgrade-package nomad-lab
+
+      # Commits any changes made to the lockfile
+      - name: Commit lock file changes
+        run: |
+          git config --global user.name github-actions
+          git config --global user.email [email protected]
+          git add uv.lock
+          if [[ `git status --porcelain` ]]; then
+            git commit -m "Update lockfile"
+            git push origin -o ci.skip # prevent triggering the pipeline again
+          fi
+
+  # Job 2: Build and Push Docker Image
+  docker-publish:
+    name: Build Docker Image
+    runs-on: ubuntu-latest
+    needs: update-lockfile
+
+    strategy:
+      fail-fast: false
+      matrix:
+        service: [app, jupyter]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref || github.ref_name }}
+          submodules: true
+
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ github.repository }}/${{ matrix.service }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          target: ${{ matrix.service == 'jupyter' && 'jupyter' || 'final' }}
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/initialize.yml

@@ -1,4 +1,3 @@
-
 name: Template Repository Initialization
 
 on:
@@ -27,18 +26,22 @@ jobs:
       - name: Update README
         run: |
           sed -i "s|GITHUB_REPOSITORY_OWNER|${{ github.repository_owner }}|g" template_README.md
+          sed -i "s|GITHUB_REPOSITORY_NAME|${{ github.event.repository.name }}|g" template_README.md
           sed -i "s|GITHUB_REPOSITORY|${{ github.repository }}|g" template_README.md
+          sed -i "s|GITHUB_REPOSITORY_NAME|${{ github.event.repository.name }}|g" template_README.md
           mv template_README.md README.md
-          
+
       # Replaces the template repository name in the docker config file with the new one
       - name: Update docker-compose.yaml
         run: |
           export image_name="${{ github.repository }}"
           sed -i "s|fairmat-nfdi/nomad-distribution-template|${image_name,,}|g" docker-compose.yaml
-          mkdir nomad-oasis
-          cp docker-compose.yaml nomad-oasis
-          zip -ur nomad-oasis.zip nomad-oasis
-          rm -r nomad-oasis
+
+        # Replaces the template repository name in the nomad config with the new one
+      - name: Update nomad.yaml
+        run: |
+          export image_name="${{ github.repository }}"
+          sed -i "s|fairmat-nfdi/nomad-distribution-template|${image_name,,}|g" configs/nomad.yaml
 
       # Deletes this workflow file to prevent it from running on branch creation
       - name: Delete initialization workflow

.volumes/fs/north/users/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

.volumes/fs/public/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

.volumes/fs/staging/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

.volumes/fs/tmp/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

.volumes/mongo/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

Dockerfile

@@ -1,8 +1,150 @@
-FROM gitlab-registry.mpcdf.mpg.de/nomad-lab/nomad-fair:develop
-USER root
-RUN apt-get update
-RUN apt-get -y install git
+# syntax=docker/dockerfile:1
+
+# Comments are provided throughout this file to help you get started.
+# If you need more help, visit the Dockerfile reference guide at
+# https://docs.docker.com/engine/reference/builder/
+
+ARG PYTHON_VERSION=3.12
+
+FROM python:${PYTHON_VERSION}-slim AS base
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+ENV VIRTUAL_ENV=/opt/venv \
+    PATH="/opt/venv/bin:$PATH" \
+    UV_LINK_MODE=copy \
+    UV_PROJECT_ENVIRONMENT=/opt/venv
+
+# Final stage to create the runnable image with minimal size
+FROM base AS base_final
+
+WORKDIR /app
+
+RUN apt-get update \
+ && apt-get install --yes --quiet --no-install-recommends \
+       libgomp1 \
+       libmagic1 \
+       curl \
+       zip \
+       unzip \
+       nodejs \
+       npm \
+       && npm install -g configurable-http-proxy@^4.2.0 \
+       # clean cache and logs
+       && rm -rf /var/lib/apt/lists/* /var/log/* /var/tmp/* ~/.npm
+
+# Activate the virtualenv in the container
+# See here for more information:
+# https://pythonspeed.com/articles/multi-stage-docker-python/
+ENV PATH="/opt/venv/bin:$PATH"
+
+# Create a non-privileged user that the frenrug will run under.
+# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+ARG UID=1000
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    nomad
+
+FROM base AS builder
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+ENV RUNTIME=docker
+
+WORKDIR /app
+
+RUN apt-get update \
+ && apt-get install --yes --quiet --no-install-recommends \
+      libgomp1 \
+      libmagic1 \
+      file \
+      gcc \
+      build-essential \
+      curl \
+      zip \
+      unzip \
+      git \
+ && rm -rf /var/lib/apt/lists/*
+
+# Create a non-privileged user that the frenrug will run under.
+# See https://docs.docker.com/develop/develop-images/dockerfile_best-practices/#user
+ARG UID=1000
+RUN adduser \
+    --disabled-password \
+    --gecos "" \
+    --home "/nonexistent" \
+    --shell "/sbin/nologin" \
+    --no-create-home \
+    --uid "${UID}" \
+    nomad
+
+
+# Install UV
+COPY --from=ghcr.io/astral-sh/uv:0.4 /uv /bin/uv
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --extra plugins --frozen --no-install-project
+
+
+COPY scripts ./scripts
+
+FROM base_final AS final
+
+COPY --chown=nomad:1000 --from=builder /opt/venv /opt/venv
+COPY --chown=nomad:1000 scripts/run.sh .
+COPY --chown=nomad:1000 scripts/run-worker.sh .
+COPY configs/nomad.yaml nomad.yaml
+
+RUN mkdir -p /app/.volumes/fs \
+ && chown -R nomad:1000 /app \
+ && chown -R nomad:1000 /opt/venv \
+ && mkdir nomad \
+ && cp /opt/venv/lib/python3.12/site-packages/nomad/jupyterhub_config.py nomad/
+
 USER nomad
-COPY plugins.txt plugins.txt
-RUN pip install -r plugins.txt
-COPY nomad.yaml nomad.yaml
+
+# The application ports
+EXPOSE 8000
+EXPOSE 9000
+
+VOLUME /app/.volumes/fs
+
+
+FROM jupyter/datascience-notebook:lab-3.6.2 AS jupyter
+
+# Fix: https://github.com/hadolint/hadolint/wiki/DL4006
+# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER root
+
+RUN apt update \
+ && apt install --yes --quiet --no-install-recommends \
+        libmagic-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+
+# Switch back to jovyan to avoid accidental container runs as root
+USER ${NB_UID}
+WORKDIR "${HOME}"
+
+COPY --from=ghcr.io/astral-sh/uv:0.4 /uv /bin/uv
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv export --extra plugins | uv pip install -r /dev/stdin --system
+
+
+# Get rid ot the following message when you open a terminal in jupyterlab:
+# groups: cannot find name for group ID 11320
+RUN touch ${HOME}/.hushlogin