Merge branch 'main' into backends

.github/workflows/pre-release.yaml

@@ -51,6 +51,8 @@ jobs:
           tags: ghcr.io/sozercan/aikit:dev
           cache-from: type=gha,scope=aikit
           cache-to: type=gha,scope=aikit,mode=max
+          sbom: true
+          provenance: true
 
       - name: Sign the images with GitHub OIDC Token
         env:

.github/workflows/release.yaml

@@ -60,6 +60,8 @@ jobs:
             ghcr.io/sozercan/aikit:latest
           cache-from: type=gha,scope=aikit
           cache-to: type=gha,scope=aikit,mode=max
+          sbom: true
+          provenance: true
 
       - name: Sign the images with GitHub OIDC Token
         env:

.github/workflows/update-models.yaml

@@ -61,6 +61,7 @@ jobs:
           docker buildx build . -t ghcr.io/sozercan/${MODEL_NAME}:${MODEL_SIZE}${MODEL_RUNTIME} \
             -f models/${{ matrix.model }}.yaml \
             --push --progress plain \
+            --sbom=true --provenance=true \
             --cache-from=type=gha,scope=${MODEL_NAME}-${MODEL_SIZE} \
             --cache-to=type=gha,scope=${MODEL_NAME}-${MODEL_SIZE},mode=max
           echo "DIGEST=$(cosign triangulate ghcr.io/sozercan/${MODEL_NAME}:${MODEL_SIZE}${MODEL_RUNTIME} --type digest)" >> $GITHUB_ENV

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21-bullseye@sha256:03c61acbdafde9d60dc4f94a75bc558ab329d9455b5f6c799182d8783798e92f as builder
+FROM golang:1.21-bullseye@sha256:ca78a56712ab89487123b3af2b30052824653730e7ff25271d2f3037153bc0df as builder
 COPY . /go/src/github.com/sozercan/aikit
 WORKDIR /go/src/github.com/sozercan/aikit
 RUN CGO_ENABLED=0 go build -o /aikit --ldflags '-extldflags "-static"' ./cmd/frontend

README.md

@@ -15,12 +15,13 @@ AIKit uses [LocalAI](https://localai.io/) under-the-hood to run inference. Local
 
 - 🐳 No GPU, Internet access or additional tools needed except for [Docker](https://docs.docker.com/desktop/install/linux-install/)!
 - 🤏 Minimal image size, resulting in less vulnerabilities and smaller attack surface with a custom [distroless](https://github.com/GoogleContainerTools/distroless)-based image
-- 🚀 Easy to use declarative configuration
+- 🚀 [Easy to use declarative configuration](./docs/specs.md)
 - ✨ OpenAI API compatible to use with any OpenAI API compatible client
+- 🖼️ [Multi-modal model support](#vision-with-llava)
 - 🚢 Kubernetes deployment ready
 - 📦 Supports multiple models with a single image
-- 🖥️ Supports GPU-accelerated inferencing with NVIDIA GPUs
-- 🔐 Signed images for `aikit` and pre-made models
+- 🖥️ [Supports GPU-accelerated inferencing with NVIDIA GPUs](#nvidia)
+- 🔐 [Signed images for `aikit` and pre-made models](./docs/cosign.md)
 
 ## Demos
 
@@ -32,6 +33,12 @@ AIKit uses [LocalAI](https://localai.io/) under-the-hood to run inference. Local
 
 [<img src="https://asciinema.org/a/DYh5bCQMNPSis1whhsfPeMOoM.svg" width="500">](https://asciinema.org/a/DYh5bCQMNPSis1whhsfPeMOoM)
 
+### Vision with LLaVA
+
+[<img src="https://asciinema.org/a/626553.svg" width="500">](https://asciinema.org/a/626553)
+
+> see [llava.yaml](./examples//llava.yaml) for the configuration used in the demo
+
 ## Pre-made Models
 
 AIKit comes with pre-made models that you can use out-of-the-box!

docs/cosign.md

@@ -0,0 +1,22 @@
+# Image verification
+
+AIKit and pre-made models are keyless signed with OIDC in GitHub Actions with [cosign](https://github.com/sigstore/cosign). You can verify the images with the following commands:
+
+```bash
+IMAGE=ghcr.io/sozercan/llama2:7b # update this with the image you want to verify
+DIGEST=$(cosign triangulate ${IMAGE} --type digest)
+cosign verify ${DIGEST} \
+    --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+    --certificate-identity-regexp 'https://github\.com/sozercan/aikit/\.github/workflows/.+'
+```
+
+You should see an output similar to the following:
+
+```bash
+Verification for ghcr.io/sozercan/llama2@sha256:d47fdba491a9a47ce4911539a77e0c0a12b2e14f5beed88cb8072924b02130b4 --
+The following checks were performed on each of these signatures:
+  - The cosign claims were validated
+  - Existence of the claims in the transparency log was verified offline
+  - The code-signing certificate was verified using trusted certificate authority certificates
+...
+```

examples/llava.yaml

@@ -0,0 +1,34 @@
+#syntax=ghcr.io/sozercan/aikit:latest
+apiVersion: v1alpha1
+debug: true
+models:
+  - name: ggml-model-q4_k.gguf
+    source:  https://huggingface.co/mys/ggml_bakllava-1/resolve/main/ggml-model-q4_k.gguf
+    sha256: 5be58c339d8762e72d482a3e071a58d2df07df4a7aaabf8869415ae2b0f088d6
+    promptTemplates:
+      - name: chat-simple
+        template: |
+          A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.
+          {{.Input}}
+          ASSISTANT:
+  - name: mmproj-model-f16.gguf
+    source: https://huggingface.co/mys/ggml_bakllava-1/resolve/main/mmproj-model-f16.gguf
+    sha256: 2e467eba710002839e0966d5e329942bb836eabd4e787bc713b07eff1d8ea13b
+config: |
+  - name: llava
+    backend: llama-cpp
+    context_size: 4096
+    threads: 4
+    mmap: true
+    mmproj: mmproj-model-f16.gguf
+    roles:
+      user: \"USER:\"
+      assistant: \"ASSISTANT:\"
+      system: \"SYSTEM:\"
+    parameters:
+      model: ggml-model-q4_k.gguf
+      temperature: 0.2
+      top_k: 40
+      top_p: 0.95
+    template:
+      chat: chat-simple