Bump github.com/sirupsen/logrus from 1.9.0 to 1.9.3

.github/dependabot.yaml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/lint.yaml

@@ -0,0 +1,30 @@
+name: lint
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+
+permissions: read-all
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+
+      - name: lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.55.2

.github/workflows/release.yaml

@@ -0,0 +1,30 @@
+name: release
+
+on:
+  push:
+    tags:
+      - v*
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+ release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+
+      - name: Run goreleaser
+        uses: goreleaser/goreleaser-action@v4
+        with:
+          version: latest
+          args: release --clean --config .goreleaser.yaml
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-docker.yaml

@@ -0,0 +1,37 @@
+name: test
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+
+permissions: read-all
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+
+      - name: build aikit
+        run: docker buildx build . -t sozercan/aikit:latest
+
+      - name: build test model
+        run: docker buildx build . -t sozercan/testmodel:latest -f test/aikitfile.yaml
+
+      - name: run test model
+        run: docker run -d -p 8080:8080 sozercan/testmodel:latest
+
+      - name: run bats tests
+        run: make test-e2e-dependencies test-e2e

.github/workflows/unit-test.yaml

@@ -0,0 +1,33 @@
+name: test
+
+on:
+  push:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches:
+      - main
+    paths-ignore:
+      - '**.md'
+
+permissions: read-all
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.21"
+
+      - name: go mod tidy
+        run: |
+          go mod tidy
+          git diff --exit-code
+
+      - name: test
+        run: make test

.gitignore

@@ -19,3 +19,6 @@
 
 # Go workspace file
 go.work
+
+bin
+coverage.txt

.golangci.yaml

@@ -0,0 +1,40 @@
+run:
+  timeout: 5m
+
+linters-settings:
+  # gocritic:
+  #   enabled-tags:
+  #   - performance
+  lll:
+    line-length: 200
+
+  misspell:
+    locale: US
+  staticcheck:
+    go: "1.21"
+
+linters:
+  disable-all: true
+  enable:
+    - errcheck
+    - errorlint
+    - exportloopref
+    - forcetypeassert
+    - gci
+    - gocritic
+    - goconst
+    - godot
+    - gofmt
+    - gofumpt
+    - goimports
+    - gosec
+    - gosimple
+    - govet
+    - ineffassign
+    - misspell
+    - revive
+    - staticcheck
+    - typecheck
+    - unconvert
+    - unused
+    - whitespace

.goreleaser.yaml

@@ -0,0 +1 @@
+# TODO

Dockerfile

@@ -0,0 +1,8 @@
+FROM golang:1.21-bullseye AS builder
+COPY . /go/src/github.com/sozercan/aikit
+WORKDIR /go/src/github.com/sozercan/aikit
+RUN CGO_ENABLED=0 go build -o /aikit --ldflags '-extldflags "-static"' ./cmd/frontend
+
+FROM scratch
+COPY --from=builder /aikit /bin/aikit
+ENTRYPOINT ["/bin/aikit"]

Makefile

@@ -0,0 +1,42 @@
+# build-aikit:
+# 	docker buildx build . -t sozercan/aikit:latest --push
+
+# build-llm:
+# 	docker buildx build . -t sozercan/myllm:latest -f test/aikitfile.yaml --push
+
+# run:
+# 	docker run -p 8080:8080 sozercan/myllm:latest
+
+# test:
+# 	go test -v ./... -race -coverprofile=coverage.txt -covermode=atomic
+
+BATS_VERSION ?= 1.10.0
+BATS_TESTS_FILE ?= test/bats/test.bats
+
+.PHONY: lint
+lint:
+	golangci-lint run -v ./... --timeout 5m
+
+.PHONY: build-aikit
+build-aikit:
+	docker buildx build . -t sozercan/aikit:latest --push --pull --no-cache
+
+.PHONY: build-test-model
+build-test-model:
+	docker buildx build . -t sozercan/testmodel:latest -f test/aikitfile.yaml --push --pull --no-cache
+
+.PHONY: run-test-model
+run-test-model:
+	docker run -p 8080:8080 --pull always sozercan/testmodel:latest
+
+.PHONY: test
+test:
+	go test -v ./... -race -coverprofile=coverage.txt -covermode=atomic
+
+.PHONY: test-e2e-dependencies
+test-e2e-dependencies:
+	curl -sSLO https://github.com/bats-core/bats-core/archive/v${BATS_VERSION}.tar.gz && tar -zxvf v${BATS_VERSION}.tar.gz && bash bats-core-${BATS_VERSION}/install.sh ${GITHUB_WORKSPACE}
+
+.PHONY: test-e2e
+test-e2e:
+	bats -t ${BATS_TESTS_FILE}

README.md

@@ -1 +1,85 @@
-# aikit
+# AIKit ✨
+
+AIKit is a quick and easy way to get started to host and deploy large language models (LLMs) for inference. No GPU or additional tools needed except for Docker! 🐳
+
+AIKit uses [LocalAI](https://localai.io/) under the hood to run inference. LocalAI is OpenAI API compatible, so you can use any OpenAI API compatible client, such as [Kubectl AI](https://github.com/sozercan/kubectl-ai), to send requests to open-source LLMs powered by AIKit!
+
+## Demo
+
+[![asciicast]()]()
+
+## Getting Started
+
+Create an `aikitfile.yaml` with the following structure:
+
+```yaml
+#syntax=sozercan/aikit:latest
+apiVersion: v1alpha1
+models:
+  - name: mistral-7b
+    source: https://huggingface.co/TheBloke/Mistral-7B-OpenOrca-GGUF/resolve/main/mistral-7b-openorca.Q6_K.gguf
+```
+
+> For full API reference, see [API](docs/api.md).
+
+> You can find more models at [model gallery](https://github.com/go-skynet/model-gallery).
+
+Then run the following command:
+
+```bash
+docker build -t my-model .
+```
+
+> If you name your file something else, you can pass it to `docker build` command with `--file` flag.
+
+This will build a docker image with your models. You can then run it with:
+
+```bash
+docker run -p 8080:8080 my-model
+```
+
+You can then send requests to `localhost:8080` to run inference from your models.
+```bash
+curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
+     "model": "mistral-7b-openorca.Q6_K.gguf",
+     "messages": [{"role": "user", "content": "explain kubernetes in less than 100 words"}],
+     "temperature": 0.5
+   }'
+{"created":1700542409,"object":"chat.completion","id":"64f8b8e6-328d-4d24-8fc2-4b7b45fbdcc0","model":"mistral-7b-openorca.Q6_K.gguf","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"\n\nKubernetes is an open-source platform for automating deployment, scaling, and management of containerized applications. It provides a way to manage and orchestrate containers across multiple hosts, ensuring that applications are always available and running efficiently.\n\nKubernetes is like a traffic cop for containers, making sure they are placed where they need to be, have the resources they need, and are working together properly. It helps developers and operations teams manage complex containerized environments with ease."}}],"usage":{"prompt_tokens":0,"completion_tokens":0,"total_tokens":0}}
+```
+
+## Kubernetes Deployment
+
+It is easy to get started to deploy your models to Kubernetes!
+
+Make sure you have a Kubernetes cluster running and `kubectl` is configured to talk to it, and your model images are accessible from the cluster. You can use [kind](https://kind.sigs.k8s.io/) to create a local Kubernetes cluster for testing purposes.
+
+```bash
+# create a test cluster using kind
+kind create cluster
+
+# load your local image to the cluster
+kind load docker-image my-model
+
+# create a deployment
+kubectl create deployment my-llm-deployment --image=my-model --image-pull-policy=IfNotPresent
+
+# expose it as a service
+kubectl expose deployment my-llm-deployment --port=8080 --target-port=8008 --name=my-llm-service
+
+# easy to scale up and down
+kubectl scale deployment my-llm-deployment --replicas=3
+
+# port-forward for testing locally
+kubectl port-forward service/my-llm-service 8080:8080
+
+# send requests to your model
+curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
+     "model": "mistral-7b-openorca.Q6_K.gguf",
+     "messages": [{"role": "user", "content": "explain kubernetes in less than 100 words"}],
+     "temperature": 0.5
+   }'
+{"created":1700542409,"object":"chat.completion","id":"64f8b8e6-328d-4d24-8fc2-4b7b45fbdcc0","model":"mistral-7b-openorca.Q6_K.gguf","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","content":"\n\nKubernetes is an open-source platform for automating deployment, scaling, and management of containerized applications. It provides a way to manage and orchestrate containers across multiple hosts, ensuring that applications are always available and running efficiently.\n\nKubernetes is like a traffic cop for containers, making sure they are placed where they need to be, have the resources they need, and are working together properly. It helps developers and operations teams manage complex containerized environments with ease."}}],"usage":{"prompt_tokens":0,"completion_tokens":0,"total_tokens":0}}
+```
+
+> For an example Kubernetes deployment YAML, see [deployment.yaml](kubernetes/deployment.yaml).

TODO

@@ -0,0 +1,9 @@
+- figure out config
+- test with cache
+- add ci tests
+- release
+- makefile
+- test with kubernetes
+- dockerhub example images
+- avx vs avx2 vs avx512 (cpu capability?)
+- record asciinema