feat: moonshine-web demo

moeru-ai · Dec 25, 2024 · 86ecb94 · 86ecb94
1 parent 70f4965
commit 86ecb94
Show file tree

Hide file tree

Showing 28 changed files with 1,318 additions and 26 deletions.
diff --git a/packages/moonshine-web/.dockerignore b/packages/moonshine-web/.dockerignore
@@ -0,0 +1,2 @@
+node_modules
+dist
diff --git a/packages/moonshine-web/Dockerfile b/packages/moonshine-web/Dockerfile
@@ -0,0 +1,18 @@
+FROM node:20-alpine as build-stage
+
+WORKDIR /app
+RUN corepack enable
+
+COPY .npmrc package.json pnpm-lock.yaml ./
+RUN --mount=type=cache,id=pnpm-store,target=/root/.pnpm-store \
+    pnpm install --frozen-lockfile
+
+COPY . .
+RUN pnpm build
+
+FROM nginx:stable-alpine as production-stage
+
+COPY --from=build-stage /app/dist /usr/share/nginx/html
+EXPOSE 80
+
+CMD ["nginx", "-g", "daemon off;"]
diff --git a/packages/moonshine-web/README.md b/packages/moonshine-web/README.md
@@ -0,0 +1,71 @@
+---
+title: Moonshine Web (Vue)
+emoji: 🌙
+colorFrom: blue
+colorTo: pink
+sdk: static
+pinned: false
+license: apache-2.0
+models:
+  - onnx-community/moonshine-base-ONNX
+short_description: Yet another Real-time in-browser speech recognition, re-implemented with Vue
+thumbnail: https://raw.githubusercontent.com/moeru-ai/airi/refs/heads/main/packages/whisper-webgpu/public/banner.png
+---
+
+<h1 align="center">Moonshine Web (Vue)</h1>
+
+<p align="center">
+  [<a href="https://moonshine-web-vue.netlify.app/">Try it</a>]
+</p>
+
+> Heavily inspired by [Realtime in-browser speech recognition](https://huggingface.co/spaces/webml-community/moonshine-web)
+
+# Moonshine Web
+
+A simple Vue + Vite application for running [Moonshine Base](https://huggingface.co/onnx-community/moonshine-base-ONNX), a powerful speech-to-text model optimized for fast and accurate automatic speech recognition (ASR) on resource-constrained devices. It runs locally in the browser using Transformers.js and WebGPU-acceleration (or WASM as a fallback).
+
+## Getting Started
+
+Follow the steps below to set up and run the application.
+
+### 1. Clone the Repository
+
+Clone the examples repository from GitHub:
+
+```sh
+git clone https://github.com/moeru-ai/airi.git
+```
+
+### 2. Navigate to the Project Directory
+
+Change your working directory to the `moonshine-web` folder:
+
+```sh
+cd packages/moonshine-web
+```
+
+### 3. Install Dependencies
+
+Install the necessary dependencies using npm:
+
+```sh
+npm i
+```
+
+### 4. Run the Development Server
+
+Start the development server:
+
+```sh
+npm run dev
+```
+
+The application should now be running locally. Open your browser and go to `http://localhost:5175` to see it in action.
+
+## Acknowledgements
+
+The audio visualizer was adapted from Wael Yasmina's [amazing tutorial](https://waelyasmina.net/articles/how-to-create-a-3d-audio-visualizer-using-three-js/).
+
+Great thanks to what Xenova have done.
+
+> [Source code](https://github.com/huggingface/transformers.js-examples/tree/38a883dd465d70d7368b86b95aa0678895ca4e83/moonshine-web)
diff --git a/packages/moonshine-web/index.html b/packages/moonshine-web/index.html
@@ -0,0 +1,136 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>Moonshine Web (Vue)</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=0" />
+    <link rel="icon" type="image/png" href="/logo.png" />
+    <script>
+      ;(function () {
+        const prefersDark = window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches
+        const setting = localStorage.getItem('vueuse-color-scheme') || 'auto'
+        if (setting === 'dark' || (prefersDark && setting !== 'light'))
+          document.documentElement.classList.toggle('dark', true)
+      })()
+    </script>
+  </head>
+  <body class="font-sans">
+    <script id="vertexshader" type="vertex">
+      uniform float u_time;
+
+      vec3 mod289(vec3 x)
+      {
+        return x - floor(x * (1.0 / 289.0)) * 289.0;
+      }
+
+      vec4 mod289(vec4 x)
+      {
+        return x - floor(x * (1.0 / 289.0)) * 289.0;
+      }
+
+      vec4 permute(vec4 x)
+      {
+        return mod289(((x*34.0)+10.0)*x);
+      }
+
+      vec4 taylorInvSqrt(vec4 r)
+      {
+        return 1.79284291400159 - 0.85373472095314 * r;
+      }
+
+      vec3 fade(vec3 t) {
+        return t*t*t*(t*(t*6.0-15.0)+10.0);
+      }
+
+      // Classic Perlin noise, periodic variant
+      float pnoise(vec3 P, vec3 rep)
+      {
+        vec3 Pi0 = mod(floor(P), rep); // Integer part, modulo period
+        vec3 Pi1 = mod(Pi0 + vec3(1.0), rep); // Integer part + 1, mod period
+        Pi0 = mod289(Pi0);
+        Pi1 = mod289(Pi1);
+        vec3 Pf0 = fract(P); // Fractional part for interpolation
+        vec3 Pf1 = Pf0 - vec3(1.0); // Fractional part - 1.0
+        vec4 ix = vec4(Pi0.x, Pi1.x, Pi0.x, Pi1.x);
+        vec4 iy = vec4(Pi0.yy, Pi1.yy);
+        vec4 iz0 = Pi0.zzzz;
+        vec4 iz1 = Pi1.zzzz;
+
+        vec4 ixy = permute(permute(ix) + iy);
+        vec4 ixy0 = permute(ixy + iz0);
+        vec4 ixy1 = permute(ixy + iz1);
+
+        vec4 gx0 = ixy0 * (1.0 / 7.0);
+        vec4 gy0 = fract(floor(gx0) * (1.0 / 7.0)) - 0.5;
+        gx0 = fract(gx0);
+        vec4 gz0 = vec4(0.5) - abs(gx0) - abs(gy0);
+        vec4 sz0 = step(gz0, vec4(0.0));
+        gx0 -= sz0 * (step(0.0, gx0) - 0.5);
+        gy0 -= sz0 * (step(0.0, gy0) - 0.5);
+
+        vec4 gx1 = ixy1 * (1.0 / 7.0);
+        vec4 gy1 = fract(floor(gx1) * (1.0 / 7.0)) - 0.5;
+        gx1 = fract(gx1);
+        vec4 gz1 = vec4(0.5) - abs(gx1) - abs(gy1);
+        vec4 sz1 = step(gz1, vec4(0.0));
+        gx1 -= sz1 * (step(0.0, gx1) - 0.5);
+        gy1 -= sz1 * (step(0.0, gy1) - 0.5);
+
+        vec3 g000 = vec3(gx0.x,gy0.x,gz0.x);
+        vec3 g100 = vec3(gx0.y,gy0.y,gz0.y);
+        vec3 g010 = vec3(gx0.z,gy0.z,gz0.z);
+        vec3 g110 = vec3(gx0.w,gy0.w,gz0.w);
+        vec3 g001 = vec3(gx1.x,gy1.x,gz1.x);
+        vec3 g101 = vec3(gx1.y,gy1.y,gz1.y);
+        vec3 g011 = vec3(gx1.z,gy1.z,gz1.z);
+        vec3 g111 = vec3(gx1.w,gy1.w,gz1.w);
+
+        vec4 norm0 = taylorInvSqrt(vec4(dot(g000, g000), dot(g010, g010), dot(g100, g100), dot(g110, g110)));
+        g000 *= norm0.x;
+        g010 *= norm0.y;
+        g100 *= norm0.z;
+        g110 *= norm0.w;
+        vec4 norm1 = taylorInvSqrt(vec4(dot(g001, g001), dot(g011, g011), dot(g101, g101), dot(g111, g111)));
+        g001 *= norm1.x;
+        g011 *= norm1.y;
+        g101 *= norm1.z;
+        g111 *= norm1.w;
+
+        float n000 = dot(g000, Pf0);
+        float n100 = dot(g100, vec3(Pf1.x, Pf0.yz));
+        float n010 = dot(g010, vec3(Pf0.x, Pf1.y, Pf0.z));
+        float n110 = dot(g110, vec3(Pf1.xy, Pf0.z));
+        float n001 = dot(g001, vec3(Pf0.xy, Pf1.z));
+        float n101 = dot(g101, vec3(Pf1.x, Pf0.y, Pf1.z));
+        float n011 = dot(g011, vec3(Pf0.x, Pf1.yz));
+        float n111 = dot(g111, Pf1);
+
+        vec3 fade_xyz = fade(Pf0);
+        vec4 n_z = mix(vec4(n000, n100, n010, n110), vec4(n001, n101, n011, n111), fade_xyz.z);
+        vec2 n_yz = mix(n_z.xy, n_z.zw, fade_xyz.y);
+        float n_xyz = mix(n_yz.x, n_yz.y, fade_xyz.x);
+        return 2.2 * n_xyz;
+      }
+
+      uniform float u_frequency;
+
+      void main() {
+          float noise = 3.0 * pnoise(position + u_time, vec3(10.0));
+          float displacement = (u_frequency / 30.) * (noise / 10.);
+          vec3 newPosition = position + normal * displacement;
+          gl_Position = projectionMatrix * modelViewMatrix * vec4(newPosition, 1.0);
+      }
+    </script>
+    <script id="fragmentshader" type="fragment">
+      uniform float u_red;
+      uniform float u_blue;
+      uniform float u_green;
+      void main() {
+          gl_FragColor = vec4(vec3(u_red, u_green, u_blue), 1. );
+      }
+    </script>
+    <div id="app"></div>
+    <script type="module" src="/src/main.ts"></script>
+    <noscript> This website requires JavaScript to function properly. Please enable JavaScript to continue. </noscript>
+  </body>
+</html>
diff --git a/packages/moonshine-web/netlify.toml b/packages/moonshine-web/netlify.toml
@@ -0,0 +1,18 @@
+[build]
+publish = "packages/moonshine-web/dist"
+command = "pnpm run packages:stub && pnpm run build"
+
+[build.environment]
+NODE_VERSION = "22"
+
+[[redirects]]
+from = "/assets/*"
+to = "/assets/:splat"
+status = 200
+force = true
+
+[[redirects]]
+from = "/*"
+to = "/index.html"
+status = 200
+force = false
diff --git a/packages/moonshine-web/package.json b/packages/moonshine-web/package.json
@@ -0,0 +1,37 @@
+{
+  "name": "@proj-airi/moonshine-web",
+  "type": "module",
+  "private": true,
+  "packageManager": "[email protected]",
+  "description": "Yet another WebGPU based STT + VAD with Moonshine model re-implemented",
+  "author": {
+    "name": "Neko Ayaka",
+    "email": "[email protected]",
+    "url": "https://github.com/nekomeowww"
+  },
+  "license": "MIT",
+  "scripts": {
+    "build": "vite build",
+    "dev": "vite  --port 5175",
+    "lint": "eslint .",
+    "preview": "vite preview",
+    "typecheck": "vue-tsc --noEmit"
+  },
+  "dependencies": {
+    "@tresjs/core": "^4.3.1",
+    "@unocss/reset": "^0.65.2",
+    "@vueuse/core": "^12.1.0",
+    "@vueuse/motion": "^2.2.6",
+    "ofetch": "^1.4.1",
+    "three": "^0.171.0",
+    "vue": "^3.5.13"
+  },
+  "devDependencies": {
+    "@huggingface/transformers": "^3.2.1",
+    "@types/audioworklet": "^0.0.65",
+    "@types/three": "^0.171.0",
+    "@vitejs/plugin-vue": "^5.2.1",
+    "@webgpu/types": "^0.1.52",
+    "vue-tsc": "^2.1.10"
+  }
+}
diff --git a/packages/moonshine-web/public/banner.png b/packages/moonshine-web/public/banner.png
diff --git a/packages/moonshine-web/public/logo.png b/packages/moonshine-web/public/logo.png