feat: idle animation

README.md

@@ -15,3 +15,8 @@ pnpm i
 ```shell
 pnpm dev
 ```
+
+## Acknowledgements
+
+- [pixiv/ChatVRM](https://github.com/pixiv/ChatVRM)
+- [josephrocca/ChatVRM-js: A JS conversion/adaptation of parts of the ChatVRM (TypeScript) code for standalone use in OpenCharacters and elsewhere](https://github.com/josephrocca/ChatVRM-js)

README.zh-CN.md

@@ -15,3 +15,8 @@ pnpm i
 ```shell
 pnpm dev
 ```
+
+## Acknowledgements
+
+- [pixiv/ChatVRM](https://github.com/pixiv/ChatVRM)
+- [josephrocca/ChatVRM-js: A JS conversion/adaptation of parts of the ChatVRM (TypeScript) code for standalone use in OpenCharacters and elsewhere](https://github.com/josephrocca/ChatVRM-js)

cspell.config.yaml

@@ -54,6 +54,7 @@ words:
   - unhead
   - unocss
   - unplugin
+  - vrma
   - vueuse
   - xsai
 ignoreWords: []

packages/stage/package.json

@@ -31,7 +31,9 @@
     "@pixi/sprite": "6",
     "@pixi/ticker": "^6.5.10",
     "@pixi/utils": "6",
-    "@pixiv/three-vrm": "^3.2.0",
+    "@pixiv/three-vrm": "^3.3.0",
+    "@pixiv/three-vrm-animation": "^3.3.0",
+    "@pixiv/three-vrm-core": "^3.3.0",
     "@tresjs/cientos": "^4.0.3",
     "@tresjs/core": "^4.3.1",
     "@types/yauzl": "^2.10.3",

packages/stage/src/components/MainStage.vue

@@ -324,7 +324,8 @@ onUnmounted(() => {
       />
       <ThreeDScene
         v-else-if="stageView === '3d'"
-        model="/assets/vrm/models/AvatarSample-A/AvatarSample_A.vrm"
+        model="/assets/vrm/models/AvatarSample-B/AvatarSample_B.vrm"
+        idle-animation="/assets/vrm/animations/idle_loop.vrma"
         w="50%" min-w="50% <lg:full" min-h="100 sm:100" h-full flex-1
         @error="console.error"
       />

packages/stage/src/components/ThreeDScene.vue

@@ -8,19 +8,20 @@ import VRMModel from './VRMModel.vue'
 
 const props = defineProps<{
   model: string
+  idleAnimation: string
 }>()
 
 const emit = defineEmits<{
   (e: 'loadModelProgress', value: number): void
   (e: 'error', value: unknown): void
 }>()
 
-const cameraPositionX = ref(0)
-const cameraPositionY = ref(0.1)
-const cameraPositionZ = ref(-1.5)
-const vrmModelPositionX = ref(0)
+const cameraPositionX = ref(-0.17)
+const cameraPositionY = ref(0)
+const cameraPositionZ = ref(-1)
+const vrmModelPositionX = ref(-0.18)
 const vrmModelPositionY = ref(-1.4)
-const vrmModelPositionZ = ref(-0.3)
+const vrmModelPositionZ = ref(-0.24)
 </script>
 
 <template>
@@ -102,14 +103,16 @@ const vrmModelPositionZ = ref(-0.3)
     </div>
     <TresCanvas :alpha="true" :antialias="true" :width="canvasWidth" :height="canvasHeight">
       <TresPerspectiveCamera :position="[cameraPositionX, cameraPositionY, cameraPositionZ]" />
+      <TresDirectionalLight :color="0xFFFFFF" :intensity="0.6" :position="[1, 1, 1]" />
       <OrbitControls />
       <VRMModel
         :model="props.model"
+        :idle-animation="props.idleAnimation"
         :position="[vrmModelPositionX, vrmModelPositionY, vrmModelPositionZ]"
         @load-model-progress="(val) => emit('loadModelProgress', val)"
         @error="(val) => emit('error', val)"
       />
-      <TresAmbientLight :intensity="1" />
+      <TresAmbientLight :color="0xFFFFFF" :intensity="0.4" />
     </TresCanvas>
   </Screen>
 </template>

packages/stage/src/components/VRMModel.vue

@@ -1,11 +1,14 @@
 <script setup lang="ts">
-import type { VRMCore } from '@pixiv/three-vrm'
-import { VRMLoaderPlugin } from '@pixiv/three-vrm'
-import { useTresContext } from '@tresjs/core'
-import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'
+import type { VRMCore } from '@pixiv/three-vrm-core'
+import { useLoop, useTresContext } from '@tresjs/core'
+import { AnimationMixer } from 'three'
+import { clipFromVRMAnimation, loadVRMAnimation } from '~/composables/vrm/animation'
+import { loadVrm } from '~/composables/vrm/core'
 
 const props = defineProps<{
   model: string
+  idleAnimation: string
+  loadAnimations?: string[]
   position: [number, number, number]
 }>()
 
@@ -14,43 +17,61 @@ const emit = defineEmits<{
   (e: 'error', value: unknown): void
 }>()
 
-let vrm: VRMCore
+const vrm = ref<VRMCore>()
+const vrmAnimationMixer = ref<AnimationMixer>()
 const { scene } = useTresContext()
+const { onBeforeRender } = useLoop()
 
-interface GLTFUserdata extends Record<string, any> {
-  vrm: VRMCore
-}
-
-onMounted(() => {
+onMounted(async () => {
   if (!scene.value) {
     return
   }
 
-  const loader = new GLTFLoader()
-  loader.register(parser => new VRMLoaderPlugin(parser))
-  loader.load(
-    props.model,
-    (gltf) => {
-      const userData = gltf.userData as GLTFUserdata
-      vrm = userData.vrm as VRMCore
-      scene.value.add(vrm.scene)
-      vrm.scene.position.set(...props.position)
-    },
-    progress => emit('loadModelProgress', Number.parseFloat((100.0 * (progress.loaded / progress.total)).toFixed(2))),
-    error => emit('error', error),
-  )
+  try {
+    const _vrm = await loadVrm(props.model, {
+      scene: scene.value,
+      lookAt: true,
+      position: props.position,
+      onProgress: progress => emit('loadModelProgress', Number.parseFloat((100.0 * (progress.loaded / progress.total)).toFixed(2))),
+    })
+    if (!_vrm) {
+      console.warn('No VRM model loaded')
+      return
+    }
+
+    const animation = await loadVRMAnimation(props.idleAnimation)
+    const clip = await clipFromVRMAnimation(_vrm, animation)
+    if (!clip) {
+      console.warn('No VRM animation loaded')
+      return
+    }
+
+    // play animation
+    vrmAnimationMixer.value = new AnimationMixer(_vrm.scene)
+    vrmAnimationMixer.value.clipAction(clip).play()
+
+    onBeforeRender(({ delta }) => {
+      vrmAnimationMixer.value?.update(delta)
+      vrm.value?.update(delta)
+    })
+
+    vrm.value = _vrm
+  }
+  catch (err) {
+    emit('error', err)
+  }
 })
 
 onUnmounted(() => {
-  if (vrm) {
+  if (vrm.value) {
     const { scene } = useTresContext()
-    scene.value.remove(vrm.scene)
+    scene.value.remove(vrm.value.scene)
   }
 })
 
 watch(() => props.position, ([x, y, z]) => {
-  if (vrm) {
-    vrm.scene.position.set(x, y, z)
+  if (vrm.value) {
+    vrm.value.scene.position.set(x, y, z)
   }
 })
 </script>

packages/stage/src/composables/vrm/animation.ts

@@ -0,0 +1,40 @@
+import type { VRMAnimation } from '@pixiv/three-vrm-animation'
+import type { VRMCore } from '@pixiv/three-vrm-core'
+import { createVRMAnimationClip } from '@pixiv/three-vrm-animation'
+import { useVRMLoader } from './loader'
+
+export interface GLTFUserdata extends Record<string, any> {
+  vrmAnimations: VRMAnimation[]
+}
+
+export async function loadVRMAnimation(url: string) {
+  const loader = useVRMLoader()
+
+  // load VRM Animation .vrma file
+  const gltf = await loader.loadAsync(url)
+
+  const userData = gltf.userData as GLTFUserdata
+  if (!userData.vrmAnimations) {
+    console.warn('No VRM animations found in the .vrma file')
+    return
+  }
+  if (userData.vrmAnimations.length === 0) {
+    console.warn('No VRM animations found in the .vrma file')
+    return
+  }
+
+  return userData.vrmAnimations[0]
+}
+
+export async function clipFromVRMAnimation(vrm?: VRMCore, animation?: VRMAnimation) {
+  if (!vrm) {
+    console.warn('No VRM found')
+    return
+  }
+  if (!animation) {
+    return
+  }
+
+  // create animation clip
+  return createVRMAnimationClip(animation, vrm)
+}

packages/stage/src/composables/vrm/core.ts

@@ -0,0 +1,52 @@
+import type { Object3D, Scene } from 'three'
+import { type VRMCore, VRMUtils } from '@pixiv/three-vrm'
+
+import { VRMLookAtQuaternionProxy } from '@pixiv/three-vrm-animation'
+import { useVRMLoader } from './loader'
+
+interface GLTFUserdata extends Record<string, any> {
+  vrmCore?: VRMCore
+}
+
+export async function loadVrm(model: string, options?: {
+  position?: [number, number, number]
+  scene?: Scene
+  lookAt?: boolean
+  onProgress?: (progress: ProgressEvent<EventTarget>) => void | Promise<void>
+}): Promise<VRMCore | undefined> {
+  const loader = useVRMLoader()
+  const gltf = await loader.loadAsync(model, progress => options?.onProgress?.(progress))
+
+  const userData = gltf.userData as GLTFUserdata
+  if (!userData.vrm) {
+    return
+  }
+
+  const _vrm = userData.vrm
+
+  // calling these functions greatly improves the performance
+  VRMUtils.removeUnnecessaryVertices(_vrm.scene)
+  VRMUtils.combineSkeletons(_vrm.scene)
+
+  // Disable frustum culling
+  _vrm.scene.traverse((object: Object3D) => {
+    object.frustumCulled = false
+  })
+
+  // Add look at quaternion proxy to the VRM; which is needed to play the look at animation
+  if (options?.lookAt && _vrm.lookAt) {
+    const lookAtQuatProxy = new VRMLookAtQuaternionProxy(_vrm.lookAt)
+    lookAtQuatProxy.name = 'lookAtQuaternionProxy'
+    _vrm.scene.add(lookAtQuatProxy)
+  }
+
+  // Add to scene
+  if (options?.scene)
+    options.scene.add(_vrm.scene)
+
+  // Set position
+  if (options?.position)
+    _vrm.scene.position.set(...options.position)
+
+  return _vrm
+}

packages/stage/src/composables/vrm/loader.ts

@@ -0,0 +1,19 @@
+import { VRMLoaderPlugin } from '@pixiv/three-vrm'
+import { VRMAnimationLoaderPlugin } from '@pixiv/three-vrm-animation'
+import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'
+
+let loader: GLTFLoader
+
+export function useVRMLoader() {
+  if (loader) {
+    return loader
+  }
+
+  loader = new GLTFLoader()
+
+  loader.crossOrigin = 'anonymous'
+  loader.register(parser => new VRMLoaderPlugin(parser))
+  loader.register(parser => new VRMAnimationLoaderPlugin(parser))
+
+  return loader
+}

packages/stage/src/libs/audio/manager.ts

@@ -0,0 +1,87 @@
+export interface AudioManagerType {
+  audioContext: AudioContext
+  analyser: AnalyserNode
+  dataBuffer: Float32Array
+  frameId: number | null
+  onVolumeChange?: (volume: number) => void
+}
+
+export function createAudioManager(): AudioManagerType {
+  const audioContext = new AudioContext()
+  const analyser = audioContext.createAnalyser()
+  const dataBuffer = new Float32Array(2048)
+
+  // Connect analyser to destination
+  analyser.connect(audioContext.destination)
+
+  return {
+    audioContext,
+    analyser,
+    dataBuffer,
+    frameId: null,
+    onVolumeChange: undefined,
+  }
+}
+
+function calculateVolume(manager: AudioManagerType): number {
+  manager.analyser.getFloatTimeDomainData(manager.dataBuffer)
+
+  // Find peak volume
+  let volume = 0.0
+  for (let i = 0; i < manager.dataBuffer.length; i++) {
+    volume = Math.max(volume, Math.abs(manager.dataBuffer[i]))
+  }
+
+  // Apply sigmoid normalization (from pixiv implementation)
+  volume = 1 / (1 + Math.exp(-45 * volume + 5))
+  return volume < 0.1 ? 0 : volume
+}
+
+function updateFrame(manager: AudioManagerType) {
+  if (manager.onVolumeChange) {
+    manager.onVolumeChange(calculateVolume(manager))
+  }
+  manager.frameId = requestAnimationFrame(() => updateFrame(manager))
+}
+
+export async function playAudio(manager: AudioManagerType, source: ArrayBuffer | string): Promise<void> {
+  try {
+    const buffer = typeof source === 'string'
+      ? await (await fetch(source)).arrayBuffer()
+      : source
+
+    const audioBuffer = await manager.audioContext.decodeAudioData(buffer)
+    const bufferSource = manager.audioContext.createBufferSource()
+
+    bufferSource.buffer = audioBuffer
+    bufferSource.connect(manager.analyser)
+    bufferSource.start()
+
+    return new Promise((resolve) => {
+      bufferSource.onended = () => resolve()
+    })
+  }
+  catch (error) {
+    console.error('Error playing audio:', error)
+    throw error
+  }
+}
+
+export function startVolumeTracking(manager: AudioManagerType, callback: (volume: number) => void) {
+  manager.onVolumeChange = callback
+  manager.audioContext.resume()
+  updateFrame(manager)
+}
+
+export function stopVolumeTracking(manager: AudioManagerType) {
+  if (manager.frameId) {
+    cancelAnimationFrame(manager.frameId)
+    manager.frameId = null
+  }
+  manager.onVolumeChange = undefined
+}
+
+export function disposeAudioManager(manager: AudioManagerType) {
+  stopVolumeTracking(manager)
+  manager.audioContext.close()
+}