Skip to content

Commit

Permalink
feat: basic 3d emotion from LLM completions
Browse files Browse the repository at this point in the history
  • Loading branch information
@nekomeowww
nekomeowww committed Dec 6, 2024
1 parent 289f822 commit b69abd2
Show file tree
Hide file tree
Showing 5 changed files with 321 additions and 71 deletions.
23 changes: 19 additions & 4 deletions packages/stage/src/components/MainStage.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { useMarkdown } from '../composables/markdown'
import { useQueue } from '../composables/queue'
import { useDelayMessageQueue, useEmotionsMessageQueue, useMessageContentQueue } from '../composables/queues'
import { llmInferenceEndToken } from '../constants'
import { EMOTION_EmotionMotionName_value, EmotionThinkMotionName } from '../constants/emotions'
import { EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../constants/emotions'
import SystemPromptV2 from '../constants/prompts/system-v2'
import { useLLM } from '../stores/llm'
import { useSettings } from '../stores/settings'
Expand All @@ -25,9 +25,13 @@ import ThreeDScene from './ThreeDScene.vue'
const nowSpeakingAvatarBorderOpacityMin = 30
const nowSpeakingAvatarBorderOpacityMax = 100
const { elevenLabsApiKey, openAiApiBaseURL, openAiApiKey } = storeToRefs(useSettings())
const {
elevenLabsApiKey,
openAiApiBaseURL,
openAiApiKey,
stageView,
} = storeToRefs(useSettings())
const openAIModel = useLocalStorage<{ id: string, name?: string }>('settings/llm/openai/model', { id: 'openai/gpt-3.5-turbo', name: 'OpenAI GPT3.5 Turbo' })
const stageView = useLocalStorage('settings/stage/view/model-renderer', '2d')
const {
streamSpeech,
Expand All @@ -39,6 +43,7 @@ const { process } = useMarkdown()
const listening = ref(false)
const live2DViewerRef = ref<{ setMotion: (motionName: string) => Promise<void> }>()
const vrmViewerRef = ref<{ setExpression: (expression: string) => void }>()
const supportedModels = ref<{ id: string, name?: string }[]>([])
const messageInput = ref<string>('')
const messages = ref<Array<Message>>([SystemPromptV2 as SystemMessage])
Expand Down Expand Up @@ -131,7 +136,16 @@ const messageContentQueue = useMessageContentQueue(ttsQueue)
const emotionsQueue = useQueue<Emotion>({
handlers: [
async (ctx) => {
await live2DViewerRef.value!.setMotion(EMOTION_EmotionMotionName_value[ctx.data])
if (stageView.value === '3d') {
const value = EMOTION_VRMExpressionName_value[ctx.data]
if (!value)
return
await vrmViewerRef.value!.setExpression(value)
}
else if (stageView.value === '2d') {
await live2DViewerRef.value!.setMotion(EMOTION_EmotionMotionName_value[ctx.data])
}
},
],
})
Expand Down Expand Up @@ -289,6 +303,7 @@ onUnmounted(() => {
/>
<ThreeDScene
v-else-if="stageView === '3d'"
ref="vrmViewerRef"
model="/assets/vrm/models/AvatarSample-B/AvatarSample_B.vrm"
idle-animation="/assets/vrm/animations/idle_loop.vrma"
w="50%" min-w="50% <lg:full" min-h="100 sm:100" h-full flex-1
Expand Down
179 changes: 112 additions & 67 deletions packages/stage/src/components/ThreeDScene.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,91 +22,136 @@ const cameraPositionZ = ref(-1)
const vrmModelPositionX = ref(-0.18)
const vrmModelPositionY = ref(-1.4)
const vrmModelPositionZ = ref(-0.24)
const modelRef = ref<{
setExpression: (expression: string) => void
}>()
defineExpose({
setExpression: (expression: string) => {
modelRef.value?.setExpression(expression)
},
})
</script>

<template>
<Screen v-slot="{ canvasHeight, canvasWidth }" relative>
<div z="10" top="2" absolute w-full gap-2 flex="~ col md:row">
<Collapsable h-fit w-full>
<template #label>
<span font-mono>Camera</span>
</template>
<div grid="~ cols-[20px_1fr_60px]" w-full gap-1 p-2 text-sm font-mono>
<div text="zinc-400 dark:zinc-500">
<span>X</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="cameraPositionX" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ cameraPositionX }}</span>
</div>
<div z="10" top="2" absolute w-full flex="~ col" gap-2>
<div w-full flex="~ col md:row" gap-2>
<Collapsable h-fit w-full>
<template #label>
<span font-mono>Camera</span>
</template>
<div grid="~ cols-[20px_1fr_60px]" w-full gap-1 p-2 text-sm font-mono>
<div text="zinc-400 dark:zinc-500">
<span>X</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="cameraPositionX" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ cameraPositionX }}</span>
</div>

<div text="zinc-400 dark:zinc-500">
<span>Y</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="cameraPositionY" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ cameraPositionY }}</span>
</div>
<div text="zinc-400 dark:zinc-500">
<span>Y</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="cameraPositionY" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ cameraPositionY }}</span>
</div>

<div text="zinc-400 dark:zinc-500">
<span>Z</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="cameraPositionZ" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ cameraPositionZ }}</span>
</div>
</div>
</Collapsable>
<Collapsable h-fit w-full>
<template #label>
<span font-mono>Model</span>
</template>
<div grid="~ cols-[20px_1fr_60px]" w-full gap-1 p-2 text-sm font-mono>
<div text="zinc-400 dark:zinc-500">
<span>X</span>
<div text="zinc-400 dark:zinc-500">
<span>Z</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="cameraPositionZ" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ cameraPositionZ }}</span>
</div>
</div>
</Collapsable>
<Collapsable h-fit w-full>
<template #label>
<span font-mono>Model</span>
</template>
<div grid="~ cols-[20px_1fr_60px]" w-full gap-1 p-2 text-sm font-mono>
<div text="zinc-400 dark:zinc-500">
<span>X</span>
</div>

<label w-full flex items-center gap-2>
<DataGuiRange v-model="vrmModelPositionX" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ vrmModelPositionX }}</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="vrmModelPositionX" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ vrmModelPositionX }}</span>
</div>

<div text="zinc-400 dark:zinc-500">
<span>Y</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="vrmModelPositionY" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ vrmModelPositionY }}</span>
</div>
<div text="zinc-400 dark:zinc-500">
<span>Y</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="vrmModelPositionY" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ vrmModelPositionY }}</span>
</div>

<div text="zinc-400 dark:zinc-500">
<span>Z</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="vrmModelPositionZ" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ vrmModelPositionZ }}</span>
<div text="zinc-400 dark:zinc-500">
<span>Z</span>
</div>
<label w-full flex items-center gap-2>
<DataGuiRange v-model="vrmModelPositionZ" :min="-10" :max="10" :step="0.01" />
</label>
<div text-right>
<span>{{ vrmModelPositionZ }}</span>
</div>
</div>
</div>
</Collapsable>
</Collapsable>
</div>
<div flex="~ row" w-full gap-2>
<button
rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
@click="modelRef?.setExpression('neutral')"
>
🙂 Neutral
</button>
<button
rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
@click="modelRef?.setExpression('surprised')"
>
🤯 Surprised
</button>
<button
rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
@click="modelRef?.setExpression('sad')"
>
😫 Sad
</button>
<button
rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
@click="modelRef?.setExpression('angry')"
>
😠 Angry
</button>
<button
rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
@click="modelRef?.setExpression('happy')"
>
😄 Happy
</button>
</div>
</div>
<TresCanvas :alpha="true" :antialias="true" :width="canvasWidth" :height="canvasHeight">
<OrbitControls />
<TresPerspectiveCamera :position="[cameraPositionX, cameraPositionY, cameraPositionZ]" />
<TresDirectionalLight :color="0xFFFFFF" :intensity="1.2" :position="[1, 1, 1]" />
<TresAmbientLight :color="0xFFFFFF" :intensity="1.5" />
<VRMModel
ref="modelRef"
:model="props.model"
:idle-animation="props.idleAnimation"
:position="[vrmModelPositionX, vrmModelPositionY, vrmModelPositionZ]"
Expand Down
11 changes: 11 additions & 0 deletions packages/stage/src/components/VRMModel.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { useLoop, useTresContext } from '@tresjs/core'
import { AnimationMixer } from 'three'
import { clipFromVRMAnimation, loadVRMAnimation, useBlink } from '~/composables/vrm/animation'
import { loadVrm } from '~/composables/vrm/core'
import { useVRMEmote } from '~/composables/vrm/expression'
const props = defineProps<{
model: string
Expand All @@ -22,6 +23,7 @@ const vrmAnimationMixer = ref<AnimationMixer>()
const { scene } = useTresContext()
const { onBeforeRender } = useLoop()
const blink = useBlink()
const vrmEmote = ref<ReturnType<typeof useVRMEmote>>()
watch(() => props.position, ([x, y, z]) => {
if (vrm.value) {
Expand Down Expand Up @@ -57,10 +59,13 @@ onMounted(async () => {
vrmAnimationMixer.value = new AnimationMixer(_vrm.scene)
vrmAnimationMixer.value.clipAction(clip).play()
vrmEmote.value = useVRMEmote(_vrm)
onBeforeRender(({ delta }) => {
vrmAnimationMixer.value?.update(delta)
vrm.value?.update(delta)
blink.update(vrm.value, delta)
vrmEmote.value?.update(delta)
})
vrm.value = _vrm
Expand All @@ -76,6 +81,12 @@ onUnmounted(() => {
scene.value.remove(vrm.value.scene)
}
})
defineExpose({
setExpression(expression: string) {
vrmEmote.value?.setEmotionWithResetAfter(expression, 1000)
},
})
</script>

<template>
Expand Down
Loading

0 comments on commit b69abd2

Please sign in to comment.