フレーズレンダラーを追加、リファクタリング

src/components/Sing/SequencerPhraseIndicator.vue

@@ -15,6 +15,7 @@ const props = defineProps<{
 
 const store = useStore();
 const classNames: Record<PhraseState, string> = {
+  SINGER_IS_NOT_SET: "singer-is-not-set",
   WAITING_TO_BE_RENDERED: "waiting-to-be-rendered",
   NOW_RENDERING: "now-rendering",
   COULD_NOT_RENDER: "could-not-render",
@@ -43,6 +44,10 @@ const className = computed(() => {
   }
 }
 
+.singer-is-not-set {
+  visibility: hidden;
+}
+
 .waiting-to-be-rendered {
   @include tint-if-in-other-track(
     "background-color",

src/components/Sing/SequencerPitch.vue

@@ -30,6 +30,7 @@ import { ExhaustiveError } from "@/type/utility";
 import { createLogger } from "@/domain/frontend/log";
 import { getLast } from "@/sing/utility";
 import { getOrThrow } from "@/helpers/mapHelper";
+import { FrameAudioQuery } from "@/openapi";
 
 type PitchLine = {
   readonly color: Color;
@@ -55,23 +56,27 @@ const pitchEditData = computed(() => {
 });
 const previewPitchEdit = computed(() => props.previewPitchEdit);
 const selectedTrackId = computed(() => store.getters.SELECTED_TRACK_ID);
+const engineFrameRateOfSelectedTrack = computed(() => {
+  const track = getOrThrow(store.state.tracks, selectedTrackId.value);
+  if (track.singer == undefined) {
+    return undefined;
+  }
+  return store.state.engineManifests[track.singer.engineId].frameRate;
+});
 const editFrameRate = computed(() => store.state.editFrameRate);
-const singingGuidesInSelectedTrack = computed(() => {
-  const singingGuides = [];
+const phraseQueriesInSelectedTrack = computed(() => {
+  const phraseQueries: { startTime: number; query: FrameAudioQuery }[] = [];
   for (const phrase of store.state.phrases.values()) {
-    if (phrase.trackId !== selectedTrackId.value) {
-      continue;
-    }
-    if (phrase.singingGuideKey == undefined) {
+    if (phrase.queryKey == undefined) {
       continue;
     }
-    const singingGuide = getOrThrow(
-      store.state.singingGuides,
-      phrase.singingGuideKey,
-    );
-    singingGuides.push(singingGuide);
+    const phraseQuery = getOrThrow(store.state.phraseQueries, phrase.queryKey);
+    phraseQueries.push({
+      startTime: phrase.startTime,
+      query: phraseQuery,
+    });
   }
-  return singingGuides;
+  return phraseQueries;
 });
 
 const originalPitchLine: PitchLine = {
@@ -259,56 +264,56 @@ const setPitchDataToPitchLine = async (
 
 const generateOriginalPitchData = () => {
   const unvoicedPhonemes = UNVOICED_PHONEMES;
-  const frameRate = editFrameRate.value; // f0（元のピッチ）は編集フレームレートで表示する
+  const engineFrameRate = engineFrameRateOfSelectedTrack.value;
 
-  // 選択中のトラックで使われている歌い方のf0を結合してピッチデータを生成する
+  // TODO: 補間を行うようにする
+  if (engineFrameRate !== editFrameRate.value) {
+    throw new Error(
+      "The frame rate between the engine and the edit does not match.",
+    );
+  }
+
+  // 選択中のトラックで使われているクエリのf0を結合してピッチデータを生成する
   const tempData = [];
-  for (const singingGuide of singingGuidesInSelectedTrack.value) {
-    // TODO: 補間を行うようにする
-    if (singingGuide.frameRate !== frameRate) {
-      throw new Error(
-        "The frame rate between the singing guide and the edit does not match.",
-      );
-    }
-    const phonemes = singingGuide.query.phonemes;
+  for (const phraseQuery of phraseQueriesInSelectedTrack.value) {
+    const phonemes = phraseQuery.query.phonemes;
     if (phonemes.length === 0) {
       throw new Error("phonemes.length is 0.");
     }
-    const f0 = singingGuide.query.f0;
+    const f0 = phraseQuery.query.f0;
 
     // 各フレームの音素の配列を生成する
     const framePhonemes = convertToFramePhonemes(phonemes);
     if (f0.length !== framePhonemes.length) {
       throw new Error("f0.length and framePhonemes.length do not match.");
     }
 
-    // 歌い方の開始フレームと終了フレームを計算する
-    const singingGuideFrameLength = f0.length;
-    const singingGuideStartFrame = Math.round(
-      singingGuide.startTime * frameRate,
+    // フレーズの開始フレームと終了フレームを計算する
+    const phraseFrameLength = f0.length;
+    const phraseStartFrame = Math.round(
+      phraseQuery.startTime * engineFrameRate,
     );
-    const singingGuideEndFrame =
-      singingGuideStartFrame + singingGuideFrameLength;
+    const phraseEndFrame = phraseStartFrame + phraseFrameLength;
 
     // 無声子音区間以外のf0をtempDataにコピーする
     // NOTE: 無声子音区間は音程が無く、f0の値が大きく上下するので表示しない
-    if (tempData.length < singingGuideEndFrame) {
-      const valuesToPush = new Array(
-        singingGuideEndFrame - tempData.length,
-      ).fill(VALUE_INDICATING_NO_DATA);
+    if (tempData.length < phraseEndFrame) {
+      const valuesToPush = new Array(phraseEndFrame - tempData.length).fill(
+        VALUE_INDICATING_NO_DATA,
+      );
       tempData.push(...valuesToPush);
     }
-    const startFrame = Math.max(0, singingGuideStartFrame);
-    const endFrame = singingGuideEndFrame;
+    const startFrame = Math.max(0, phraseStartFrame);
+    const endFrame = phraseEndFrame;
     for (let i = startFrame; i < endFrame; i++) {
-      const phoneme = framePhonemes[i - singingGuideStartFrame];
+      const phoneme = framePhonemes[i - phraseStartFrame];
       const unvoiced = unvoicedPhonemes.includes(phoneme);
       if (!unvoiced) {
-        tempData[i] = f0[i - singingGuideStartFrame];
+        tempData[i] = f0[i - phraseStartFrame];
       }
     }
   }
-  return toPitchData(tempData, frameRate);
+  return toPitchData(tempData, engineFrameRate);
 };
 
 const generatePitchEditData = () => {
@@ -350,11 +355,14 @@ const generatePitchEditData = () => {
 const asyncLock = new AsyncLock({ maxPending: 1 });
 
 watch(
-  [singingGuidesInSelectedTrack, tempos, tpqn],
+  [phraseQueriesInSelectedTrack, tempos, tpqn],
   async () => {
     asyncLock.acquire(
       "originalPitch",
       async () => {
+        if (engineFrameRateOfSelectedTrack.value == undefined) {
+          return;
+        }
         const originalPitchData = generateOriginalPitchData();
         await setPitchDataToPitchLine(originalPitchData, originalPitchLine);
         renderInNextFrame = true;

src/sing/domain.ts

@@ -4,17 +4,12 @@ import {
   Note,
   Phrase,
   PhraseSource,
-  SingingGuide,
-  SingingGuideSource,
-  SingingVoiceSource,
   Tempo,
   TimeSignature,
-  phraseSourceHashSchema,
+  PhraseKey,
   Track,
-  singingGuideSourceHashSchema,
-  singingVoiceSourceHashSchema,
 } from "@/store/type";
-import { FramePhoneme } from "@/openapi";
+import { FrameAudioQuery, FramePhoneme } from "@/openapi";
 import { TrackId } from "@/type/preload";
 
 const BEAT_TYPES = [2, 4, 8, 16];
@@ -379,23 +374,9 @@ export function isValidPitchEditData(pitchEditData: number[]) {
   );
 }
 
-export const calculatePhraseSourceHash = async (phraseSource: PhraseSource) => {
+export const calculatePhraseKey = async (phraseSource: PhraseSource) => {
   const hash = await calculateHash(phraseSource);
-  return phraseSourceHashSchema.parse(hash);
-};
-
-export const calculateSingingGuideSourceHash = async (
-  singingGuideSource: SingingGuideSource,
-) => {
-  const hash = await calculateHash(singingGuideSource);
-  return singingGuideSourceHashSchema.parse(hash);
-};
-
-export const calculateSingingVoiceSourceHash = async (
-  singingVoiceSource: SingingVoiceSource,
-) => {
-  const hash = await calculateHash(singingVoiceSource);
-  return singingVoiceSourceHashSchema.parse(hash);
+  return PhraseKey(hash);
 };
 
 export function getStartTicksOfPhrase(phrase: Phrase) {
@@ -469,7 +450,11 @@ export function convertToFramePhonemes(phonemes: FramePhoneme[]) {
 }
 
 export function applyPitchEdit(
-  singingGuide: SingingGuide,
+  singingGuide: {
+    query: FrameAudioQuery;
+    frameRate: number;
+    startTime: number;
+  },
   pitchEditData: number[],
   editFrameRate: number,
 ) {