Skip to content

Commit

Permalink
EditorFrameAudioQueryを使うようにした
Browse files Browse the repository at this point in the history
  • Loading branch information
@sigprogramming
sigprogramming committed Aug 31, 2024
1 parent 837026d commit c6407f0
Show file tree
Hide file tree
Showing 5 changed files with 71 additions and 79 deletions.
13 changes: 3 additions & 10 deletions src/components/Sing/SequencerPitch.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import { ExhaustiveError } from "@/type/utility";
import { createLogger } from "@/domain/frontend/log";
import { getLast } from "@/sing/utility";
import { getOrThrow } from "@/helpers/mapHelper";
import { FrameAudioQuery } from "@/openapi";
import { EditorFrameAudioQuery } from "@/store/type";
type PitchLine = {
readonly color: Color;
Expand Down Expand Up @@ -59,8 +59,7 @@ const selectedTrackId = computed(() => store.getters.SELECTED_TRACK_ID);
const editFrameRate = computed(() => store.state.editFrameRate);
const singingGuidesInSelectedTrack = computed(() => {
const singingGuides: {
query: FrameAudioQuery;
frameRate: number;
query: EditorFrameAudioQuery;
startTime: number;
}[] = [];
for (const phrase of store.state.phrases.values()) {
Expand All @@ -70,16 +69,10 @@ const singingGuidesInSelectedTrack = computed(() => {
if (phrase.queryKey == undefined) {
continue;
}
const track = store.state.tracks.get(phrase.trackId);
if (track == undefined || track.singer == undefined) {
continue;
}
const phraseQuery = getOrThrow(store.state.phraseQueries, phrase.queryKey);
const engineManifest = store.state.engineManifests[track.singer.engineId];
singingGuides.push({
startTime: phrase.startTime,
query: phraseQuery,
frameRate: engineManifest.frameRate,
});
}
return singingGuides;
Expand Down Expand Up @@ -276,7 +269,7 @@ const generateOriginalPitchData = () => {
const tempData = [];
for (const singingGuide of singingGuidesInSelectedTrack.value) {
// TODO: 補間を行うようにする
if (singingGuide.frameRate !== frameRate) {
if (singingGuide.query.frameRate !== frameRate) {
throw new Error(
"The frame rate between the singing guide and the edit does not match.",
);
Expand Down
38 changes: 18 additions & 20 deletions src/sing/domain.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import {
TimeSignature,
PhraseKey,
Track,
EditorFrameAudioQuery,
} from "@/store/type";
import { FrameAudioQuery, FramePhoneme } from "@/openapi";
import { FramePhoneme } from "@/openapi";
import { TrackId } from "@/type/preload";

const BEAT_TYPES = [2, 4, 8, 16];
Expand Down Expand Up @@ -450,46 +451,43 @@ export function convertToFramePhonemes(phonemes: FramePhoneme[]) {
}

export function applyPitchEdit(
singingGuide: {
query: FrameAudioQuery;
frameRate: number;
startTime: number;
},
phraseQuery: EditorFrameAudioQuery,
phraseStartTime: number,
pitchEditData: number[],
editFrameRate: number,
) {
// 歌い方のフレームレートと編集フレームレートが一致しない場合はエラー
// フレーズのクエリのフレームレートと編集フレームレートが一致しない場合はエラー
// TODO: 補間するようにする
if (singingGuide.frameRate !== editFrameRate) {
if (phraseQuery.frameRate !== editFrameRate) {
throw new Error(
"The frame rate between the singing guide and the edit data does not match.",
"The frame rate between the phrase query and the edit data does not match.",
);
}
const unvoicedPhonemes = UNVOICED_PHONEMES;
const f0 = singingGuide.query.f0;
const phonemes = singingGuide.query.phonemes;
const f0 = phraseQuery.f0;
const phonemes = phraseQuery.phonemes;

// 各フレームの音素の配列を生成する
const framePhonemes = convertToFramePhonemes(phonemes);
if (f0.length !== framePhonemes.length) {
throw new Error("f0.length and framePhonemes.length do not match.");
}

// 歌い方の開始フレームと終了フレームを計算する
const singingGuideFrameLength = f0.length;
const singingGuideStartFrame = Math.round(
singingGuide.startTime * singingGuide.frameRate,
// フレーズのクエリの開始フレームと終了フレームを計算する
const phraseQueryFrameLength = f0.length;
const phraseQueryStartFrame = Math.round(
phraseStartTime * phraseQuery.frameRate,
);
const singingGuideEndFrame = singingGuideStartFrame + singingGuideFrameLength;
const phraseQueryEndFrame = phraseQueryStartFrame + phraseQueryFrameLength;

// ピッチ編集をf0に適用する
const startFrame = Math.max(0, singingGuideStartFrame);
const endFrame = Math.min(pitchEditData.length, singingGuideEndFrame);
const startFrame = Math.max(0, phraseQueryStartFrame);
const endFrame = Math.min(pitchEditData.length, phraseQueryEndFrame);
for (let i = startFrame; i < endFrame; i++) {
const phoneme = framePhonemes[i - singingGuideStartFrame];
const phoneme = framePhonemes[i - phraseQueryStartFrame];
const voiced = !unvoicedPhonemes.includes(phoneme);
if (voiced && pitchEditData[i] !== VALUE_INDICATING_NO_DATA) {
f0[i - singingGuideStartFrame] = pitchEditData[i];
f0[i - phraseQueryStartFrame] = pitchEditData[i];
}
}
}
Expand Down
54 changes: 22 additions & 32 deletions src/sing/phraseRendering.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import {
FrameAudioQueryKey,
Note,
PhraseKey,
Singer,
Expand All @@ -9,6 +8,8 @@ import {
Track,
SingingVolumeKey,
SingingVolume,
EditorFrameAudioQueryKey,
EditorFrameAudioQuery,
} from "@/store/type";
import {
FrameAudioQuery,
Expand Down Expand Up @@ -98,7 +99,7 @@ const shiftKeyOfNotes = (notes: NoteForRequestToEngine[], keyShift: number) => {
}
};

const getPhonemes = (query: FrameAudioQuery) => {
const getPhonemes = (query: EditorFrameAudioQuery) => {
return query.phonemes.map((value) => value.phoneme).join(" ");
};

Expand Down Expand Up @@ -182,8 +183,8 @@ type Phrase = Readonly<{
notes: Note[];
startTime: number;
queryKey: {
get: () => FrameAudioQueryKey | undefined;
set: (value: FrameAudioQueryKey | undefined) => void;
get: () => EditorFrameAudioQueryKey | undefined;
set: (value: EditorFrameAudioQueryKey | undefined) => void;
};
singingVolumeKey: {
get: () => SingingVolumeKey | undefined;
Expand All @@ -199,17 +200,20 @@ type Phrase = Readonly<{
* フレーズレンダリングで必要となる外部のキャッシュや関数
*/
type ExternalDependencies = Readonly<{
queryCache: Map<FrameAudioQueryKey, FrameAudioQuery>;
queryCache: Map<EditorFrameAudioQueryKey, EditorFrameAudioQuery>;
singingVolumeCache: Map<SingingVolumeKey, SingingVolume>;
singingVoiceCache: Map<SingingVoiceKey, SingingVoice>;

phrases: {
get: (phraseKey: PhraseKey) => Phrase;
};
phraseQueries: {
get: (queryKey: FrameAudioQueryKey) => FrameAudioQuery;
set: (queryKey: FrameAudioQueryKey, query: FrameAudioQuery) => void;
delete: (queryKey: FrameAudioQueryKey) => void;
get: (queryKey: EditorFrameAudioQueryKey) => EditorFrameAudioQuery;
set: (
queryKey: EditorFrameAudioQueryKey,
query: EditorFrameAudioQuery,
) => void;
delete: (queryKey: EditorFrameAudioQueryKey) => void;
};
phraseSingingVolumes: {
get: (singingVolumeKey: SingingVolumeKey) => SingingVolume;
Expand Down Expand Up @@ -316,13 +320,13 @@ const generateQuerySource = (context: Context): QuerySource => {

const calculateQueryKey = async (querySource: QuerySource) => {
const hash = await calculateHash(querySource);
return FrameAudioQueryKey(hash);
return EditorFrameAudioQueryKey(hash);
};

const generateQuery = async (
querySource: QuerySource,
externalDependencies: ExternalDependencies,
) => {
): Promise<EditorFrameAudioQuery> => {
const notesForRequestToEngine = createNotesForRequestToEngine(
querySource.firstRestDuration,
lastRestDurationSeconds,
Expand All @@ -340,7 +344,7 @@ const generateQuery = async (
);

shiftPitch(query.f0, querySource.keyRangeAdjustment);
return query;
return { ...query, frameRate: querySource.engineFrameRate };
};

const queryGenerationStage: Stage = {
Expand Down Expand Up @@ -411,7 +415,7 @@ type SingingVolumeSource = Readonly<{
notes: Note[];
keyRangeAdjustment: number;
volumeRangeAdjustment: number;
queryForVolumeGeneration: FrameAudioQuery;
queryForVolumeGeneration: EditorFrameAudioQuery;
}>;

const generateSingingVolumeSource = (context: Context): SingingVolumeSource => {
Expand All @@ -422,10 +426,6 @@ const generateSingingVolumeSource = (context: Context): SingingVolumeSource => {
if (track.singer == undefined) {
throw new Error("track.singer is undefined.");
}
const engineFrameRate = getOrThrow(
context.snapshot.engineFrameRates,
track.singer.engineId,
);
const phrase = phrases.get(context.phraseKey);
const phraseQueryKey = phrase.queryKey.get();
if (phraseQueryKey == undefined) {
Expand All @@ -434,17 +434,14 @@ const generateSingingVolumeSource = (context: Context): SingingVolumeSource => {
const query = phraseQueries.get(phraseQueryKey);
const clonedQuery = cloneWithUnwrapProxy(query);
applyPitchEdit(
{
query: clonedQuery,
startTime: phrase.startTime,
frameRate: engineFrameRate,
},
clonedQuery,
phrase.startTime,
track.pitchEditData,
context.snapshot.editFrameRate,
);
return {
engineId: track.singer.engineId,
engineFrameRate,
engineFrameRate: query.frameRate,
tpqn: context.snapshot.tpqn,
tempos: context.snapshot.tempos,
firstRestDuration: phrase.firstRestDuration,
Expand Down Expand Up @@ -572,7 +569,7 @@ const singingVolumeGenerationStage: Stage = {
*/
type SingingVoiceSource = Readonly<{
singer: Singer;
queryForSingingVoiceSynthesis: FrameAudioQuery;
queryForSingingVoiceSynthesis: EditorFrameAudioQuery;
}>;

const generateSingingVoiceSource = (context: Context): SingingVoiceSource => {
Expand All @@ -585,10 +582,6 @@ const generateSingingVoiceSource = (context: Context): SingingVoiceSource => {
if (track.singer == undefined) {
throw new Error("track.singer is undefined.");
}
const engineFrameRate = getOrThrow(
context.snapshot.engineFrameRates,
track.singer.engineId,
);
const phrase = phrases.get(context.phraseKey);
const phraseQueryKey = phrase.queryKey.get();
const phraseSingingVolumeKey = phrase.singingVolumeKey.get();
Expand All @@ -603,11 +596,8 @@ const generateSingingVoiceSource = (context: Context): SingingVoiceSource => {
const clonedQuery = cloneWithUnwrapProxy(query);
const clonedSingingVolume = cloneWithUnwrapProxy(singingVolume);
applyPitchEdit(
{
query: clonedQuery,
startTime: phrase.startTime,
frameRate: engineFrameRate,
},
clonedQuery,
phrase.startTime,
track.pitchEditData,
context.snapshot.editFrameRate,
);
Expand Down
13 changes: 7 additions & 6 deletions src/store/singing.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,11 @@ import {
PhraseKey,
Track,
SequenceId,
FrameAudioQueryKey,
SingingVolumeKey,
SingingVolume,
SingingVoiceKey,
EditorFrameAudioQueryKey,
EditorFrameAudioQuery,
} from "./type";
import { DEFAULT_PROJECT_NAME, sanitizeFileName } from "./utility";
import {
Expand Down Expand Up @@ -247,7 +248,7 @@ const phraseSingingVoices = new Map<SingingVoiceKey, SingingVoice>();
const sequences = new Map<SequenceId, Sequence & { trackId: TrackId }>();
const animationTimer = new AnimationTimer();

const queryCache = new Map<FrameAudioQueryKey, FrameAudioQuery>();
const queryCache = new Map<EditorFrameAudioQueryKey, EditorFrameAudioQuery>();
const singingVolumeCache = new Map<SingingVolumeKey, SingingVolume>();
const singingVoiceCache = new Map<SingingVoiceKey, SingingVoice>();

Expand Down Expand Up @@ -880,7 +881,7 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
queryKey,
}: {
phraseKey: PhraseKey;
queryKey: FrameAudioQueryKey | undefined;
queryKey: EditorFrameAudioQueryKey | undefined;
},
) {
const phrase = getOrThrow(state.phrases, phraseKey);
Expand Down Expand Up @@ -947,16 +948,16 @@ export const singingStore = createPartialStore<SingingStoreTypes>({
queryKey,
query,
}: {
queryKey: FrameAudioQueryKey;
query: FrameAudioQuery;
queryKey: EditorFrameAudioQueryKey;
query: EditorFrameAudioQuery;
},
) {
state.phraseQueries.set(queryKey, query);
},
},

DELETE_PHRASE_QUERY: {
mutation(state, { queryKey }: { queryKey: FrameAudioQueryKey }) {
mutation(state, { queryKey }: { queryKey: EditorFrameAudioQueryKey }) {
state.phraseQueries.delete(queryKey);
},
},
Expand Down
Loading

0 comments on commit c6407f0

Please sign in to comment.