diff --git a/src/components/Sing/ScoreSequencer.vue b/src/components/Sing/ScoreSequencer.vue index 8cf0459c84..b8eb1ff61c 100644 --- a/src/components/Sing/ScoreSequencer.vue +++ b/src/components/Sing/ScoreSequencer.vue @@ -668,10 +668,8 @@ const previewDrawPitch = () => { for (let i = cursorFrame; i <= prevCursorPos.frame; i++) { tempPitchEdit.data[i - tempPitchEdit.startFrame] = Math.exp( Interpolate.linear( - cursorFrame, - Math.log(cursorFrequency), - prevCursorPos.frame, - Math.log(prevCursorPos.frequency), + { x: cursorFrame, y: Math.log(cursorFrequency) }, + { x: prevCursorPos.frame, y: Math.log(prevCursorPos.frequency) }, i, ), ); @@ -680,10 +678,8 @@ const previewDrawPitch = () => { for (let i = prevCursorPos.frame; i <= cursorFrame; i++) { tempPitchEdit.data[i - tempPitchEdit.startFrame] = Math.exp( Interpolate.linear( - prevCursorPos.frame, - Math.log(prevCursorPos.frequency), - cursorFrame, - Math.log(cursorFrequency), + { x: prevCursorPos.frame, y: Math.log(prevCursorPos.frequency) }, + { x: cursorFrame, y: Math.log(cursorFrequency) }, i, ), ); diff --git a/src/components/Sing/SequencerPitch.vue b/src/components/Sing/SequencerPitch.vue index 9e14202b26..6eeb94a487 100644 --- a/src/components/Sing/SequencerPitch.vue +++ b/src/components/Sing/SequencerPitch.vue @@ -29,7 +29,7 @@ import { } from "@/composables/onMountOrActivate"; import { ExhaustiveError } from "@/type/utility"; import { createLogger } from "@/domain/frontend/log"; -import { Interpolate } from "@/sing/utility"; +import { Interpolate, iterativeEndPointFit } from "@/sing/utility"; import { Color } from "@/sing/graphics/color"; import { Points } from "@/sing/graphics/points"; import { getLast } from "@/sing/utility"; @@ -120,14 +120,14 @@ const pitchEditLine: PitchLine = { lineStripMap: new Map(), }; const interpOriginalPitchLine: PitchLine = { - color: new Color(171, 199, 201, 255), + color: ref(new Color(171, 199, 201, 255)), width: 1.5, pitchDataMap: new Map(), lineStripMap: new Map(), }; const interpOriginalPitchKnots: PitchKnots = { - color: new Color(149, 188, 207, 255), - radius: 2, + color: new Color(212, 154, 148, 255), + radius: 3, }; const canvasContainer = ref(null); @@ -137,12 +137,14 @@ let canvasHeight: number | undefined; let renderer: PIXI.Renderer | undefined; let stage: PIXI.Container | undefined; +let lineStripsContainer: PIXI.Container | undefined; +let pointsContainer: PIXI.Container | undefined; let requestId: number | undefined; let renderInNextFrame = false; const updateLineStrips = (pitchLine: PitchLine) => { - if (stage == undefined) { - throw new Error("stage is undefined."); + if (lineStripsContainer == undefined) { + throw new Error("lineStripsContainer is undefined."); } if (canvasWidth == undefined) { throw new Error("canvasWidth is undefined."); @@ -159,7 +161,7 @@ const updateLineStrips = (pitchLine: PitchLine) => { // 無くなったピッチデータを調べて、そのピッチデータに対応するLineStripを削除する for (const [key, lineStrip] of pitchLine.lineStripMap) { if (!pitchLine.pitchDataMap.has(key)) { - stage.removeChild(lineStrip.displayObject); + lineStripsContainer.removeChild(lineStrip.displayObject); removedLineStrips.push(lineStrip); pitchLine.lineStripMap.delete(key); } @@ -171,7 +173,7 @@ const updateLineStrips = (pitchLine: PitchLine) => { const currentLineStrip = pitchLine.lineStripMap.get(key)!; // テーマなど色が変更された場合、LineStripを再作成する if (!currentLineStrip.color.equals(pitchLine.color.value)) { - stage.removeChild(currentLineStrip.displayObject); + lineStripsContainer.removeChild(currentLineStrip.displayObject); currentLineStrip.destroy(); pitchLine.lineStripMap.delete(key); } else { @@ -199,10 +201,10 @@ const updateLineStrips = (pitchLine: PitchLine) => { } // pitchEditLineの場合は最後に追加する(originalより前面に表示) if (pitchLine === pitchEditLine) { - stage.addChild(lineStrip.displayObject); + lineStripsContainer.addChild(lineStrip.displayObject); } else { // originalLineは最初に追加する(EditLineの背面に表示) - stage.addChildAt(lineStrip.displayObject, 0); + lineStripsContainer.addChildAt(lineStrip.displayObject, 0); } pitchLine.lineStripMap.set(key, lineStrip); } @@ -248,8 +250,8 @@ const updateLineStrips = (pitchLine: PitchLine) => { }; const updatePoints = (pitchKnots: PitchKnots) => { - if (stage == undefined) { - throw new Error("stage is undefined."); + if (pointsContainer == undefined) { + throw new Error("pointsContainer is undefined."); } const tpqn = store.state.tpqn; const zoomX = store.state.sequencerZoomX; @@ -257,7 +259,10 @@ const updatePoints = (pitchKnots: PitchKnots) => { const offsetX = props.offsetX; const offsetY = props.offsetY; - if (pitchKnots.knotsData == undefined) { + if ( + pitchKnots.knotsData == undefined || + pitchKnots.knotsData.xArray.length === 0 + ) { return; } @@ -272,7 +277,7 @@ const updatePoints = (pitchKnots: PitchKnots) => { pitchKnots.radius, 8, ); - stage.addChild(pitchKnots.points.displayObject); + pointsContainer.addChild(pitchKnots.points.displayObject); } // ポイントを計算してlineStripに設定&更新 @@ -296,33 +301,33 @@ const render = () => { throw new Error("stage is undefined."); } - // シンガーが未設定の場合はピッチラインをすべて非表示にして終了 const singer = store.getters.SELECTED_TRACK.singer; - if (!singer) { - for (const lineStrip of originalPitchLine.lineStripMap.values()) { - lineStrip.renderable = false; - } - for (const lineStrip of pitchEditLine.lineStripMap.values()) { - lineStrip.renderable = false; - } - renderer.render(stage); - return; + if (singer) { + stage.renderable = true; + updateLineStrips(originalPitchLine); + updateLineStrips(pitchEditLine); + updateLineStrips(interpOriginalPitchLine); + updatePoints(interpOriginalPitchKnots); + } else { + // シンガーが未設定の場合はピッチラインをすべて非表示にして終了 + stage.renderable = false; } - - // ピッチラインのLineStripを更新する - updateLineStrips(originalPitchLine); - updateLineStrips(pitchEditLine); - updateLineStrips(interpOriginalPitchLine); - updatePoints(interpOriginalPitchKnots); - renderer.render(stage); }; -const toPitchData = (framewiseData: number[], frameRate: number): PitchData => { +const toPitchData = ( + startFrame: number, + framewiseData: number[], + frameRate: number, +): PitchData => { const data = framewiseData; const ticksArray: number[] = []; for (let i = 0; i < data.length; i++) { - const ticks = secondToTick(i / frameRate, tempos.value, tpqn.value); + const ticks = secondToTick( + (startFrame + i) / frameRate, + tempos.value, + tpqn.value, + ); ticksArray.push(ticks); } return { ticksArray, data }; @@ -345,19 +350,14 @@ const splitPitchData = (pitchData: PitchData, delimiter: number) => { return pitchDataArray; }; -const setPitchDataToPitchLine = async ( - pitchData: PitchData, +const setPitchDataArrayToPitchLine = async ( + pitchDataArray: PitchData[], pitchLine: PitchLine, ) => { - const partialPitchDataArray = splitPitchData( - pitchData, - VALUE_INDICATING_NO_DATA, - ).filter((value) => value.data.length >= 2); - pitchLine.pitchDataMap.clear(); - for (const partialPitchData of partialPitchDataArray) { - const hash = await calculatePitchDataHash(partialPitchData); - pitchLine.pitchDataMap.set(hash, partialPitchData); + for (const pitchData of pitchDataArray) { + const hash = await calculatePitchDataHash(pitchData); + pitchLine.pitchDataMap.set(hash, pitchData); } }; @@ -412,7 +412,7 @@ const generateOriginalPitchData = () => { } } } - return toPitchData(tempData, frameRate); + return tempData; }; const generatePitchEditData = () => { @@ -448,40 +448,11 @@ const generatePitchEditData = () => { throw new ExhaustiveError(previewPitchEditType); } } - return toPitchData(tempData, frameRate); -}; - -const generateInterpOriginalPitchData = ( - points: { x: number; y: number }[], -) => { - const frameRate = editFrameRate.value; - - const xArray: number[] = []; - for (let i = 0; i < 400; i++) { - xArray.push(i); - } - const tempData = Interpolate.catmullRom(points, xArray).map((value) => - noteNumberToFrequency(value), - ); - return toPitchData(tempData, frameRate); + return tempData; }; -const setKnotsDataToPitchKnots = (points: { x: number; y: number }[]) => { - const frameRate = editFrameRate.value; - - const ticksArray: number[] = []; - for (let i = 0; i < points.length; i++) { - const ticks = secondToTick( - points[i].x / frameRate, - tempos.value, - tpqn.value, - ); - ticksArray.push(ticks); - } - interpOriginalPitchKnots.knotsData = { - xArray: ticksArray, - yArray: points.map((value) => value.y), - }; +const setKnotsDataToPitchKnots = (knotsData: KnotsData) => { + interpOriginalPitchKnots.knotsData = knotsData; }; const asyncLock = new AsyncLock({ maxPending: 1 }); @@ -492,23 +463,14 @@ watch( asyncLock.acquire( "originalPitch", async () => { - const originalPitchData = generateOriginalPitchData(); - await setPitchDataToPitchLine(originalPitchData, originalPitchLine); - - const points = [ - { x: 100, y: 60 }, - { x: 160, y: 62 }, - { x: 165, y: 62 }, - { x: 180, y: 65 }, - { x: 220, y: 63 }, - { x: 290, y: 69 }, - ]; - const interpOriginalPitchData = generateInterpOriginalPitchData(points); - await setPitchDataToPitchLine( - interpOriginalPitchData, - interpOriginalPitchLine, - ); - setKnotsDataToPitchKnots(points); + const frameRate = editorFrameRate.value; + const framewiseData = generateOriginalPitchData(); + const pitchData = toPitchData(0, framewiseData, frameRate); + const pitchDataArray = splitPitchData( + pitchData, + VALUE_INDICATING_NO_DATA, + ).filter((value) => value.data.length >= 2); + await setPitchDataArrayToPitchLine(pitchDataArray, originalPitchLine); renderInNextFrame = true; }, @@ -528,8 +490,49 @@ watch( asyncLock.acquire( "pitchEdit", async () => { - const pitchEditData = generatePitchEditData(); - await setPitchDataToPitchLine(pitchEditData, pitchEditLine); + const frameRate = editorFrameRate.value; + const framewiseData = generatePitchEditData(); + const pitchData = toPitchData(0, framewiseData, frameRate); + const pitchDataArray = splitPitchData( + pitchData, + VALUE_INDICATING_NO_DATA, + ).filter((value) => value.data.length >= 2); + await setPitchDataArrayToPitchLine(pitchDataArray, pitchEditLine); + + const interpPitchDataArray: PitchData[] = []; + const knotsData: KnotsData = { xArray: [], yArray: [] }; + for (const pitchData of pitchDataArray) { + const points1: { x: number; y: number }[] = []; + for (let i = 0; i < pitchData.ticksArray.length; i++) { + const ticks = pitchData.ticksArray[i]; + const freq = pitchData.data[i]; + const noteNumber = frequencyToNoteNumber(freq); + points1.push({ x: ticks, y: noteNumber }); + } + const points2 = iterativeEndPointFit(points1, 0.12); + knotsData.xArray.push(...points2.map((value) => value.x)); + knotsData.yArray.push(...points2.map((value) => value.y)); + const period = 10; + const minX = points2[0].x; + const maxX = getLast(points2).x; + const xValues: number[] = []; + for (let i = minX; i < maxX; i += period) { + xValues.push(i); + } + xValues.push(maxX); + const yValues = Interpolate.catmullRom(points2, xValues); + const interpPitchData: PitchData = { + ticksArray: xValues, + data: yValues.map((value) => noteNumberToFrequency(value)), + }; + interpPitchDataArray.push(interpPitchData); + } + await setPitchDataArrayToPitchLine( + interpPitchDataArray, + interpOriginalPitchLine, + ); + setKnotsDataToPitchKnots(knotsData); + renderInNextFrame = true; }, (err) => { @@ -593,6 +596,11 @@ onMountedOrActivated(() => { autoDensity: true, }); stage = new PIXI.Container(); + lineStripsContainer = new PIXI.Container(); + pointsContainer = new PIXI.Container(); + + stage.addChild(lineStripsContainer); + stage.addChild(pointsContainer); // webGLVersionをチェックする // 2未満の場合、ピッチの表示ができないのでエラーとしてロギングする diff --git a/src/sing/utility.ts b/src/sing/utility.ts index e0dc0bea23..ed9ee117fa 100644 --- a/src/sing/utility.ts +++ b/src/sing/utility.ts @@ -10,77 +10,211 @@ export function getLast(array: T[]) { return array[array.length - 1]; } +export function calculateDistanceFromPointToLine( + p0: { x: number; y: number }, + p1: { x: number; y: number }, + p2: { x: number; y: number }, +) { + const lineM = (p1.y - p0.y) / (p1.x - p0.x); + return ( + Math.abs(lineM * p2.x - p2.y + p0.y - lineM * p0.x) / + Math.sqrt(lineM ** 2 + 1) + ); +} + export class Interpolate { - static linear(x0: number, y0: number, x1: number, y1: number, x: number) { - if (x1 <= x0) { - throw new Error("x1 must be greater than x0."); + static linear( + p0: { x: number; y: number }, + p1: { x: number; y: number }, + x: number, + ) { + if (p1.x <= p0.x) { + throw new Error("p1.x must be greater than p0.x."); } - return y0 + ((y1 - y0) * (x - x0)) / (x1 - x0); + const m = (p1.y - p0.y) / (p1.x - p0.x); + return p0.y + (x - p0.x) * m; } static cubicHermite( - x0: number, - y0: number, + p0: { x: number; y: number }, m0: number, - x1: number, - y1: number, + p1: { x: number; y: number }, m1: number, x: number, ) { - const t = (x - x0) / (x1 - x0); + const dx = p1.x - p0.x; + const t = (x - p0.x) / dx; const h0 = 2 * t ** 3 - 3 * t ** 2 + 1; const h1 = t ** 3 - 2 * t ** 2 + t; const h2 = -2 * t ** 3 + 3 * t ** 2; const h3 = t ** 3 - t ** 2; - return y0 * h0 + m0 * (x1 - x0) * h1 + y1 * h2 + m1 * (x1 - x0) * h3; + return p0.y * h0 + m0 * dx * h1 + p1.y * h2 + m1 * dx * h3; } - static catmullRom(pArray: { x: number; y: number }[], xArray: number[]) { - if (pArray.length < 2) { - throw new Error("pArray.length must be at least 2."); + static catmullRom(points: { x: number; y: number }[], xValues: number[]) { + if (points.length < 2) { + throw new Error("points.length must be at least 2."); } - const n = pArray.length; - const firstP = pArray[0]; - const lastP = pArray[n - 1]; + const n = points.length; + const firstP = points[0]; + const lastP = points[n - 1]; - const calcM = (i: number) => { - const p0 = pArray[Math.max(0, i - 1)]; - const p1 = pArray[Math.min(n - 1, i + 1)]; - return (p1.y - p0.y) / (p1.x - p0.x); - }; + const mValues: number[] = []; + for (let i = 0; i < n; i++) { + const p0 = points[Math.max(0, i - 1)]; + const p1 = points[Math.min(n - 1, i + 1)]; + const m = (p1.y - p0.y) / (p1.x - p0.x); + mValues.push(m); + } + + const yValues: number[] = []; + for (const x of xValues) { + if (x < firstP.x) { + const m = mValues[0]; + const y = firstP.y + (x - firstP.x) * m; + yValues.push(y); + } else if (x >= lastP.x) { + const m = mValues[n - 1]; + const y = lastP.y + (x - lastP.x) * m; + yValues.push(y); + } else { + for (let i = 0; i < n - 1; i++) { + if (x < points[i + 1].x) { + const p0 = points[i]; + const p1 = points[i + 1]; + const m0 = mValues[i]; + const m1 = mValues[i + 1]; + const y = this.cubicHermite(p0, m0, p1, m1, x); + yValues.push(y); + break; + } + } + } + } + return yValues; + } + + static pchip(points: { x: number; y: number }[], xValues: number[]) { + if (points.length < 2) { + throw new Error("points.length must be at least 2."); + } + const n = points.length; + const firstP = points[0]; + const lastP = points[n - 1]; - const mArray: number[] = []; + const mValues: number[] = []; for (let i = 0; i < n; i++) { - const m = calcM(i); - mArray.push(m); + const p0 = points[Math.max(0, i - 1)]; + const p1 = points[i]; + const p2 = points[Math.min(n - 1, i + 1)]; + const dx = p2.x - p0.x; + const dy0 = p1.y - p0.y; + const dy1 = p2.y - p1.y; + const m = dy0 * dy1 <= 0 ? 0 : (dy0 + dy1) / dx; + mValues.push(m); + } + for (let i = 0; i < n - 1; i++) { + const m0 = mValues[i]; + const m1 = mValues[i + 1]; + const p0 = points[i]; + const p1 = points[i + 1]; + const dx = p1.x - p0.x; + const dy = p1.y - p0.y; + if (dy !== 0) { + const d = dy / dx; + const a = m0 / d; + const b = m1 / d; + const t = 3 / Math.sqrt(a * a + b * b); + if (t < 1) { + mValues[i] = t * a * d; + mValues[i + 1] = t * b * d; + } + } } - const yArray: number[] = []; - for (const x of xArray) { + const yValues: number[] = []; + for (const x of xValues) { if (x < firstP.x) { - const m = calcM(0); + const m = mValues[0]; const y = firstP.y + (x - firstP.x) * m; - yArray.push(y); + yValues.push(y); } else if (x >= lastP.x) { - const m = calcM(n - 1); + const m = mValues[n - 1]; const y = lastP.y + (x - lastP.x) * m; - yArray.push(y); + yValues.push(y); } else { for (let i = 0; i < n - 1; i++) { - if (x < pArray[i + 1].x) { - const p0 = pArray[i]; - const p1 = pArray[i + 1]; - const m0 = calcM(i); - const m1 = calcM(i + 1); - const y = this.cubicHermite(p0.x, p0.y, m0, p1.x, p1.y, m1, x); - yArray.push(y); + if (x < points[i + 1].x) { + const p0 = points[i]; + const p1 = points[i + 1]; + const m0 = mValues[i]; + const m1 = mValues[i + 1]; + const y = this.cubicHermite(p0, m0, p1, m1, x); + yValues.push(y); break; } } } } - return yArray; + return yValues; + } +} + +export function differentiate(yValues: number[]) { + const n = yValues.length; + const diffArray: number[] = []; + for (let i = 0; i < n; i++) { + const y0 = yValues[Math.max(0, i - 1)]; + const y1 = yValues[Math.min(n - 1, i + 1)]; + diffArray.push((y1 - y0) / 2); + } + return diffArray; +} + +export function iterativeEndPointFit( + points: { x: number; y: number }[], + epsilon: number, +) { + for (let i = 1; i < points.length; i++) { + if (points[i - 1].x > points[i].x) { + throw new Error("Points must be sorted by x coordinate."); + } + } + const markedPoints = [points[0], getLast(points)]; + const pointsWaitingToBeProcessed = [points]; + while (true) { + const pointsToProcess = pointsWaitingToBeProcessed.pop(); + if (pointsToProcess == undefined) { + break; + } + if (pointsToProcess.length <= 2) { + continue; + } + const p0 = pointsToProcess[0]; + const p1 = getLast(pointsToProcess); + let farthestPointIndex = 1; + let farthestPointD = 0; + for (let i = 1; i < pointsToProcess.length - 1; i++) { + const p2 = pointsToProcess[i]; + const d = Math.abs(p2.y - Interpolate.linear(p0, p1, p2.x)); + if (d > farthestPointD) { + farthestPointD = d; + farthestPointIndex = i; + } + } + if (farthestPointD >= epsilon) { + const farthestPoint = pointsToProcess[farthestPointIndex]; + markedPoints.push(farthestPoint); + const slicedPoints1 = pointsToProcess.slice(0, farthestPointIndex + 1); + const slicedPoints2 = pointsToProcess.slice( + farthestPointIndex, + pointsToProcess.length, + ); + pointsWaitingToBeProcessed.push(slicedPoints1); + pointsWaitingToBeProcessed.push(slicedPoints2); + } } + return markedPoints.sort((a, b) => a.x - b.x); } function ceilToOdd(value: number) { diff --git a/src/store/singing.ts b/src/store/singing.ts index 30b1c57de5..2752128be6 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -379,11 +379,9 @@ const muteLastPauSection = ( } else { for (let i = 0; i < fadeOutFrameLength; i++) { volume[lastPauStartFrame + i] *= Interpolate.linear( - 0, - 1, - fadeOutFrameLength - 1, - 0, - i, + { x: 0, y: 1 }, + { x: fadeOutFrameLength - 1, y: 0 }, + i ); } }