Skip to content

Commit

Permalink
Fixed issue: object interpolated incorrectly if a frame with the obje…
Browse files Browse the repository at this point in the history 
…ct keyframe is deleted (#8951)
  • Loading branch information
@bsekachev
bsekachev authored Jan 20, 2025
1 parent d3be5b6 commit 6fead08
Show file tree
Hide file tree
Showing 9 changed files with 186 additions and 146 deletions.
4 changes: 4 additions & 0 deletions changelog.d/20250116_184520_sekachev.bs_fixed_deleted_frames.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
### Fixed

- A job cannot be opened if to remove an image with the latest keyframe of a track
(<https://github.com/cvat-ai/cvat/pull/8952>)
4 changes: 4 additions & 0 deletions changelog.d/20250116_191626_sekachev.bs_fixed_deleted_frames.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
### Fixed

- A track will be interpolated incorrectly if to delete an image containing the object keyframe
(<https://github.com/cvat-ai/cvat/pull/8951>)
64 changes: 34 additions & 30 deletions cvat-core/src/annotations-collection.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
// Copyright (C) 2019-2022 Intel Corporation
// Copyright (C) 2022-2024 CVAT.ai Corporation
// Copyright (C) 2022-2025 CVAT.ai Corporation
//
// SPDX-License-Identifier: MIT

import {
shapeFactory, trackFactory, Track, Shape, Tag,
MaskShape, BasicInjection,
SkeletonShape, SkeletonTrack, PolygonShape, CuboidShape,
MaskShape, BasicInjection, SkeletonShape,
SkeletonTrack, PolygonShape, CuboidShape,
RectangleShape, PolylineShape, PointsShape, EllipseShape,
InterpolationNotPossibleError,
} from './annotations-objects';
import { SerializedCollection, SerializedShape, SerializedTrack } from './server-response-types';
import AnnotationsFilter from './annotations-filter';
Expand All @@ -22,7 +23,6 @@ import {
HistoryActions, ShapeType, ObjectType, colors, Source, DimensionType, JobType,
} from './enums';
import AnnotationHistory from './annotations-history';
import { Job } from './session';

const validateAttributesList = (
attributes: { spec_id: number, value: string }[],
Expand All @@ -48,14 +48,9 @@ const labelAttributesAsDict = (label: Label): Record<number, Attribute> => (
}, {})
);

export type FrameMeta = Record<number, Awaited<ReturnType<Job['frames']['get']>>> & {
deleted_frames: Record<number, boolean>
};

export default class Collection {
public flush: boolean;
private stopFrame: number;
private frameMeta: FrameMeta;
private labels: Record<number, Label>;
private annotationsFilter: AnnotationsFilter;
private history: AnnotationHistory;
Expand All @@ -71,11 +66,10 @@ export default class Collection {
history: AnnotationHistory;
stopFrame: number;
dimension: DimensionType;
frameMeta: Collection['frameMeta'];
framesInfo: BasicInjection['framesInfo'];
jobType: JobType;
}) {
this.stopFrame = data.stopFrame;
this.frameMeta = data.frameMeta;

this.labels = data.labels.reduce((labelAccumulator, label) => {
labelAccumulator[label.id] = label;
Expand All @@ -96,15 +90,16 @@ export default class Collection {
this.groups = {
max: 0,
}; // it is an object to we can pass it as an argument by a reference

this.injection = {
labels: this.labels,
groups: this.groups,
frameMeta: this.frameMeta,
framesInfo: data.framesInfo,
history: this.history,
dimension: data.dimension,
jobType: data.jobType,
nextClientID: () => ++config.globalObjectsCounter,
groupColors: {},
nextClientID: () => ++config.globalObjectsCounter,
getMasksOnFrame: (frame: number) => (this.shapes[frame] as MaskShape[])
.filter((object) => object instanceof MaskShape),
};
Expand Down Expand Up @@ -239,9 +234,13 @@ export default class Collection {
}

public get(frame: number, allTracks: boolean, filters: object[]): ObjectState[] {
if (this.injection.framesInfo.isFrameDeleted(frame)) {
return [];
}

const { tracks } = this;
const shapes = this.shapes[frame] || [];
const tags = this.tags[frame] || [];
const shapes = this.shapes[frame] ?? [];
const tags = this.tags[frame] ?? [];

const objects = [].concat(tracks, shapes, tags);
const visible = [];
Expand All @@ -251,12 +250,17 @@ export default class Collection {
continue;
}

const stateData = object.get(frame);
if (stateData.outside && !stateData.keyframe && !allTracks && object instanceof Track) {
continue;
try {
const stateData = object.get(frame);
if (stateData.outside && !stateData.keyframe && !allTracks && object instanceof Track) {
continue;
}
visible.push(stateData);
} catch (error: unknown) {
if (!(error instanceof InterpolationNotPossibleError)) {
throw error;
}
}

visible.push(stateData);
}

const objectStates = [];
Expand Down Expand Up @@ -771,7 +775,7 @@ export default class Collection {
);
}

const { width, height } = this.frameMeta[slicedObject.frame];
const { width, height } = this.injection.framesInfo[slicedObject.frame];
if (slicedObject instanceof MaskShape) {
points1.push(slicedObject.left, slicedObject.top, slicedObject.right, slicedObject.bottom);
points2.push(slicedObject.left, slicedObject.top, slicedObject.right, slicedObject.bottom);
Expand Down Expand Up @@ -923,7 +927,7 @@ export default class Collection {
count -= 1;
}
for (let i = start + 1; lastIsKeyframe ? i < stop : i <= stop; i++) {
if (this.frameMeta.deleted_frames[i]) {
if (this.injection.framesInfo.isFrameDeleted(i)) {
count--;
}
}
Expand All @@ -936,7 +940,7 @@ export default class Collection {
const keyframes = Object.keys(track.shapes)
.sort((a, b) => +a - +b)
.map((el) => +el)
.filter((frame) => !this.frameMeta.deleted_frames[frame]);
.filter((frame) => !this.injection.framesInfo.isFrameDeleted(frame));

let prevKeyframe = keyframes[0];
let visible = false;
Expand Down Expand Up @@ -987,19 +991,19 @@ export default class Collection {
}

const { name: label } = object.label;
if (objectType === 'tag' && !this.frameMeta.deleted_frames[object.frame]) {
if (objectType === 'tag' && !this.injection.framesInfo.isFrameDeleted(object.frame)) {
labels[label].tag++;
labels[label].manually++;
labels[label].total++;
} else if (objectType === 'track') {
scanTrack(object);
} else if (!this.frameMeta.deleted_frames[object.frame]) {
} else if (!this.injection.framesInfo.isFrameDeleted(object.frame)) {
const { shapeType } = object as Shape;
labels[label][shapeType].shape++;
labels[label].manually++;
labels[label].total++;
if (shapeType === ShapeType.SKELETON) {
(object as SkeletonShape).elements.forEach((element) => {
(object as unknown as SkeletonShape).elements.forEach((element) => {
const combinedName = [label, element.label.name].join(sep);
labels[combinedName][element.shapeType].shape++;
labels[combinedName].manually++;
Expand Down Expand Up @@ -1086,7 +1090,7 @@ export default class Collection {
outside: state.outside || false,
occluded: state.occluded || false,
points: state.shapeType === 'mask' ? (() => {
const { width, height } = this.frameMeta[state.frame];
const { width, height } = this.injection.framesInfo[state.frame];
return cropMask(state.points, width, height);
})() : state.points,
rotation: state.rotation || 0,
Expand Down Expand Up @@ -1292,7 +1296,7 @@ export default class Collection {
const predicate = sign > 0 ? (frame) => frame <= frameTo : (frame) => frame >= frameTo;
const update = sign > 0 ? (frame) => frame + 1 : (frame) => frame - 1;
for (let frame = frameFrom; predicate(frame); frame = update(frame)) {
if (!allowDeletedFrames && this.frameMeta[frame].deleted) {
if (!allowDeletedFrames && this.injection.framesInfo.isFrameDeleted(frame)) {
continue;
}

Expand Down Expand Up @@ -1359,7 +1363,7 @@ export default class Collection {
if (!annotationsFilters) {
let frame = frameFrom;
while (predicate(frame)) {
if (!allowDeletedFrames && this.frameMeta[frame].deleted) {
if (!allowDeletedFrames && this.injection.framesInfo.isFrameDeleted(frame)) {
frame = update(frame);
continue;
}
Expand All @@ -1374,7 +1378,7 @@ export default class Collection {
const linearSearch = filtersStr.match(/"var":"width"/) || filtersStr.match(/"var":"height"/);

for (let frame = frameFrom; predicate(frame); frame = update(frame)) {
if (!allowDeletedFrames && this.frameMeta[frame].deleted) {
if (!allowDeletedFrames && this.injection.framesInfo.isFrameDeleted(frame)) {
continue;
}

Expand Down
50 changes: 23 additions & 27 deletions cvat-core/src/annotations-objects.ts
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Copyright (C) 2019-2022 Intel Corporation
// Copyright (C) 2022-2024 CVAT.ai Corporation
// Copyright (C) 2022-2025 CVAT.ai Corporation
//
// SPDX-License-Identifier: MIT

Expand Down Expand Up @@ -58,12 +58,18 @@ function computeNewSource(currentSource: Source): Source {
return Source.MANUAL;
}

type FrameInfo = {
width: number;
height: number;
};

export interface BasicInjection {
labels: Record<number, Label>;
groups: { max: number };
frameMeta: {
deleted_frames: Record<number, boolean>;
};
framesInfo: Readonly<{
[index: number]: Readonly<FrameInfo>;
isFrameDeleted: (frame: number) => boolean;
}>;
history: AnnotationHistory;
groupColors: Record<number, string>;
parentID?: number;
Expand All @@ -79,6 +85,8 @@ type AnnotationInjection = BasicInjection & {
readOnlyFields?: string[];
};

export class InterpolationNotPossibleError extends Error {}

class Annotation {
public clientID: number;
protected taskLabels: Record<number, Label>;
Expand Down Expand Up @@ -394,15 +402,15 @@ class Annotation {
}

class Drawn extends Annotation {
protected frameMeta: AnnotationInjection['frameMeta'];
protected framesInfo: AnnotationInjection['framesInfo'];
protected descriptions: string[];
public hidden: boolean;
protected pinned: boolean;
public shapeType: ShapeType;

constructor(data, clientID: number, color: string, injection: AnnotationInjection) {
super(data, clientID, color, injection);
this.frameMeta = injection.frameMeta;
this.framesInfo = injection.framesInfo;
this.descriptions = data.descriptions || [];
this.hidden = false;
this.pinned = true;
Expand Down Expand Up @@ -487,16 +495,10 @@ class Drawn extends Annotation {
checkObjectType('points', data.points, null, Array);
checkNumberOfPoints(this.shapeType, data.points);
// cut points
const { width, height, filename } = this.frameMeta[frame];
const { width, height } = this.framesInfo[frame];
fittedPoints = this.fitPoints(data.points, data.rotation, width, height);
let check = true;
if (filename && filename.slice(filename.length - 3) === 'pcd') {
check = false;
}
if (check) {
if (!checkShapeArea(this.shapeType, fittedPoints)) {
fittedPoints = [];
}
if (this.dimension === DimensionType.DIMENSION_2D && !checkShapeArea(this.shapeType, fittedPoints)) {
fittedPoints = [];
}
}

Expand Down Expand Up @@ -960,7 +962,7 @@ export class Track extends Drawn {
let last = Number.MIN_SAFE_INTEGER;

for (const frame of frames) {
if (frame in this.frameMeta.deleted_frames) {
if (this.framesInfo.isFrameDeleted(frame)) {
continue;
}

Expand Down Expand Up @@ -1414,10 +1416,7 @@ export class Track extends Drawn {
};
}

throw new DataError(
'No one left position or right position was found. ' +
`Interpolation impossible. Client ID: ${this.clientID}`,
);
throw new InterpolationNotPossibleError();
}
}

Expand Down Expand Up @@ -2216,7 +2215,7 @@ export class MaskShape extends Shape {
constructor(data: SerializedShape, clientID: number, color: string, injection: AnnotationInjection) {
super(data, clientID, color, injection);
const [left, top, right, bottom] = this.points.slice(-4);
const { width, height } = this.frameMeta[this.frame];
const { width, height } = this.framesInfo[this.frame];
if (left >= width || top >= height || right >= width || bottom >= height) {
this.points = cropMask(this.points, width, height);
}
Expand All @@ -2229,7 +2228,7 @@ export class MaskShape extends Shape {
protected validateStateBeforeSave(data: ObjectState, updated: ObjectState['updateFlags'], frame?: number): number[] {
super.validateStateBeforeSave(data, updated, frame);
if (updated.points) {
const { width, height } = this.frameMeta[frame];
const { width, height } = this.framesInfo[frame];
return cropMask(data.points, width, height);
}

Expand Down Expand Up @@ -2610,7 +2609,7 @@ class PolyTrack extends Track {
return Math.sqrt((point1.x - point2.x) ** 2 + (point1.y - point2.y) ** 2);
}

function minimizeSegment(baseLength: number, N: number, startInterpolated, stopInterpolated): void {
function minimizeSegment(baseLength: number, N: number, startInterpolated, stopInterpolated): Point2D[] {
const threshold = baseLength / (2 * N);
const minimized = [interpolatedPoints[startInterpolated]];
let latestPushed = startInterpolated;
Expand Down Expand Up @@ -3275,10 +3274,7 @@ export class SkeletonTrack extends Track {
};
}

throw new DataError(
'No one left position or right position was found. ' +
`Interpolation impossible. Client ID: ${this.clientID}`,
);
throw new InterpolationNotPossibleError();
}
}

Expand Down
Loading

0 comments on commit 6fead08

Please sign in to comment.