fix: LEAP-1490: Fix AudioUltra working with relations and comments

web/libs/editor/src/components/InteractiveOverlays/BoundingBox.js

@@ -1,3 +1,4 @@
+import { FF_DEV_2715, isFF } from "../../utils/feature-flags";
 import { wrapArray } from "../../utils/utilities";
 import { Geometry } from "./Geometry";
 
@@ -80,11 +81,20 @@ const _detect = (region) => {
   if (isResult) {
     return Geometry.getDOMBBox(region.from_name.elementRef?.current);
   }
-  switch (region.type) {
+
+  let type = region.type;
+  if (type === "audioregion") {
+    if (isFF(FF_DEV_2715)) {
+      type = "audioregion::ultra";
+    } else {
+      type = "audioregion::old";
+    }
+  }
+  switch (type) {
     case "textrange":
     case "richtextregion":
     case "textarearegion":
-    case "audioregion":
+    case "audioregion::old":
     case "paragraphs":
     case "timeseriesregion": {
       const regionBbox = Geometry.getDOMBBox(region.getRegionElement());
@@ -104,6 +114,22 @@ const _detect = (region) => {
 
       return regionBbox;
     }
+    case "audioregion::ultra": {
+      const bbox = region.bboxCoordsCanvas;
+      const stageEl = region.parent?.stageRef?.current;
+      const stageBbox = Geometry.getDOMBBox(stageEl, true);
+
+      return bbox
+        ? stageBbox
+          ? {
+              x: stageBbox.x + bbox.left,
+              y: stageBbox.y + bbox.top,
+              width: bbox.right - bbox.left,
+              height: bbox.bottom - bbox.top,
+            }
+          : bbox
+        : DEFAULT_BBOX;
+    }
     case "rectangleregion":
     case "ellipseregion":
     case "polygonregion":

web/libs/editor/src/components/InteractiveOverlays/NodesConnector.js

@@ -1,5 +1,5 @@
 import { debounce } from "../../utils/debounce";
-import { FF_PER_FIELD_COMMENTS, isFF } from "../../utils/feature-flags";
+import { FF_DEV_2715, FF_PER_FIELD_COMMENTS, isFF } from "../../utils/feature-flags";
 import { wrapArray } from "../../utils/utilities";
 import { Geometry } from "./Geometry";
 import { RelationShape } from "./RelationShape";
@@ -19,9 +19,17 @@ const parentImagePropsWatch = {
 const obtainWatcher = (node) => {
   switch (node.type) {
     case "richtextregion":
-    case "audioregion":
     case "paragraphs":
       return DOMWatcher;
+    case "audioregion": {
+      if (isFF(FF_DEV_2715)) {
+        return createPropertyWatcher(["bboxTriggers"]);
+      }
+      if (node.getRegionElement) {
+        return DOMWatcher;
+      }
+      return null;
+    }
     case "rectangleregion":
       return createPropertyWatcher(["x", "y", "width", "height", "hidden", parentImagePropsWatch]);
     case "ellipseregion":

web/libs/editor/src/lib/AudioUltra/Regions/Regions.ts

@@ -86,7 +86,7 @@ export class Regions {
     const currentTime = this.waveform.currentTime;
 
     this.regions.forEach((region) => {
-      region.highlighted = region.start <= currentTime && region.end >= currentTime;
+      region.active = region.start <= currentTime && region.end >= currentTime;
       region.render();
     });
   }

web/libs/editor/src/lib/AudioUltra/Regions/Segment.ts

@@ -48,6 +48,8 @@ export class Segment extends Events<SegmentEvents> {
   color: RgbaColorArray = rgba("#afafaf");
   selected = false;
   highlighted = false;
+  // active means that segment intersects with the cursor
+  active = false;
   updateable = true;
   locked = false;
   deleteable = true;
@@ -168,6 +170,21 @@ export class Segment extends Events<SegmentEvents> {
     return this.xStart + this.width;
   }
 
+  get yStart() {
+    const { timelinePlacement, timelineHeight } = this;
+    const timelineLayer = this.visualizer.getLayer("timeline");
+    const timelineTop = timelinePlacement === defaults.timelinePlacement;
+    const top = timelineLayer?.isVisible && timelineTop ? timelineHeight : 0;
+
+    return top;
+  }
+
+  get yEnd() {
+    const { height } = this.visualizer;
+    const { timelineHeight } = this;
+    return this.yStart + (height - timelineHeight);
+  }
+
   get width() {
     const { start, end } = this;
     const { width } = this.visualizer;
@@ -203,7 +220,7 @@ export class Segment extends Events<SegmentEvents> {
 
   private get inViewport() {
     const { xStart: startX, xEnd: endX } = this;
-    const width = this.visualizer.width * this.zoom;
+    const width = this.visualizer.width;
 
     // Both coordinates are less than or equal to 0
     if (startX <= 0 && endX <= 0) return false;
@@ -322,16 +339,14 @@ export class Segment extends Events<SegmentEvents> {
       return;
     }
 
-    const { color: _color, selected, highlighted, timelinePlacement, timelineHeight } = this;
+    const { color: _color, selected, highlighted, active } = this;
     const { height } = this.visualizer;
 
     const color = _color.clone();
-    const timelineLayer = this.visualizer.getLayer("timeline");
-    const timelineTop = timelinePlacement === defaults.timelinePlacement;
-    const top = timelineLayer?.isVisible && timelineTop ? timelineHeight : 0;
+    const top = this.yStart;
     const layer = this.controller.layerGroup;
 
-    if (selected || highlighted) {
+    if (selected || highlighted || active) {
       color.darken(0.4);
     }
 
@@ -359,7 +374,7 @@ export class Segment extends Events<SegmentEvents> {
   };
 
   handleHighlighted = (highlighted?: boolean) => {
-    if (!this.updateable || this.selected) return;
+    if (!this.updateable || (this.isDragging && this.selected)) return;
     this.highlighted = highlighted ?? !this.highlighted;
     this.invoke("update", [this]);
     this.waveform.invoke("regionUpdated", [this]);

web/libs/editor/src/lib/AudioUltra/Waveform.ts

@@ -160,6 +160,14 @@ export interface WaveformOptions {
     denoize: boolean;
   };
 }
+
+export type WaveformFrameState = {
+  width: number;
+  height: number;
+  zoom: number;
+  scroll: number;
+};
+
 interface WaveformEventTypes extends RegionsGlobalEvents, RegionGlobalEvents {
   load: () => void;
   error: (error: Error) => void;
@@ -176,6 +184,7 @@ interface WaveformEventTypes extends RegionsGlobalEvents, RegionGlobalEvents {
   durationChanged: (duration: number) => void;
   scroll: (scroll: number) => void;
   layersUpdated: (layers: Map<string, Layer>) => void;
+  frameDrawn: (frameState: WaveformFrameState) => void;
 }
 
 export class Waveform extends Events<WaveformEventTypes> {
@@ -549,8 +558,19 @@ export class Waveform extends Events<WaveformEventTypes> {
   private initEvents() {
     this.cursor.on("mouseMove", this.handleCursorMove);
     this.visualizer.on("layersUpdated", () => this.invoke("layersUpdated", [this.getLayers()]));
+    this.visualizer.on("draw", () => this.handleDrawn());
   }
 
+  private handleDrawn = () => {
+    const frameState = {
+      width: this.visualizer.width,
+      height: this.visualizer.height,
+      zoom: this.zoom,
+      scroll: this.visualizer.getScrollLeftPx(),
+    };
+    this.invoke("frameDrawn", [frameState]);
+  };
+
   /**
    * Handle cursor move event
    */

web/libs/editor/src/lib/AudioUltra/react/index.ts

@@ -1,8 +1,8 @@
-import { type MutableRefObject, useEffect, useRef, useState } from "react";
+import { type MutableRefObject, useEffect, useMemo, useRef, useState } from "react";
 
 import { isTimeRelativelySimilar } from "../Common/Utils";
 import type { Layer } from "../Visual/Layer";
-import { Waveform, type WaveformOptions } from "../Waveform";
+import { Waveform, type WaveformFrameState, type WaveformOptions } from "../Waveform";
 
 export const useWaveform = (
   containter: MutableRefObject<HTMLElement | null | undefined>,
@@ -14,6 +14,7 @@ export const useWaveform = (
     onError?: (error: Error) => void;
     autoLoad?: boolean;
     showLabels?: boolean;
+    onFrameChanged?: (frame: { width: number; height: number; zoom: number; scroll: number }) => void;
   },
 ) => {
   const waveform = useRef<Waveform>();
@@ -29,6 +30,30 @@ export const useWaveform = (
   const [layers, setLayers] = useState<Layer[]>([]);
   const [layerVisibility, setLayerVisibility] = useState(new Map());
 
+  const onFrameChangedRef = useRef(options?.onFrameChanged);
+  onFrameChangedRef.current = options?.onFrameChanged;
+
+  const updateAfterRegionDraw = useMemo(() => {
+    let prevFrameState: WaveformFrameState | null = null;
+    let requestId = -1;
+    const updateAfterRegionDraw = (frameState: WaveformFrameState) => {
+      cancelAnimationFrame(requestId);
+      requestId = requestAnimationFrame(() => {
+        if (
+          !prevFrameState ||
+          frameState.width !== prevFrameState.width ||
+          frameState.height !== prevFrameState.height ||
+          frameState.zoom !== prevFrameState.zoom ||
+          frameState.scroll !== prevFrameState.scroll
+        ) {
+          onFrameChangedRef.current?.(frameState);
+          prevFrameState = frameState;
+        }
+      });
+    };
+    return updateAfterRegionDraw;
+  }, []);
+
   useEffect(() => {
     const wf = new Waveform({
       ...(options ?? {}),
@@ -64,6 +89,7 @@ export const useWaveform = (
       }
     });
     wf.on("zoom", setZoom);
+    wf.on("frameDrawn", updateAfterRegionDraw);
     wf.on("muted", setMuted);
     wf.on("durationChanged", setDuration);
     wf.on("volumeChanged", setVolume);

web/libs/editor/src/regions/AudioRegion/AudioUltraRegionModel.js

@@ -2,6 +2,7 @@ import { types } from "mobx-state-tree";
 import { AudioModel } from "../../tags/object/AudioNext";
 import Utils from "../../utils";
 import Constants from "../../core/Constants";
+import { clamp } from "../../utils/utilities";
 
 export const AudioUltraRegionModel = types
   .model("AudioUltraRegionModel", {
@@ -16,8 +17,30 @@ export const AudioUltraRegionModel = types
   })
   .volatile(() => ({
     hideable: true,
+    _ws_region: null,
   }))
   .views((self) => ({
+    get bboxTriggers() {
+      return [self.start, self.end, self._ws_region, self.object?._ws, self.object?._wf_frame];
+    },
+    get bboxCoordsCanvas() {
+      if (!self.bboxTriggers) {
+        return null;
+      }
+
+      const { _ws_region } = self;
+      if (!_ws_region) return null;
+      if (!_ws_region.inViewport) return null;
+
+      const { xStart, xEnd, yStart, yEnd, visualizer } = _ws_region;
+      return {
+        left: clamp(xStart, 0, visualizer.width),
+        top: yStart,
+        right: clamp(xEnd, 0, visualizer.width),
+        bottom: yEnd,
+      };
+    },
+
     wsRegionOptions() {
       const reg = {
         id: self.id,
@@ -143,5 +166,14 @@ export const AudioUltraRegionModel = types
           self.updatePosition();
         }
       },
+
+      setWSRegion(wsRegion) {
+        self._ws_region = wsRegion;
+
+        if (wsRegion) {
+          wsRegion.on("mouseOver", self.onMouseOver);
+          wsRegion.on("mouseLeave", self.onMouseLeave);
+        }
+      },
     };
   });

web/libs/editor/src/tags/object/AudioUltra/model.js

@@ -1,5 +1,6 @@
 import { observe } from "mobx";
 import { getEnv, getRoot, getType, types } from "mobx-state-tree";
+import { createRef } from "react";
 import { customTypes } from "../../../core/CustomTypes";
 import { guidGenerator } from "../../../core/Helpers.ts";
 import { AnnotationMixin } from "../../../mixins/AnnotationMixin";
@@ -131,6 +132,9 @@ export const AudioModel = types.compose(
     })
     .volatile(() => ({
       errors: [],
+      stageRef: createRef(),
+      _ws: null,
+      _wf_frame: null,
     }))
     .views((self) => ({
       get hasStates() {
@@ -402,7 +406,7 @@ export const AudioModel = types.compose(
             states,
           });
 
-          r._ws_region = wsRegion;
+          r.setWSRegion(wsRegion);
 
           self.regions.push(r);
           self.annotation.addRegion(r);
@@ -415,7 +419,7 @@ export const AudioModel = types.compose(
           const find_r = self.annotation.areas.get(wsRegion.id);
 
           if (find_r) {
-            find_r._ws_region = wsRegion;
+            find_r.setWSRegion(wsRegion);
             find_r.updateColor();
             return find_r;
           }
@@ -436,7 +440,7 @@ export const AudioModel = types.compose(
           const r = self.annotation.createResult(wsRegion, labels, control, self);
           const updatedRegion = wsRegion.convertToRegion(labels.labels);
 
-          r._ws_region = updatedRegion;
+          r.setWSRegion(updatedRegion);
           r.updateColor();
           return r;
         },
@@ -459,7 +463,7 @@ export const AudioModel = types.compose(
 
           const r = self._ws.addRegion(options, false);
 
-          region._ws_region = r;
+          region.setWSRegion(r);
         },
 
         updateWsRegion(region) {
@@ -474,7 +478,7 @@ export const AudioModel = types.compose(
 
         clearRegionMappings() {
           self.regs.forEach((r) => {
-            r._ws_region = null;
+            r.setWSRegion(null);
           });
         },
 
@@ -534,6 +538,9 @@ export const AudioModel = types.compose(
             console.warn("Already destroyed");
           }
         },
+        setWFFrame(frame) {
+          self._wf_frame = frame;
+        },
       };
     }),
 );