Merge pull request #316 from Jw705/refactor/240312-compression

Refactor(#309): 화이트보드 정보를 압축해서 전송하도록 개선

backend/src/lecture/dto/whiteboard-event.dto.ts

@@ -1,5 +1,5 @@
 export class WhiteboardEventDto {
-  objects: fabric.Object[];
+  objects: Uint8Array;
 
   viewport: number[];
 

backend/src/lecture/lecture.service.ts

@@ -68,7 +68,7 @@ export class LectureService {
 
   async saveWhiteBoardLog(lecture: Lecture, whiteboardEventDto: WhiteboardEventDto) {
     const whiteboardLog = new this.whiteboardLogModel({
-      canvasJSON: whiteboardEventDto.canvasJSON,
+      objects: whiteboardEventDto.objects,
       viewport: whiteboardEventDto.viewport,
       eventTime: whiteboardEventDto.eventTime,
       width: whiteboardEventDto.width,

backend/src/lecture/schema/whiteboard-log.schema.ts

@@ -7,7 +7,7 @@ export type WhiteBoardLogDocument = HydratedDocument<WhiteboardLog>;
 @Schema()
 export class WhiteboardLog {
   @Prop({ required: true })
-  objects: fabric.Object[];
+  objects: Uint8Array;
 
   @Prop({ required: true })
   viewport: number[];

frontend/package.json

@@ -14,6 +14,7 @@
     "dotenv": "^16.3.1",
     "fabric": "^5.3.0",
     "modernizr": "^3.13.0",
+    "pako": "^2.1.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-router-dom": "^6.18.0",
@@ -23,6 +24,7 @@
   },
   "devDependencies": {
     "@types/fabric": "^5.3.6",
+    "@types/pako": "^2.0.3",
     "@types/react": "^18.2.15",
     "@types/react-dom": "^18.2.7",
     "@typescript-eslint/eslint-plugin": "^6.0.0",

frontend/src/components/Header/components/HeaderInstructorControls.tsx

@@ -196,11 +196,18 @@ const HeaderInstructorControls = ({ setLectureCode, setLectureTitle }: HeaderIns
         offerToReceiveVideo: false
       });
       saveCanvasData(fabricCanvasRef!, canvasData, startTime);
+      const reducedCanvasData: object = {
+        //objects: new Uint8Array(), // Uint8Array 형태의 데이터는 서버로 전송시 오류가 나서 일단 보내지 않습니다.
+        viewport: canvasData.viewport,
+        eventTime: canvasData.eventTime,
+        width: canvasData.width,
+        height: canvasData.height
+      };
       socketRef.current.emit("presenterOffer", {
-        whiteboard: canvasData,
         socketId: socketRef.current.id,
         roomId: roomid,
-        SDP: SDP
+        SDP: SDP,
+        whiteboard: reducedCanvasData // TODO: 현재 whiteboard의 objects 데이터를 보내는 경우 서버에 SDP 값이 누락되는 문제를 해결해야 합니다.
       });
       pcRef.current.setLocalDescription(SDP);
       getPresenterCandidate();
@@ -250,23 +257,14 @@ const HeaderInstructorControls = ({ setLectureCode, setLectureTitle }: HeaderIns
     const onFrame = () => {
       saveCanvasData(fabricCanvasRef!, canvasData, startTime).then(
         ([isCanvasDataChanged, isViewportChanged, isSizeChanged]) => {
-          // 캔버스 내부 객체가 변경되지 않은 경우에는 oobjects를 제외한 나머지 데이터만 전송
-          if (!isCanvasDataChanged && (isViewportChanged || isSizeChanged)) {
-            const reducedCanvasData: ICanvasData = {
-              objects: [],
-              viewport: canvasData.viewport,
-              eventTime: canvasData.eventTime,
-              width: canvasData.width,
-              height: canvasData.width
-            };
-            submitData(reducedCanvasData);
-          } else if (isCanvasDataChanged || isViewportChanged || isSizeChanged) {
+          if (isCanvasDataChanged || isViewportChanged || isSizeChanged) {
+            // 캔버스 내부 객체 변화 여부를 확인하기 위한 canvasJSON 데이터는 제외하고 전송
             const reducedCanvasData: ICanvasData = {
               objects: canvasData.objects,
               viewport: canvasData.viewport,
               eventTime: canvasData.eventTime,
               width: canvasData.width,
-              height: canvasData.width
+              height: canvasData.height
             };
             submitData(reducedCanvasData);
           }
@@ -323,7 +321,7 @@ const HeaderInstructorControls = ({ setLectureCode, setLectureTitle }: HeaderIns
   };
 
   let canvasData: ICanvasData = {
-    objects: [],
+    objects: new Uint8Array(),
     viewport: [0, 0, 0, 0, 0, 0],
     eventTime: 0,
     width: 0,
@@ -361,15 +359,6 @@ const HeaderInstructorControls = ({ setLectureCode, setLectureTitle }: HeaderIns
       }
     });
     setInstructorSocket(lectureSocketRef.current);
-
-    // 지연 시간 체크를 위해 큰 크기의 더미 데이터를 화이트보드에 로드합니다. 개선이 끝나면 제거 예정입니다.
-    axios("./dummy70.json")
-      .then(({ data }) => {
-        fabricCanvasRef!.loadFromJSON(data.canvasJSON, () => {});
-      })
-      .catch((error) => {
-        console.log("화이트보드 데이터 로딩 실패", error);
-      });
   };
   const handleServerError = (err: any) => {
     console.error(err.message);

frontend/src/components/Header/components/HeaderParticipantControls.tsx

@@ -20,7 +20,7 @@ import calcNormalizedVolume from "@/utils/calcNormalizedVolume";
 import selectedSpeakerState from "@/stores/stateSelectedSpeaker";
 import speakerVolumeState from "@/stores/stateSpeakerVolume";
 import micVolumeState from "@/stores/stateMicVolume";
-import participantCavasInstanceState from "@/stores/stateParticipantCanvasInstance";
+import participantCanvasInstanceState from "@/stores/stateParticipantCanvasInstance";
 import participantSocketRefState from "@/stores/stateParticipantSocketRef";
 
 interface HeaderParticipantControlsProps {
@@ -36,7 +36,7 @@ const HeaderParticipantControls = ({ setLectureCode, setLectureTitle }: HeaderPa
 
   const selectedSpeaker = useRecoilValue(selectedSpeakerState);
   const speakerVolume = useRecoilValue(speakerVolumeState);
-  const fabricCanvasRef = useRecoilValue(participantCavasInstanceState);
+  const fabricCanvasRef = useRecoilValue(participantCanvasInstanceState);
   const setSpeakerVolume = useSetRecoilState(speakerVolumeState);
   const setMicVolumeState = useSetRecoilState(micVolumeState);
   const setParticipantSocket = useSetRecoilState(participantSocketRefState);
@@ -93,11 +93,6 @@ const HeaderParticipantControls = ({ setLectureCode, setLectureTitle }: HeaderPa
     (audioContextRef.current as any).setSinkId(selectedSpeaker);
   }, [selectedSpeaker]);
 
-  /* ------------------------------------------------------------------------------- */
-  // 화이트보드 지연 시간 체크를 위해 임시로 추가한 코드입니다. 추후 삭제될 예정입니다.
-  let startTime = Date.now();
-  /* ------------------------------------------------------------------------------- */
-
   const enterLecture = async () => {
     await checkAuth();
     await initConnection();
@@ -122,7 +117,7 @@ const HeaderParticipantControls = ({ setLectureCode, setLectureTitle }: HeaderPa
   };
 
   let canvasData: ICanvasData = {
-    objects: [],
+    objects: new Uint8Array(),
     viewport: [1, 0, 0, 1, 0, 0],
     eventTime: 0,
     width: 0,
@@ -272,8 +267,7 @@ const HeaderParticipantControls = ({ setLectureCode, setLectureTitle }: HeaderPa
 
   const handleServerAnswer = (data: any) => {
     if (!pcRef.current) return;
-    // const startTime = new Date(data.startTime).getTime();
-    startTime = new Date(data.startTime).getTime();
+    const startTime = new Date(data.startTime).getTime();
     const updateElapsedTime = () => {
       const elapsedTime = Math.floor((Date.now() - startTime) / 1000);
       setElapsedTime(elapsedTime);
@@ -284,8 +278,7 @@ const HeaderParticipantControls = ({ setLectureCode, setLectureTitle }: HeaderPa
     loadCanvasData({
       fabricCanvas: fabricCanvasRef!,
       currentData: canvasData,
-      newData: data.whiteboard,
-      debugData: {}
+      newData: data.whiteboard
     });
     canvasData = data.whiteboard;
     pcRef.current.setRemoteDescription(data.SDP);
@@ -299,12 +292,10 @@ const HeaderParticipantControls = ({ setLectureCode, setLectureTitle }: HeaderPa
     showToast({ message: "서버 연결에 실패했습니다", type: "alert" });
   };
   const handleWhiteboardUpdate = (data: any) => {
-    const debugData = { startTime: startTime, arriveTime: Date.now() };
     loadCanvasData({
       fabricCanvas: fabricCanvasRef!,
       currentData: canvasData,
-      newData: data.content,
-      debugData: debugData
+      newData: data.content
     });
     canvasData = data.content;
   };

frontend/src/pages/Participant/Participant.tsx

@@ -3,7 +3,7 @@ import { useSetRecoilState, useRecoilValue } from "recoil";
 import { fabric } from "fabric";
 
 import Header from "@/components/Header/Header";
-import participantCavasInstanceState from "@/stores/stateParticipantCanvasInstance";
+import participantCanvasInstanceState from "@/stores/stateParticipantCanvasInstance";
 import CloseIcon from "@/assets/svgs/close.svg?react";
 import QuestionIcon from "@/assets/svgs/whiteboard/question.svg?react";
 import LogToggleButton from "@/components/Button/LogToggleButton";
@@ -13,7 +13,7 @@ import isQuestionLogOpenState from "@/stores/stateIsQuestionLogOpen";
 
 const Participant = () => {
   const canvasContainerRef = useRef<HTMLDivElement>(null);
-  const setCanvas = useSetRecoilState(participantCavasInstanceState);
+  const setCanvas = useSetRecoilState(participantCanvasInstanceState);
   const canvasRef = useRef<HTMLCanvasElement>(null);
   const isQuestionLogOpen = useRecoilValue(isQuestionLogOpenState);
   useEffect(() => {

frontend/src/pages/Review/Review.tsx

@@ -37,7 +37,7 @@ const Review = () => {
   const lectureId = new URLSearchParams(useLocation().search).get("id") || "nodata";
   let startTime = Date.now();
   let canvasData: ICanvasData = {
-    objects: [],
+    objects: new Uint8Array(0),
     viewport: [0, 0, 0, 0, 0, 0],
     eventTime: 0,
     width: 0,

frontend/src/stores/stateParticipantCanvasInstance.ts

@@ -1,10 +1,10 @@
 import { atom } from "recoil";
 import { fabric } from "fabric";
 
-const participantCavasInstanceState = atom<fabric.Canvas | null>({
-  key: "participantCavasInstanceState",
+const participantCanvasInstanceState = atom<fabric.Canvas | null>({
+  key: "participantCanvasInstanceState",
   default: null,
   dangerouslyAllowMutability: true
 });
 
-export default participantCavasInstanceState;
+export default participantCanvasInstanceState;