Merge pull request #322 from platinouss/fix/240325-edit-whiteboard

Fix(#314, #321): 화이트보드 편집이 반영되지 않는 이슈

frontend/src/components/Header/components/HeaderInstructorControls.tsx

@@ -196,18 +196,21 @@ const HeaderInstructorControls = ({ setLectureCode, setLectureTitle }: HeaderIns
         offerToReceiveVideo: false
       });
       saveCanvasData(fabricCanvasRef!, canvasData, startTime);
+      socketRef.current.emit("presenterOffer", {
+        socketId: socketRef.current.id,
+        roomId: roomid,
+        SDP: SDP
+      });
       const reducedCanvasData: object = {
-        //objects: new Uint8Array(), // Uint8Array 형태의 데이터는 서버로 전송시 오류가 나서 일단 보내지 않습니다.
+        objects: canvasData.objects,
         viewport: canvasData.viewport,
         eventTime: canvasData.eventTime,
         width: canvasData.width,
         height: canvasData.height
       };
-      socketRef.current.emit("presenterOffer", {
-        socketId: socketRef.current.id,
+      socketRef.current.emit("whiteboardInit", {
         roomId: roomid,
-        SDP: SDP,
-        whiteboard: reducedCanvasData // TODO: 현재 whiteboard의 objects 데이터를 보내는 경우 서버에 SDP 값이 누락되는 문제를 해결해야 합니다.
+        whiteboardDetails: reducedCanvasData
       });
       pcRef.current.setLocalDescription(SDP);
       getPresenterCandidate();

frontend/src/utils/fabricCanvasUtil.ts

@@ -43,12 +43,6 @@ export const loadCanvasData = ({
   currentData: ICanvasData;
   newData: ICanvasData;
 }) => {
-  // 서버에 저장되어있던 currentWhiteboardData 의 newData.objects는 ArrayBuffer가 아닌 Node.js Buffer 형태로 전달되어 와서 변환이 필요함
-  // TODO: 서버에서 ArrayBuffer로 전달되도록 수정 필요
-  if (newData.objects?.byteLength === undefined) {
-    // @ts-ignore : newData.objects가 Uint8Array로 변환되어있지 않은 상태라서 임시로 처리했습니다.
-    newData.objects = new Uint8Array(newData.objects.data);
-  }
   const isCanvasDataChanged = newData.objects?.byteLength !== 0;
   const isViewportChanged = JSON.stringify(currentData.viewport) !== JSON.stringify(newData.viewport);
   const isSizeChanged = currentData.width !== newData.width || currentData.height !== newData.height;

mediaServer/src/constants/client-status.constant.ts

@@ -0,0 +1,4 @@
+export enum ClientStatus {
+  ONLINE,
+  OFFLINE
+}

mediaServer/src/dto/room-info.dto.ts → mediaServer/src/dto/room-info-request.dto.ts

@@ -1,6 +1,6 @@
 import { ICanvasData } from '../types/canvas-data.interface';
 
-export class RoomInfoDto {
+export class RoomInfoRequestDto {
   presenterEmail: string;
   startTime: Date;
   currentWhiteboardData: string;

mediaServer/src/dto/room-info-response.dto.ts

@@ -0,0 +1,25 @@
+import { convertBoardObjectToBuffer } from '../services/lecture.service';
+
+export class RoomInfoResponseDto {
+  private readonly _presenterEmail: string;
+  private readonly _startTime: string;
+  private readonly _currentWhiteboardData: Record<string, string | Buffer>;
+
+  constructor({ presenterEmail, startTime, currentWhiteboardData }: Record<string, string>) {
+    this._presenterEmail = presenterEmail;
+    this._startTime = startTime;
+    this._currentWhiteboardData = convertBoardObjectToBuffer(currentWhiteboardData);
+  }
+
+  get presenterEmail() {
+    return this._presenterEmail;
+  }
+
+  get startTime() {
+    return this._startTime;
+  }
+
+  get currentWhiteboardData() {
+    return this._currentWhiteboardData;
+  }
+}

mediaServer/src/dto/server-answer.dto.ts

@@ -1,11 +1,13 @@
+import { RoomInfoResponseDto } from './room-info-response.dto';
+
 export class ServerAnswerDto {
-  whiteboard: Record<string, string>;
+  whiteboard: Record<string, string | Buffer>;
   startTime: string;
   SDP: RTCSessionDescriptionInit;
 
-  constructor(whiteboard: Record<string, string>, startTime: string, SDP: RTCSessionDescriptionInit) {
-    this.whiteboard = whiteboard;
-    this.startTime = startTime;
+  constructor(roomInfo: RoomInfoResponseDto, SDP: RTCSessionDescriptionInit) {
+    this.whiteboard = roomInfo.currentWhiteboardData;
+    this.startTime = roomInfo.startTime;
     this.SDP = SDP;
   }
 }

mediaServer/src/listeners/client.listener.ts

@@ -1,16 +1,22 @@
 import { Socket } from 'socket.io';
 import { getClientIdAndClientType } from '../services/participant.service';
 import { saveClientInfo } from '../repositories/client.repsitory';
-import { findRoomInfoById } from '../repositories/room.repository';
+import { findRoomInfoById, saveRoomInfo } from '../repositories/room.repository';
 import { ClientConnectionInfo } from '../models/ClientConnectionInfo';
 import { relayServer } from '../main';
 import { getEmailByJwtPayload } from '../utils/auth';
 import { isNotEqualPresenterEmail, isReconnectPresenter } from '../validation/client.validation';
 import { RTCPeerConnection } from 'wrtc';
 import { pc_config } from '../config/pc.config';
-import { sendPrevLectureData, setPresenterConnection } from '../services/presenter.service';
+import {
+  sendPrevLectureData,
+  setPresenterConnection,
+  updatePresenterConnectionInfo
+} from '../services/presenter.service';
 import { setParticipantWebRTCConnection, setPresenterWebRTCConnection } from '../services/webrtc-connection.service';
-import { hasCurrentBoardDataInLecture } from '../validation/lecture.validation';
+import { canEnterRoom } from '../validation/lecture.validation';
+import { ClientStatus } from '../constants/client-status.constant';
+import { RoomInfoRequestDto } from '../dto/room-info-request.dto';
 
 export class ClientListener {
   createRoom = (socket: Socket) => {
@@ -24,15 +30,23 @@ export class ClientListener {
           return;
         }
         socket.join(email);
-        relayServer.clientConnectionInfoList.set(email, new ClientConnectionInfo(RTCPC, socket));
         if (isReconnectPresenter(roomInfo.presenterEmail, email)) {
           relayServer.clearScheduledEndLecture(data.roomId);
+          updatePresenterConnectionInfo(email, RTCPC, socket);
           await sendPrevLectureData(data.roomId, email, roomInfo);
         } else {
-          await setPresenterConnection(data.roomId, email, RTCPC, data.whiteboard);
+          await setPresenterConnection(data.roomId, email, RTCPC, socket);
         }
         await setPresenterWebRTCConnection(data.roomId, email, RTCPC, socket, data.SDP);
       });
+      socket.on('whiteboardInit', async (data) => {
+        const clientConnectionInfo = relayServer.clientConnectionInfoList.get(email);
+        if (clientConnectionInfo && clientConnectionInfo.status == ClientStatus.OFFLINE) {
+          clientConnectionInfo.setOnlineStatus();
+          return;
+        }
+        saveRoomInfo(data.roomId, new RoomInfoRequestDto(email, data.whiteboardDetails));
+      });
     } catch (e) {
       console.log(e);
     }
@@ -47,7 +61,7 @@ export class ClientListener {
         await saveClientInfo(clientId, clientType, data.roomId);
         relayServer.clientConnectionInfoList.set(clientId, new ClientConnectionInfo(RTCPC, socket));
         const roomInfo = await findRoomInfoById(data.roomId);
-        if (!hasCurrentBoardDataInLecture(roomInfo.currentWhiteboardData)) {
+        if (!canEnterRoom(roomInfo)) {
           return;
         }
         await setParticipantWebRTCConnection(data.roomId, clientId, RTCPC, roomInfo, socket, data.SDP);

mediaServer/src/listeners/lecture.listener.ts

@@ -82,6 +82,7 @@ export class LectureListener {
           return;
         }
         presenterStreamInfo.pauseRecording();
+        clientConnectionInfo.setOfflineStatus();
         clientConnectionInfo.disconnectWebRTCConnection();
       }
       if (isParticipant(clientInfo.type, clientInfo.roomId, clientInfo.roomId)) {

mediaServer/src/main.ts

@@ -2,7 +2,7 @@ import { RelayServer } from './RelayServer';
 import { ClientListener } from './listeners/client.listener';
 import { LectureListener } from './listeners/lecture.listener';
 
-const PORT = 3000;
+const PORT = 3001;
 
 const relayServer = new RelayServer(PORT);
 const clientListener = new ClientListener();

mediaServer/src/models/ClientConnectionInfo.ts

@@ -1,25 +1,37 @@
 import { RTCPeerConnection } from 'wrtc';
 import { Socket } from 'socket.io';
+import { ClientStatus } from '../constants/client-status.constant';
 
 export class ClientConnectionInfo {
-  private readonly _RTCPC: RTCPeerConnection;
+  private _RTCPC: RTCPeerConnection;
   private _enterSocket: Socket | null;
   private _lectureSocket: Socket | null;
+  private _status: ClientStatus;
 
   constructor(RTCPC: RTCPeerConnection, enterSocket?: Socket) {
     this._RTCPC = RTCPC;
     this._enterSocket = enterSocket ?? null;
     this._lectureSocket = null;
+    this._status = ClientStatus.ONLINE;
   }
 
   get RTCPC(): RTCPeerConnection {
     return this._RTCPC;
   }
 
+  get status() {
+    return this._status;
+  }
+
   set lectureSocket(socket: Socket) {
     this._lectureSocket = socket;
   }
 
+  updateConnection = (RTCPC: RTCPeerConnection, socket: Socket) => {
+    this._RTCPC = RTCPC;
+    this._enterSocket = socket;
+  };
+
   disconnectWebRTCConnection = () => {
     this._RTCPC.close();
   };
@@ -30,4 +42,12 @@ export class ClientConnectionInfo {
     this._lectureSocket?.leave(roomId);
     this._lectureSocket?.disconnect();
   };
+
+  setOfflineStatus = () => {
+    this._status = ClientStatus.OFFLINE;
+  };
+
+  setOnlineStatus = () => {
+    this._status = ClientStatus.ONLINE;
+  };
 }

mediaServer/src/repositories/room.repository.ts

@@ -1,13 +1,14 @@
-import { RoomInfoDto } from '../dto/room-info.dto';
+import { RoomInfoRequestDto } from '../dto/room-info-request.dto';
 import { redis } from '../config/redis.config';
 import { ROOM_INFO_KEY_PREFIX } from '../constants/redis-key.constant';
 import { ICanvasData } from '../types/canvas-data.interface';
+import { RoomInfoResponseDto } from '../dto/room-info-response.dto';
 
-const findRoomInfoById = (roomId: string) => {
-  return redis.hgetall(ROOM_INFO_KEY_PREFIX + roomId);
+const findRoomInfoById = async (roomId: string) => {
+  return new RoomInfoResponseDto(await redis.hgetall(ROOM_INFO_KEY_PREFIX + roomId));
 };
 
-const saveRoomInfo = async (roomId: string, roomInfo: RoomInfoDto) => {
+const saveRoomInfo = async (roomId: string, roomInfo: RoomInfoRequestDto) => {
   await redis.hset(ROOM_INFO_KEY_PREFIX + roomId, roomInfo);
 };
 

mediaServer/src/services/lecture.service.ts

@@ -41,4 +41,17 @@ const scheduleEndLecture = (roomId: string, presenterId: string) => {
   relayServer.scheduledEndLectureList.set(roomId, timerId);
 };
 
-export { startLecture, scheduleEndLecture };
+const convertBoardObjectToBuffer = (currentBoardDetails: string | undefined) => {
+  if (!currentBoardDetails) {
+    return currentBoardDetails;
+  }
+  const boardDetails = JSON.parse(currentBoardDetails);
+  if (!boardDetails.objects) {
+    boardDetails.objects = Buffer.alloc(0);
+  } else {
+    boardDetails.objects = Buffer.from(boardDetails.objects);
+  }
+  return boardDetails;
+};
+
+export { startLecture, scheduleEndLecture, convertBoardObjectToBuffer };

mediaServer/src/services/presenter.service.ts

@@ -5,34 +5,37 @@ import {
   setQuestionStreamAndGroup
 } from '../repositories/question-repository';
 import { StreamReadRaw } from '../types/redis-stream.type';
-import { sendDataToClient } from './socket.service';
+import { sendDataToClient, sendRoomDetailsToReconnectedPresenter } from './socket.service';
 import { Message } from '../models/Message';
 import { mediaConverter } from '../utils/media-converter';
-import { deleteRoomInfoById, saveRoomInfo, updateWhiteboardData } from '../repositories/room.repository';
+import { deleteRoomInfoById, updateWhiteboardData } from '../repositories/room.repository';
 import { MessageType } from '../constants/message-type.constant';
 import { relayServer } from '../main';
 import { ICanvasData } from '../types/canvas-data.interface';
 import { RoomConnectionInfo } from '../models/RoomConnectionInfo';
 import { saveClientInfo } from '../repositories/client.repsitory';
 import { ClientType } from '../constants/client-type.constant';
-import { RoomInfoDto } from '../dto/room-info.dto';
 import { RTCPeerConnection } from 'wrtc';
+import { RoomInfoResponseDto } from '../dto/room-info-response.dto';
+import { ClientConnectionInfo } from '../models/ClientConnectionInfo';
+import { Socket } from 'socket.io';
 
-const setPresenterConnection = async (
-  roomId: string,
-  email: string,
-  RTCPC: RTCPeerConnection,
-  initBoardData: ICanvasData
-) => {
+const setPresenterConnection = async (roomId: string, email: string, RTCPC: RTCPeerConnection, socket: Socket) => {
+  relayServer.clientConnectionInfoList.set(email, new ClientConnectionInfo(RTCPC, socket));
   relayServer.roomConnectionInfoList.set(roomId, new RoomConnectionInfo(RTCPC));
   if (await isQuestionStreamExisted(roomId)) {
     await deleteQuestionStream(roomId);
   }
-  await setQuestionStreamAndGroup(roomId);
-  await Promise.all([
-    saveClientInfo(email, ClientType.PRESENTER, roomId),
-    saveRoomInfo(roomId, new RoomInfoDto(email, initBoardData))
-  ]);
+  await Promise.all([setQuestionStreamAndGroup(roomId), saveClientInfo(email, ClientType.PRESENTER, roomId)]);
+};
+
+const updatePresenterConnectionInfo = (email: string, RTCPC: RTCPeerConnection, socket: Socket) => {
+  const presenterConnectionInfo = relayServer.clientConnectionInfoList.get(email);
+  if (!presenterConnectionInfo) {
+    relayServer.clientConnectionInfoList.set(email, new ClientConnectionInfo(RTCPC, socket));
+    return;
+  }
+  presenterConnectionInfo.updateConnection(RTCPC, socket);
 };
 
 const editWhiteboard = async (roomId: string, content: ICanvasData) => {
@@ -52,13 +55,9 @@ const endLecture = async (roomId: string, email: string) => {
   await Promise.all([deleteRoomInfoById(roomId), deleteQuestionStream(roomId)]);
 };
 
-const sendPrevLectureData = async (roomId: string, email: string, roomInfo: Record<string, string>) => {
+const sendPrevLectureData = async (roomId: string, email: string, roomInfo: RoomInfoResponseDto) => {
   const unsolvedQuestions = (await findUnsolvedQuestions(roomId, email)) as StreamReadRaw;
-  sendDataToClient('/create-room', email, 'reconnectPresenter', {
-    whiteboard: JSON.parse(roomInfo.currentWhiteboardData),
-    startTime: roomInfo.startTime,
-    questions: unsolvedQuestions[0][1]
-  });
+  sendRoomDetailsToReconnectedPresenter(email, roomInfo, unsolvedQuestions);
 };
 
-export { setPresenterConnection, editWhiteboard, endLecture, sendPrevLectureData };
+export { setPresenterConnection, updatePresenterConnectionInfo, editWhiteboard, endLecture, sendPrevLectureData };

mediaServer/src/services/socket.service.ts

@@ -1,7 +1,21 @@
 import { relayServer } from '../main';
+import { StreamReadRaw } from '../types/redis-stream.type';
+import { RoomInfoResponseDto } from '../dto/room-info-response.dto';
 
 const sendDataToClient = (namespace: string, target: string, eventName: string, data: any) => {
   relayServer.socket.of(namespace).to(target).emit(eventName, data);
 };
 
-export { sendDataToClient };
+const sendRoomDetailsToReconnectedPresenter = (
+  email: string,
+  roomInfo: RoomInfoResponseDto,
+  unsolvedQuestions: StreamReadRaw
+) => {
+  sendDataToClient('/create-room', email, 'reconnectPresenter', {
+    whiteboard: roomInfo.currentWhiteboardData,
+    startTime: roomInfo.startTime,
+    questions: unsolvedQuestions[0][1]
+  });
+};
+
+export { sendDataToClient, sendRoomDetailsToReconnectedPresenter };

mediaServer/src/services/webrtc-connection.service.ts

@@ -6,6 +6,7 @@ import { ServerAnswerDto } from '../dto/server-answer.dto';
 import { setPresenterMediaStream } from './participant.service';
 import { sendDataToClient } from './socket.service';
 import { RoomConnectionInfo } from '../models/RoomConnectionInfo';
+import { RoomInfoResponseDto } from '../dto/room-info-response.dto';
 
 const setTrackEvent = (RTCPC: RTCPeerConnection, roomId: string) => {
   RTCPC.ontrack = (event) => {
@@ -63,15 +64,15 @@ const setParticipantWebRTCConnection = async (
   roomId: string,
   clientId: string,
   RTCPC: RTCPeerConnection,
-  roomInfo: Record<string, string>,
+  roomInfo: RoomInfoResponseDto,
   socket: Socket,
   offer: RTCSessionDescriptionInit
 ) => {
   setPresenterMediaStream(RTCPC, roomId);
   exchangeCandidate('/enter-room', clientId, socket);
   RTCPC.setRemoteDescription(offer);
   const answer = await RTCPC.createAnswer();
-  const answerData = new ServerAnswerDto(JSON.parse(roomInfo.currentWhiteboardData), roomInfo.startTime, answer);
+  const answerData = new ServerAnswerDto(roomInfo, answer);
   sendDataToClient('/enter-room', clientId, 'serverAnswer', answerData);
   RTCPC.setLocalDescription(answer);
 };