From d1dec321e21f88cfd2894ca59c1af4b8b87fa792 Mon Sep 17 00:00:00 2001 From: igor Date: Fri, 24 Nov 2023 13:10:21 +0200 Subject: [PATCH] Complete similar int array serialization algorithm. --- .../src/binary-serialization.ts | 115 ++++++++++++------ packages/p2p-media-loader-core/src/core.ts | 36 +----- .../p2p-media-loader-core/src/p2p/command.ts | 30 ++--- 3 files changed, 93 insertions(+), 88 deletions(-) diff --git a/packages/p2p-media-loader-core/src/binary-serialization.ts b/packages/p2p-media-loader-core/src/binary-serialization.ts index a70e6ae2..5083bbb1 100644 --- a/packages/p2p-media-loader-core/src/binary-serialization.ts +++ b/packages/p2p-media-loader-core/src/binary-serialization.ts @@ -56,11 +56,13 @@ export function serializeInt(num: bigint): Uint8Array { export function deserializeInt(bytes: Uint8Array) { const metadata = bytes[0]; - const code = (metadata >> 4) & 0b00001111; + const code = metadata >> 4; if (code !== SerializedItem.Int) { - throw new Error("error"); + throw new Error( + "Trying to deserialize integer with invalid serialized item code" + ); } - const numberBytesLength = metadata & 0b00001111; + const numberBytesLength = metadata & 0b1111; const start = 1; const end = start + numberBytesLength; return { @@ -69,6 +71,63 @@ export function deserializeInt(bytes: Uint8Array) { }; } +export function serializeSimilarIntArray(numbers: bigint[]) { + const commonPartNumbersMap = new Map(); + + for (const number of numbers) { + const common = number & ~0xffn; + const diffByte = number & 0xffn; + const bytes = commonPartNumbersMap.get(common) ?? new ResizableUint8Array(); + if (!bytes.length) commonPartNumbersMap.set(common, bytes); + bytes.push(Number(diffByte)); + } + + const arrayMetadata = [ + SerializedItem.SimilarIntArray << 4, + commonPartNumbersMap.size, + ]; + const result = new ResizableUint8Array(); + result.unshift(arrayMetadata); + + for (const [commonPart, binaryArray] of commonPartNumbersMap) { + const { length } = binaryArray.getBytesChunks(); + const commonPartWithLength = commonPart | (BigInt(length) & 0xffn); + binaryArray.unshift(serializeInt(commonPartWithLength)); + result.push(binaryArray.getBytes()); + } + + return result.getBytes(); +} + +export function deserializeSimilarIntArray(bytes: Uint8Array) { + const [codeByte, commonPartArraysAmount] = bytes; + const code = codeByte >> 4; + if (code !== SerializedItem.SimilarIntArray) { + throw new Error( + "Trying to deserialize similar int array with invalid serialized item code" + ); + } + + let offset = 2; + const originalIntArr: bigint[] = []; + for (let i = 0; i < commonPartArraysAmount; i++) { + const { number: commonPartWithLength, byteLength } = deserializeInt( + bytes.slice(offset) + ); + offset += byteLength; + const arrayLength = commonPartWithLength & 0xffn; + const commonPart = commonPartWithLength & ~0xffn; + + for (let j = 0; j < arrayLength; j++) { + const diffPart = BigInt(bytes[offset]); + originalIntArr.push(commonPart | diffPart); + offset++; + } + } + + return { numbers: originalIntArr, byteLength: offset }; +} + function joinUint8Arrays(arrays: Uint8Array[], length?: number) { const byteLength = length ?? arrays.reduce((sum, arr) => sum + arr.length, 0); const bytes = new Uint8Array(byteLength); @@ -86,6 +145,17 @@ export class ResizableUint8Array { private _length = 0; push(bytes: Uint8Array | number | number[]) { + this.addBytes(bytes, "end"); + } + + unshift(bytes: Uint8Array | number | number[]) { + this.addBytes(bytes, "start"); + } + + private addBytes( + bytes: Uint8Array | number | number[], + position: "start" | "end" + ) { let bytesToAdd: Uint8Array; if (bytes instanceof Uint8Array) { bytesToAdd = bytes; @@ -95,7 +165,7 @@ export class ResizableUint8Array { bytesToAdd = new Uint8Array([bytes]); } this._length += bytesToAdd.length; - this.bytes.push(bytesToAdd); + this.bytes[position === "start" ? "unshift" : "push"](bytesToAdd); } getBytesChunks(): ReadonlyArray { @@ -105,41 +175,8 @@ export class ResizableUint8Array { getBytes(): Uint8Array { return joinUint8Arrays(this.bytes, this._length); } -} - -function serializeSimilarIntArray(numbers: bigint[]) { - const map = new Map(); - for (const number of numbers) { - const common = number & ~0xffn; - const diffByte = number & 0xffn; - let bytes = map.get(common); - if (!bytes) { - bytes = new ResizableUint8Array(); - const commonWithLength = common & - const commonBytes = intToBytes(common); - bytes.push(commonBytes); - map.set(common, bytes); - } - bytes.push(Number(diffByte)); - } - - const arrayMetadata = (SerializedItem.SimilarIntArray << 4) | map.size; - const result = new ResizableUint8Array(); - result.push(arrayMetadata); - - for (const binaryArray of map.values()) { - result.push(binaryArray.getBytes()); - } - - return result.getBytes(); -} - -function deserializeSimilarIntArray(bytes: Uint8Array) { - const metadata = bytes[0]; - const code = (metadata >> 4) & 0b00001111; - const - if (code !== SerializedItem.SimilarIntArray) { - throw new Error("error"); + get length() { + return this._length; } } diff --git a/packages/p2p-media-loader-core/src/core.ts b/packages/p2p-media-loader-core/src/core.ts index e197b61d..69700d8b 100644 --- a/packages/p2p-media-loader-core/src/core.ts +++ b/packages/p2p-media-loader-core/src/core.ts @@ -12,9 +12,6 @@ import { LinkedMap } from "./linked-map"; import { BandwidthApproximator } from "./bandwidth-approximator"; import { EngineCallbacks } from "./request-container"; import { SegmentsMemoryStorage } from "./segments-storage"; -// import * as Bits from "./p2p/bits"; -import * as Command from "./p2p/command"; -import * as Serialization from "./binary-serialization"; export class Core { private manifestResponseUrl?: string; @@ -36,38 +33,7 @@ export class Core { private mainStreamLoader?: HybridLoader; private secondaryStreamLoader?: HybridLoader; - constructor(private readonly eventHandlers?: CoreEventHandlers) { - // const numbers = [ - // 9999991, 9999992, 9999993, 9999994, 9999995, 9999996, 9999997, 9999998, - // 9999999, 9999990, 9999981, 9999982, 9999983, 9999984, 9999985, 9999986, - // ]; - - const numbers = [-112, -113, -114, -115, -116]; - // const stringified = JSON.stringify(numbers); - // const encoded = new TextEncoder().encode(stringified); - - // const serialized = Command.serializeSegmentAnnouncementCommand({ - // c: Command.PeerCommandType.SegmentsAnnouncement, - // p: numbers, - // l: numbers, - // }); - // const command = Command.deserializeCommand(serialized); - - // const number = 546515; - // const serializedNumber = Serialization.serializeInt(BigInt(number)); - // const deserializedNumber = Serialization.deserializeInt(serializedNumber); - - const serialized = Serialization.serializeSimilarIntArr( - numbers.map((number) => BigInt(number)) - ); - const command = Serialization.deserializeSimilarIntArr(serialized); - - // console.log("Encoded bytes: ", encoded.length); - console.log("Serialized: ", serialized); - console.log("Deserialized", command); - // console.log("serializedNumber: ", serializedNumber); - // console.log("deserializedNumber", deserializedNumber); - } + constructor(private readonly eventHandlers?: CoreEventHandlers) {} setManifestResponseUrl(url: string): void { this.manifestResponseUrl = url.split("?")[0]; diff --git a/packages/p2p-media-loader-core/src/p2p/command.ts b/packages/p2p-media-loader-core/src/p2p/command.ts index 21f3f673..4ed33f8d 100644 --- a/packages/p2p-media-loader-core/src/p2p/command.ts +++ b/packages/p2p-media-loader-core/src/p2p/command.ts @@ -40,15 +40,15 @@ export function serializeSegmentAnnouncementCommand( command: PeerSegmentAnnouncementCommand ) { const creator = new BinaryCommandCreator(command.c); - creator.addSimilarUIntArr("l", command.l); - creator.addSimilarUIntArr("p", command.p); + creator.addSimilarIntArr("l", command.l); + creator.addSimilarIntArr("p", command.p); creator.complete(); return creator.getResultBuffer(); } export function serializePeerSegmentCommand(command: PeerSegmentCommand) { const creator = new BinaryCommandCreator(command.c); - creator.addUInteger("i", command.i); + creator.addInteger("i", command.i); creator.complete(); return creator.getResultBuffer(); } @@ -57,7 +57,7 @@ export function serializePeerSendSegmentCommand( command: PeerSendSegmentCommand ) { const creator = new BinaryCommandCreator(command.c); - creator.addUInteger("i", command.i); + creator.addInteger("i", command.i); creator.complete(); return creator.getResultBuffer(); } @@ -74,11 +74,13 @@ function isBufferCommand(bytes: Uint8Array) { } export function deserializeCommand(bytes: Uint8Array) { - if (!isBufferCommand) { + if (!isBufferCommand(bytes)) { throw new Error("Given bytes don't represent peer command."); } const [, commandCode] = bytes; - const data: { [key: string]: any } = {}; + const deserializedCommand: { [key: string]: any } = { + c: commandCode, + }; let offset = 2; do { @@ -92,26 +94,26 @@ export function deserializeCommand(bytes: Uint8Array) { const { number, byteLength } = Serialization.deserializeInt( bytes.slice(offset) ); - data[name] = Number(number); + deserializedCommand[name] = Number(number); offset += byteLength; } break; case Serialization.SerializedItem.SimilarIntArray: { const { numbers, byteLength } = - Serialization.deserializeSimilarIntArr(bytes.slice(offset)); - data[name] = numbers.map((n) => Number(n)); + Serialization.deserializeSimilarIntArray(bytes.slice(offset)); + deserializedCommand[name] = numbers.map((n) => Number(n)); offset += byteLength; } break; } } while (bytes[offset] !== "}".charCodeAt(0) && offset < bytes.length); - return { commandCode, data }; + return deserializedCommand; } function getDataTypeFromByte(byte: number): Serialization.SerializedItem { - const typeCode = (byte & 0b11100000) >> 5; + const typeCode = byte >> 4; if (!Object.values(Serialization.SerializedItem).includes(typeCode)) { throw new Error("Not existing type"); } @@ -128,15 +130,15 @@ class BinaryCommandCreator { this.bytes.push(commandType); } - addUInteger(name: string, value: number) { + addInteger(name: string, value: number) { this.bytes.push(name.charCodeAt(0)); const bytes = Serialization.intToBytes(BigInt(value)); this.bytes.push(bytes); } - addSimilarUIntArr(name: string, arr: number[]) { + addSimilarIntArr(name: string, arr: number[]) { this.bytes.push(name.charCodeAt(0)); - const bytes = Serialization.serializeSimilarIntArr( + const bytes = Serialization.serializeSimilarIntArray( arr.map((num) => BigInt(num)) ); this.bytes.push(bytes);