Skip to content

Commit

Permalink
Complete similar int array serialization algorithm.
Browse files Browse the repository at this point in the history
  • Loading branch information
@i-zolotarenko
i-zolotarenko committed Nov 24, 2023
1 parent bc32531 commit d1dec32
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 88 deletions.
115 changes: 76 additions & 39 deletions packages/p2p-media-loader-core/src/binary-serialization.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,13 @@ export function serializeInt(num: bigint): Uint8Array {

export function deserializeInt(bytes: Uint8Array) {
const metadata = bytes[0];
const code = (metadata >> 4) & 0b00001111;
const code = metadata >> 4;
if (code !== SerializedItem.Int) {
throw new Error("error");
throw new Error(
"Trying to deserialize integer with invalid serialized item code"
);
}
const numberBytesLength = metadata & 0b00001111;
const numberBytesLength = metadata & 0b1111;
const start = 1;
const end = start + numberBytesLength;
return {
Expand All @@ -69,6 +71,63 @@ export function deserializeInt(bytes: Uint8Array) {
};
}

export function serializeSimilarIntArray(numbers: bigint[]) {
const commonPartNumbersMap = new Map<bigint, ResizableUint8Array>();

for (const number of numbers) {
const common = number & ~0xffn;
const diffByte = number & 0xffn;
const bytes = commonPartNumbersMap.get(common) ?? new ResizableUint8Array();
if (!bytes.length) commonPartNumbersMap.set(common, bytes);
bytes.push(Number(diffByte));
}

const arrayMetadata = [
SerializedItem.SimilarIntArray << 4,
commonPartNumbersMap.size,
];
const result = new ResizableUint8Array();
result.unshift(arrayMetadata);

for (const [commonPart, binaryArray] of commonPartNumbersMap) {
const { length } = binaryArray.getBytesChunks();
const commonPartWithLength = commonPart | (BigInt(length) & 0xffn);
binaryArray.unshift(serializeInt(commonPartWithLength));
result.push(binaryArray.getBytes());
}

return result.getBytes();
}

export function deserializeSimilarIntArray(bytes: Uint8Array) {
const [codeByte, commonPartArraysAmount] = bytes;
const code = codeByte >> 4;
if (code !== SerializedItem.SimilarIntArray) {
throw new Error(
"Trying to deserialize similar int array with invalid serialized item code"
);
}

let offset = 2;
const originalIntArr: bigint[] = [];
for (let i = 0; i < commonPartArraysAmount; i++) {
const { number: commonPartWithLength, byteLength } = deserializeInt(
bytes.slice(offset)
);
offset += byteLength;
const arrayLength = commonPartWithLength & 0xffn;
const commonPart = commonPartWithLength & ~0xffn;

for (let j = 0; j < arrayLength; j++) {
const diffPart = BigInt(bytes[offset]);
originalIntArr.push(commonPart | diffPart);
offset++;
}
}

return { numbers: originalIntArr, byteLength: offset };
}

function joinUint8Arrays(arrays: Uint8Array[], length?: number) {
const byteLength = length ?? arrays.reduce((sum, arr) => sum + arr.length, 0);
const bytes = new Uint8Array(byteLength);
Expand All @@ -86,6 +145,17 @@ export class ResizableUint8Array {
private _length = 0;

push(bytes: Uint8Array | number | number[]) {
this.addBytes(bytes, "end");
}

unshift(bytes: Uint8Array | number | number[]) {
this.addBytes(bytes, "start");
}

private addBytes(
bytes: Uint8Array | number | number[],
position: "start" | "end"
) {
let bytesToAdd: Uint8Array;
if (bytes instanceof Uint8Array) {
bytesToAdd = bytes;
Expand All @@ -95,7 +165,7 @@ export class ResizableUint8Array {
bytesToAdd = new Uint8Array([bytes]);
}
this._length += bytesToAdd.length;
this.bytes.push(bytesToAdd);
this.bytes[position === "start" ? "unshift" : "push"](bytesToAdd);
}

getBytesChunks(): ReadonlyArray<Uint8Array> {
Expand All @@ -105,41 +175,8 @@ export class ResizableUint8Array {
getBytes(): Uint8Array {
return joinUint8Arrays(this.bytes, this._length);
}
}

function serializeSimilarIntArray(numbers: bigint[]) {
const map = new Map<bigint, ResizableUint8Array>();

for (const number of numbers) {
const common = number & ~0xffn;
const diffByte = number & 0xffn;
let bytes = map.get(common);
if (!bytes) {
bytes = new ResizableUint8Array();
const commonWithLength = common &
const commonBytes = intToBytes(common);
bytes.push(commonBytes);
map.set(common, bytes);
}
bytes.push(Number(diffByte));
}

const arrayMetadata = (SerializedItem.SimilarIntArray << 4) | map.size;
const result = new ResizableUint8Array();
result.push(arrayMetadata);

for (const binaryArray of map.values()) {
result.push(binaryArray.getBytes());
}

return result.getBytes();
}

function deserializeSimilarIntArray(bytes: Uint8Array) {
const metadata = bytes[0];
const code = (metadata >> 4) & 0b00001111;
const
if (code !== SerializedItem.SimilarIntArray) {
throw new Error("error");
get length() {
return this._length;
}
}
36 changes: 1 addition & 35 deletions packages/p2p-media-loader-core/src/core.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@ import { LinkedMap } from "./linked-map";
import { BandwidthApproximator } from "./bandwidth-approximator";
import { EngineCallbacks } from "./request-container";
import { SegmentsMemoryStorage } from "./segments-storage";
// import * as Bits from "./p2p/bits";
import * as Command from "./p2p/command";
import * as Serialization from "./binary-serialization";

export class Core<TStream extends Stream = Stream> {
private manifestResponseUrl?: string;
Expand All @@ -36,38 +33,7 @@ export class Core<TStream extends Stream = Stream> {
private mainStreamLoader?: HybridLoader;
private secondaryStreamLoader?: HybridLoader;

constructor(private readonly eventHandlers?: CoreEventHandlers) {
// const numbers = [
// 9999991, 9999992, 9999993, 9999994, 9999995, 9999996, 9999997, 9999998,
// 9999999, 9999990, 9999981, 9999982, 9999983, 9999984, 9999985, 9999986,
// ];

const numbers = [-112, -113, -114, -115, -116];
// const stringified = JSON.stringify(numbers);
// const encoded = new TextEncoder().encode(stringified);

// const serialized = Command.serializeSegmentAnnouncementCommand({
// c: Command.PeerCommandType.SegmentsAnnouncement,
// p: numbers,
// l: numbers,
// });
// const command = Command.deserializeCommand(serialized);

// const number = 546515;
// const serializedNumber = Serialization.serializeInt(BigInt(number));
// const deserializedNumber = Serialization.deserializeInt(serializedNumber);

const serialized = Serialization.serializeSimilarIntArr(
numbers.map((number) => BigInt(number))
);
const command = Serialization.deserializeSimilarIntArr(serialized);

// console.log("Encoded bytes: ", encoded.length);
console.log("Serialized: ", serialized);
console.log("Deserialized", command);
// console.log("serializedNumber: ", serializedNumber);
// console.log("deserializedNumber", deserializedNumber);
}
constructor(private readonly eventHandlers?: CoreEventHandlers) {}

setManifestResponseUrl(url: string): void {
this.manifestResponseUrl = url.split("?")[0];
Expand Down
30 changes: 16 additions & 14 deletions packages/p2p-media-loader-core/src/p2p/command.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ export function serializeSegmentAnnouncementCommand(
command: PeerSegmentAnnouncementCommand
) {
const creator = new BinaryCommandCreator(command.c);
creator.addSimilarUIntArr("l", command.l);
creator.addSimilarUIntArr("p", command.p);
creator.addSimilarIntArr("l", command.l);
creator.addSimilarIntArr("p", command.p);
creator.complete();
return creator.getResultBuffer();
}

export function serializePeerSegmentCommand(command: PeerSegmentCommand) {
const creator = new BinaryCommandCreator(command.c);
creator.addUInteger("i", command.i);
creator.addInteger("i", command.i);
creator.complete();
return creator.getResultBuffer();
}
Expand All @@ -57,7 +57,7 @@ export function serializePeerSendSegmentCommand(
command: PeerSendSegmentCommand
) {
const creator = new BinaryCommandCreator(command.c);
creator.addUInteger("i", command.i);
creator.addInteger("i", command.i);
creator.complete();
return creator.getResultBuffer();
}
Expand All @@ -74,11 +74,13 @@ function isBufferCommand(bytes: Uint8Array) {
}

export function deserializeCommand(bytes: Uint8Array) {
if (!isBufferCommand) {
if (!isBufferCommand(bytes)) {
throw new Error("Given bytes don't represent peer command.");
}
const [, commandCode] = bytes;
const data: { [key: string]: any } = {};
const deserializedCommand: { [key: string]: any } = {
c: commandCode,
};

let offset = 2;
do {
Expand All @@ -92,26 +94,26 @@ export function deserializeCommand(bytes: Uint8Array) {
const { number, byteLength } = Serialization.deserializeInt(
bytes.slice(offset)
);
data[name] = Number(number);
deserializedCommand[name] = Number(number);
offset += byteLength;
}
break;
case Serialization.SerializedItem.SimilarIntArray:
{
const { numbers, byteLength } =
Serialization.deserializeSimilarIntArr(bytes.slice(offset));
data[name] = numbers.map((n) => Number(n));
Serialization.deserializeSimilarIntArray(bytes.slice(offset));
deserializedCommand[name] = numbers.map((n) => Number(n));
offset += byteLength;
}
break;
}
} while (bytes[offset] !== "}".charCodeAt(0) && offset < bytes.length);

return { commandCode, data };
return deserializedCommand;
}

function getDataTypeFromByte(byte: number): Serialization.SerializedItem {
const typeCode = (byte & 0b11100000) >> 5;
const typeCode = byte >> 4;
if (!Object.values(Serialization.SerializedItem).includes(typeCode)) {
throw new Error("Not existing type");
}
Expand All @@ -128,15 +130,15 @@ class BinaryCommandCreator {
this.bytes.push(commandType);
}

addUInteger(name: string, value: number) {
addInteger(name: string, value: number) {
this.bytes.push(name.charCodeAt(0));
const bytes = Serialization.intToBytes(BigInt(value));
this.bytes.push(bytes);
}

addSimilarUIntArr(name: string, arr: number[]) {
addSimilarIntArr(name: string, arr: number[]) {
this.bytes.push(name.charCodeAt(0));
const bytes = Serialization.serializeSimilarIntArr(
const bytes = Serialization.serializeSimilarIntArray(
arr.map((num) => BigInt(num))
);
this.bytes.push(bytes);
Expand Down

0 comments on commit d1dec32

Please sign in to comment.