From 807927dbd2d77e08d0cd7e7b2b062ced422fba37 Mon Sep 17 00:00:00 2001 From: monyone Date: Sat, 22 Jun 2024 08:59:18 +0900 Subject: [PATCH 1/5] Feature: Parse MultiTrack Audio for Enhanced FLV --- src/demux/flv-demuxer.js | 336 ++++++++++++++++++++++++--------------- 1 file changed, 205 insertions(+), 131 deletions(-) diff --git a/src/demux/flv-demuxer.js b/src/demux/flv-demuxer.js index a954c9e..a97a9fd 100644 --- a/src/demux/flv-demuxer.js +++ b/src/demux/flv-demuxer.js @@ -92,6 +92,8 @@ class FLVDemuxer { fps_den: 1000 }; + this._enhanedFlvAudioMultitrackMode = null; + this._flvSoundRateTable = [5500, 11025, 22050, 44100, 48000]; this._mpegSamplingRates = [ @@ -493,18 +495,54 @@ class FLVDemuxer { let soundFormat = soundSpec >>> 4; if (soundFormat === 9) { // Enhanced FLV + let packetType = soundSpec & 0x0F; + if (packetType === 5) { // AudioPacketType.Multitrack + if (dataSize <= 1) { + Log.w(this.TAG, 'Flv: Invalid audio packet, missing audioMultitrackType in Ehnanced FLV payload!'); + return; + } + this._enhanedFlvAudioMultitrackMode = (v.getUint8(1) & 0xF0) >> 4; + let packetType = v.getUint8(1) & 0x0F; + + let meta = this._audioMetadata; + let track = this._audioTrack; + + if (!meta) { + if (this._hasAudio === false && this._hasAudioFlagOverrided === false) { + this._hasAudio = true; + this._mediaInfo.hasAudio = true; + } + + // initial metadata + meta = this._audioMetadata = {}; + meta.type = 'audio'; + meta.id = track.id; + meta.timescale = this._timescale; + meta.duration = this._duration; + } + + let fourcc = null; + if (this._enhanedFlvAudioMultitrackMode !== 2) { // not AvMultitrackType.ManyTracksManyCodecs + if (dataSize <= 6) { + Log.w(this.TAG, 'Flv: Invalid audio packet, missing Audio fourcc of OneTrack/ManyTracks in Ehnanced FLV payload!'); + return; + } + fourcc = String.fromCharCode(... (new Uint8Array(arrayBuffer, dataOffset, dataSize)).slice(2, 6)); + this._parseEnhanedAudioPacket(arrayBuffer, dataOffset + 6, dataSize - 6, tagTimestamp, fourcc, packetType); + return; + } else { + this._parseEnhanedAudioPacket(arrayBuffer, dataOffset + 1, dataSize - 1, tagTimestamp, fourcc, packetType); + return; + } + } + + // other Enhanced Audio Payload if (dataSize <= 5) { - Log.w(this.TAG, 'Flv: Invalid audio packet, missing AudioFourCC in Ehnanced FLV payload!'); + Log.w(this.TAG, 'Flv: Invalid audio packet, missing AudioFourCC in Ehnanced Single Track FLV payload!'); return; } - let packetType = soundSpec & 0x0F; let fourcc = String.fromCharCode(... (new Uint8Array(arrayBuffer, dataOffset, dataSize)).slice(1, 5)); - - if (fourcc === 'Opus') { - this._parseOpusAudioPacket(arrayBuffer, dataOffset + 5, dataSize - 5, tagTimestamp, packetType); - } else { - this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec: ' + fourcc); - } + this._parseEnhanedAudioPacket(arrayBuffer, dataOffset + 5, dataSize - 5, tagTimestamp, fourcc, packetType); return; } @@ -547,130 +585,84 @@ class FLVDemuxer { meta.channelCount = (soundType === 0 ? 1 : 2); } + let packetType = v.getUint8(1); if (soundFormat === 10) { // AAC - let aacData = this._parseAACAudioData(arrayBuffer, dataOffset + 1, dataSize - 1); - if (aacData == undefined) { - return; - } + this._parseAACAudioPacket(arrayBuffer, dataOffset + 2, dataSize - 2, tagTimestamp, packetType); + } else if (soundFormat === 2) { // MP3 + this._parseMP3AudioPacket(arrayBuffer, dataOffset + 1, dataSize - 1, tagTimestamp, packetType); + } + } - if (aacData.packetType === 0) { // AAC sequence header (AudioSpecificConfig) - if (meta.config) { - if (buffersAreEqual(aacData.data.config, meta.config)) { - // If AudioSpecificConfig is not changed, ignore it to avoid generating initialization segment repeatedly - return; - } else { - Log.w(this.TAG, 'AudioSpecificConfig has been changed, re-generate initialization segment'); - } - } - let misc = aacData.data; - meta.audioSampleRate = misc.samplingRate; - meta.channelCount = misc.channelCount; - meta.codec = misc.codec; - meta.originalCodec = misc.originalCodec; - meta.config = misc.config; - // The decode result of an aac sample is 1024 PCM samples - meta.refSampleDuration = 1024 / meta.audioSampleRate * meta.timescale; - Log.v(this.TAG, 'Parsed AudioSpecificConfig'); - - if (this._isInitialMetadataDispatched()) { - // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer - if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) { - this._onDataAvailable(this._audioTrack, this._videoTrack); - } - } else { - this._audioInitialMetadataDispatched = true; - } - // then notify new metadata - this._dispatch = false; - this._onTrackMetadata('audio', meta); - - let mi = this._mediaInfo; - mi.audioCodec = meta.originalCodec; - mi.audioSampleRate = meta.audioSampleRate; - mi.audioChannelCount = meta.channelCount; - if (mi.hasVideo) { - if (mi.videoCodec != null) { - mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; - } + _parseAACAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType) { + let aacData = this._parseAACAudioData(arrayBuffer, dataOffset, dataSize, packetType); + if (aacData == undefined) { + return; + } + + let meta = this._audioMetadata; + let track = this._audioTrack; + + if (packetType === 0) { // AAC sequence header (AudioSpecificConfig) + if (meta.config) { + if (buffersAreEqual(aacData.data.config, meta.config)) { + // If AudioSpecificConfig is not changed, ignore it to avoid generating initialization segment repeatedly + return; } else { - mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; + Log.w(this.TAG, 'AudioSpecificConfig has been changed, re-generate initialization segment'); } - if (mi.isComplete()) { - this._onMediaInfo(mi); + } + let misc = aacData.data; + meta.audioSampleRate = misc.samplingRate; + meta.channelCount = misc.channelCount; + meta.codec = misc.codec; + meta.originalCodec = misc.originalCodec; + meta.config = misc.config; + // The decode result of an aac sample is 1024 PCM samples + meta.refSampleDuration = 1024 / meta.audioSampleRate * meta.timescale; + Log.v(this.TAG, 'Parsed AudioSpecificConfig'); + + if (this._isInitialMetadataDispatched()) { + // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer + if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) { + this._onDataAvailable(this._audioTrack, this._videoTrack); } - } else if (aacData.packetType === 1) { // AAC raw frame data - let dts = this._timestampBase + tagTimestamp; - let aacSample = {unit: aacData.data, length: aacData.data.byteLength, dts: dts, pts: dts}; - track.samples.push(aacSample); - track.length += aacData.data.length; } else { - Log.e(this.TAG, `Flv: Unsupported AAC data type ${aacData.packetType}`); + this._audioInitialMetadataDispatched = true; } - } else if (soundFormat === 2) { // MP3 - if (!meta.codec) { - // We need metadata for mp3 audio track, extract info from frame header - let misc = this._parseMP3AudioData(arrayBuffer, dataOffset + 1, dataSize - 1, true); - if (misc == undefined) { - return; - } - meta.audioSampleRate = misc.samplingRate; - meta.channelCount = misc.channelCount; - meta.codec = misc.codec; - meta.originalCodec = misc.originalCodec; - // The decode result of an mp3 sample is 1152 PCM samples - meta.refSampleDuration = 1152 / meta.audioSampleRate * meta.timescale; - Log.v(this.TAG, 'Parsed MPEG Audio Frame Header'); + // then notify new metadata + this._dispatch = false; + this._onTrackMetadata('audio', meta); - this._audioInitialMetadataDispatched = true; - this._onTrackMetadata('audio', meta); - - let mi = this._mediaInfo; - mi.audioCodec = meta.codec; - mi.audioSampleRate = meta.audioSampleRate; - mi.audioChannelCount = meta.channelCount; - mi.audioDataRate = misc.bitRate; - if (mi.hasVideo) { - if (mi.videoCodec != null) { - mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; - } - } else { - mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; - } - if (mi.isComplete()) { - this._onMediaInfo(mi); + let mi = this._mediaInfo; + mi.audioCodec = meta.originalCodec; + mi.audioSampleRate = meta.audioSampleRate; + mi.audioChannelCount = meta.channelCount; + if (mi.hasVideo) { + if (mi.videoCodec != null) { + mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; } + } else { + mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; } - - // This packet is always a valid audio packet, extract it - let data = this._parseMP3AudioData(arrayBuffer, dataOffset + 1, dataSize - 1, false); - if (data == undefined) { - return; + if (mi.isComplete()) { + this._onMediaInfo(mi); } + } else if (packetType === 1) { // AAC raw frame data let dts = this._timestampBase + tagTimestamp; - let mp3Sample = {unit: data, length: data.byteLength, dts: dts, pts: dts}; - track.samples.push(mp3Sample); - track.length += data.length; + let aacSample = {unit: aacData.data, length: aacData.data.byteLength, dts: dts, pts: dts}; + track.samples.push(aacSample); + track.length += aacData.data.length; + } else { + Log.e(this.TAG, `Flv: Unsupported AAC data type ${packetType}`); } } - _parseAACAudioData(arrayBuffer, dataOffset, dataSize) { - if (dataSize <= 1) { - Log.w(this.TAG, 'Flv: Invalid AAC packet, missing AACPacketType or/and Data!'); - return; - } - - let result = {}; - let array = new Uint8Array(arrayBuffer, dataOffset, dataSize); - - result.packetType = array[0]; - - if (array[0] === 0) { - result.data = this._parseAACAudioSpecificConfig(arrayBuffer, dataOffset + 1, dataSize - 1); + _parseAACAudioData(arrayBuffer, dataOffset, dataSize, packetType) { + if (packetType === 0) { + return { data: this._parseAACAudioSpecificConfig(arrayBuffer, dataOffset, dataSize) }; } else { - result.data = array.subarray(1); + return { data: new Uint8Array(arrayBuffer, dataOffset, dataSize) } } - - return result; } _parseAACAudioSpecificConfig(arrayBuffer, dataOffset, dataSize) { @@ -774,7 +766,56 @@ class FLVDemuxer { }; } - _parseMP3AudioData(arrayBuffer, dataOffset, dataSize, requestHeader) { + _parseMP3AudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType) { + let meta = this._audioMetadata; + let track = this._audioTrack; + + if (!meta.codec) { + // We need metadata for mp3 audio track, extract info from frame header + let misc = this._parseMP3AudioData(arrayBuffer, dataOffset, dataSize, packetType, true); + if (misc == undefined) { + return; + } + meta.audioSampleRate = misc.samplingRate; + meta.channelCount = misc.channelCount; + meta.codec = misc.codec; + meta.originalCodec = misc.originalCodec; + // The decode result of an mp3 sample is 1152 PCM samples + meta.refSampleDuration = 1152 / meta.audioSampleRate * meta.timescale; + Log.v(this.TAG, 'Parsed MPEG Audio Frame Header'); + + this._audioInitialMetadataDispatched = true; + this._onTrackMetadata('audio', meta); + + let mi = this._mediaInfo; + mi.audioCodec = meta.codec; + mi.audioSampleRate = meta.audioSampleRate; + mi.audioChannelCount = meta.channelCount; + mi.audioDataRate = misc.bitRate; + if (mi.hasVideo) { + if (mi.videoCodec != null) { + mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; + } + } else { + mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; + } + if (mi.isComplete()) { + this._onMediaInfo(mi); + } + } + + // This packet is always a valid audio packet, extract it + let data = this._parseMP3AudioData(arrayBuffer, dataOffset, dataSize, packetType, false); + if (data == undefined) { + return; + } + let dts = this._timestampBase + tagTimestamp; + let mp3Sample = {unit: data, length: data.byteLength, dts: dts, pts: dts}; + track.samples.push(mp3Sample); + track.length += data.length; + } + + _parseMP3AudioData(arrayBuffer, dataOffset, dataSize, packetType, requestHeader) { if (dataSize < 4) { Log.w(this.TAG, 'Flv: Invalid MP3 packet, header missing!'); return; @@ -850,6 +891,54 @@ class FLVDemuxer { return result; } + _parseEnhanedAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, fourcc, packetType) { + let v = new DataView(arrayBuffer, dataOffset, dataSize); + let enhanced_offset = 0, enhanced_datasize = dataSize; + while (enhanced_offset < dataSize) { + if (this._enhanedFlvAudioMultitrackMode === 2) { // is not MultiTrackMultiCodec + if (enhanced_offset + 4 >= dataSize) { + Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, fourcc in ManyTrackManyCodec Missing!'); + return; + } + fourcc = String.fromCharCode(... (new Uint8Array(arrayBuffer, dataOffset, dataSize)).slice(enhanced_offset + 0, enhanced_offset + 4)); + enhanced_offset += 4; + } + + let track_id = null; + if (this._enhanedFlvAudioMultitrackMode != null) { + if (enhanced_offset + 1 >= dataSize) { + Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, TrackId for MultiTrack Audio Missing!'); + return; + } + track_id = v.getUint8(enhanced_offset); + enhanced_offset += 1; + } + + if (this._enhanedFlvAudioMultitrackMode !== 0) { // has ManyTrack + if (enhanced_offset + 3 >= dataSize) { + Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, DataSize for MultiTrack Audio Missing!'); + return; + } + enhanced_datasize = (v.getUint8(enhanced_offset + 0) << 16) | (v.getUint8(enhanced_offset + 1) << 8) | (v.getUint8(enhanced_offset + 2) << 0); + enhanced_offset += 3; + } else { + enhanced_datasize = dataSize - enhanced_offset; + } + + /*if (fourcc === 'mp4a') { + this._parseAACAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); + } else if (fourcc === '.mp3') { + this._parseMP3AudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); + } else */if (fourcc === 'Opus') { + this._parseOpusAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); + } else { + this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec: ' + fourcc); + } + + enhanced_offset += enhanced_datasize; + } + } + _parseOpusAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType) { if (packetType === 0) { // OpusSequenceHeader this._parseOpusSequenceHeader(arrayBuffer, dataOffset, dataSize); @@ -869,21 +958,6 @@ class FLVDemuxer { return; } let meta = this._audioMetadata; - let track = this._audioTrack; - - if (!meta) { - if (this._hasAudio === false && this._hasAudioFlagOverrided === false) { - this._hasAudio = true; - this._mediaInfo.hasAudio = true; - } - - // initial metadata - meta = this._audioMetadata = {}; - meta.type = 'audio'; - meta.id = track.id; - meta.timescale = this._timescale; - meta.duration = this._duration; - } // Identification Header let v = new DataView(arrayBuffer, dataOffset, dataSize); From d391c942cfd5f97119ac8b1e58576f5172ce7046 Mon Sep 17 00:00:00 2001 From: monyone Date: Sat, 22 Jun 2024 16:33:27 +0900 Subject: [PATCH 2/5] Feature: AudioTrack Selection --- src/core/transmuxer.js | 8 ++--- src/core/transmuxing-controller.js | 5 +-- src/core/transmuxing-worker.js | 2 +- src/demux/flv-demuxer.js | 35 ++++++++++++++++++-- src/player/mse-player.ts | 4 +++ src/player/player-engine-dedicated-thread.ts | 11 +++++- src/player/player-engine-main-thread.ts | 12 ++++++- src/player/player-engine-worker-cmd-def.ts | 1 + src/player/player-engine-worker.ts | 4 ++- 9 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/core/transmuxer.js b/src/core/transmuxer.js index f313cfa..e963aab 100644 --- a/src/core/transmuxer.js +++ b/src/core/transmuxer.js @@ -27,7 +27,7 @@ import MediaInfo from './media-info.js'; class Transmuxer { - constructor(mediaDataSource, config) { + constructor(mediaDataSource, audioTrackIndex, config) { this.TAG = 'Transmuxer'; this._emitter = new EventEmitter(); @@ -36,7 +36,7 @@ class Transmuxer { this._worker = work(require.resolve('./transmuxing-worker')); this._workerDestroying = false; this._worker.addEventListener('message', this._onWorkerMessage.bind(this)); - this._worker.postMessage({cmd: 'init', param: [mediaDataSource, config]}); + this._worker.postMessage({cmd: 'init', param: [mediaDataSource, audioTrackIndex, config]}); this.e = { onLoggingConfigChanged: this._onLoggingConfigChanged.bind(this) }; @@ -45,10 +45,10 @@ class Transmuxer { } catch (error) { Log.e(this.TAG, 'Error while initialize transmuxing worker, fallback to inline transmuxing'); this._worker = null; - this._controller = new TransmuxingController(mediaDataSource, config); + this._controller = new TransmuxingController(mediaDataSource, audioTrackIndex, config); } } else { - this._controller = new TransmuxingController(mediaDataSource, config); + this._controller = new TransmuxingController(mediaDataSource, audioTrackIndex, config); } if (this._controller) { diff --git a/src/core/transmuxing-controller.js b/src/core/transmuxing-controller.js index f1b6bcc..4d22634 100644 --- a/src/core/transmuxing-controller.js +++ b/src/core/transmuxing-controller.js @@ -31,11 +31,12 @@ import {LoaderStatus, LoaderErrors} from '../io/loader.js'; // Transmuxing (IO, Demuxing, Remuxing) controller, with multipart support class TransmuxingController { - constructor(mediaDataSource, config) { + constructor(mediaDataSource, audioTrackIndex, config) { this.TAG = 'TransmuxingController'; this._emitter = new EventEmitter(); this._config = config; + this._audioTrackIndex = audioTrackIndex; // treat single part media as multipart media, which has only one segment if (!mediaDataSource.segments) { @@ -280,7 +281,7 @@ class TransmuxingController { } _setupFLVDemuxerRemuxer(probeData) { - this._demuxer = new FLVDemuxer(probeData, this._config); + this._demuxer = new FLVDemuxer(probeData, this._audioTrackIndex, this._config); if (!this._remuxer) { this._remuxer = new MP4Remuxer(this._config); diff --git a/src/core/transmuxing-worker.js b/src/core/transmuxing-worker.js index 525cc8a..5e79caf 100644 --- a/src/core/transmuxing-worker.js +++ b/src/core/transmuxing-worker.js @@ -46,7 +46,7 @@ let TransmuxingWorker = function (self) { self.addEventListener('message', function (e) { switch (e.data.cmd) { case 'init': - controller = new TransmuxingController(e.data.param[0], e.data.param[1]); + controller = new TransmuxingController(e.data.param[0], e.data.param[1], e.data.param[2]); controller.on(TransmuxingEvents.IO_ERROR, onIOError.bind(this)); controller.on(TransmuxingEvents.DEMUX_ERROR, onDemuxError.bind(this)); controller.on(TransmuxingEvents.INIT_SEGMENT, onInitSegment.bind(this)); diff --git a/src/demux/flv-demuxer.js b/src/demux/flv-demuxer.js index a97a9fd..15d13ba 100644 --- a/src/demux/flv-demuxer.js +++ b/src/demux/flv-demuxer.js @@ -48,7 +48,7 @@ function ReadBig32(array, index) { class FLVDemuxer { - constructor(probeData, config) { + constructor(probeData, audioTrackIndex, config) { this.TAG = 'FLVDemuxer'; this._config = config; @@ -93,6 +93,9 @@ class FLVDemuxer { }; this._enhanedFlvAudioMultitrackMode = null; + this._enhanedFlvAudioTrackIds = []; + this._currentAudioTrackIndex = audioTrackIndex; + this._currentAudioTrackId = null; this._flvSoundRateTable = [5500, 11025, 22050, 44100, 48000]; @@ -256,6 +259,15 @@ class FLVDemuxer { this._mediaInfo.hasVideo = hasVideo; } + _updateAudioTrackIds(trackIndex, newTrackIds) { + let newTrackId = null; + if (trackIndex >= 1) { + newTrackId = newTrackIds[trackIndex - 1] || null; + } + this._enhanedFlvAudioTrackIds = newTrackIds; + this._currentAudioTrackId = newTrackId; + } + resetMediaInfo() { this._mediaInfo = new MediaInfo(); } @@ -546,7 +558,11 @@ class FLVDemuxer { return; } + // Legacy FLV + if (this._currentAudioTrackId != null) { + return; + } if (soundFormat !== 2 && soundFormat !== 10) { // MP3 or AAC this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec idx: ' + soundFormat); @@ -912,6 +928,13 @@ class FLVDemuxer { } track_id = v.getUint8(enhanced_offset); enhanced_offset += 1; + + console.log(track_id); + if (!this._enhanedFlvAudioTrackIds.includes(track_id)) { + let newTrackIds = [... this._enhanedFlvAudioTrackIds, track_id]; + console.log(newTrackIds, this._currentAudioTrackIndex); + this._updateAudioTrackIds(this._currentAudioTrackIndex, newTrackIds); + } } if (this._enhanedFlvAudioMultitrackMode !== 0) { // has ManyTrack @@ -925,11 +948,17 @@ class FLVDemuxer { enhanced_datasize = dataSize - enhanced_offset; } - /*if (fourcc === 'mp4a') { + // ignore not current track + if (this._currentAudioTrackId !== track_id) { + enhanced_offset += enhanced_datasize; + continue; + } + + if (fourcc === 'mp4a') { this._parseAACAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); } else if (fourcc === '.mp3') { this._parseMP3AudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); - } else */if (fourcc === 'Opus') { + } else if (fourcc === 'Opus') { this._parseOpusAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); } else { this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec: ' + fourcc); diff --git a/src/player/mse-player.ts b/src/player/mse-player.ts index a6a2d0f..1be2b59 100644 --- a/src/player/mse-player.ts +++ b/src/player/mse-player.ts @@ -94,6 +94,10 @@ class MSEPlayer { this._player_engine.pause(); } + public set audioTrack(index: number) { + this._player_engine.selectAudioTrack(index); + } + public get type(): string { return this._type; } diff --git a/src/player/player-engine-dedicated-thread.ts b/src/player/player-engine-dedicated-thread.ts index 6ab32c5..80a886f 100644 --- a/src/player/player-engine-dedicated-thread.ts +++ b/src/player/player-engine-dedicated-thread.ts @@ -37,7 +37,7 @@ import { WorkerCommandPacketLoggingConfig, WorkerCommandPacketTimeUpdate, WorkerCommandPacketReadyStateChange, - WorkerCommandPacketUnbufferedSeek + WorkerCommandPacketUnbufferedSeek, } from './player-engine-worker-cmd-def.js'; import { WorkerMessagePacket, @@ -63,6 +63,8 @@ class PlayerEngineDedicatedThread implements PlayerEngine { private _media_element?: HTMLMediaElement = null; + private _audio_track_index = 0; + private _worker: Worker; private _worker_destroying: boolean = false; @@ -113,6 +115,7 @@ class PlayerEngineDedicatedThread implements PlayerEngine { this._worker.postMessage({ cmd: 'init', media_data_source: this._media_data_source, + audio_track_index: this._audio_track_index, config: this._config } as WorkerCommandPacketInit); @@ -278,6 +281,12 @@ class PlayerEngineDedicatedThread implements PlayerEngine { } } + public selectAudioTrack(index: number): void { + this._audio_track_index = index; + this.unload(); + this.load(); + } + public get mediaInfo(): MediaInfo { return Object.assign({}, this._media_info); } diff --git a/src/player/player-engine-main-thread.ts b/src/player/player-engine-main-thread.ts index 91e39bd..1816039 100644 --- a/src/player/player-engine-main-thread.ts +++ b/src/player/player-engine-main-thread.ts @@ -44,6 +44,8 @@ class PlayerEngineMainThread implements PlayerEngine { private _media_element?: HTMLMediaElement = null; + private _audio_track_index = 0; + private _mse_controller?: MSEController = null; private _transmuxer?: Transmuxer = null; @@ -184,7 +186,7 @@ class PlayerEngineMainThread implements PlayerEngine { return; } - this._transmuxer = new Transmuxer(this._media_data_source, this._config); + this._transmuxer = new Transmuxer(this._media_data_source, this._audio_track_index, this._config); this._transmuxer.on(TransmuxingEvents.INIT_SEGMENT, (type: string, is: any) => { this._mse_controller.appendInitSegment(is); @@ -333,6 +335,14 @@ class PlayerEngineMainThread implements PlayerEngine { } } + public selectAudioTrack(index: number): void { + let currentTime = this._media_element?.currentTime || 0; + this._audio_track_index = index; + this.unload(); + this.load(); + this._seeking_handler.directSeek(currentTime); + } + public get mediaInfo(): MediaInfo { return Object.assign({}, this._media_info); } diff --git a/src/player/player-engine-worker-cmd-def.ts b/src/player/player-engine-worker-cmd-def.ts index 56d4ae2..e67292d 100644 --- a/src/player/player-engine-worker-cmd-def.ts +++ b/src/player/player-engine-worker-cmd-def.ts @@ -37,6 +37,7 @@ export type WorkerCommandPacket = { export type WorkerCommandPacketInit = WorkerCommandPacket & { cmd: 'init', media_data_source: any, + audio_track_index: number, config: any, }; diff --git a/src/player/player-engine-worker.ts b/src/player/player-engine-worker.ts index 9e4da03..1c1e96e 100644 --- a/src/player/player-engine-worker.ts +++ b/src/player/player-engine-worker.ts @@ -53,6 +53,7 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { const logcat_callback: (type: string, str: string) => void = onLogcatCallback.bind(this); let media_data_source: any = null; + let audio_track_index: number = 0; let config: any = null; let mse_controller: MSEController = null; @@ -89,6 +90,7 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { case 'init': { const packet = command_packet as WorkerCommandPacketInit; media_data_source = packet.media_data_source; + audio_track_index = packet.audio_track_index; config = packet.config; break; } @@ -188,7 +190,7 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { return; } - transmuxer = new Transmuxer(media_data_source, config); + transmuxer = new Transmuxer(media_data_source, audio_track_index, config); transmuxer.on(TransmuxingEvents.INIT_SEGMENT, (type: string, is: any) => { mse_controller.appendInitSegment(is); From 4d5ead3c0afb5d9b575de8762e9734c91946fba2 Mon Sep 17 00:00:00 2001 From: monyone Date: Sat, 22 Jun 2024 16:38:53 +0900 Subject: [PATCH 3/5] Fix: Single Enhanced Track --- src/demux/flv-demuxer.js | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/src/demux/flv-demuxer.js b/src/demux/flv-demuxer.js index 15d13ba..391c235 100644 --- a/src/demux/flv-demuxer.js +++ b/src/demux/flv-demuxer.js @@ -507,6 +507,24 @@ class FLVDemuxer { let soundFormat = soundSpec >>> 4; if (soundFormat === 9) { // Enhanced FLV + let meta = this._audioMetadata; + let track = this._audioTrack; + + if (!meta) { + if (this._hasAudio === false && this._hasAudioFlagOverrided === false) { + this._hasAudio = true; + this._mediaInfo.hasAudio = true; + } + + // initial metadata + meta = this._audioMetadata = {}; + meta.type = 'audio'; + meta.id = track.id; + meta.timescale = this._timescale; + meta.duration = this._duration; + } + + let packetType = soundSpec & 0x0F; if (packetType === 5) { // AudioPacketType.Multitrack if (dataSize <= 1) { @@ -516,23 +534,6 @@ class FLVDemuxer { this._enhanedFlvAudioMultitrackMode = (v.getUint8(1) & 0xF0) >> 4; let packetType = v.getUint8(1) & 0x0F; - let meta = this._audioMetadata; - let track = this._audioTrack; - - if (!meta) { - if (this._hasAudio === false && this._hasAudioFlagOverrided === false) { - this._hasAudio = true; - this._mediaInfo.hasAudio = true; - } - - // initial metadata - meta = this._audioMetadata = {}; - meta.type = 'audio'; - meta.id = track.id; - meta.timescale = this._timescale; - meta.duration = this._duration; - } - let fourcc = null; if (this._enhanedFlvAudioMultitrackMode !== 2) { // not AvMultitrackType.ManyTracksManyCodecs if (dataSize <= 6) { @@ -937,7 +938,7 @@ class FLVDemuxer { } } - if (this._enhanedFlvAudioMultitrackMode !== 0) { // has ManyTrack + if (this._enhanedFlvAudioMultitrackMode != null && this._enhanedFlvAudioMultitrackMode !== 0) { // has ManyTrack if (enhanced_offset + 3 >= dataSize) { Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, DataSize for MultiTrack Audio Missing!'); return; From fafa3ad8485d792dcea848f7b3ed8ef48489d900 Mon Sep 17 00:00:00 2001 From: monyone Date: Sat, 22 Jun 2024 22:40:17 +0900 Subject: [PATCH 4/5] Fix: ChangeType instead of unload/load --- d.ts/src/player/mse-player.d.ts | 1 + .../player-engine-dedicated-thread.d.ts | 1 + .../src/player/player-engine-main-thread.d.ts | 1 + docs/api.md | 1 + src/core/mse-controller.js | 1 + src/core/transmuxer.js | 16 +- src/core/transmuxing-controller.js | 36 +++- src/core/transmuxing-worker.js | 3 + src/demux/flv-demuxer.js | 157 +++++++++++++----- src/player/player-engine-dedicated-thread.ts | 25 +-- src/player/player-engine-main-thread.ts | 17 +- src/player/player-engine-worker-cmd-def.ts | 8 +- src/player/player-engine-worker.ts | 9 +- 13 files changed, 180 insertions(+), 96 deletions(-) diff --git a/d.ts/src/player/mse-player.d.ts b/d.ts/src/player/mse-player.d.ts index ec1161d..6b434c9 100644 --- a/d.ts/src/player/mse-player.d.ts +++ b/d.ts/src/player/mse-player.d.ts @@ -14,6 +14,7 @@ declare class MSEPlayer { unload(): void; play(): Promise; pause(): void; + selectAudioTrack(index: number): void; get type(): string; get buffered(): TimeRanges; get duration(): number; diff --git a/d.ts/src/player/player-engine-dedicated-thread.d.ts b/d.ts/src/player/player-engine-dedicated-thread.d.ts index b2c7484..f6933ba 100644 --- a/d.ts/src/player/player-engine-dedicated-thread.d.ts +++ b/d.ts/src/player/player-engine-dedicated-thread.d.ts @@ -29,6 +29,7 @@ declare class PlayerEngineDedicatedThread implements PlayerEngine { play(): Promise; pause(): void; seek(seconds: number): void; + selectAudioTrack(track: number): void; get mediaInfo(): MediaInfo; get statisticsInfo(): any; _onLoggingConfigChanged(config: any): void; diff --git a/d.ts/src/player/player-engine-main-thread.d.ts b/d.ts/src/player/player-engine-main-thread.d.ts index d43e6fa..2deda5a 100644 --- a/d.ts/src/player/player-engine-main-thread.d.ts +++ b/d.ts/src/player/player-engine-main-thread.d.ts @@ -31,6 +31,7 @@ declare class PlayerEngineMainThread implements PlayerEngine { play(): Promise; pause(): void; seek(seconds: number): void; + selectAudioTrack(track: number): void; get mediaInfo(): MediaInfo; get statisticsInfo(): any; private _onMSESourceOpen; diff --git a/docs/api.md b/docs/api.md index 41db38f..78a10b8 100644 --- a/docs/api.md +++ b/docs/api.md @@ -158,6 +158,7 @@ interface Player { unload(): void; play(): Promise; pause(): void; + selectAudioTrack(track: number): void; type: string; buffered: TimeRanges; duration: number; diff --git a/src/core/mse-controller.js b/src/core/mse-controller.js index 7aaaa41..d657206 100644 --- a/src/core/mse-controller.js +++ b/src/core/mse-controller.js @@ -250,6 +250,7 @@ class MSEController { return; } } else { + this._sourceBuffers[is.type].changeType(mimeType); Log.v(this.TAG, `Notice: ${is.type} mimeType changed, origin: ${this._mimeTypes[is.type]}, target: ${mimeType}`); } this._mimeTypes[is.type] = mimeType; diff --git a/src/core/transmuxer.js b/src/core/transmuxer.js index e963aab..1e90f3d 100644 --- a/src/core/transmuxer.js +++ b/src/core/transmuxer.js @@ -27,7 +27,7 @@ import MediaInfo from './media-info.js'; class Transmuxer { - constructor(mediaDataSource, audioTrackIndex, config) { + constructor(mediaDataSource, config) { this.TAG = 'Transmuxer'; this._emitter = new EventEmitter(); @@ -36,7 +36,7 @@ class Transmuxer { this._worker = work(require.resolve('./transmuxing-worker')); this._workerDestroying = false; this._worker.addEventListener('message', this._onWorkerMessage.bind(this)); - this._worker.postMessage({cmd: 'init', param: [mediaDataSource, audioTrackIndex, config]}); + this._worker.postMessage({cmd: 'init', param: [mediaDataSource, config]}); this.e = { onLoggingConfigChanged: this._onLoggingConfigChanged.bind(this) }; @@ -45,10 +45,10 @@ class Transmuxer { } catch (error) { Log.e(this.TAG, 'Error while initialize transmuxing worker, fallback to inline transmuxing'); this._worker = null; - this._controller = new TransmuxingController(mediaDataSource, audioTrackIndex, config); + this._controller = new TransmuxingController(mediaDataSource, config); } } else { - this._controller = new TransmuxingController(mediaDataSource, audioTrackIndex, config); + this._controller = new TransmuxingController(mediaDataSource, config); } if (this._controller) { @@ -142,6 +142,14 @@ class Transmuxer { } } + selectAudioTrack(track) { + if (this._worker) { + this._worker.postMessage({cmd: 'select_audio_track', param: track}); + } else { + this._controller.selectAudioTrack(track); + } + } + _onInitSegment(type, initSegment) { // do async invoke Promise.resolve().then(() => { diff --git a/src/core/transmuxing-controller.js b/src/core/transmuxing-controller.js index 4d22634..81ac0ec 100644 --- a/src/core/transmuxing-controller.js +++ b/src/core/transmuxing-controller.js @@ -31,12 +31,11 @@ import {LoaderStatus, LoaderErrors} from '../io/loader.js'; // Transmuxing (IO, Demuxing, Remuxing) controller, with multipart support class TransmuxingController { - constructor(mediaDataSource, audioTrackIndex, config) { + constructor(mediaDataSource, config) { this.TAG = 'TransmuxingController'; this._emitter = new EventEmitter(); this._config = config; - this._audioTrackIndex = audioTrackIndex; // treat single part media as multipart media, which has only one segment if (!mediaDataSource.segments) { @@ -119,8 +118,8 @@ class TransmuxingController { this._emitter.removeListener(event, listener); } - start() { - this._loadSegment(0); + start(optionalFrom) { + this._loadSegment(0, optionalFrom); this._enableStatisticsReporter(); } @@ -135,10 +134,10 @@ class TransmuxingController { ioctl.onRedirect = this._onIORedirect.bind(this); ioctl.onRecoveredEarlyEof = this._onIORecoveredEarlyEof.bind(this); - if (optionalFrom) { - this._demuxer.bindDataSource(this._ioctl); - } else { + if (typeof optionalFrom === 'undefined') { ioctl.onDataArrival = this._onInitChunkArrival.bind(this); + } else { + ioctl.onDataArrival = this._onExistsChunkArrival.bind(this); } ioctl.open(optionalFrom); @@ -221,6 +220,19 @@ class TransmuxingController { this._enableStatisticsReporter(); } + selectAudioTrack(track) { + if (this._config.isLive) { + this._demuxer.selectAudioTrack(track); + } else { // FIXME: Seek needed? + this.stop(); + this._remuxer.insertDiscontinuity(); + this._demuxer.resetMediaInfo(); + this._demuxer._firstParse = true; + this._demuxer.selectAudioTrack(track); + this.start(0); + } + } + _searchSegmentIndexContains(milliseconds) { let segments = this._mediaDataSource.segments; let idx = segments.length - 1; @@ -234,6 +246,14 @@ class TransmuxingController { return idx; } + _onExistsChunkArrival(data, byteStart) { + // IOController seeked immediately after opened, byteStart > 0 callback may received + this._demuxer.bindDataSource(this._ioctl); + this._demuxer.timestampBase = this._mediaDataSource.segments[this._currentSegmentIndex].timestampBase; + + return this._demuxer.parseChunks(data, byteStart); + } + _onInitChunkArrival(data, byteStart) { let consumed = 0; @@ -281,7 +301,7 @@ class TransmuxingController { } _setupFLVDemuxerRemuxer(probeData) { - this._demuxer = new FLVDemuxer(probeData, this._audioTrackIndex, this._config); + this._demuxer = new FLVDemuxer(probeData, this._config); if (!this._remuxer) { this._remuxer = new MP4Remuxer(this._config); diff --git a/src/core/transmuxing-worker.js b/src/core/transmuxing-worker.js index 5e79caf..99997c3 100644 --- a/src/core/transmuxing-worker.js +++ b/src/core/transmuxing-worker.js @@ -88,6 +88,9 @@ let TransmuxingWorker = function (self) { case 'resume': controller.resume(); break; + case 'select_audio_track': + controller.selectAudioTrack(e.data.param); + break; case 'logging_config': { let config = e.data.param; LoggingControl.applyConfig(config); diff --git a/src/demux/flv-demuxer.js b/src/demux/flv-demuxer.js index e588fd6..3a5812c 100644 --- a/src/demux/flv-demuxer.js +++ b/src/demux/flv-demuxer.js @@ -49,7 +49,7 @@ function ReadBig32(array, index) { class FLVDemuxer { - constructor(probeData, audioTrackIndex, config) { + constructor(probeData, config) { this.TAG = 'FLVDemuxer'; this._config = config; @@ -93,10 +93,11 @@ class FLVDemuxer { fps_den: 1000 }; - this._enhanedFlvAudioMultitrackMode = null; + this._enhancedFlvAudioMultitrackMode = null; this._enhanedFlvAudioTrackIds = []; - this._currentAudioTrackIndex = audioTrackIndex; + this._currentAudioTrackIndex = 0; this._currentAudioTrackId = null; + this._audioTrackInitSegments = new Map(); this._flvSoundRateTable = [5500, 11025, 22050, 44100, 48000]; @@ -260,13 +261,50 @@ class FLVDemuxer { this._mediaInfo.hasVideo = hasVideo; } + selectAudioTrack(track) { + let newTrackId = null; + if (track >= 1) { + newTrackId = this._enhanedFlvAudioTrackIds[track - 1] || null; + } + if (this._currentAudioTrackId === newTrackId) { return; } + + this._currentAudioTrackId = newTrackId; + let meta = this._audioTrackInitSegments.get(this._currentAudioTrackId); + + // flush segment + this._onDataAvailable(this._audioTrack, this._videoTrack); + this._videoTrack = {type: 'video', id: 1, sequenceNumber: 0, samples: [], length: 0}; + this._audioTrack = {type: 'audio', id: 2, sequenceNumber: 0, samples: [], length: 0}; + + // then notify new metadata + this._dispatch = false; + this._onTrackMetadata('audio', meta); + + let mi = this._mediaInfo; + mi.audioCodec = meta.originalCodec; + mi.audioSampleRate = meta.audioSampleRate; + mi.audioChannelCount = meta.channelCount; + if (mi.hasVideo) { + if (mi.videoCodec != null) { + mi.mimeType = 'video/x-flv; codecs="' + mi.videoCodec + ',' + mi.audioCodec + '"'; + } + } else { + mi.mimeType = 'video/x-flv; codecs="' + mi.audioCodec + '"'; + } + if (mi.isComplete()) { + this._onMediaInfo(mi); + } + } + _updateAudioTrackIds(trackIndex, newTrackIds) { + this._enhanedFlvAudioTrackIds = newTrackIds; let newTrackId = null; if (trackIndex >= 1) { - newTrackId = newTrackIds[trackIndex - 1] || null; + newTrackId = this._enhanedFlvAudioTrackIds[trackIndex - 1] || null; + } + if (this._currentAudioTrackId !== newTrackId) { + this.selectAudioTrack(trackIndex); } - this._enhanedFlvAudioTrackIds = newTrackIds; - this._currentAudioTrackId = newTrackId; } resetMediaInfo() { @@ -532,11 +570,11 @@ class FLVDemuxer { Log.w(this.TAG, 'Flv: Invalid audio packet, missing audioMultitrackType in Ehnanced FLV payload!'); return; } - this._enhanedFlvAudioMultitrackMode = (v.getUint8(1) & 0xF0) >> 4; + this._enhancedFlvAudioMultitrackMode = (v.getUint8(1) & 0xF0) >> 4; let packetType = v.getUint8(1) & 0x0F; let fourcc = null; - if (this._enhanedFlvAudioMultitrackMode !== 2) { // not AvMultitrackType.ManyTracksManyCodecs + if (this._enhancedFlvAudioMultitrackMode !== 2) { // not AvMultitrackType.ManyTracksManyCodecs if (dataSize <= 6) { Log.w(this.TAG, 'Flv: Invalid audio packet, missing Audio fourcc of OneTrack/ManyTracks in Ehnanced FLV payload!'); return; @@ -605,27 +643,27 @@ class FLVDemuxer { let packetType = v.getUint8(1); if (soundFormat === 10) { // AAC - this._parseAACAudioPacket(arrayBuffer, dataOffset + 2, dataSize - 2, tagTimestamp, packetType); + this._parseAACAudioPacket(arrayBuffer, dataOffset + 2, dataSize - 2, tagTimestamp, packetType, null); } else if (soundFormat === 2) { // MP3 - this._parseMP3AudioPacket(arrayBuffer, dataOffset + 1, dataSize - 1, tagTimestamp, packetType); + this._parseMP3AudioPacket(arrayBuffer, dataOffset + 1, dataSize - 1, tagTimestamp, packetType, null); } } - _parseAACAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType) { + _parseAACAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType, trackId) { let aacData = this._parseAACAudioData(arrayBuffer, dataOffset, dataSize, packetType); if (aacData == undefined) { return; } - let meta = this._audioMetadata; + let meta = { ... this._audioMetadata }; let track = this._audioTrack; if (packetType === 0) { // AAC sequence header (AudioSpecificConfig) if (meta.config) { - if (buffersAreEqual(aacData.data.config, meta.config)) { + if (this._currentAudioTrackId === trackId && buffersAreEqual(aacData.data.config, meta.config)) { // If AudioSpecificConfig is not changed, ignore it to avoid generating initialization segment repeatedly return; - } else { + } else if (this._currentAudioTrackId === trackId && !buffersAreEqual(aacData.data.config, meta.config)) { Log.w(this.TAG, 'AudioSpecificConfig has been changed, re-generate initialization segment'); } } @@ -639,6 +677,12 @@ class FLVDemuxer { meta.refSampleDuration = 1024 / meta.audioSampleRate * meta.timescale; Log.v(this.TAG, 'Parsed AudioSpecificConfig'); + this._audioTrackInitSegments.set(trackId, { ... meta }); + // ignore not current track + if (this._currentAudioTrackId !== trackId) { + return; + } + if (this._isInitialMetadataDispatched()) { // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) { @@ -649,6 +693,7 @@ class FLVDemuxer { } // then notify new metadata this._dispatch = false; + this._audioMetadata = meta; this._onTrackMetadata('audio', meta); let mi = this._mediaInfo; @@ -666,6 +711,9 @@ class FLVDemuxer { this._onMediaInfo(mi); } } else if (packetType === 1) { // AAC raw frame data + // ignore not current track + if (this._currentAudioTrackId !== trackId) { return; } + let dts = this._timestampBase + tagTimestamp; let aacSample = {unit: aacData.data, length: aacData.data.byteLength, dts: dts, pts: dts}; track.samples.push(aacSample); @@ -784,8 +832,8 @@ class FLVDemuxer { }; } - _parseMP3AudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType) { - let meta = this._audioMetadata; + _parseMP3AudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType, trackId) { + let meta = { ... this._audioMetadata }; let track = this._audioTrack; if (!meta.codec) { @@ -802,7 +850,14 @@ class FLVDemuxer { meta.refSampleDuration = 1152 / meta.audioSampleRate * meta.timescale; Log.v(this.TAG, 'Parsed MPEG Audio Frame Header'); + this._audioTrackInitSegments.set(trackId, { ... meta }); + // ignore not current track + if (this._currentAudioTrackId !== trackId) { + return; + } + this._audioInitialMetadataDispatched = true; + this._audioMetadata = meta; this._onTrackMetadata('audio', meta); let mi = this._mediaInfo; @@ -913,7 +968,7 @@ class FLVDemuxer { let v = new DataView(arrayBuffer, dataOffset, dataSize); let enhanced_offset = 0, enhanced_datasize = dataSize; while (enhanced_offset < dataSize) { - if (this._enhanedFlvAudioMultitrackMode === 2) { // is not MultiTrackMultiCodec + if (this._enhancedFlvAudioMultitrackMode === 2) { // is not MultiTrackMultiCodec if (enhanced_offset + 4 >= dataSize) { Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, fourcc in ManyTrackManyCodec Missing!'); return; @@ -922,22 +977,22 @@ class FLVDemuxer { enhanced_offset += 4; } - let track_id = null; - if (this._enhanedFlvAudioMultitrackMode != null) { + let trackId = null; + if (this._enhancedFlvAudioMultitrackMode != null) { if (enhanced_offset + 1 >= dataSize) { Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, TrackId for MultiTrack Audio Missing!'); return; } - track_id = v.getUint8(enhanced_offset); + trackId = v.getUint8(enhanced_offset); enhanced_offset += 1; - if (!this._enhanedFlvAudioTrackIds.includes(track_id)) { - let newTrackIds = [... this._enhanedFlvAudioTrackIds, track_id]; + if (!this._enhanedFlvAudioTrackIds.includes(trackId)) { + let newTrackIds = [... this._enhanedFlvAudioTrackIds, trackId]; this._updateAudioTrackIds(this._currentAudioTrackIndex, newTrackIds); } } - if (this._enhanedFlvAudioMultitrackMode != null && this._enhanedFlvAudioMultitrackMode !== 0) { // has ManyTrack + if (this._enhancedFlvAudioMultitrackMode != null && this._enhancedFlvAudioMultitrackMode !== 0) { // has ManyTrack if (enhanced_offset + 3 >= dataSize) { Log.w(this.TAG, 'Flv: Invalid Enhanced Audio packet, DataSize for MultiTrack Audio Missing!'); return; @@ -948,20 +1003,14 @@ class FLVDemuxer { enhanced_datasize = dataSize - enhanced_offset; } - // ignore not current track - if (this._currentAudioTrackId !== track_id) { - enhanced_offset += enhanced_datasize; - continue; - } - if (fourcc === 'mp4a') { - this._parseAACAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); + this._parseAACAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType, trackId); } else if (fourcc === '.mp3') { - this._parseMP3AudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); + this._parseMP3AudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType, trackId); } else if (fourcc === 'Opus') { - this._parseOpusAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); + this._parseOpusAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType, trackId); } else if (fourcc === 'fLaC') { - this._parseFlacAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType); + this._parseFlacAudioPacket(arrayBuffer, dataOffset + enhanced_offset, enhanced_datasize, tagTimestamp, packetType, trackId); } else { this._onError(DemuxErrors.CODEC_UNSUPPORTED, 'Flv: Unsupported audio codec: ' + fourcc); } @@ -970,10 +1019,11 @@ class FLVDemuxer { } } - _parseOpusAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType) { + _parseOpusAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType, trackId) { if (packetType === 0) { // OpusSequenceHeader - this._parseOpusSequenceHeader(arrayBuffer, dataOffset, dataSize); + this._parseOpusSequenceHeader(arrayBuffer, dataOffset, dataSize, trackId); } else if (packetType === 1) { // OpusCodedData + if (this._currentAudioTrackId !== trackId) { return; } this._parseOpusAudioData(arrayBuffer, dataOffset, dataSize, tagTimestamp); } else if (packetType === 2) { // empty, Opus end of sequence @@ -983,12 +1033,12 @@ class FLVDemuxer { } } - _parseOpusSequenceHeader(arrayBuffer, dataOffset, dataSize) { + _parseOpusSequenceHeader(arrayBuffer, dataOffset, dataSize, trackId) { if (dataSize <= 16) { Log.w(this.TAG, 'Flv: Invalid OpusSequenceHeader, lack of data!'); return; } - let meta = this._audioMetadata; + let meta = { ... this._audioMetadata }; // Identification Header let v = new DataView(arrayBuffer, dataOffset, dataSize); @@ -1007,10 +1057,10 @@ class FLVDemuxer { originalCodec: 'opus', }; if (meta.config) { - if (buffersAreEqual(misc.config, meta.config)) { + if (this._currentAudioTrackId === trackId && buffersAreEqual(misc.config, meta.config)) { // If OpusSequenceHeader is not changed, ignore it to avoid generating initialization segment repeatedly return; - } else { + } else if (this._currentAudioTrackId === trackId && !buffersAreEqual(misc.config, meta.config)) { Log.w(this.TAG, 'OpusSequenceHeader has been changed, re-generate initialization segment'); } } @@ -1023,6 +1073,12 @@ class FLVDemuxer { meta.refSampleDuration = 20; Log.v(this.TAG, 'Parsed OpusSequenceHeader'); + this._audioTrackInitSegments.set(trackId, { ... meta }); + // ignore not current track + if (this._currentAudioTrackId !== trackId) { + return; + } + if (this._isInitialMetadataDispatched()) { // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) { @@ -1033,6 +1089,7 @@ class FLVDemuxer { } // then notify new metadata this._dispatch = false; + this._audioMetadata = meta; this._onTrackMetadata('audio', meta); let mi = this._mediaInfo; @@ -1062,10 +1119,11 @@ class FLVDemuxer { track.length += data.length; } - _parseFlacAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType) { + _parseFlacAudioPacket(arrayBuffer, dataOffset, dataSize, tagTimestamp, packetType, trackId) { if (packetType === 0) { // FlacSequenceHeader - this._parseFlacSequenceHeader(arrayBuffer, dataOffset, dataSize); + this._parseFlacSequenceHeader(arrayBuffer, dataOffset, dataSize, trackId); } else if (packetType === 1) { // FlacCodedData + if (this._currentAudioTrackId !== trackId) { return; } this._parseFlacAudioData(arrayBuffer, dataOffset, dataSize, tagTimestamp); } else if (packetType === 2) { // empty, Flac end of sequence @@ -1075,8 +1133,8 @@ class FLVDemuxer { } } - _parseFlacSequenceHeader(arrayBuffer, dataOffset, dataSize) { - let meta = this._audioMetadata; + _parseFlacSequenceHeader(arrayBuffer, dataOffset, dataSize, trackId) { + let meta = { ... this._audioMetadata }; let track = this._audioTrack; if (!meta) { @@ -1094,7 +1152,7 @@ class FLVDemuxer { } // METADATA_BLOCK_HEADER - let header = new Uint8Array(arrayBuffer, dataOffset + 4, dataSize - 4); + let header = new Uint8Array(arrayBuffer, dataOffset, dataSize); let gb = new ExpGolomb(header); let minimum_block_size = gb.readBits(16); // minimum_block_size let maximum_block_size = gb.readBits(16); // maximum_block_size @@ -1122,10 +1180,10 @@ class FLVDemuxer { originalCodec: 'flac', }; if (meta.config) { - if (buffersAreEqual(misc.config, meta.config)) { + if (this._currentAudioTrackId === trackId && buffersAreEqual(misc.config, meta.config)) { // If FlacSequenceHeader is not changed, ignore it to avoid generating initialization segment repeatedly return; - } else { + } else if (this._currentAudioTrackId === trackId && !buffersAreEqual(misc.config, meta.config)) { Log.w(this.TAG, 'FlacSequenceHeader has been changed, re-generate initialization segment'); } } @@ -1139,6 +1197,12 @@ class FLVDemuxer { Log.v(this.TAG, 'Parsed FlacSequenceHeader'); + this._audioTrackInitSegments.set(trackId, { ... meta }); + // ignore not current track + if (this._currentAudioTrackId !== trackId) { + return; + } + if (this._isInitialMetadataDispatched()) { // Non-initial metadata, force dispatch (or flush) parsed frames to remuxer if (this._dispatch && (this._audioTrack.length || this._videoTrack.length)) { @@ -1149,6 +1213,7 @@ class FLVDemuxer { } // then notify new metadata this._dispatch = false; + this._audioMetadata = meta; this._onTrackMetadata('audio', meta); let mi = this._mediaInfo; diff --git a/src/player/player-engine-dedicated-thread.ts b/src/player/player-engine-dedicated-thread.ts index 1b2bbd2..af61053 100644 --- a/src/player/player-engine-dedicated-thread.ts +++ b/src/player/player-engine-dedicated-thread.ts @@ -64,8 +64,6 @@ class PlayerEngineDedicatedThread implements PlayerEngine { private _media_element?: HTMLMediaElement = null; - private _audio_track_index = 0; - private _worker: Worker; private _worker_destroying: boolean = false; @@ -116,7 +114,6 @@ class PlayerEngineDedicatedThread implements PlayerEngine { this._worker.postMessage({ cmd: 'init', media_data_source: this._media_data_source, - audio_track_index: this._audio_track_index, config: this._config } as WorkerCommandPacketInit); @@ -282,22 +279,16 @@ class PlayerEngineDedicatedThread implements PlayerEngine { } } - public selectAudioTrack(index: number): void { - let currentTime = this._media_element?.currentTime || 0; - let paused = this._media_element?.paused || false; - this._audio_track_index = index; - this.unload(); - this._worker?.postMessage({ - cmd: 'select_audio_track', - audio_track: index - } as WorkerCommandPacketSelectAudioTrack); - this.load(); + public selectAudioTrack(track: number): void { if (!this._config.isLive) { - this._seeking_handler.directSeek(currentTime); - } - if (!paused) { - this.play(); + this._worker.postMessage({ + cmd: 'flush', + }); } + this._worker.postMessage({ + cmd: 'select_audio_track', + track, + } as WorkerCommandPacketSelectAudioTrack) } public get mediaInfo(): MediaInfo { diff --git a/src/player/player-engine-main-thread.ts b/src/player/player-engine-main-thread.ts index 48e7bd6..95c3c9c 100644 --- a/src/player/player-engine-main-thread.ts +++ b/src/player/player-engine-main-thread.ts @@ -44,8 +44,6 @@ class PlayerEngineMainThread implements PlayerEngine { private _media_element?: HTMLMediaElement = null; - private _audio_track_index = 0; - private _mse_controller?: MSEController = null; private _transmuxer?: Transmuxer = null; @@ -186,7 +184,7 @@ class PlayerEngineMainThread implements PlayerEngine { return; } - this._transmuxer = new Transmuxer(this._media_data_source, this._audio_track_index, this._config); + this._transmuxer = new Transmuxer(this._media_data_source, this._config); this._transmuxer.on(TransmuxingEvents.INIT_SEGMENT, (type: string, is: any) => { this._mse_controller.appendInitSegment(is); @@ -335,18 +333,11 @@ class PlayerEngineMainThread implements PlayerEngine { } } - public selectAudioTrack(index: number): void { - let currentTime = this._media_element?.currentTime || 0; - let paused = this._media_element?.paused || false; - this._audio_track_index = index; - this.unload(); - this.load(); + public selectAudioTrack(track: number): void { if (!this._config.isLive) { - this._seeking_handler.directSeek(currentTime); - } - if (!paused) { - this.play(); + this._mse_controller?.flush(); } + this._transmuxer.selectAudioTrack(track); } public get mediaInfo(): MediaInfo { diff --git a/src/player/player-engine-worker-cmd-def.ts b/src/player/player-engine-worker-cmd-def.ts index 218a0f6..940e085 100644 --- a/src/player/player-engine-worker-cmd-def.ts +++ b/src/player/player-engine-worker-cmd-def.ts @@ -24,12 +24,13 @@ export type WorkerCommandOp = | 'shutdown_mse' | 'load' | 'unload' + | 'flush' | 'unbuffered_seek' | 'timeupdate' | 'readystatechange' + | 'select_audio_track' | 'pause_transmuxer' - | 'resume_transmuxer' - | 'select_audio_track'; + | 'resume_transmuxer'; export type WorkerCommandPacket = { cmd: WorkerCommandOp, @@ -38,7 +39,6 @@ export type WorkerCommandPacket = { export type WorkerCommandPacketInit = WorkerCommandPacket & { cmd: 'init', media_data_source: any, - audio_track_index: number, config: any, }; @@ -64,5 +64,5 @@ export type WorkerCommandPacketReadyStateChange = WorkerCommandPacket & { export type WorkerCommandPacketSelectAudioTrack = WorkerCommandPacket & { cmd: 'select_audio_track', - audio_track: number, + track: number, }; diff --git a/src/player/player-engine-worker.ts b/src/player/player-engine-worker.ts index b2d5dc8..885f3e3 100644 --- a/src/player/player-engine-worker.ts +++ b/src/player/player-engine-worker.ts @@ -54,7 +54,6 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { const logcat_callback: (type: string, str: string) => void = onLogcatCallback.bind(this); let media_data_source: any = null; - let audio_track_index: number = 0; let config: any = null; let mse_controller: MSEController = null; @@ -91,7 +90,6 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { case 'init': { const packet = command_packet as WorkerCommandPacketInit; media_data_source = packet.media_data_source; - audio_track_index = packet.audio_track_index; config = packet.config; break; } @@ -100,7 +98,7 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { break; case 'select_audio_track': const packet = command_packet as WorkerCommandPacketSelectAudioTrack; - audio_track_index = packet.audio_track; + transmuxer.selectAudioTrack(packet.track); break; case 'initialize_mse': initializeMSE(); @@ -114,6 +112,9 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { case 'unload': unload(); break; + case 'flush': + mse_controller.flush(); + break; case 'unbuffered_seek': { const packet = command_packet as WorkerCommandPacketUnbufferedSeek; mse_controller.flush(); @@ -195,7 +196,7 @@ const PlayerEngineWorker = (self: DedicatedWorkerGlobalScope) => { return; } - transmuxer = new Transmuxer(media_data_source, audio_track_index, config); + transmuxer = new Transmuxer(media_data_source, config); transmuxer.on(TransmuxingEvents.INIT_SEGMENT, (type: string, is: any) => { mse_controller.appendInitSegment(is); From 24c7358b0edf8aa96e13e103c91a9ef8a54c888e Mon Sep 17 00:00:00 2001 From: monyone Date: Sun, 23 Jun 2024 11:07:01 +0900 Subject: [PATCH 5/5] feature: multitrack audio for mpegts --- src/demux/pat-pmt-pes.ts | 20 +- src/demux/ts-demuxer.ts | 453 ++++++++++++++++++++++++--------------- 2 files changed, 287 insertions(+), 186 deletions(-) diff --git a/src/demux/pat-pmt-pes.ts b/src/demux/pat-pmt-pes.ts index b1c0974..5eac141 100644 --- a/src/demux/pat-pmt-pes.ts +++ b/src/demux/pat-pmt-pes.ts @@ -37,23 +37,19 @@ export class PMT { common_pids: { h264: number | undefined, h265: number | undefined; - adts_aac: number | undefined, - loas_aac: number | undefined, - opus: number | undefined, - ac3: number | undefined, - eac3: number | undefined, - mp3: number | undefined } = { h264: undefined, h265: undefined, - adts_aac: undefined, - loas_aac: undefined, - opus: undefined, - ac3: undefined, - eac3: undefined, - mp3: undefined }; + audio_pids: { + [pid: number]: boolean + } = {}; + + opus_pids: { + [pid: number]: boolean; + } = {}; + pes_private_data_pids: { [pid: number]: boolean } = {}; diff --git a/src/demux/ts-demuxer.ts b/src/demux/ts-demuxer.ts index 74431ee..c4952d8 100644 --- a/src/demux/ts-demuxer.ts +++ b/src/demux/ts-demuxer.ts @@ -75,6 +75,7 @@ type MP3AudioMetadata = { sample_rate: number, channel_count: number; }; +type AudioMetadata = AACAudioMetadata | AC3AudioMetadata | EAC3AudioMetadata | OpusAudioMetadata | MP3AudioMetadata; type AudioData = { codec: 'aac'; data: AACFrame; @@ -112,6 +113,12 @@ class TSDemuxer extends BaseDemuxer { private pmt_: PMT; private program_pmt_map_: ProgramToPMTMap = {}; + private audioTrackPids: number[] = []; + private currentAudioTrackIndex = 0; + private currentAudioTrackPid = null; + private audioTrackInitSegments = new Map(); + private audioTrackMetadata = new Map(); + private pes_slice_queues_: PIDToSliceQueues = {}; private section_slice_queues_: PIDToSliceQueues = {}; @@ -127,7 +134,7 @@ class TSDemuxer extends BaseDemuxer { details: undefined }; - private audio_metadata_: AACAudioMetadata | AC3AudioMetadata | EAC3AudioMetadata | OpusAudioMetadata | MP3AudioMetadata = { + private audio_metadata_: AudioMetadata = { codec: undefined, audio_object_type: undefined, sampling_freq_index: undefined, @@ -244,6 +251,37 @@ class TSDemuxer extends BaseDemuxer { this.media_info_ = new MediaInfo(); } + selectAudioTrack(track: number) { + this.currentAudioTrackIndex = track; + let previous_meta = this.audioTrackMetadata.get(this.currentAudioTrackPid); + let newTrackPid = this.audioTrackPids[track]; + if (this.currentAudioTrackPid === newTrackPid) { return; } + + this.currentAudioTrackPid = newTrackPid; + let sample = this.audioTrackInitSegments.get(this.currentAudioTrackPid); + if (!sample) { return; } + let meta = this.audioTrackMetadata.get(this.currentAudioTrackPid); + if (!meta) { return; } + + // flush segment + this.dispatchAudioMediaSegment(); + this.video_track_ = {type: 'video', id: 1, sequenceNumber: 0, samples: [], length: 0}; + this.audio_track_ = {type: 'audio', id: 2, sequenceNumber: 0, samples: [], length: 0}; + + // then notify new metadata + if (previous_meta == null || this.detectAudioMetadataChange(sample, previous_meta, this.currentAudioTrackPid)) { + this.dispatchAudioInitSegment(sample, meta, this.currentAudioTrackPid); + } + } + + _updateAudioTrackIds(trackIndex: number, newTrackIds: number[]) { + this.audioTrackPids = newTrackIds; + let newTrackPid = this.audioTrackPids[trackIndex]; + if (this.currentAudioTrackPid !== newTrackPid) { + this.selectAudioTrack(trackIndex); + } + } + public parseChunks(chunk: ArrayBuffer, byte_start: number): number { if (!this.onError || !this.onMediaInfo @@ -340,12 +378,8 @@ class TSDemuxer extends BaseDemuxer { // process PES only for known common_pids if (pid === this.pmt_.common_pids.h264 || pid === this.pmt_.common_pids.h265 - || pid === this.pmt_.common_pids.adts_aac - || pid === this.pmt_.common_pids.loas_aac - || pid === this.pmt_.common_pids.ac3 - || pid === this.pmt_.common_pids.eac3 - || pid === this.pmt_.common_pids.opus - || pid === this.pmt_.common_pids.mp3 + || this.pmt_.audio_pids[pid] === true + || this.pmt_.opus_pids[pid] === true || this.pmt_.pes_private_data_pids[pid] === true || this.pmt_.timed_id3_pids[pid] === true || this.pmt_.synchronous_klv_pids[pid] === true @@ -594,16 +628,16 @@ class TSDemuxer extends BaseDemuxer { switch (pes_data.stream_type) { case StreamType.kMPEG1Audio: case StreamType.kMPEG2Audio: - this.parseMP3Payload(payload, pts); + this.parseMP3Payload(payload, pts, pes_data.pid); break; case StreamType.kPESPrivateData: - if (this.pmt_.common_pids.opus === pes_data.pid) { - this.parseOpusPayload(payload, pts); - } else if (this.pmt_.common_pids.ac3 === pes_data.pid) { - this.parseAC3Payload(payload, pts); + if (this.pmt_.opus_pids[pes_data.pid]) { + this.parseOpusPayload(payload, pts, pes_data.pid); + } /*else if (this.pmt_.common_pids.ac3 === pes_data.pid) { + this.parseAC3Payload(payload, pts, pes_data.pid); } else if (this.pmt_.common_pids.eac3 === pes_data.pid) { - this.parseEAC3Payload(payload, pts); - } else if (this.pmt_.asynchronous_klv_pids[pes_data.pid]) { + this.parseEAC3Payload(payload, pts, pes_data.pid); + } */else if (this.pmt_.asynchronous_klv_pids[pes_data.pid]) { this.parseAsynchronousKLVMetadataPayload(payload, pes_data.pid, stream_id); } else if (this.pmt_.smpte2038_pids[pes_data.pid]) { this.parseSMPTE2038MetadataPayload(payload, pts, dts, pes_data.pid, stream_id); @@ -612,16 +646,16 @@ class TSDemuxer extends BaseDemuxer { } break; case StreamType.kADTSAAC: - this.parseADTSAACPayload(payload, pts); + this.parseADTSAACPayload(payload, pts, pes_data.pid); break; case StreamType.kLOASAAC: - this.parseLOASAACPayload(payload, pts); + this.parseLOASAACPayload(payload, pts, pes_data.pid); break; case StreamType.kAC3: - this.parseAC3Payload(payload, pts); + this.parseAC3Payload(payload, pts, pes_data.pid); break; case StreamType.kEAC3: - this.parseEAC3Payload(payload, pts); + this.parseEAC3Payload(payload, pts, pes_data.pid); break; case StreamType.kMetadata: if (this.pmt_.timed_id3_pids[pes_data.pid]) { @@ -760,6 +794,8 @@ class TSDemuxer extends BaseDemuxer { let info_start_index = 12 + program_info_length; let info_bytes = section_length - 9 - program_info_length - 4; + let audioPids = []; + let hasVideo = false, hasAudio = false; for (let i = info_start_index; i < info_start_index + info_bytes; ) { let stream_type = data[i] as StreamType; @@ -768,23 +804,34 @@ class TSDemuxer extends BaseDemuxer { pmt.pid_stream_type[elementary_PID] = stream_type; - let already_has_video = pmt.common_pids.h264 || pmt.common_pids.h265; - let already_has_audio = pmt.common_pids.adts_aac || pmt.common_pids.loas_aac || pmt.common_pids.ac3 || pmt.common_pids.eac3 || pmt.common_pids.opus || pmt.common_pids.mp3; + let already_has_video = pmt.common_pids.h264 || pmt.common_pids.h265; if (stream_type === StreamType.kH264 && !already_has_video) { pmt.common_pids.h264 = elementary_PID; + hasVideo = true; } else if (stream_type === StreamType.kH265 && !already_has_video) { pmt.common_pids.h265 = elementary_PID; - } else if (stream_type === StreamType.kADTSAAC && !already_has_audio) { - pmt.common_pids.adts_aac = elementary_PID; - } else if (stream_type === StreamType.kLOASAAC && !already_has_audio) { - pmt.common_pids.loas_aac = elementary_PID; - } else if (stream_type === StreamType.kAC3 && !already_has_audio) { - pmt.common_pids.ac3 = elementary_PID; // ATSC AC-3 - } else if (stream_type === StreamType.kEAC3 && !already_has_audio) { - pmt.common_pids.eac3 = elementary_PID; // ATSC EAC-3 - } else if ((stream_type === StreamType.kMPEG1Audio || stream_type === StreamType.kMPEG2Audio) && !already_has_audio) { - pmt.common_pids.mp3 = elementary_PID; + hasVideo = true; + } else if (stream_type === StreamType.kADTSAAC) { + pmt.audio_pids[elementary_PID] = true; // ADTS AAC + audioPids.push(elementary_PID); + hasAudio = true; + } else if (stream_type === StreamType.kLOASAAC) { + pmt.audio_pids[elementary_PID] = true; // LOAS AAC + audioPids.push(elementary_PID); + hasAudio = true; + } else if (stream_type === StreamType.kAC3) { + pmt.audio_pids[elementary_PID] = true; // ATSC AC-3 + audioPids.push(elementary_PID); + hasAudio = true; + } else if (stream_type === StreamType.kEAC3) { + pmt.audio_pids[elementary_PID] = true; // ATSC EAC-3 + audioPids.push(elementary_PID); + hasAudio = true; + } else if (stream_type === StreamType.kMPEG1Audio || stream_type === StreamType.kMPEG2Audio) { + pmt.audio_pids[elementary_PID] = true; // MP3 + audioPids.push(elementary_PID); + hasAudio = true; } else if (stream_type === StreamType.kPESPrivateData) { pmt.pes_private_data_pids[elementary_PID] = true; if (ES_info_length > 0) { @@ -802,12 +849,16 @@ class TSDemuxer extends BaseDemuxer { } */ /* else if (registration === 'EC-3' && !alrady_has_audio) { pmt.common_pids.eac3 = elementary_PID; // DVB EAC-3 (FIXME: NEED VERIFY) } */ else if (registration === 'Opus') { - pmt.common_pids.opus = elementary_PID; + pmt.opus_pids[elementary_PID] = true; + audioPids.push(elementary_PID); + hasAudio = true; } else if (registration === 'KLVA') { pmt.asynchronous_klv_pids[elementary_PID] = true; } } else if (tag === 0x7F) { // DVB extension descriptor - if (elementary_PID === pmt.common_pids.opus) { + if (pmt.opus_pids[elementary_PID]) { + let audio_metadata_ = this.audioTrackMetadata.get(elementary_PID); + let ext_desc_tag = data[offset + 2]; let channel_config_code: number | null = null; if (ext_desc_tag === 0x80) { // User defined (provisional Opus) @@ -830,14 +881,16 @@ class TSDemuxer extends BaseDemuxer { meta } as const; - if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = meta; - this.dispatchAudioInitSegment(sample); - } else if (this.detectAudioMetadataChange(sample)) { + if (this.currentAudioTrackPid !== elementary_PID) { + this.audioTrackInitSegments.set(elementary_PID, { ... sample }); + this.audioTrackMetadata.set(elementary_PID, { ... meta }); + } else if (this.audio_init_segment_dispatched_ == false) { + this.dispatchAudioInitSegment(sample, meta, elementary_PID); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(sample, audio_metadata_, elementary_PID)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(sample); + this.dispatchAudioInitSegment(sample, meta, elementary_PID); } } } @@ -884,18 +937,15 @@ class TSDemuxer extends BaseDemuxer { i += 5 + ES_info_length; } + this._updateAudioTrackIds(this.currentAudioTrackIndex, audioPids); if (program_number === this.current_program_) { if (this.pmt_ == undefined) { Log.v(this.TAG, `Parsed first PMT: ${JSON.stringify(pmt)}`); } this.pmt_ = pmt; - if (pmt.common_pids.h264 || pmt.common_pids.h265) { - this.has_video_ = true; - } - if (pmt.common_pids.adts_aac || pmt.common_pids.loas_aac || pmt.common_pids.ac3 || pmt.common_pids.opus || pmt.common_pids.mp3) { - this.has_audio_ = true; - } + this.has_video_ = hasVideo; + this.has_audio_ = hasAudio; } } @@ -1198,7 +1248,7 @@ class TSDemuxer extends BaseDemuxer { } } - private parseADTSAACPayload(data: Uint8Array, pts: number) { + private parseADTSAACPayload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched @@ -1212,15 +1262,17 @@ class TSDemuxer extends BaseDemuxer { data = buf; } + let audio_metadata_ = this.audioTrackMetadata.get(pid); + let ref_sample_duration: number; let base_pts_ms: number; if (pts != undefined) { base_pts_ms = pts / this.timescale_; } - if (this.audio_metadata_.codec === 'aac') { + if (audio_metadata_ && audio_metadata_.codec === 'aac') { if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = 1024 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1024 / audio_metadata_.sampling_frequency * 1000; base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; } else if (pts == undefined){ Log.w(this.TAG, `AAC: Unknown pts`); @@ -1228,7 +1280,7 @@ class TSDemuxer extends BaseDemuxer { } if (this.aac_last_incomplete_data_ && this.audio_last_sample_pts_) { - ref_sample_duration = 1024 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1024 / audio_metadata_.sampling_frequency * 1000; let new_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; if (Math.abs(new_pts_ms - base_pts_ms) > 1) { @@ -1245,26 +1297,33 @@ class TSDemuxer extends BaseDemuxer { let last_sample_pts_ms: number; while ((aac_frame = adts_parser.readNextAACFrame()) != null) { + const meta = { + codec: 'aac', + audio_object_type: aac_frame.audio_object_type, + sampling_freq_index: aac_frame.sampling_freq_index, + sampling_frequency: aac_frame.sampling_frequency, + channel_config: aac_frame.channel_config + }; + ref_sample_duration = 1024 / aac_frame.sampling_frequency * 1000; const audio_sample = { codec: 'aac', data: aac_frame } as const; + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + continue; + } + if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'aac', - audio_object_type: aac_frame.audio_object_type, - sampling_freq_index: aac_frame.sampling_freq_index, - sampling_frequency: aac_frame.sampling_frequency, - channel_config: aac_frame.channel_config - }; - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + this.dispatchAudioInitSegment(audio_sample, meta, pid); } last_sample_pts_ms = sample_pts_ms; @@ -1291,7 +1350,7 @@ class TSDemuxer extends BaseDemuxer { } } - private parseLOASAACPayload(data: Uint8Array, pts: number) { + private parseLOASAACPayload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched @@ -1305,15 +1364,17 @@ class TSDemuxer extends BaseDemuxer { data = buf; } + let audio_metadata_ = this.audioTrackMetadata.get(pid); + let ref_sample_duration: number; let base_pts_ms: number; if (pts != undefined) { base_pts_ms = pts / this.timescale_; } - if (this.audio_metadata_.codec === 'aac') { + if (audio_metadata_ && audio_metadata_.codec === 'aac') { if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = 1024 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1024 / audio_metadata_.sampling_frequency * 1000; base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; } else if (pts == undefined){ Log.w(this.TAG, `AAC: Unknown pts`); @@ -1321,7 +1382,7 @@ class TSDemuxer extends BaseDemuxer { } if (this.aac_last_incomplete_data_ && this.audio_last_sample_pts_) { - ref_sample_duration = 1024 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1024 / audio_metadata_.sampling_frequency * 1000; let new_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; if (Math.abs(new_pts_ms - base_pts_ms) > 1) { @@ -1338,6 +1399,14 @@ class TSDemuxer extends BaseDemuxer { let last_sample_pts_ms: number; while ((aac_frame = loas_parser.readNextAACFrame(this.loas_previous_frame ?? undefined)) != null) { + const meta = { + codec: 'aac', + audio_object_type: aac_frame.audio_object_type, + sampling_freq_index: aac_frame.sampling_freq_index, + sampling_frequency: aac_frame.sampling_frequency, + channel_config: aac_frame.channel_config + }; + this.loas_previous_frame = aac_frame; ref_sample_duration = 1024 / aac_frame.sampling_frequency * 1000; const audio_sample = { @@ -1345,20 +1414,19 @@ class TSDemuxer extends BaseDemuxer { data: aac_frame } as const; + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + continue; + } + if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'aac', - audio_object_type: aac_frame.audio_object_type, - sampling_freq_index: aac_frame.sampling_freq_index, - sampling_frequency: aac_frame.sampling_frequency, - channel_config: aac_frame.channel_config - }; - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + this.dispatchAudioInitSegment(audio_sample, meta, pid); } last_sample_pts_ms = sample_pts_ms; @@ -1385,13 +1453,15 @@ class TSDemuxer extends BaseDemuxer { } } - private parseAC3Payload(data: Uint8Array, pts: number) { + private parseAC3Payload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched return; } + let audio_metadata_ = this.audioTrackMetadata.get(pid); + let ref_sample_duration: number; let base_pts_ms: number; @@ -1399,9 +1469,9 @@ class TSDemuxer extends BaseDemuxer { base_pts_ms = pts / this.timescale_; } - if (this.audio_metadata_.codec === 'ac-3') { + if (audio_metadata_ && audio_metadata_.codec === 'ac-3') { if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = 1536 / this.audio_metadata_.sampling_frequency * 1000; + ref_sample_duration = 1536 / audio_metadata_.sampling_frequency * 1000; base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; } else if (pts == undefined){ Log.w(this.TAG, `AC3: Unknown pts`); @@ -1415,27 +1485,34 @@ class TSDemuxer extends BaseDemuxer { let last_sample_pts_ms: number; while ((ac3_frame = adts_parser.readNextAC3Frame()) != null) { + const meta = { + codec: 'ac-3', + sampling_frequency: ac3_frame.sampling_frequency, + bit_stream_identification: ac3_frame.bit_stream_identification, + bit_stream_mode: ac3_frame.bit_stream_mode, + low_frequency_effects_channel_on: ac3_frame.low_frequency_effects_channel_on, + channel_mode: ac3_frame.channel_mode, + }; + ref_sample_duration = 1536 / ac3_frame.sampling_frequency * 1000; const audio_sample = { codec: 'ac-3', data: ac3_frame } as const; + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + continue; + } + if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'ac-3', - sampling_frequency: ac3_frame.sampling_frequency, - bit_stream_identification: ac3_frame.bit_stream_identification, - bit_stream_mode: ac3_frame.bit_stream_mode, - low_frequency_effects_channel_on: ac3_frame.low_frequency_effects_channel_on, - channel_mode: ac3_frame.channel_mode, - }; - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + this.dispatchAudioInitSegment(audio_sample, meta, pid); } last_sample_pts_ms = sample_pts_ms; @@ -1459,7 +1536,7 @@ class TSDemuxer extends BaseDemuxer { } } - private parseEAC3Payload(data: Uint8Array, pts: number) { + private parseEAC3Payload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched @@ -1469,47 +1546,54 @@ class TSDemuxer extends BaseDemuxer { let ref_sample_duration: number; let base_pts_ms: number; - if (pts != undefined) { - base_pts_ms = pts / this.timescale_; - } + let audio_metadata_ = this.audioTrackMetadata.get(pid); - if (this.audio_metadata_.codec === 'ec-3') { + let adts_parser = new EAC3Parser(data); + let eac3_frame: EAC3Frame = null; + let sample_pts_ms = base_pts_ms; + let last_sample_pts_ms: number; + + if (audio_metadata_ && audio_metadata_.codec === 'ec-3') { if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = (256 * this.audio_metadata_.num_blks) / this.audio_metadata_.sampling_frequency * 1000; // TODO: AEC3 BLK + ref_sample_duration = (256 * audio_metadata_.num_blks) / audio_metadata_.sampling_frequency * 1000; // TODO: AEC3 BLK base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; } else if (pts == undefined){ Log.w(this.TAG, `EAC3: Unknown pts`); return; } } - - let adts_parser = new EAC3Parser(data); - let eac3_frame: EAC3Frame = null; - let sample_pts_ms = base_pts_ms; - let last_sample_pts_ms: number; + if (pts != undefined) { + base_pts_ms = pts / this.timescale_; + } while ((eac3_frame = adts_parser.readNextEAC3Frame()) != null) { + const meta = { + codec: 'ec-3', + sampling_frequency: eac3_frame.sampling_frequency, + bit_stream_identification: eac3_frame.bit_stream_identification, + low_frequency_effects_channel_on: eac3_frame.low_frequency_effects_channel_on, + num_blks: eac3_frame.num_blks, + channel_mode: eac3_frame.channel_mode, + }; + ref_sample_duration = 1536 / eac3_frame.sampling_frequency * 1000; // TODO: EAC3 BLK const audio_sample = { codec: 'ec-3', data: eac3_frame } as const; + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } + if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'ec-3', - sampling_frequency: eac3_frame.sampling_frequency, - bit_stream_identification: eac3_frame.bit_stream_identification, - low_frequency_effects_channel_on: eac3_frame.low_frequency_effects_channel_on, - num_blks: eac3_frame.num_blks, - channel_mode: eac3_frame.channel_mode, - }; - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + this.dispatchAudioInitSegment(audio_sample, meta, pid); } last_sample_pts_ms = sample_pts_ms; @@ -1533,12 +1617,16 @@ class TSDemuxer extends BaseDemuxer { } } - private parseOpusPayload(data: Uint8Array, pts: number) { + private parseOpusPayload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched return; } + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, discard it + return; + } let ref_sample_duration: number; let base_pts_ms: number; @@ -1546,14 +1634,13 @@ class TSDemuxer extends BaseDemuxer { if (pts != undefined) { base_pts_ms = pts / this.timescale_; } - if (this.audio_metadata_.codec === 'opus') { - if (pts == undefined && this.audio_last_sample_pts_ != undefined) { - ref_sample_duration = 20; - base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; - } else if (pts == undefined){ - Log.w(this.TAG, `Opus: Unknown pts`); - return; - } + + if (pts == undefined && this.audio_last_sample_pts_ != undefined) { + ref_sample_duration = 20; + base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; + } else if (pts == undefined){ + Log.w(this.TAG, `Opus: Unknown pts`); + return; } let sample_pts_ms = base_pts_ms; @@ -1568,8 +1655,8 @@ class TSDemuxer extends BaseDemuxer { let size = 0; while (data[index] === 0xFF) { - size += 255; - index += 1; + size += 255; + index += 1; } size += data[index]; index += 1; @@ -1598,7 +1685,7 @@ class TSDemuxer extends BaseDemuxer { } } - private parseMP3Payload(data: Uint8Array, pts: number) { + private parseMP3Payload(data: Uint8Array, pts: number, pid: number) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, // Wait for first IDR frame and video init segment being dispatched @@ -1657,6 +1744,8 @@ class TSDemuxer extends BaseDemuxer { break; } + let audio_metadata_ = this.audioTrackMetadata.get(pid); + const sample = new MP3Data(); sample.object_type = object_type; sample.sample_rate = sample_rate; @@ -1667,20 +1756,26 @@ class TSDemuxer extends BaseDemuxer { data: sample } as const; + const meta = { + codec: 'mp3', + object_type, + sample_rate, + channel_count + }; + + if (this.currentAudioTrackPid !== pid) { + // If not current Audio Track, memoized and discard it + this.dispatchAudioInitSegment(audio_sample, meta, pid); + return; + } if (this.audio_init_segment_dispatched_ == false) { - this.audio_metadata_ = { - codec: 'mp3', - object_type, - sample_rate, - channel_count - } - this.dispatchAudioInitSegment(audio_sample); - } else if (this.detectAudioMetadataChange(audio_sample)) { + this.dispatchAudioInitSegment(audio_sample, meta, pid); + } else if (audio_metadata_ == null || this.detectAudioMetadataChange(audio_sample, audio_metadata_, pid)) { // flush stashed frames before notify new AudioSpecificConfig this.dispatchAudioMediaSegment(); - // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(audio_sample); + // notify new MP3 AudioSpecificConfig + this.dispatchAudioInitSegment(audio_sample, meta, pid); } let mp3_sample = { @@ -1693,94 +1788,98 @@ class TSDemuxer extends BaseDemuxer { this.audio_track_.length += data.byteLength; } - private detectAudioMetadataChange(sample: AudioData): boolean { - if (sample.codec !== this.audio_metadata_.codec) { + private detectAudioMetadataChange(sample: AudioData, previous_audio_metadata_: any, pid: number): boolean { + if (this.currentAudioTrackPid !== pid) { + return false; + } + + if (sample.codec !== previous_audio_metadata_.codec) { Log.v(this.TAG, `Audio: Audio Codecs changed from ` + - `${this.audio_metadata_.codec} to ${sample.codec}`); + `${previous_audio_metadata_.codec} to ${sample.codec}`); return true; } - if (sample.codec === 'aac' && this.audio_metadata_.codec === 'aac') { + if (sample.codec === 'aac' && previous_audio_metadata_.codec === 'aac') { const frame = sample.data; - if (frame.audio_object_type !== this.audio_metadata_.audio_object_type) { + if (frame.audio_object_type !== previous_audio_metadata_.audio_object_type) { Log.v(this.TAG, `AAC: AudioObjectType changed from ` + - `${this.audio_metadata_.audio_object_type} to ${frame.audio_object_type}`); + `${previous_audio_metadata_.audio_object_type} to ${frame.audio_object_type}`); return true; } - if (frame.sampling_freq_index !== this.audio_metadata_.sampling_freq_index) { + if (frame.sampling_freq_index !== previous_audio_metadata_.sampling_freq_index) { Log.v(this.TAG, `AAC: SamplingFrequencyIndex changed from ` + - `${this.audio_metadata_.sampling_freq_index} to ${frame.sampling_freq_index}`); + `${previous_audio_metadata_.sampling_freq_index} to ${frame.sampling_freq_index}`); return true; } - if (frame.channel_config !== this.audio_metadata_.channel_config) { + if (frame.channel_config !== previous_audio_metadata_.channel_config) { Log.v(this.TAG, `AAC: Channel configuration changed from ` + - `${this.audio_metadata_.channel_config} to ${frame.channel_config}`); + `${previous_audio_metadata_.channel_config} to ${frame.channel_config}`); return true; } - } else if (sample.codec === 'ac-3' && this.audio_metadata_.codec === 'ac-3') { + } else if (sample.codec === 'ac-3' && previous_audio_metadata_.codec === 'ac-3') { const frame = sample.data; - if (frame.sampling_frequency !== this.audio_metadata_.sampling_frequency) { + if (frame.sampling_frequency !== previous_audio_metadata_.sampling_frequency) { Log.v(this.TAG, `AC3: Sampling Frequency changed from ` + - `${this.audio_metadata_.sampling_frequency} to ${frame.sampling_frequency}`); + `${previous_audio_metadata_.sampling_frequency} to ${frame.sampling_frequency}`); return true; } - if (frame.bit_stream_identification !== this.audio_metadata_.bit_stream_identification) { + if (frame.bit_stream_identification !== previous_audio_metadata_.bit_stream_identification) { Log.v(this.TAG, `AC3: Bit Stream Identification changed from ` + - `${this.audio_metadata_.bit_stream_identification} to ${frame.bit_stream_identification}`); + `${previous_audio_metadata_.bit_stream_identification} to ${frame.bit_stream_identification}`); return true; } - if (frame.bit_stream_mode !== this.audio_metadata_.bit_stream_mode) { + if (frame.bit_stream_mode !== previous_audio_metadata_.bit_stream_mode) { Log.v(this.TAG, `AC3: BitStream Mode changed from ` + - `${this.audio_metadata_.bit_stream_mode} to ${frame.bit_stream_mode}`); + `${previous_audio_metadata_.bit_stream_mode} to ${frame.bit_stream_mode}`); return true; } - if (frame.channel_mode !== this.audio_metadata_.channel_mode) { + if (frame.channel_mode !== previous_audio_metadata_.channel_mode) { Log.v(this.TAG, `AC3: Channel Mode changed from ` + - `${this.audio_metadata_.channel_mode} to ${frame.channel_mode}`); + `${previous_audio_metadata_.channel_mode} to ${frame.channel_mode}`); return true; } - if (frame.low_frequency_effects_channel_on !== this.audio_metadata_.low_frequency_effects_channel_on) { + if (frame.low_frequency_effects_channel_on !== previous_audio_metadata_.low_frequency_effects_channel_on) { Log.v(this.TAG, `AC3: Low Frequency Effects Channel On changed from ` + - `${this.audio_metadata_.low_frequency_effects_channel_on} to ${frame.low_frequency_effects_channel_on}`); + `${previous_audio_metadata_.low_frequency_effects_channel_on} to ${frame.low_frequency_effects_channel_on}`); return true; } - } else if (sample.codec === 'opus' && this.audio_metadata_.codec === 'opus') { + } else if (sample.codec === 'opus' && previous_audio_metadata_.codec === 'opus') { const data = sample.meta; - if (data.sample_rate !== this.audio_metadata_.sample_rate) { + if (data.sample_rate !== previous_audio_metadata_.sample_rate) { Log.v(this.TAG, `Opus: SamplingFrequencyIndex changed from ` + - `${this.audio_metadata_.sample_rate} to ${data.sample_rate}`); + `${previous_audio_metadata_.sample_rate} to ${data.sample_rate}`); return true; } - if (data.channel_count !== this.audio_metadata_.channel_count) { + if (data.channel_count !== previous_audio_metadata_.channel_count) { Log.v(this.TAG, `Opus: Channel count changed from ` + - `${this.audio_metadata_.channel_count} to ${data.channel_count}`); + `${previous_audio_metadata_.channel_count} to ${data.channel_count}`); return true; } - } else if (sample.codec === 'mp3' && this.audio_metadata_.codec === 'mp3') { + } else if (sample.codec === 'mp3' && previous_audio_metadata_.codec === 'mp3') { const data = sample.data; - if (data.object_type !== this.audio_metadata_.object_type) { + if (data.object_type !== previous_audio_metadata_.object_type) { Log.v(this.TAG, `MP3: AudioObjectType changed from ` + - `${this.audio_metadata_.object_type} to ${data.object_type}`); + `${previous_audio_metadata_.object_type} to ${data.object_type}`); return true; } - if (data.sample_rate !== this.audio_metadata_.sample_rate) { + if (data.sample_rate !== previous_audio_metadata_.sample_rate) { Log.v(this.TAG, `MP3: SamplingFrequencyIndex changed from ` + - `${this.audio_metadata_.sample_rate} to ${data.sample_rate}`); + `${previous_audio_metadata_.sample_rate} to ${data.sample_rate}`); return true; } - if (data.channel_count !== this.audio_metadata_.channel_count) { + if (data.channel_count !== previous_audio_metadata_.channel_count) { Log.v(this.TAG, `MP3: Channel count changed from ` + - `${this.audio_metadata_.channel_count} to ${data.channel_count}`); + `${previous_audio_metadata_.channel_count} to ${data.channel_count}`); return true; } } @@ -1788,14 +1887,14 @@ class TSDemuxer extends BaseDemuxer { return false; } - private dispatchAudioInitSegment(sample: AudioData) { + private dispatchAudioInitSegment(sample: AudioData, audio_metadata_: any, pid: number) { let meta: any = {}; meta.type = 'audio'; meta.id = this.audio_track_.id; meta.timescale = 1000; meta.duration = this.duration_; - if (this.audio_metadata_.codec === 'aac') { + if (audio_metadata_.codec === 'aac') { let aac_frame = sample.codec === 'aac' ? sample.data : null; let audio_specific_config = new AudioSpecificConfig(aac_frame); @@ -1805,7 +1904,7 @@ class TSDemuxer extends BaseDemuxer { meta.originalCodec = audio_specific_config.original_codec_mimetype; meta.config = audio_specific_config.config; meta.refSampleDuration = 1024 / meta.audioSampleRate * meta.timescale; - } else if (this.audio_metadata_.codec === 'ac-3') { + } else if (audio_metadata_.codec === 'ac-3') { let ac3_frame = sample.codec === 'ac-3' ? sample.data : null; let ac3_config = new AC3Config(ac3_frame); meta.audioSampleRate = ac3_config.sampling_rate @@ -1814,7 +1913,7 @@ class TSDemuxer extends BaseDemuxer { meta.originalCodec = ac3_config.original_codec_mimetype; meta.config = ac3_config.config; meta.refSampleDuration = 1536 / meta.audioSampleRate * meta.timescale; - } else if (this.audio_metadata_.codec === 'ec-3') { + } else if (audio_metadata_.codec === 'ec-3') { let ec3_frame = sample.codec === 'ec-3' ? sample.data : null; let ec3_config = new EAC3Config(ec3_frame); meta.audioSampleRate = ec3_config.sampling_rate @@ -1823,29 +1922,35 @@ class TSDemuxer extends BaseDemuxer { meta.originalCodec = ec3_config.original_codec_mimetype; meta.config = ec3_config.config; meta.refSampleDuration = (256 * ec3_config.num_blks) / meta.audioSampleRate * meta.timescale; // TODO: blk size - } else if (this.audio_metadata_.codec === 'opus') { - meta.audioSampleRate = this.audio_metadata_.sample_rate; - meta.channelCount = this.audio_metadata_.channel_count; - meta.channelConfigCode = this.audio_metadata_.channel_config_code; + } else if (audio_metadata_.codec === 'opus') { + meta.audioSampleRate = audio_metadata_.sample_rate; + meta.channelCount = audio_metadata_.channel_count; + meta.channelConfigCode = audio_metadata_.channel_config_code; meta.codec = 'opus'; meta.originalCodec = 'opus'; meta.config = undefined; meta.refSampleDuration = 20; - } else if (this.audio_metadata_.codec === 'mp3') { - meta.audioSampleRate = this.audio_metadata_.sample_rate; - meta.channelCount = this.audio_metadata_.channel_count; + } else if (audio_metadata_.codec === 'mp3') { + meta.audioSampleRate = audio_metadata_.sample_rate; + meta.channelCount = audio_metadata_.channel_count; meta.codec = 'mp3'; meta.originalCodec = 'mp3'; meta.config = undefined; } + this.audioTrackInitSegments.set(pid, { ... sample }); + this.audioTrackMetadata.set(pid, { ... audio_metadata_ }); + if (this.currentAudioTrackPid !== pid) { + return; + } + if (this.audio_init_segment_dispatched_ == false) { Log.v(this.TAG, `Generated first AudioSpecificConfig for mimeType: ${meta.codec}`); } this.onTrackMetadata('audio', meta); this.audio_init_segment_dispatched_ = true; - this.video_metadata_changed_ = false; + this.audio_metadata_changed_ = false; // notify new MediaInfo let mi = this.media_info_;