fix(core): fix to remove the VPS, SPS and PPS prefix of each key fram…

…e (for AVC and HEVC)

core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/MediaCodecEncoder.kt

@@ -27,8 +27,6 @@ import io.github.thibaultbee.streampack.internal.data.Frame
 import io.github.thibaultbee.streampack.internal.events.EventHandler
 import io.github.thibaultbee.streampack.internal.utils.extensions.isAudio
 import io.github.thibaultbee.streampack.internal.utils.extensions.slices
-import io.github.thibaultbee.streampack.internal.utils.extensions.startsWith
-import io.github.thibaultbee.streampack.internal.utils.extensions.toByteArray
 import io.github.thibaultbee.streampack.logger.Logger
 import io.github.thibaultbee.streampack.utils.TAG
 import java.nio.ByteBuffer
@@ -81,18 +79,8 @@ abstract class MediaCodecEncoder<T : Config>(
                                 null
                             }
 
-                            // Remove startCode + sps + startCode + pps
-                            var frameBuffer = buffer
-                            extra?.let { it ->
-                                var prefix = ByteArray(0)
-                                it.forEach { csd -> prefix = prefix.plus(csd.toByteArray()) }
-                                if (buffer.startsWith(prefix)) {
-                                    buffer.position(prefix.size)
-                                    frameBuffer = buffer.slice()
-                                }
-                            }
                             Frame(
-                                frameBuffer,
+                                processBuffer(buffer, extra),
                                 mimeType,
                                 info.presentationTimeUs, // pts
                                 null, // dts
@@ -259,10 +247,21 @@ abstract class MediaCodecEncoder<T : Config>(
         mediaCodec = null
     }
 
+    /**
+     * Process buffer before sending it to [IEncoderListener.onOutputFrame].
+     * Use it to remove headers from the frame.
+     *
+     * @param buffer buffer to process
+     * @param extra extra buffers (SPS, PPS, VPS,...)
+     * @return processed buffer
+     */
+    protected open fun processBuffer(buffer: ByteBuffer, extra: List<ByteBuffer>?): ByteBuffer {
+        return buffer
+    }
+
     private fun generateExtra(format: MediaFormat): List<ByteBuffer> {
         val extra = mutableListOf<ByteBuffer>()
 
-
         format.getByteBuffer("csd-0")?.let {
             /**
              * For HEVC, vps, sps amd pps are all in csd-0.

core/src/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt

@@ -29,7 +29,9 @@ import io.github.thibaultbee.streampack.internal.gl.EGlSurface
 import io.github.thibaultbee.streampack.internal.gl.FullFrameRect
 import io.github.thibaultbee.streampack.internal.gl.Texture2DProgram
 import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider
+import io.github.thibaultbee.streampack.internal.utils.extensions.removePrefixes
 import io.github.thibaultbee.streampack.listeners.OnErrorListener
+import java.nio.ByteBuffer
 import java.util.concurrent.Executors
 
 /**
@@ -103,6 +105,15 @@ class VideoMediaCodecEncoder(
         super.stopStream()
     }
 
+    override fun processBuffer(buffer: ByteBuffer, extra: List<ByteBuffer>?): ByteBuffer {
+        // Remove startCode + sps + startCode + pps
+        return if (extra != null) {
+            buffer.removePrefixes(extra)
+        } else {
+            buffer
+        }
+    }
+
     val inputSurface: Surface?
         get() = codecSurface?.inputSurface
 

core/src/main/java/io/github/thibaultbee/streampack/internal/utils/av/video/hevc/HEVCDecoderConfigurationRecord.kt

@@ -24,6 +24,7 @@ import io.github.thibaultbee.streampack.internal.utils.extensions.shl
 import io.github.thibaultbee.streampack.internal.utils.extensions.shr
 import io.github.thibaultbee.streampack.internal.utils.extensions.startCodeSize
 import java.nio.ByteBuffer
+import kotlin.experimental.and
 
 data class HEVCDecoderConfigurationRecord(
     private val configurationVersion: Int = 0x01,
@@ -69,9 +70,9 @@ data class HEVCDecoderConfigurationRecord(
         buffer.putShort(averageFrameRate) // avgFrameRate
         buffer.put(
             (constantFrameRate shl 6)
-                    or (numTemporalLayers shl 3)
+                    or ((numTemporalLayers and 0x7) shl 3)
                     or (temporalIdNested shl 2)
-                    or lengthSizeMinusOne.toInt()
+                    or (lengthSizeMinusOne.toInt() and 0x3)
         ) // constantFrameRate 2 bits = 1 for stable / numTemporalLayers 3 bits /  temporalIdNested 1 bit / lengthSizeMinusOne 2 bits
 
         buffer.put(parameterSets.size) // numOfArrays

core/src/main/java/io/github/thibaultbee/streampack/internal/utils/extensions/ByteBufferExtensions.kt

@@ -131,7 +131,6 @@ fun ByteBuffer.indicesOf(prefix: ByteArray): List<Int> {
 
 /**
  * Get all [ByteBuffer] occurrences that start with [prefix].
- *
  */
 fun ByteBuffer.slices(prefix: ByteArray): List<ByteBuffer> {
     val slices = mutableListOf<Pair<Int, Int>>()
@@ -154,22 +153,49 @@ fun ByteBuffer.slices(prefix: ByteArray): List<ByteBuffer> {
 }
 
 /**
- * Check if [ByteBuffer] starts with a ByteArray [prefix].
+ * Whether the [ByteBuffer] starts with a [ByteArray] [prefix] from the current position.
  */
 fun ByteBuffer.startsWith(prefix: ByteArray): Boolean {
+    if (this.remaining() < prefix.size) {
+        return false
+    }
+
+    val position = this.position()
     for (i in prefix.indices) {
-        if (this.get(i) != prefix[i]) {
+        if (this.get(position + i) != prefix[i]) {
             return false
         }
     }
     return true
 }
 
 /**
- * Check if [ByteBuffer] starts with a String [prefix].
+ * Whether the [ByteBuffer] starts with a [String] [prefix] from the current position.
  */
 fun ByteBuffer.startWith(prefix: String) = startsWith(prefix.toByteArray())
 
+
+/**
+ * Whether the [ByteBuffer] starts with a [ByteBuffer] [prefix] from the current position.
+ */
+fun ByteBuffer.startWith(prefix: ByteBuffer) = startsWith(prefix.toByteArray())
+
+/**
+ * Whether the [ByteBuffer] starts with a list of [ByteBuffer] [prefixes] from the current position.
+ *
+ * @param prefixes [List] of [ByteBuffer]
+ * @return [Pair] of [Boolean] (whether the [ByteBuffer] start with the prefix) and [Int] that is the index of the prefix found (-1 if not found).
+ */
+fun ByteBuffer.startsWith(prefixes: List<ByteBuffer>): Pair<Boolean, Int> {
+    prefixes.forEachIndexed { index, byteBuffer ->
+        if (this.startWith(byteBuffer)) {
+            return Pair(true, index)
+        }
+    }
+    return Pair(false, -1)
+}
+
+
 /**
  * Returns ByteBuffer array even if [ByteBuffer.hasArray] returns false.
  *
@@ -243,3 +269,45 @@ fun ByteBuffer.extractRbsp(headerLength: Int): ByteBuffer {
     rbsp.rewind()
     return rbsp
 }
+
+/**
+ * Remove all [prefixes] from [ByteBuffer] whatever their order.
+ * It slices [ByteBuffer] so it does not copy data.
+ *
+ * Once a prefix is found, it is removed from the [prefixes] list.
+ *
+ * @param prefixes [List] of [ByteBuffer] to remove
+ * @return [ByteBuffer] without prefixes
+ */
+fun ByteBuffer.removePrefixes(prefixes: List<ByteBuffer>): ByteBuffer {
+    val mutablePrefixes = prefixes.toMutableList()
+    var hasPrefix = true
+    while (hasPrefix) {
+        val result = this.startsWith(mutablePrefixes)
+        hasPrefix = result.first
+        if (hasPrefix) {
+            this.position(this.position() + mutablePrefixes[result.second].limit())
+            mutablePrefixes.removeAt(result.second)
+        }
+    }
+
+    return this.slice()
+}
+
+/**
+ * Whether [ByteBuffer] is Annex B or not.
+ * Annex B frames start with a start code (0x00000001 or 0x000001).
+ */
+val ByteBuffer.isAnnexB: Boolean
+    get() = this.startCodeSize != 0
+
+
+/**
+ * Whether [ByteBuffer] is AVCC/HVCC or not.
+ * AVCC/HVCC frames start with a the frame size.
+ */
+val ByteBuffer.isAvcc: Boolean
+    get() {
+        val size = this.getInt(0)
+        return size == (this.remaining() - 4)
+    }