diff --git a/Moblin/RtmpServer/RtmpServer.swift b/Moblin/RtmpServer/RtmpServer.swift index 14358b922..0782eabc3 100644 --- a/Moblin/RtmpServer/RtmpServer.swift +++ b/Moblin/RtmpServer/RtmpServer.swift @@ -1,3 +1,4 @@ +import AVFAudio import CoreMedia import Foundation import HaishinKit @@ -21,6 +22,7 @@ class RtmpServer { var onPublishStart: (String) -> Void var onPublishStop: (String) -> Void var onFrame: (String, CMSampleBuffer) -> Void + var onAudioBuffer: (String, AVAudioPCMBuffer) -> Void var settings: SettingsRtmpServer private var periodicTimer: DispatchSourceTimer? var totalBytesReceived: UInt64 = 0 @@ -29,12 +31,14 @@ class RtmpServer { init(settings: SettingsRtmpServer, onPublishStart: @escaping (String) -> Void, onPublishStop: @escaping (String) -> Void, - onFrame: @escaping (String, CMSampleBuffer) -> Void) + onFrame: @escaping (String, CMSampleBuffer) -> Void, + onAudioBuffer: @escaping (String, AVAudioPCMBuffer) -> Void) { self.settings = settings self.onPublishStart = onPublishStart self.onPublishStop = onPublishStop self.onFrame = onFrame + self.onAudioBuffer = onAudioBuffer clients = [] } diff --git a/Moblin/RtmpServer/RtmpServerChunkStream.swift b/Moblin/RtmpServer/RtmpServerChunkStream.swift index dfb52c3f2..84ab1656b 100644 --- a/Moblin/RtmpServer/RtmpServerChunkStream.swift +++ b/Moblin/RtmpServer/RtmpServerChunkStream.swift @@ -28,9 +28,12 @@ class RtmpServerChunkStream: VideoCodecDelegate { private var videoTimestampZero: Double private var videoTimestamp: Double private var formatDescription: CMVideoFormatDescription? - private var videoCodec: VideoCodec? + private var videoDecoder: VideoCodec? private var numberOfFrames: UInt64 = 0 private var videoCodecLockQueue = DispatchQueue(label: "com.eerimoq.Moblin.VideoCodec") + private var audioBuffer: AVAudioCompressedBuffer? + private var audioDecoder: AVAudioConverter? + private var pcmAudioFormat: AVAudioFormat? init(client: RtmpServerClient, streamId: UInt16) { self.client = client @@ -49,8 +52,8 @@ class RtmpServerChunkStream: VideoCodecDelegate { } func stop() { - videoCodec?.stopRunning() - videoCodec = nil + videoDecoder?.stopRunning() + videoDecoder = nil client = nil } @@ -324,20 +327,95 @@ class RtmpServerChunkStream: VideoCodecDelegate { PTS: \(timing.presentationTimeStamp.seconds), \ DTS: \(timing.decodeTimeStamp.seconds) """) */ - switch messageData[1] { - case FLVAACPacketType.seq.rawValue: - if let config = - AudioSpecificConfig(bytes: [UInt8](messageData[codec.headerSize ..< messageData.count])) - { - logger.info("rtmp-server: client: \(config.audioStreamBasicDescription())") - } - case FLVAACPacketType.raw.rawValue: - break + switch FLVAACPacketType(rawValue: messageData[1]) { + case .seq: + processMessageAudioTypeSeq(client: client, codec: codec) + case .raw: + processMessageAudioTypeRaw(client: client, codec: codec) default: break } } + private func processMessageAudioTypeSeq(client _: RtmpServerClient, codec: FLVAudioCodec) { + if let config = + AudioSpecificConfig(bytes: [UInt8](messageData[codec.headerSize ..< messageData.count])) + { + var streamDescription = config.audioStreamBasicDescription() + logger.info("rtmp-server: client: \(streamDescription)") + if let audioFormat = AVAudioFormat(streamDescription: &streamDescription) { + logger.info("rtmp-server: client: \(audioFormat)") + audioBuffer = AVAudioCompressedBuffer( + format: audioFormat, + packetCapacity: 1, + maximumPacketSize: 1024 * Int(audioFormat.channelCount) + ) + pcmAudioFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: audioFormat.sampleRate, + channels: audioFormat.channelCount, + interleaved: audioFormat.isInterleaved + ) + guard let pcmAudioFormat else { + logger.info("rtmp-server: client: Failed to create PCM audio format") + return + } + audioDecoder = AVAudioConverter(from: audioFormat, to: pcmAudioFormat) + guard let audioDecoder else { + logger.info("rtmp-server: client: Failed to create audio decdoer") + return + } + } else { + logger.info("rtmp-server: client: Failed to create audio format") + audioBuffer = nil + audioDecoder = nil + } + } + } + + private func processMessageAudioTypeRaw(client _: RtmpServerClient, codec: FLVAudioCodec) { + guard let audioBuffer else { + return + } + let length = messageData.count - codec.headerSize + messageData.withUnsafeBytes { (buffer: UnsafeRawBufferPointer) in + guard let baseAddress = buffer.baseAddress else { + return + } + audioBuffer.packetDescriptions?.pointee = AudioStreamPacketDescription( + mStartOffset: 0, + mVariableFramesInPacket: 0, + mDataByteSize: UInt32(length) + ) + audioBuffer.packetCount = 1 + audioBuffer.byteLength = UInt32(length) + audioBuffer.data.copyMemory(from: baseAddress.advanced(by: codec.headerSize), byteCount: length) + } + guard let audioDecoder, let pcmAudioFormat else { + return + } + var error: NSError? + guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: pcmAudioFormat, frameCapacity: 1024) else { + return + } + let outputStatus = audioDecoder.convert(to: outputBuffer, error: &error) { _, inputStatus in + inputStatus.pointee = .haveData + return self.audioBuffer + } + switch outputStatus { + case .haveData: + client?.handleAudioBuffer(audioBuffer: outputBuffer) + case .error: + if let error { + logger.info("rtmp-server: client: Error \(error)") + } else { + logger.info("rtmp-server: client: Unknown error") + } + default: + logger.info("rtmp-server: client: Output status \(outputStatus.rawValue)") + } + } + private func processMessageVideo() { guard let client else { return @@ -381,17 +459,17 @@ class RtmpServerChunkStream: VideoCodecDelegate { ) return } - guard videoCodec == nil else { + guard videoDecoder == nil else { return } var config = AVCDecoderConfigurationRecord() config.data = messageData.subdata(in: FLVTagType.video.headerSize ..< messageData.count) let status = config.makeFormatDescription(&formatDescription) if status == noErr { - videoCodec = VideoCodec(lockQueue: videoCodecLockQueue) - videoCodec!.formatDescription = formatDescription - videoCodec!.delegate = self - videoCodec!.startRunning() + videoDecoder = VideoCodec(lockQueue: videoCodecLockQueue) + videoDecoder!.formatDescription = formatDescription + videoDecoder!.delegate = self + videoDecoder!.startRunning() } else { client.stopInternal(reason: "Format description error \(status)") } @@ -403,7 +481,7 @@ class RtmpServerChunkStream: VideoCodecDelegate { return } if let sampleBuffer = makeSampleBuffer(client: client) { - videoCodec?.appendSampleBuffer(sampleBuffer) + videoDecoder?.appendSampleBuffer(sampleBuffer) } else { client.stopInternal(reason: "Make sample buffer failed") } diff --git a/Moblin/RtmpServer/RtmpServerClient.swift b/Moblin/RtmpServer/RtmpServerClient.swift index d45e2ee4d..7fc0d83fc 100644 --- a/Moblin/RtmpServer/RtmpServerClient.swift +++ b/Moblin/RtmpServer/RtmpServerClient.swift @@ -1,3 +1,4 @@ +import AVFAudio import CoreMedia import Foundation import HaishinKit @@ -88,6 +89,10 @@ class RtmpServerClient { server?.onFrame(streamKey, sampleBuffer) } + func handleAudioBuffer(audioBuffer: AVAudioPCMBuffer) { + server?.onAudioBuffer(streamKey, audioBuffer) + } + private func handleData(data: Data) { switch state { case .uninitialized: diff --git a/Moblin/Various/Media.swift b/Moblin/Various/Media.swift index b20b1ede4..665ad3da9 100644 --- a/Moblin/Various/Media.swift +++ b/Moblin/Various/Media.swift @@ -516,6 +516,11 @@ final class Media: NSObject { netStream.addReplaceVideoSampleBuffer(id: cameraId, sampleBuffer) } + func addRtmpAudioBuffer(cameraId _: UUID, audioBuffer _: AVAudioPCMBuffer) { + // logger.info("RTMP audio buffer \(audioBuffer)") + // netStream.addReplaceVideoSampleBuffer(id: cameraId, sampleBuffer) + } + func addRtmpCamera(cameraId: UUID, latency: Double) { netStream.addReplaceVideo(cameraId: cameraId, latency: latency) } diff --git a/Moblin/Various/Model.swift b/Moblin/Various/Model.swift index 88e891800..66a50f97a 100644 --- a/Moblin/Various/Model.swift +++ b/Moblin/Various/Model.swift @@ -1072,7 +1072,8 @@ final class Model: NSObject, ObservableObject { rtmpServer = RtmpServer(settings: database.rtmpServer!.clone(), onPublishStart: handleRtmpServerPublishStart, onPublishStop: handleRtmpServerPublishStop, - onFrame: handleRtmpServerFrame) + onFrame: handleRtmpServerFrame, + onAudioBuffer: handleRtmpServerAudioBuffer) rtmpServer!.start() } } @@ -1106,6 +1107,13 @@ final class Model: NSObject, ObservableObject { media.addRtmpSampleBuffer(cameraId: cameraId, sampleBuffer: sampleBuffer) } + func handleRtmpServerAudioBuffer(streamKey: String, audioBuffer: AVAudioPCMBuffer) { + guard let cameraId = getRtmpStream(streamKey: streamKey)?.id else { + return + } + media.addRtmpAudioBuffer(cameraId: cameraId, audioBuffer: audioBuffer) + } + private func listCameras(position: AVCaptureDevice.Position) -> [Camera] { var deviceTypes: [AVCaptureDevice.DeviceType] = [ .builtInTripleCamera,