From a3aac4da75a31cce0dddd66746468101a1beb4c9 Mon Sep 17 00:00:00 2001 From: advplyr Date: Sun, 1 May 2022 12:19:31 -0500 Subject: [PATCH] Update: iOS AudioPlayer direct play with multi-track using AVQueuePlayer --- ios/App/Podfile | 14 +- ios/App/Shared/player/AudioPlayer.swift | 194 ++++++++++++++++------ ios/App/Shared/player/PlayerHandler.swift | 18 +- ios/App/Shared/util/ApiClient.swift | 3 +- 4 files changed, 162 insertions(+), 67 deletions(-) diff --git a/ios/App/Podfile b/ios/App/Podfile index 44119188..584acebb 100644 --- a/ios/App/Podfile +++ b/ios/App/Podfile @@ -9,13 +9,13 @@ install! 'cocoapods', :disable_input_output_paths => true def capacitor_pods pod 'Capacitor', :path => '../../node_modules/@capacitor/ios' pod 'CapacitorCordova', :path => '../../node_modules/@capacitor/ios' - pod 'CapacitorApp', :path => '..\..\node_modules\@capacitor\app' - pod 'CapacitorDialog', :path => '..\..\node_modules\@capacitor\dialog' - pod 'CapacitorHaptics', :path => '..\..\node_modules\@capacitor\haptics' - pod 'CapacitorNetwork', :path => '..\..\node_modules\@capacitor\network' - pod 'CapacitorStatusBar', :path => '..\..\node_modules\@capacitor\status-bar' - pod 'CapacitorStorage', :path => '..\..\node_modules\@capacitor\storage' - pod 'RobingenzCapacitorAppUpdate', :path => '..\..\node_modules\@robingenz\capacitor-app-update' + pod 'CapacitorApp', :path => '../../node_modules/@capacitor/app' + pod 'CapacitorDialog', :path => '../../node_modules/@capacitor/dialog' + pod 'CapacitorHaptics', :path => '../../node_modules/@capacitor/haptics' + pod 'CapacitorNetwork', :path => '../../node_modules/@capacitor/network' + pod 'CapacitorStatusBar', :path => '../../node_modules/@capacitor/status-bar' + pod 'CapacitorStorage', :path => '../../node_modules/@capacitor/storage' + pod 'RobingenzCapacitorAppUpdate', :path => '../../node_modules/@robingenz/capacitor-app-update' end target 'App' do diff --git a/ios/App/Shared/player/AudioPlayer.swift b/ios/App/Shared/player/AudioPlayer.swift index d088400e..823047ab 100644 --- a/ios/App/Shared/player/AudioPlayer.swift +++ b/ios/App/Shared/player/AudioPlayer.swift @@ -24,29 +24,25 @@ class AudioPlayer: NSObject { private var playWhenReady: Bool private var initialPlaybackRate: Float - private var audioPlayer: AVPlayer + private var audioPlayer: AVQueuePlayer private var playbackSession: PlaybackSession - private var activeAudioTrack: AudioTrack + + private var queueObserver:NSKeyValueObservation? + private var queueItemStatusObserver:NSKeyValueObservation? + + private var currentTrackIndex = 0 + private var allPlayerItems:[AVPlayerItem] = [] // MARK: - Constructor init(playbackSession: PlaybackSession, playWhenReady: Bool = false, playbackRate: Float = 1) { self.playWhenReady = playWhenReady self.initialPlaybackRate = playbackRate - self.audioPlayer = AVPlayer() + self.audioPlayer = AVQueuePlayer() self.playbackSession = playbackSession self.status = -1 self.rate = 0.0 self.tmpRate = playbackRate - if playbackSession.audioTracks.count != 1 || playbackSession.audioTracks[0].mimeType != "application/vnd.apple.mpegurl" { - NSLog("The player only support HLS streams right now") - self.activeAudioTrack = AudioTrack(index: 0, startOffset: -1, duration: -1, title: "", contentUrl: nil, mimeType: "", metadata: nil, serverIndex: 0) - - super.init() - return - } - self.activeAudioTrack = playbackSession.audioTracks[0] - super.init() initAudioSession() @@ -56,14 +52,29 @@ class AudioPlayer: NSObject { self.audioPlayer.addObserver(self, forKeyPath: #keyPath(AVPlayer.rate), options: .new, context: &playerContext) self.audioPlayer.addObserver(self, forKeyPath: #keyPath(AVPlayer.currentItem), options: .new, context: &playerContext) - let playerItem = AVPlayerItem(asset: createAsset()) - playerItem.addObserver(self, forKeyPath: #keyPath(AVPlayerItem.status), options: .new, context: &playerItemContext) + for track in playbackSession.audioTracks { + let playerItem = AVPlayerItem(asset: createAsset(itemId: playbackSession.libraryItemId!, track: track)) + self.allPlayerItems.append(playerItem) + } - self.audioPlayer.replaceCurrentItem(with: playerItem) + self.currentTrackIndex = getItemIndexForTime(time: playbackSession.currentTime) + NSLog("TEST: Starting track index \(self.currentTrackIndex) for start time \(playbackSession.currentTime)") + let playerItems = self.allPlayerItems[self.currentTrackIndex.. Int { + for index in 0.. 0.0 + + public func seek(_ to: Double, from:String) { + let continuePlaying = rate > 0.0 pause() - self.audioPlayer.seek(to: CMTime(seconds: to, preferredTimescale: 1000)) { completed in - if !completed { - NSLog("WARNING: seeking not completed (to \(to)") + + NSLog("TEST: Seek to \(to) from \(from)") + + let currentTrack = self.playbackSession.audioTracks[self.currentTrackIndex] + let ctso = currentTrack.startOffset ?? 0.0 + let trackEnd = ctso + currentTrack.duration + NSLog("TEST: Seek current track END = \(trackEnd)") + + + let indexOfSeek = getItemIndexForTime(time: to) + NSLog("TEST: Seek to index \(indexOfSeek) | Current index \(self.currentTrackIndex)") + + // Reconstruct queue if seeking to a different track + if (self.currentTrackIndex != indexOfSeek) { + self.currentTrackIndex = indexOfSeek + + self.playbackSession.currentTime = to + + self.playWhenReady = continuePlaying // Only playWhenReady if already playing + self.status = -1 + let playerItems = self.allPlayerItems[indexOfSeek.. 0.0 && !(observed && rate == 1) { @@ -160,20 +252,31 @@ class AudioPlayer: NSObject { } public func getCurrentTime() -> Double { - self.audioPlayer.currentTime().seconds + let currentTrackTime = self.audioPlayer.currentTime().seconds + let audioTrack = playbackSession.audioTracks[currentTrackIndex] + let startOffset = audioTrack.startOffset ?? 0.0 + return startOffset + currentTrackTime } public func getDuration() -> Double { - self.audioPlayer.currentItem?.duration.seconds ?? 0 + return playbackSession.duration } // MARK: - Private - private func createAsset() -> AVAsset { - let headers: [String: String] = [ - "Authorization": "Bearer \(Store.serverConfig!.token)" - ] + private func createAsset(itemId:String, track:AudioTrack) -> AVAsset { + let filename = track.metadata?.filename ?? "" + let filenameEncoded = filename.addingPercentEncoding(withAllowedCharacters: NSCharacterSet.urlQueryAllowed) + let urlstr = "\(Store.serverConfig!.address)/s/item/\(itemId)/\(filenameEncoded ?? "")?token=\(Store.serverConfig!.token)" + let url = URL(string: urlstr)! + return AVURLAsset(url: url) - return AVURLAsset(url: URL(string: "\(Store.serverConfig!.address)\(activeAudioTrack.contentUrl ?? "")")!, options: ["AVURLAssetHTTPHeaderFieldsKey": headers]) + // Method for HLS +// let headers: [String: String] = [ +// "Authorization": "Bearer \(Store.serverConfig!.token)" +// ] +// +// return AVURLAsset(url: URL(string: "\(Store.serverConfig!.address)\(activeAudioTrack.contentUrl ?? "")")!, options: ["AVURLAssetHTTPHeaderFieldsKey": headers]) } + private func initAudioSession() { do { try AVAudioSession.sharedInstance().setCategory(.playback, mode: .spokenAudio, options: [.allowAirPlay]) @@ -209,7 +312,7 @@ class AudioPlayer: NSObject { return .noSuchContent } - seek(getCurrentTime() + command.preferredIntervals[0].doubleValue) + seek(getCurrentTime() + command.preferredIntervals[0].doubleValue, from: "remote") return .success } commandCenter.skipBackwardCommand.isEnabled = true @@ -219,7 +322,7 @@ class AudioPlayer: NSObject { return .noSuchContent } - seek(getCurrentTime() - command.preferredIntervals[0].doubleValue) + seek(getCurrentTime() - command.preferredIntervals[0].doubleValue, from: "remote") return .success } @@ -229,7 +332,7 @@ class AudioPlayer: NSObject { return .noSuchContent } - self.seek(event.positionTime) + self.seek(event.positionTime, from: "remote") return .success } @@ -251,26 +354,9 @@ class AudioPlayer: NSObject { // MARK: - Observer public override func observeValue(forKeyPath keyPath: String?, of object: Any?, change: [NSKeyValueChangeKey : Any]?, context: UnsafeMutableRawPointer?) { - if context == &playerItemContext { - if keyPath == #keyPath(AVPlayer.status) { - guard let playerStatus = AVPlayerItem.Status(rawValue: (change?[.newKey] as? Int ?? -1)) else { return } - - if playerStatus == .readyToPlay { - self.updateNowPlaying() - - let firstReady = self.status < 0 - self.status = 0 - if self.playWhenReady { - seek(playbackSession.currentTime) - self.playWhenReady = false - self.play() - } else if (firstReady) { // Only seek on first readyToPlay - seek(playbackSession.currentTime) - } - } - } - } else if context == &playerContext { + if context == &playerContext { if keyPath == #keyPath(AVPlayer.rate) { + NSLog("TEST: playerContext observer player rate") self.setPlaybackRate(change?[.newKey] as? Float ?? 1.0, observed: true) } else if keyPath == #keyPath(AVPlayer.currentItem) { NSLog("WARNING: Item ended") diff --git a/ios/App/Shared/player/PlayerHandler.swift b/ios/App/Shared/player/PlayerHandler.swift index c2009e7b..7f3f5f57 100644 --- a/ios/App/Shared/player/PlayerHandler.swift +++ b/ios/App/Shared/player/PlayerHandler.swift @@ -13,6 +13,7 @@ class PlayerHandler { private static var timer: Timer? private static var listeningTimePassedSinceLastSync = 0.0 + private static var lastSyncReport:PlaybackReport? public static func startPlayback(session: PlaybackSession, playWhenReady: Bool, playbackRate: Float) { if player != nil { @@ -68,7 +69,7 @@ class PlayerHandler { } let destinationTime = player!.getCurrentTime() + amount - player!.seek(destinationTime) + player!.seek(destinationTime, from: "handler") } public static func seekBackward(amount: Double) { if player == nil { @@ -76,10 +77,10 @@ class PlayerHandler { } let destinationTime = player!.getCurrentTime() - amount - player!.seek(destinationTime) + player!.seek(destinationTime, from: "handler") } public static func seek(amount: Double) { - player?.seek(amount) + player?.seek(amount, from: "handler") } public static func paused() -> Bool { @@ -113,10 +114,17 @@ class PlayerHandler { return } - let report = PlaybackReport(currentTime: player!.getCurrentTime(), duration: player!.getDuration(), timeListened: listeningTimePassedSinceLastSync) + let playerCurrentTime = player!.getCurrentTime() + if (lastSyncReport != nil && lastSyncReport?.currentTime == playerCurrentTime) { + // No need to syncProgress + return + } - session!.currentTime = player!.getCurrentTime() + let report = PlaybackReport(currentTime: playerCurrentTime, duration: player!.getDuration(), timeListened: listeningTimePassedSinceLastSync) + + session!.currentTime = playerCurrentTime listeningTimePassedSinceLastSync = 0 + lastSyncReport = report // TODO: check if online NSLog("sending playback report") diff --git a/ios/App/Shared/util/ApiClient.swift b/ios/App/Shared/util/ApiClient.swift index 2b3f06bd..dd5993d5 100644 --- a/ios/App/Shared/util/ApiClient.swift +++ b/ios/App/Shared/util/ApiClient.swift @@ -61,7 +61,8 @@ class ApiClient { } ApiClient.postResource(endpoint: endpoint, parameters: [ - "forceTranscode": "true", // TODO: direct play + "forceDirectPlay": "true", + "forceTranscode": "false", // TODO: direct play "mediaPlayer": "AVPlayer", ], decodable: PlaybackSession.self) { obj in var session = obj