From b0905d0270e253f436df45e062291aee9ca64054 Mon Sep 17 00:00:00 2001 From: ronaldheft Date: Tue, 16 Aug 2022 12:32:22 -0400 Subject: [PATCH] Fix crashes related to Realm threading --- ios/App/App/plugins/AbsAudioPlayer.swift | 21 ++- .../models/LocalLibraryExtensions.swift | 54 ++++---- ios/App/Shared/models/PlaybackSession.swift | 125 +++++++++++++++--- .../Shared/player/ActivePlaybackSession.swift | 41 ++++++ ios/App/Shared/player/AudioPlayer.swift | 34 +++-- ios/App/Shared/player/PlayerHandler.swift | 26 ++-- ios/App/Shared/util/ApiClient.swift | 5 +- ios/App/Shared/util/Database.swift | 5 + 8 files changed, 236 insertions(+), 75 deletions(-) create mode 100644 ios/App/Shared/player/ActivePlaybackSession.swift diff --git a/ios/App/App/plugins/AbsAudioPlayer.swift b/ios/App/App/plugins/AbsAudioPlayer.swift index 536fd84f..4c5c6db0 100644 --- a/ios/App/App/plugins/AbsAudioPlayer.swift +++ b/ios/App/App/plugins/AbsAudioPlayer.swift @@ -7,6 +7,7 @@ import Foundation import Capacitor +import RealmSwift @objc(AbsAudioPlayer) public class AbsAudioPlayer: CAPPlugin { @@ -47,7 +48,11 @@ public class AbsAudioPlayer: CAPPlugin { if (isLocalItem) { let item = Database.shared.getLocalLibraryItem(localLibraryItemId: libraryItemId!) let episode = item?.getPodcastEpisode(episodeId: episodeId) - let playbackSession = item?.getPlaybackSession(episode: episode) + guard let playbackSession = item?.getPlaybackSession(episode: episode) else { + NSLog("Failed to get local playback session") + return call.resolve([:]) + } + playbackSession.save() sendPrepareMetadataEvent(itemId: libraryItemId!, playWhenReady: playWhenReady) do { self.sendPlaybackSession(session: try playbackSession.asDictionary()) @@ -57,7 +62,7 @@ public class AbsAudioPlayer: CAPPlugin { debugPrint(exception) call.resolve([:]) } - PlayerHandler.startPlayback(session: playbackSession!, playWhenReady: playWhenReady, playbackRate: playbackRate) + PlayerHandler.startPlayback(sessionId: playbackSession.id, playWhenReady: playWhenReady, playbackRate: playbackRate) self.sendMetadata() } else { // Playing from the server sendPrepareMetadataEvent(itemId: libraryItemId!, playWhenReady: playWhenReady) @@ -71,7 +76,8 @@ public class AbsAudioPlayer: CAPPlugin { call.resolve([:]) } - PlayerHandler.startPlayback(session: session, playWhenReady: playWhenReady, playbackRate: playbackRate) + session.save() + PlayerHandler.startPlayback(sessionId: session.id, playWhenReady: playWhenReady, playbackRate: playbackRate) self.sendMetadata() } } @@ -197,14 +203,15 @@ public class AbsAudioPlayer: CAPPlugin { @objc func onPlaybackFailed() { if (PlayerHandler.getPlayMethod() == PlayMethod.directplay.rawValue) { - let playbackSession = PlayerHandler.getPlaybackSession() - let libraryItemId = playbackSession?.libraryItemId ?? "" - let episodeId = playbackSession?.episodeId ?? nil + let session = PlayerHandler.getPlaybackSession() + let libraryItemId = session?.libraryItemId ?? "" + let episodeId = session?.episodeId ?? nil NSLog("Forcing Transcode") // If direct playing then fallback to transcode ApiClient.startPlaybackSession(libraryItemId: libraryItemId, episodeId: episodeId, forceTranscode: true) { session in - PlayerHandler.startPlayback(session: session, playWhenReady: self.initialPlayWhenReady, playbackRate: self.initialPlaybackRate) + session.save() + PlayerHandler.startPlayback(sessionId: session.id, playWhenReady: self.initialPlayWhenReady, playbackRate: self.initialPlaybackRate) do { self.sendPlaybackSession(session: try session.asDictionary()) diff --git a/ios/App/Shared/models/LocalLibraryExtensions.swift b/ios/App/Shared/models/LocalLibraryExtensions.swift index cacfa09e..71b49f44 100644 --- a/ios/App/Shared/models/LocalLibraryExtensions.swift +++ b/ios/App/Shared/models/LocalLibraryExtensions.swift @@ -74,14 +74,14 @@ extension LocalLibraryItem { let mediaProgress = Database.shared.getLocalMediaProgress(localMediaProgressId: mediaProgressId) let mediaMetadata = self.media?.metadata - let chapters = Array(self.media?.chapters ?? List()) + let chapters = self.media?.chapters ?? List() let authorName = mediaMetadata?.authorDisplayName - var audioTracks = [AudioTrack]() + let audioTracks = List() if let episode = episode, let track = episode.audioTrack { audioTracks.append(track) } else if let tracks = self.media?.tracks { - audioTracks.append(contentsOf: tracks) + audioTracks.append(objectsIn: tracks) } let dateNow = Date().timeIntervalSince1970 @@ -175,35 +175,41 @@ extension LocalMediaProgress { } func updateIsFinished(_ finished: Bool) { - if self.isFinished != finished { - self.progress = finished ? 1.0 : 0.0 - } + try! Realm().write { + if self.isFinished != finished { + self.progress = finished ? 1.0 : 0.0 + } - if self.startedAt == 0 && finished { - self.startedAt = Int(Date().timeIntervalSince1970) + if self.startedAt == 0 && finished { + self.startedAt = Int(Date().timeIntervalSince1970) + } + + self.isFinished = finished + self.lastUpdate = Int(Date().timeIntervalSince1970) + self.finishedAt = finished ? lastUpdate : nil } - - self.isFinished = finished - self.lastUpdate = Int(Date().timeIntervalSince1970) - self.finishedAt = finished ? lastUpdate : nil } func updateFromPlaybackSession(_ playbackSession: PlaybackSession) { - self.currentTime = playbackSession.currentTime - self.progress = playbackSession.progress - self.lastUpdate = Int(Date().timeIntervalSince1970) - self.isFinished = playbackSession.progress >= 100.0 - self.finishedAt = self.isFinished ? self.lastUpdate : nil + try! Realm().write { + self.currentTime = playbackSession.currentTime + self.progress = playbackSession.progress + self.lastUpdate = Int(Date().timeIntervalSince1970) + self.isFinished = playbackSession.progress >= 100.0 + self.finishedAt = self.isFinished ? self.lastUpdate : nil + } } func updateFromServerMediaProgress(_ serverMediaProgress: MediaProgress) { - self.isFinished = serverMediaProgress.isFinished - self.progress = serverMediaProgress.progress - self.currentTime = serverMediaProgress.currentTime - self.duration = serverMediaProgress.duration - self.lastUpdate = serverMediaProgress.lastUpdate - self.finishedAt = serverMediaProgress.finishedAt - self.startedAt = serverMediaProgress.startedAt + try! Realm().write { + self.isFinished = serverMediaProgress.isFinished + self.progress = serverMediaProgress.progress + self.currentTime = serverMediaProgress.currentTime + self.duration = serverMediaProgress.duration + self.lastUpdate = serverMediaProgress.lastUpdate + self.finishedAt = serverMediaProgress.finishedAt + self.startedAt = serverMediaProgress.startedAt + } } static func fetchOrCreateLocalMediaProgress(localMediaProgressId: String?, localLibraryItemId: String?, localEpisodeId: String?) -> LocalMediaProgress? { diff --git a/ios/App/Shared/models/PlaybackSession.swift b/ios/App/Shared/models/PlaybackSession.swift index ae9dfb56..eadee179 100644 --- a/ios/App/Shared/models/PlaybackSession.swift +++ b/ios/App/Shared/models/PlaybackSession.swift @@ -6,29 +6,30 @@ // import Foundation +import RealmSwift -struct PlaybackSession: Codable { - var id: String +class PlaybackSession: Object, Codable { + @Persisted(primaryKey: true) var id: String = "" var userId: String? - var libraryItemId: String? - var episodeId: String? - var mediaType: String + @Persisted var libraryItemId: String? + @Persisted var episodeId: String? + @Persisted var mediaType: String = "" // var mediaMetadata: MediaTypeMetadata - It is not implemented in android? - var chapters: [Chapter] - var displayTitle: String? - var displayAuthor: String? - var coverPath: String? - var duration: Double - var playMethod: Int - var startedAt: Double? - var updatedAt: Double? - var timeListening: Double - var audioTracks: [AudioTrack] - var currentTime: Double - var libraryItem: LibraryItem? - var localLibraryItem: LocalLibraryItem? - var serverConnectionConfigId: String? - var serverAddress: String? + @Persisted var chapters = List() + @Persisted var displayTitle: String? + @Persisted var displayAuthor: String? + @Persisted var coverPath: String? + @Persisted var duration: Double = 0 + @Persisted var playMethod: Int = 1 + @Persisted var startedAt: Double? + @Persisted var updatedAt: Double? + @Persisted var timeListening: Double = 0 + @Persisted var audioTracks = List() + @Persisted var currentTime: Double = 0 + @Persisted var libraryItem: LibraryItem? + @Persisted var localLibraryItem: LocalLibraryItem? + @Persisted var serverConnectionConfigId: String? + @Persisted var serverAddress: String? var isLocal: Bool { self.localLibraryItem != nil } @@ -47,4 +48,88 @@ struct PlaybackSession: Codable { } var progress: Double { self.currentTime / self.totalDuration } + + internal init(id: String, userId: String? = nil, libraryItemId: String? = nil, episodeId: String? = nil, mediaType: String, chapters: List = List(), displayTitle: String? = nil, displayAuthor: String? = nil, coverPath: String? = nil, duration: Double, playMethod: Int, startedAt: Double? = nil, updatedAt: Double? = nil, timeListening: Double, audioTracks: List = List(), currentTime: Double, libraryItem: LibraryItem? = nil, localLibraryItem: LocalLibraryItem? = nil, serverConnectionConfigId: String? = nil, serverAddress: String? = nil) { + self.id = id + self.userId = userId + self.libraryItemId = libraryItemId + self.episodeId = episodeId + self.mediaType = mediaType + self.chapters = chapters + self.displayTitle = displayTitle + self.displayAuthor = displayAuthor + self.coverPath = coverPath + self.duration = duration + self.playMethod = playMethod + self.startedAt = startedAt + self.updatedAt = updatedAt + self.timeListening = timeListening + self.audioTracks = audioTracks + self.currentTime = currentTime + self.libraryItem = libraryItem + self.localLibraryItem = localLibraryItem + self.serverConnectionConfigId = serverConnectionConfigId + self.serverAddress = serverAddress + } + + private enum CodingKeys : String, CodingKey { + case id, userId, libraryItemId, episodeId, mediaType, chapters, displayTitle, displayAuthor, coverPath, duration, playMethod, startedAt, updatedAt, timeListening, audioTracks, currentTime, libraryItem, localLibraryItem, serverConnectionConfigId, serverAddress, isLocal, localMediaProgressId + } + + override init() {} + + required init(from decoder: Decoder) throws { + super.init() + let values = try decoder.container(keyedBy: CodingKeys.self) + id = try values.decode(String.self, forKey: .id) + userId = try values.decodeIfPresent(String.self, forKey: .userId) + libraryItemId = try values.decodeIfPresent(String.self, forKey: .libraryItemId) + episodeId = try values.decodeIfPresent(String.self, forKey: .episodeId) + mediaType = try values.decode(String.self, forKey: .mediaType) + if let chapterList = try values.decodeIfPresent([Chapter].self, forKey: .chapters) { + chapters.append(objectsIn: chapterList) + } + displayTitle = try values.decodeIfPresent(String.self, forKey: .displayTitle) + displayAuthor = try values.decodeIfPresent(String.self, forKey: .displayAuthor) + coverPath = try values.decodeIfPresent(String.self, forKey: .coverPath) + duration = try values.decode(Double.self, forKey: .duration) + playMethod = try values.decode(Int.self, forKey: .playMethod) + startedAt = try values.decodeIfPresent(Double.self, forKey: .startedAt) + updatedAt = try values.decodeIfPresent(Double.self, forKey: .updatedAt) + timeListening = try values.decode(Double.self, forKey: .timeListening) + if let trackList = try values.decodeIfPresent([AudioTrack].self, forKey: .audioTracks) { + audioTracks.append(objectsIn: trackList) + } + currentTime = try values.decode(Double.self, forKey: .currentTime) + libraryItem = try values.decodeIfPresent(LibraryItem.self, forKey: .libraryItem) + localLibraryItem = try values.decodeIfPresent(LocalLibraryItem.self, forKey: .localLibraryItem) + serverConnectionConfigId = try values.decodeIfPresent(String.self, forKey: .serverConnectionConfigId) + serverAddress = try values.decodeIfPresent(String.self, forKey: .serverAddress) + } + + func encode(to encoder: Encoder) throws { + var container = encoder.container(keyedBy: CodingKeys.self) + try container.encode(id, forKey: .id) + try container.encode(userId, forKey: .userId) + try container.encode(libraryItemId, forKey: .libraryItemId) + try container.encode(episodeId, forKey: .episodeId) + try container.encode(mediaType, forKey: .mediaType) + try container.encode(Array(chapters), forKey: .chapters) + try container.encode(displayTitle, forKey: .displayTitle) + try container.encode(displayAuthor, forKey: .displayAuthor) + try container.encode(coverPath, forKey: .coverPath) + try container.encode(duration, forKey: .duration) + try container.encode(playMethod, forKey: .playMethod) + try container.encode(startedAt, forKey: .startedAt) + try container.encode(updatedAt, forKey: .updatedAt) + try container.encode(timeListening, forKey: .timeListening) + try container.encode(Array(audioTracks), forKey: .audioTracks) + try container.encode(currentTime, forKey: .currentTime) + try container.encode(libraryItem, forKey: .libraryItem) + try container.encode(localLibraryItem, forKey: .localLibraryItem) + try container.encode(serverConnectionConfigId, forKey: .serverConnectionConfigId) + try container.encode(serverAddress, forKey: .serverAddress) + try container.encode(isLocal, forKey: .isLocal) + try container.encode(localMediaProgressId, forKey: .localMediaProgressId) + } } diff --git a/ios/App/Shared/player/ActivePlaybackSession.swift b/ios/App/Shared/player/ActivePlaybackSession.swift new file mode 100644 index 00000000..debbd7ae --- /dev/null +++ b/ios/App/Shared/player/ActivePlaybackSession.swift @@ -0,0 +1,41 @@ +// +// ActivePlaybackSession.swift +// App +// +// Created by Ron Heft on 8/16/22. +// + +import Foundation +import RealmSwift + +class ActivePlaybackSession { + + static let shared = ActivePlaybackSession() + + private let queue = DispatchQueue(label: "ABSActivePlaybackSession") + private var _session: PlaybackSession? + + private init() { + // Singleton + } + + func startSession(_ session: ThreadSafeReference) { + queue.sync { + _session = try? Realm().resolve(session) + } + } + + // This is a funky method, but it ensures the accessing thread gets a live reference to session properly resolved + func get() -> PlaybackSession? { + var activeSession: ThreadSafeReference? + queue.sync { + let realm = try! Realm() + guard let session = _session else { return } + r + activeSession = ThreadSafeReference(to: session) + } + guard let activeSession = activeSession else { return nil } + return try? Realm().resolve(activeSession) + } + +} diff --git a/ios/App/Shared/player/AudioPlayer.swift b/ios/App/Shared/player/AudioPlayer.swift index 61398f2a..81765b13 100644 --- a/ios/App/Shared/player/AudioPlayer.swift +++ b/ios/App/Shared/player/AudioPlayer.swift @@ -32,7 +32,7 @@ class AudioPlayer: NSObject { private var initialPlaybackRate: Float private var audioPlayer: AVQueuePlayer - private var playbackSession: PlaybackSession + private var sessionId: String private var queueObserver:NSKeyValueObservation? private var queueItemStatusObserver:NSKeyValueObservation? @@ -41,12 +41,12 @@ class AudioPlayer: NSObject { private var allPlayerItems:[AVPlayerItem] = [] // MARK: - Constructor - init(playbackSession: PlaybackSession, playWhenReady: Bool = false, playbackRate: Float = 1) { + init(sessionId: String, playWhenReady: Bool = false, playbackRate: Float = 1) { self.playWhenReady = playWhenReady self.initialPlaybackRate = playbackRate self.audioPlayer = AVQueuePlayer() self.audioPlayer.automaticallyWaitsToMinimizeStalling = false - self.playbackSession = playbackSession + self.sessionId = sessionId self.status = -1 self.rate = 0.0 self.tmpRate = playbackRate @@ -56,6 +56,8 @@ class AudioPlayer: NSObject { initAudioSession() setupRemoteTransportControls() + let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)! + // Listen to player events self.audioPlayer.addObserver(self, forKeyPath: #keyPath(AVPlayer.rate), options: .new, context: &playerContext) self.audioPlayer.addObserver(self, forKeyPath: #keyPath(AVPlayer.currentItem), options: .new, context: &playerContext) @@ -106,6 +108,7 @@ class AudioPlayer: NSObject { } func getItemIndexForTime(time:Double) -> Int { + let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)! for index in 0.. Double { + let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)! let currentTrackTime = self.audioPlayer.currentTime().seconds let audioTrack = playbackSession.audioTracks[currentTrackIndex] let startOffset = audioTrack.startOffset ?? 0.0 @@ -269,17 +278,20 @@ class AudioPlayer: NSObject { } public func getPlayMethod() -> Int { - return self.playbackSession.playMethod + let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)! + return playbackSession.playMethod } - public func getPlaybackSession() -> PlaybackSession { - return self.playbackSession + public func getPlaybackSessionId() -> String { + return self.sessionId } public func getDuration() -> Double { + let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)! return playbackSession.duration } // MARK: - Private private func createAsset(itemId:String, track:AudioTrack) -> AVAsset { + let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)! if (playbackSession.playMethod == PlayMethod.directplay.rawValue) { // The only reason this is separate is because the filename needs to be encoded let filename = track.metadata?.filename ?? "" diff --git a/ios/App/Shared/player/PlayerHandler.swift b/ios/App/Shared/player/PlayerHandler.swift index a3030d29..f52ae787 100644 --- a/ios/App/Shared/player/PlayerHandler.swift +++ b/ios/App/Shared/player/PlayerHandler.swift @@ -6,10 +6,10 @@ // import Foundation +import RealmSwift class PlayerHandler { private static var player: AudioPlayer? - private static var session: PlaybackSession? private static var timer: Timer? private static var lastSyncTime: Double = 0.0 @@ -68,7 +68,8 @@ class PlayerHandler { timer = nil } - public static func startPlayback(session: PlaybackSession, playWhenReady: Bool, playbackRate: Float) { + public static func startPlayback(sessionId: String, playWhenReady: Bool, playbackRate: Float) { + guard let session = Database.shared.getPlaybackSession(id: sessionId) else { return } if player != nil { player?.destroy() player = nil @@ -76,8 +77,7 @@ class PlayerHandler { NowPlayingInfo.shared.setSessionMetadata(metadata: NowPlayingMetadata(id: session.id, itemId: session.libraryItemId!, artworkUrl: session.coverPath, title: session.displayTitle ?? "Unknown title", author: session.displayAuthor, series: nil)) - self.session = session - player = AudioPlayer(playbackSession: session, playWhenReady: playWhenReady, playbackRate: playbackRate) + player = AudioPlayer(sessionId: sessionId, playWhenReady: playWhenReady, playbackRate: playbackRate) startTickTimer() } @@ -108,7 +108,9 @@ class PlayerHandler { } public static func getPlaybackSession() -> PlaybackSession? { - self.player?.getPlaybackSession() + guard let player = player else { return nil } + guard let session = Database.shared.getPlaybackSession(id: player.getPlaybackSessionId()) else { return nil } + return session } public static func seekForward(amount: Double) { @@ -164,8 +166,8 @@ class PlayerHandler { } public static func syncProgress() { - if session == nil { return } guard let player = player else { return } + guard let session = getPlaybackSession() else { return } // Prevent a sync at the current time let playerCurrentTime = player.getCurrentTime() @@ -185,15 +187,17 @@ class PlayerHandler { lastSyncTime = Date().timeIntervalSince1970 // seconds let report = PlaybackReport(currentTime: playerCurrentTime, duration: player.getDuration(), timeListened: listeningTimePassedSinceLastSync) - session!.currentTime = playerCurrentTime + session.update { + session.currentTime = playerCurrentTime + } listeningTimePassedSinceLastSync = 0 lastSyncReport = report - let sessionIsLocal = session!.isLocal + let sessionIsLocal = session.isLocal if !sessionIsLocal { if Connectivity.isConnectedToInternet { NSLog("sending playback report") - ApiClient.reportPlaybackProgress(report: report, sessionId: session!.id) + ApiClient.reportPlaybackProgress(report: report, sessionId: session.id) } } else { if let localMediaProgress = syncLocalProgress() { @@ -207,10 +211,10 @@ class PlayerHandler { } private static func syncLocalProgress() -> LocalMediaProgress? { - guard let session = session else { return nil } + guard let session = getPlaybackSession() else { return nil } let localMediaProgress = LocalMediaProgress.fetchOrCreateLocalMediaProgress(localMediaProgressId: session.localMediaProgressId, localLibraryItemId: session.localLibraryItem?.id, localEpisodeId: session.episodeId) - guard var localMediaProgress = localMediaProgress else { + guard let localMediaProgress = localMediaProgress else { // Local media progress should have been created // If we're here, it means a library id is invalid return nil diff --git a/ios/App/Shared/util/ApiClient.swift b/ios/App/Shared/util/ApiClient.swift index 7addc065..cbf3068b 100644 --- a/ios/App/Shared/util/ApiClient.swift +++ b/ios/App/Shared/util/ApiClient.swift @@ -167,13 +167,14 @@ class ApiClient { } public static func reportLocalMediaProgress(_ localMediaProgress: LocalMediaProgress, callback: @escaping (_ success: Bool) -> Void) { - postResource(endpoint: "api/session/local", parameters: localMediaProgress, callback: callback) + let progress = localMediaProgress.freeze() + postResource(endpoint: "api/session/local", parameters: progress, callback: callback) } public static func syncMediaProgress(callback: @escaping (_ results: LocalMediaProgressSyncResultsPayload) -> Void) { let localMediaProgressList = Database.shared.getAllLocalMediaProgress().filter { $0.serverConnectionConfigId == Store.serverConfig?.id - } + }.map { $0.freeze() } if ( !localMediaProgressList.isEmpty ) { let payload = LocalMediaProgressSyncPayload(localMediaProgress: localMediaProgressList) diff --git a/ios/App/Shared/util/Database.swift b/ios/App/Shared/util/Database.swift index 7548333c..12716c22 100644 --- a/ios/App/Shared/util/Database.swift +++ b/ios/App/Shared/util/Database.swift @@ -197,4 +197,9 @@ class Database { realm.delete(progress!) } } + + public func getPlaybackSession(id: String) -> PlaybackSession? { + let realm = try! Realm() + return realm.object(ofType: PlaybackSession.self, forPrimaryKey: id) + } }