From 480df58ce4ec303698e699310a3213093e4f1cea Mon Sep 17 00:00:00 2001 From: advplyr Date: Fri, 3 Jun 2022 18:58:07 -0500 Subject: [PATCH] Fix:Check with server after pause of 1 minute or longer for updated media progress & show toast on client if progress sync is failing --- .../app/data/PlaybackSession.kt | 2 + .../app/player/MediaProgressSyncer.kt | 26 +++++- .../app/player/MediaSessionCallback.kt | 20 +++-- .../app/player/PlayerListener.kt | 12 ++- .../app/player/PlayerNotificationService.kt | 84 ++++++++++++++++++- .../app/plugins/AbsAudioPlayer.kt | 11 +-- .../audiobookshelf/app/server/ApiHandler.kt | 38 ++++++++- components/app/AudioPlayerContainer.vue | 10 ++- 8 files changed, 177 insertions(+), 26 deletions(-) diff --git a/android/app/src/main/java/com/audiobookshelf/app/data/PlaybackSession.kt b/android/app/src/main/java/com/audiobookshelf/app/data/PlaybackSession.kt index e764289a..6af1c70c 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/data/PlaybackSession.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/data/PlaybackSession.kt @@ -62,6 +62,8 @@ class PlaybackSession( val localMediaProgressId get() = if (episodeId.isNullOrEmpty()) localLibraryItemId else "$localLibraryItemId-$localEpisodeId" @get:JsonIgnore val progress get() = currentTime / getTotalDuration() + @get:JsonIgnore + val isLocalLibraryItemOnly get() = localLibraryItemId != "" && libraryItemId == null @JsonIgnore fun getCurrentTrackIndex():Int { diff --git a/android/app/src/main/java/com/audiobookshelf/app/player/MediaProgressSyncer.kt b/android/app/src/main/java/com/audiobookshelf/app/player/MediaProgressSyncer.kt index 8833ad5f..4aa370f3 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/player/MediaProgressSyncer.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/player/MediaProgressSyncer.kt @@ -4,6 +4,7 @@ import android.os.Handler import android.os.Looper import android.util.Log import com.audiobookshelf.app.data.LocalMediaProgress +import com.audiobookshelf.app.data.MediaProgress import com.audiobookshelf.app.data.PlaybackSession import com.audiobookshelf.app.device.DeviceManager import com.audiobookshelf.app.server.ApiHandler @@ -24,6 +25,7 @@ class MediaProgressSyncer(val playerNotificationService:PlayerNotificationServic var listeningTimerRunning:Boolean = false private var lastSyncTime:Long = 0 + private var failedSyncs:Int = 0 var currentPlaybackSession: PlaybackSession? = null // copy of pb session currently syncing var currentLocalMediaProgress: LocalMediaProgress? = null @@ -41,6 +43,7 @@ class MediaProgressSyncer(val playerNotificationService:PlayerNotificationServic currentLocalMediaProgress = null listeningTimerTask?.cancel() lastSyncTime = 0L + failedSyncs = 0 } else { return } @@ -68,6 +71,16 @@ class MediaProgressSyncer(val playerNotificationService:PlayerNotificationServic reset() } + fun syncFromServerProgress(mediaProgress: MediaProgress) { + currentPlaybackSession?.let { + it.updatedAt = mediaProgress.lastUpdate + it.currentTime = mediaProgress.currentTime + + DeviceManager.dbManager.saveLocalPlaybackSession(it) + saveLocalProgress(it) + } + } + fun sync(currentTime:Double) { val diffSinceLastSync = System.currentTimeMillis() - lastSyncTime if (diffSinceLastSync < 1000L) { @@ -100,7 +113,17 @@ class MediaProgressSyncer(val playerNotificationService:PlayerNotificationServic } } else { apiHandler.sendProgressSync(currentSessionId, syncData) { - Log.d(tag, "Progress sync data sent to server $currentDisplayTitle for time $currentTime") + if (it) { + Log.d(tag, "Progress sync data sent to server $currentDisplayTitle for time $currentTime") + failedSyncs = 0 + } else { + failedSyncs++ + if (failedSyncs == 2) { + playerNotificationService.alertSyncFailing() // Show alert in client + failedSyncs = 0 + } + Log.d(tag, "Progress sync failed ($failedSyncs) to send to server $currentDisplayTitle for time $currentTime") + } } } } @@ -130,5 +153,6 @@ class MediaProgressSyncer(val playerNotificationService:PlayerNotificationServic currentPlaybackSession = null currentLocalMediaProgress = null lastSyncTime = 0L + failedSyncs = 0 } } diff --git a/android/app/src/main/java/com/audiobookshelf/app/player/MediaSessionCallback.kt b/android/app/src/main/java/com/audiobookshelf/app/player/MediaSessionCallback.kt index e86f4ed2..3798a55e 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/player/MediaSessionCallback.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/player/MediaSessionCallback.kt @@ -9,12 +9,8 @@ import android.os.Message import android.support.v4.media.session.MediaSessionCompat import android.util.Log import android.view.KeyEvent -import com.audiobookshelf.app.data.LibraryItem import com.audiobookshelf.app.data.LibraryItemWrapper import com.audiobookshelf.app.data.PodcastEpisode -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.GlobalScope -import kotlinx.coroutines.launch import java.util.* import kotlin.concurrent.schedule @@ -119,10 +115,13 @@ class MediaSessionCallback(var playerNotificationService:PlayerNotificationServi fun handleCallMediaButton(intent: Intent): Boolean { if(Intent.ACTION_MEDIA_BUTTON == intent.action) { - var keyEvent = intent.getParcelableExtra(Intent.EXTRA_KEY_EVENT) - if (keyEvent?.getAction() == KeyEvent.ACTION_UP) { - when (keyEvent?.getKeyCode()) { + val keyEvent = intent.getParcelableExtra(Intent.EXTRA_KEY_EVENT) + + if (keyEvent?.action == KeyEvent.ACTION_UP) { + Log.d(tag, "handleCallMediaButton: key action_up for ${keyEvent.keyCode}") + when (keyEvent.keyCode) { KeyEvent.KEYCODE_HEADSETHOOK -> { + Log.d(tag, "handleCallMediaButton: Headset Hook") if (0 == mediaButtonClickCount) { if (playerNotificationService.mPlayer.isPlaying) playerNotificationService.pause() @@ -132,6 +131,7 @@ class MediaSessionCallback(var playerNotificationService:PlayerNotificationServi handleMediaButtonClickCount() } KeyEvent.KEYCODE_MEDIA_PLAY -> { + Log.d(tag, "handleCallMediaButton: Media Play") if (0 == mediaButtonClickCount) { playerNotificationService.play() playerNotificationService.sleepTimerManager.checkShouldExtendSleepTimer() @@ -139,6 +139,7 @@ class MediaSessionCallback(var playerNotificationService:PlayerNotificationServi handleMediaButtonClickCount() } KeyEvent.KEYCODE_MEDIA_PAUSE -> { + Log.d(tag, "handleCallMediaButton: Media Pause") if (0 == mediaButtonClickCount) playerNotificationService.pause() handleMediaButtonClickCount() } @@ -152,6 +153,7 @@ class MediaSessionCallback(var playerNotificationService:PlayerNotificationServi playerNotificationService.closePlayback() } KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> { + Log.d(tag, "handleCallMediaButton: Media Play/Pause") if (playerNotificationService.mPlayer.isPlaying) { if (0 == mediaButtonClickCount) playerNotificationService.pause() handleMediaButtonClickCount() @@ -164,7 +166,7 @@ class MediaSessionCallback(var playerNotificationService:PlayerNotificationServi } } else -> { - Log.d(tag, "KeyCode:${keyEvent.getKeyCode()}") + Log.d(tag, "KeyCode:${keyEvent.keyCode}") return false } } @@ -173,7 +175,7 @@ class MediaSessionCallback(var playerNotificationService:PlayerNotificationServi return true } - fun handleMediaButtonClickCount() { + private fun handleMediaButtonClickCount() { mediaButtonClickCount++ if (1 == mediaButtonClickCount) { Timer().schedule(mediaButtonClickTimeout) { diff --git a/android/app/src/main/java/com/audiobookshelf/app/player/PlayerListener.kt b/android/app/src/main/java/com/audiobookshelf/app/player/PlayerListener.kt index 17aff650..59a73f21 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/player/PlayerListener.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/player/PlayerListener.kt @@ -5,6 +5,8 @@ import com.audiobookshelf.app.data.PlayerState import com.google.android.exoplayer2.PlaybackException import com.google.android.exoplayer2.Player +const val PAUSE_LEN_BEFORE_RECHECK = 60000 // 1 minute + class PlayerListener(var playerNotificationService:PlayerNotificationService) : Player.Listener { var tag = "PlayerListener" @@ -81,6 +83,12 @@ class PlayerListener(var playerNotificationService:PlayerNotificationService) : Log.d(tag, "SeekBackTime: back time is 0") } } + + // Check if playback session still exists or sync media progress if updated + if (lastPauseTime > PAUSE_LEN_BEFORE_RECHECK) { + val shouldCarryOn = playerNotificationService.checkCurrentSessionProgress() + if (!shouldCarryOn) return + } } } else { Log.d(tag, "SeekBackTime: Player not playing set last pause time") @@ -102,8 +110,8 @@ class PlayerListener(var playerNotificationService:PlayerNotificationService) : private fun calcPauseSeekBackTime() : Long { if (lastPauseTime <= 0) return 0 - var time: Long = System.currentTimeMillis() - lastPauseTime - var seekback: Long + val time: Long = System.currentTimeMillis() - lastPauseTime + val seekback: Long if (time < 3000) seekback = 0 else if (time < 300000) seekback = 10000 // 3s to 5m = jump back 10s else if (time < 1800000) seekback = 20000 // 5m to 30m = jump back 20s diff --git a/android/app/src/main/java/com/audiobookshelf/app/player/PlayerNotificationService.kt b/android/app/src/main/java/com/audiobookshelf/app/player/PlayerNotificationService.kt index 848b28bc..6bfb5e6f 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/player/PlayerNotificationService.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/player/PlayerNotificationService.kt @@ -48,12 +48,12 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { fun onPlaybackClosed() fun onPlayingUpdate(isPlaying: Boolean) fun onMetadata(metadata: PlaybackMetadata) - fun onPrepare(audiobookId: String, playWhenReady: Boolean) fun onSleepTimerEnded(currentPosition: Long) fun onSleepTimerSet(sleepTimeRemaining: Int) fun onLocalMediaProgressUpdate(localMediaProgress: LocalMediaProgress) fun onPlaybackFailed(errorMessage:String) fun onMediaPlayerChanged(mediaPlayer:String) + fun onProgressSyncFailing() } private val tag = "PlayerService" @@ -68,7 +68,7 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { private lateinit var transportControls:MediaControllerCompat.TransportControls lateinit var mediaManager: MediaManager - private lateinit var apiHandler: ApiHandler + lateinit var apiHandler: ApiHandler lateinit var mPlayer: ExoPlayer lateinit var currentPlayer:Player @@ -392,6 +392,21 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { } } + fun startNewPlaybackSession() { + currentPlaybackSession?.let { playbackSession -> + val forceTranscode = playbackSession.isHLS // If already HLS then force + val playItemRequestPayload = getPlayItemRequestPayload(forceTranscode) + + val libraryItemId = playbackSession.libraryItemId ?: "" // Must be true since direct play + val episodeId = playbackSession.episodeId + apiHandler.playLibraryItem(libraryItemId, episodeId, playItemRequestPayload) { + Handler(Looper.getMainLooper()).post { + preparePlayer(it, true, null) + } + } + } + } + fun switchToPlayer(useCastPlayer: Boolean) { val wasPlaying = currentPlayer.isPlaying if (useCastPlayer) { @@ -483,6 +498,67 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { return currentPlaybackSession?.id } + // Called from PlayerListener play event + // check with server if progress has updated since last play and sync progress update + fun checkCurrentSessionProgress():Boolean { + if (currentPlaybackSession == null) return true + + currentPlaybackSession?.let { playbackSession -> + if (!apiHandler.isOnline() || playbackSession.isLocalLibraryItemOnly) { + return true // carry on + } + + if (playbackSession.isLocal) { + // Local playback session check if server has updated media progress + Log.d(tag, "checkCurrentSessionProgress: Checking if local media progress was updated on server") + apiHandler.getMediaProgress(playbackSession.libraryItemId!!, playbackSession.episodeId) { mediaProgress -> + if (mediaProgress.lastUpdate > playbackSession.updatedAt && mediaProgress.currentTime != playbackSession.currentTime) { + Log.d(tag, "checkCurrentSessionProgress: Media progress was updated since last play time updating from ${playbackSession.currentTime} to ${mediaProgress.currentTime}") + mediaProgressSyncer.syncFromServerProgress(mediaProgress) + + Handler(Looper.getMainLooper()).post { + seekPlayer(playbackSession.currentTimeMs) + } + } + + Handler(Looper.getMainLooper()).post { + // Should already be playing + currentPlayer.volume = 1F // Volume on sleep timer might have decreased this + mediaProgressSyncer.start() + clientEventEmitter?.onPlayingUpdate(true) + } + } + } else { + // Streaming from server so check if playback session still exists on server + Log.d( + tag, + "checkCurrentSessionProgress: Checking if playback session for server stream is still available" + ) + apiHandler.getPlaybackSession(playbackSession.id) { + if (it == null) { + Log.d( + tag, + "checkCurrentSessionProgress: Playback session does not exist on server - start new playback session" + ) + + Handler(Looper.getMainLooper()).post { + currentPlayer.pause() + startNewPlaybackSession() + } + } else { + Log.d(tag, "checkCurrentSessionProgress: Playback session still available on server") + Handler(Looper.getMainLooper()).post { + currentPlayer.volume = 1F // Volume on sleep timer might have decreased this + mediaProgressSyncer.start() + clientEventEmitter?.onPlayingUpdate(true) + } + } + } + } + } + return false + } + fun play() { if (currentPlayer.isPlaying) { Log.d(tag, "Already playing") @@ -570,6 +646,10 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { return ctx } + fun alertSyncFailing() { + clientEventEmitter?.onProgressSyncFailing() + } + // // MEDIA BROWSER STUFF (ANDROID AUTO) // diff --git a/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsAudioPlayer.kt b/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsAudioPlayer.kt index 807bfe9e..44c1491c 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsAudioPlayer.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsAudioPlayer.kt @@ -59,13 +59,6 @@ class AbsAudioPlayer : Plugin() { notifyListeners("onMetadata", JSObject(jacksonMapper.writeValueAsString(metadata))) } - override fun onPrepare(audiobookId: String, playWhenReady: Boolean) { - val jsobj = JSObject() - jsobj.put("audiobookId", audiobookId) - jsobj.put("playWhenReady", playWhenReady) - notifyListeners("onPrepareMedia", jsobj) - } - override fun onSleepTimerEnded(currentPosition: Long) { emit("onSleepTimerEnded", currentPosition) } @@ -85,6 +78,10 @@ class AbsAudioPlayer : Plugin() { override fun onMediaPlayerChanged(mediaPlayer:String) { emit("onMediaPlayerChanged", mediaPlayer) } + + override fun onProgressSyncFailing() { + emit("onProgressSyncFailing", "") + } }) } mainActivity.pluginCallback = foregroundServiceReady diff --git a/android/app/src/main/java/com/audiobookshelf/app/server/ApiHandler.kt b/android/app/src/main/java/com/audiobookshelf/app/server/ApiHandler.kt index 01cd165b..0f0a93b9 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/server/ApiHandler.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/server/ApiHandler.kt @@ -86,9 +86,15 @@ class ApiHandler(var ctx:Context) { override fun onResponse(call: Call, response: Response) { response.use { - if (!it.isSuccessful) throw IOException("Unexpected code $response") + if (!it.isSuccessful) { +// throw IOException("Unexpected code $response") + val jsobj = JSObject() + jsobj.put("error", "Unexpected code $response") + cb(jsobj) + return + } - val bodyString = it.body!!.string() + val bodyString = it.body!!.string() if (bodyString == "OK") { cb(JSObject()) } else { @@ -184,11 +190,15 @@ class ApiHandler(var ctx:Context) { } } - fun sendProgressSync(sessionId:String, syncData: MediaProgressSyncData, cb: () -> Unit) { + fun sendProgressSync(sessionId:String, syncData: MediaProgressSyncData, cb: (Boolean) -> Unit) { val payload = JSObject(jacksonMapper.writeValueAsString(syncData)) postRequest("/api/session/$sessionId/sync", payload) { - cb() + if (!it.getString("error").isNullOrEmpty()) { + cb(false) + } else { + cb(true) + } } } @@ -252,4 +262,24 @@ class ApiHandler(var ctx:Context) { cb() } } + + fun getMediaProgress(libraryItemId:String, episodeId:String?, cb: (MediaProgress) -> Unit) { + val endpoint = if(episodeId.isNullOrEmpty()) "/api/me/progress/$libraryItemId" else "/api/me/progress/$libraryItemId/$episodeId" + getRequest(endpoint) { + val progress = jacksonMapper.readValue(it.toString()) + cb(progress) + } + } + + fun getPlaybackSession(playbackSessionId:String, cb: (PlaybackSession?) -> Unit) { + val endpoint = "/api/session/$playbackSessionId" + getRequest(endpoint) { + val err = it.getString("error") + if (!err.isNullOrEmpty()) { + cb(null) + } else { + cb(jacksonMapper.readValue(it.toString())) + } + } + } } diff --git a/components/app/AudioPlayerContainer.vue b/components/app/AudioPlayerContainer.vue index 3b4ea5ee..6a25eec7 100644 --- a/components/app/AudioPlayerContainer.vue +++ b/components/app/AudioPlayerContainer.vue @@ -30,9 +30,11 @@ export default { onSleepTimerEndedListener: null, onSleepTimerSetListener: null, onMediaPlayerChangedListener: null, + onProgressSyncFailing: null, sleepInterval: null, currentEndOfChapterTime: 0, - serverLibraryItemId: null + serverLibraryItemId: null, + syncFailedToast: null } }, watch: { @@ -241,6 +243,10 @@ export default { onMediaPlayerChanged(data) { var mediaPlayer = data.value this.$store.commit('setMediaPlayer', mediaPlayer) + }, + showProgressSyncIsFailing() { + if (!isNaN(this.syncFailedToast)) this.$toast.dismiss(this.syncFailedToast) + this.syncFailedToast = this.$toast('Progress is not being synced', { timeout: false, type: 'error' }) } }, mounted() { @@ -248,6 +254,7 @@ export default { this.onSleepTimerEndedListener = AbsAudioPlayer.addListener('onSleepTimerEnded', this.onSleepTimerEnded) this.onSleepTimerSetListener = AbsAudioPlayer.addListener('onSleepTimerSet', this.onSleepTimerSet) this.onMediaPlayerChangedListener = AbsAudioPlayer.addListener('onMediaPlayerChanged', this.onMediaPlayerChanged) + this.onProgressSyncFailing = AbsAudioPlayer.addListener('onProgressSyncFailing', this.showProgressSyncIsFailing) this.playbackSpeed = this.$store.getters['user/getUserSetting']('playbackRate') console.log(`[AudioPlayerContainer] Init Playback Speed: ${this.playbackSpeed}`) @@ -264,6 +271,7 @@ export default { if (this.onSleepTimerEndedListener) this.onSleepTimerEndedListener.remove() if (this.onSleepTimerSetListener) this.onSleepTimerSetListener.remove() if (this.onMediaPlayerChangedListener) this.onMediaPlayerChangedListener.remove() + if (this.onProgressSyncFailing) this.onProgressSyncFailing.remove() // if (this.$server.socket) { // this.$server.socket.off('stream_open', this.streamOpen)