From f70f707100b0a3f979f426ad6a56350d3a551b24 Mon Sep 17 00:00:00 2001 From: advplyr Date: Sat, 2 Apr 2022 12:12:00 -0500 Subject: [PATCH] Rebuilding audio player and handling playback sessions in android. Moving all logic natively --- .../com/audiobookshelf/app/MyNativeAudio.kt | 14 +- .../app/PlayerNotificationService.kt | 32 +- .../audiobookshelf/app/SleepTimerManager.kt | 14 +- .../audiobookshelf/app/data/DataClasses.kt | 11 +- .../com/audiobookshelf/app/data/DbManager.kt | 2 - .../audiobookshelf/app/data/DeviceClasses.kt | 4 +- .../app/data/PlaybackSession.kt | 41 +-- .../app/device/FolderScanner.kt | 13 +- .../audiobookshelf/app/server/ApiHandler.kt | 7 +- components/app/AudioPlayer.vue | 270 ++++---------- components/app/AudioPlayerContainer.vue | 328 +----------------- components/covers/BookCover.vue | 2 +- components/ui/Dropdown.vue | 2 +- layouts/default.vue | 2 +- pages/localMedia/folders/_id.vue | 6 +- pages/localMedia/folders/index.vue | 4 +- plugins/constants.js | 10 +- store/globals.js | 10 +- store/user.js | 4 + 19 files changed, 180 insertions(+), 596 deletions(-) diff --git a/android/app/src/main/java/com/audiobookshelf/app/MyNativeAudio.kt b/android/app/src/main/java/com/audiobookshelf/app/MyNativeAudio.kt index de655757..32798c2a 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/MyNativeAudio.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/MyNativeAudio.kt @@ -5,6 +5,7 @@ import android.os.Handler import android.os.Looper import android.util.Log import androidx.core.content.ContextCompat +import com.audiobookshelf.app.data.PlaybackSession import com.audiobookshelf.app.device.DeviceManager import com.audiobookshelf.app.server.ApiHandler import com.capacitorjs.plugins.app.AppPlugin @@ -30,7 +31,15 @@ class MyNativeAudio : Plugin() { playerNotificationService.setBridge(bridge) - playerNotificationService.setCustomObjectListener(object : PlayerNotificationService.MyCustomObjectListener { + playerNotificationService.clientEventEmitter = (object : PlayerNotificationService.ClientEventEmitter { + override fun onPlaybackSession(playbackSession: PlaybackSession) { + notifyListeners("onPlaybackSession", JSObject(jacksonObjectMapper().writeValueAsString(playbackSession))) + } + + override fun onPlaybackClosed() { + emit("onPlaybackClosed", true) + } + override fun onPlayingUpdate(isPlaying: Boolean) { emit("onPlayingUpdate", isPlaying) } @@ -67,10 +76,9 @@ class MyNativeAudio : Plugin() { @PluginMethod fun prepareLibraryItem(call: PluginCall) { var libraryItemId = call.getString("libraryItemId", "").toString() - var mediaEntityId = call.getString("mediaEntityId", "").toString() var playWhenReady = call.getBoolean("playWhenReady") == true - apiHandler.playLibraryItem(libraryItemId) { + apiHandler.playLibraryItem(libraryItemId, false) { Handler(Looper.getMainLooper()).post() { Log.d(tag, "Preparing Player TEST ${jacksonObjectMapper().writeValueAsString(it)}") diff --git a/android/app/src/main/java/com/audiobookshelf/app/PlayerNotificationService.kt b/android/app/src/main/java/com/audiobookshelf/app/PlayerNotificationService.kt index 6787c4fb..7739c05d 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/PlayerNotificationService.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/PlayerNotificationService.kt @@ -54,7 +54,9 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { var isStarted = false } - interface MyCustomObjectListener { + interface ClientEventEmitter { + fun onPlaybackSession(playbackSession:PlaybackSession) + fun onPlaybackClosed() fun onPlayingUpdate(isPlaying: Boolean) fun onMetadata(metadata: JSObject) fun onPrepare(audiobookId: String, playWhenReady: Boolean) @@ -65,7 +67,7 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { private val tag = "PlayerService" private val binder = LocalBinder() - var listener:MyCustomObjectListener? = null + var clientEventEmitter:ClientEventEmitter? = null private lateinit var ctx:Context private lateinit var mediaSessionConnector: MediaSessionConnector @@ -106,9 +108,6 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { private var mShakeDetector: ShakeDetector? = null private var shakeSensorUnregisterTask:TimerTask? = null - fun setCustomObjectListener(mylistener: MyCustomObjectListener) { - listener = mylistener - } fun setBridge(bridge: Bridge) { webviewBridge = bridge } @@ -380,8 +379,8 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { override fun getMediaDescription(player: Player, windowIndex: Int): MediaDescriptionCompat { var builder = MediaDescriptionCompat.Builder() .setMediaId(currentPlaybackSession!!.id) - .setTitle(currentPlaybackSession!!.getTitle()) - .setSubtitle(currentPlaybackSession!!.getAuthor()) + .setTitle(currentPlaybackSession!!.displayTitle) + .setSubtitle(currentPlaybackSession!!.displayAuthor) .setMediaUri(currentPlaybackSession!!.getContentUri()) .setIconUri(currentPlaybackSession!!.getCoverUri()) return builder.build() @@ -656,7 +655,7 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { audiobookProgressSyncer.stop() } - listener?.onPlayingUpdate(player.isPlaying) + clientEventEmitter?.onPlayingUpdate(player.isPlaying) } } } @@ -668,6 +667,9 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { */ fun preparePlayer(playbackSession: PlaybackSession, playWhenReady:Boolean) { currentPlaybackSession = playbackSession + + clientEventEmitter?.onPlaybackSession(playbackSession) + var metadata = playbackSession.getMediaMetadataCompat() mediaSession.setMetadata(metadata) var mediaMetadata = playbackSession.getExoMediaMetadata() @@ -679,7 +681,7 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { if (mPlayer == currentPlayer) { var mediaSource:MediaSource - if (currentPlaybackSession?.isLocal == true) { + if (!playbackSession.isHLS) { Log.d(tag, "Playing Local File") var dataSourceFactory = DefaultDataSourceFactory(ctx, channelId) mediaSource = ProgressiveMediaSource.Factory(dataSourceFactory).createMediaSource(mediaItem) @@ -881,11 +883,13 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { } fun terminateStream() { - if (currentPlayer.playbackState == Player.STATE_READY) { - currentPlayer.clearMediaItems() - } - currentAudiobookStreamData?.id = "" +// if (currentPlayer.playbackState == Player.STATE_READY) { +// currentPlayer.clearMediaItems() +// } + currentPlayer.clearMediaItems() + currentPlaybackSession = null lastPauseTime = 0 + clientEventEmitter?.onPlaybackClosed() } fun sendClientMetadata(stateName: String) { @@ -895,7 +899,7 @@ class PlayerNotificationService : MediaBrowserServiceCompat() { metadata.put("duration", duration) metadata.put("currentTime", mPlayer.currentPosition) metadata.put("stateName", stateName) - listener?.onMetadata(metadata) + clientEventEmitter?.onMetadata(metadata) } diff --git a/android/app/src/main/java/com/audiobookshelf/app/SleepTimerManager.kt b/android/app/src/main/java/com/audiobookshelf/app/SleepTimerManager.kt index 7d5e3dae..e0d0ec60 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/SleepTimerManager.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/SleepTimerManager.kt @@ -87,7 +87,7 @@ class SleepTimerManager constructor(playerNotificationService:PlayerNotification } } - playerNotificationService.listener?.onSleepTimerSet(getSleepTimerTimeRemainingSeconds()) + playerNotificationService.clientEventEmitter?.onSleepTimerSet(getSleepTimerTimeRemainingSeconds()) sleepTimerRunning = true sleepTimerTask = Timer("SleepTimer", false).schedule(0L, 1000L) { @@ -99,14 +99,14 @@ class SleepTimerManager constructor(playerNotificationService:PlayerNotification Log.d(tag, "Timer Elapsed $sleepTimerElapsed | Sleep TIMER time remaining $sleepTimeSecondsRemaining s") if (sleepTimeSecondsRemaining > 0) { - playerNotificationService.listener?.onSleepTimerSet(sleepTimeSecondsRemaining) + playerNotificationService.clientEventEmitter?.onSleepTimerSet(sleepTimeSecondsRemaining) } if (sleepTimeSecondsRemaining <= 0) { Log.d(tag, "Sleep Timer Pausing Player on Chapter") pause() - playerNotificationService.listener?.onSleepTimerEnded(getCurrentTime()) + playerNotificationService.clientEventEmitter?.onSleepTimerEnded(getCurrentTime()) clearSleepTimer() sleepTimerFinishedAt = System.currentTimeMillis() } else if (sleepTimeSecondsRemaining <= 30) { @@ -136,7 +136,7 @@ class SleepTimerManager constructor(playerNotificationService:PlayerNotification fun cancelSleepTimer() { Log.d(tag, "Canceling Sleep Timer") clearSleepTimer() - playerNotificationService.listener?.onSleepTimerSet(0) + playerNotificationService.clientEventEmitter?.onSleepTimerSet(0) } private fun extendSleepTime() { @@ -150,7 +150,7 @@ class SleepTimerManager constructor(playerNotificationService:PlayerNotification if (sleepTimerEndTime > getDuration()) sleepTimerEndTime = getDuration() } - playerNotificationService.listener?.onSleepTimerSet(getSleepTimerTimeRemainingSeconds()) + playerNotificationService.clientEventEmitter?.onSleepTimerSet(getSleepTimerTimeRemainingSeconds()) } fun checkShouldExtendSleepTimer() { @@ -197,7 +197,7 @@ class SleepTimerManager constructor(playerNotificationService:PlayerNotification } setVolume(1F) - playerNotificationService.listener?.onSleepTimerSet(getSleepTimerTimeRemainingSeconds()) + playerNotificationService.clientEventEmitter?.onSleepTimerSet(getSleepTimerTimeRemainingSeconds()) } fun decreaseSleepTime(time: Long) { @@ -219,6 +219,6 @@ class SleepTimerManager constructor(playerNotificationService:PlayerNotification } setVolume(1F) - playerNotificationService.listener?.onSleepTimerSet(getSleepTimerTimeRemainingSeconds()) + playerNotificationService.clientEventEmitter?.onSleepTimerSet(getSleepTimerTimeRemainingSeconds()) } } diff --git a/android/app/src/main/java/com/audiobookshelf/app/data/DataClasses.kt b/android/app/src/main/java/com/audiobookshelf/app/data/DataClasses.kt index 1e30846a..b24b1b18 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/data/DataClasses.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/data/DataClasses.kt @@ -46,7 +46,8 @@ data class Book( var metadata:BookMetadata, var coverPath:String?, var tags:MutableList, - var audioFiles:MutableList + var audioFiles:MutableList, + var chapters:MutableList ) : MediaType() // This auto-detects whether it is a Book or Podcast @@ -154,3 +155,11 @@ data class AudioTrack( var localFileId:String?, var audioProbeResult:AudioProbeResult? ) + +@JsonIgnoreProperties(ignoreUnknown = true) +data class BookChapter( + var id:Int, + var start:Double, + var end:Double, + var title:String? +) diff --git a/android/app/src/main/java/com/audiobookshelf/app/data/DbManager.kt b/android/app/src/main/java/com/audiobookshelf/app/data/DbManager.kt index 181320bc..6358b435 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/data/DbManager.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/data/DbManager.kt @@ -52,11 +52,9 @@ class DbManager : Plugin() { } fun saveLocalMediaItems(localMediaItems:List) { - GlobalScope.launch(Dispatchers.IO) { localMediaItems.map { Paper.book("localMediaItems").write(it.id, it) } - } } fun saveLocalFolder(localFolder:LocalFolder) { diff --git a/android/app/src/main/java/com/audiobookshelf/app/data/DeviceClasses.kt b/android/app/src/main/java/com/audiobookshelf/app/data/DeviceClasses.kt index 7c61fecd..2fcc3669 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/data/DeviceClasses.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/data/DeviceClasses.kt @@ -29,7 +29,7 @@ data class LocalMediaItem( var absolutePath:String, var audioTracks:MutableList, var localFiles:MutableList, - var coverPath:String? + var coverContentUrl:String? ) { @JsonIgnore @@ -53,7 +53,7 @@ data class LocalMediaItem( var sessionId = "play-${UUID.randomUUID()}" var mediaMetadata = getMediaMetadata() - return PlaybackSession(sessionId,null,null,null,null,mediaType,mediaMetadata,null,getDuration(),PLAYMETHOD_LOCAL,audioTracks,0.0,null,this,null,null) + return PlaybackSession(sessionId,null,null,null, mediaType, mediaMetadata, mutableListOf(), name, "author name here",null,getDuration(),PLAYMETHOD_LOCAL,audioTracks,0.0,null,this,null,null) } } diff --git a/android/app/src/main/java/com/audiobookshelf/app/data/PlaybackSession.kt b/android/app/src/main/java/com/audiobookshelf/app/data/PlaybackSession.kt index 777e1487..dcafef24 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/data/PlaybackSession.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/data/PlaybackSession.kt @@ -19,9 +19,11 @@ class PlaybackSession( var userId:String?, var libraryItemId:String?, var episodeId:String?, - var mediaEntityId:String?, var mediaType:String, var mediaMetadata:MediaTypeMetadata, + var chapters:MutableList, + var displayTitle: String?, + var displayAuthor: String?, var coverPath:String?, var duration:Double, var playMethod:Int, @@ -37,23 +39,9 @@ class PlaybackSession( val isLocal get() = playMethod == PLAYMETHOD_LOCAL val currentTimeMs get() = (currentTime * 1000L).toLong() - @JsonIgnore - fun getTitle():String { - if (mediaMetadata == null) return "Unset" - var metadata = mediaMetadata as BookMetadata - return metadata.title - } - - @JsonIgnore - fun getAuthor():String { - if (mediaMetadata == null) return "Unset" - var metadata = mediaMetadata as BookMetadata - return metadata.authorName ?: "Unset" - } - @JsonIgnore fun getCoverUri(): Uri { - if (localMediaItem?.coverPath != null) return Uri.parse(localMediaItem?.coverPath) ?: Uri.parse("android.resource://com.audiobookshelf.app/" + R.drawable.icon) + if (localMediaItem?.coverContentUrl != null) return Uri.parse(localMediaItem?.coverContentUrl) ?: Uri.parse("android.resource://com.audiobookshelf.app/" + R.drawable.icon) if (coverPath == null) return Uri.parse("android.resource://com.audiobookshelf.app/" + R.drawable.icon) return Uri.parse("$serverUrl/api/items/$libraryItemId/cover?token=$token") @@ -75,11 +63,11 @@ class PlaybackSession( @JsonIgnore fun getMediaMetadataCompat(): MediaMetadataCompat { var metadataBuilder = MediaMetadataCompat.Builder() - .putString(MediaMetadataCompat.METADATA_KEY_TITLE, this.getTitle()) - .putString(MediaMetadataCompat.METADATA_KEY_DISPLAY_TITLE, getTitle()) - .putString(MediaMetadataCompat.METADATA_KEY_DISPLAY_SUBTITLE, this.getAuthor()) - .putString(MediaMetadataCompat.METADATA_KEY_AUTHOR, this.getAuthor()) - .putString(MediaMetadataCompat.METADATA_KEY_ARTIST, this.getAuthor()) + .putString(MediaMetadataCompat.METADATA_KEY_TITLE, displayTitle) + .putString(MediaMetadataCompat.METADATA_KEY_DISPLAY_TITLE, displayTitle) + .putString(MediaMetadataCompat.METADATA_KEY_DISPLAY_SUBTITLE, displayAuthor) + .putString(MediaMetadataCompat.METADATA_KEY_AUTHOR, displayAuthor) + .putString(MediaMetadataCompat.METADATA_KEY_ARTIST, displayAuthor) .putString(MediaMetadataCompat.METADATA_KEY_ALBUM, "series") .putString(MediaMetadataCompat.METADATA_KEY_MEDIA_ID, id) return metadataBuilder.build() @@ -87,13 +75,12 @@ class PlaybackSession( @JsonIgnore fun getExoMediaMetadata(): MediaMetadata { - var authorName = this.getAuthor() var metadataBuilder = MediaMetadata.Builder() - .setTitle(this.getTitle()) - .setDisplayTitle(this.getTitle()) - .setArtist(authorName) - .setAlbumArtist(authorName) - .setSubtitle(authorName) + .setTitle(displayTitle) + .setDisplayTitle(displayTitle) + .setArtist(displayAuthor) + .setAlbumArtist(displayAuthor) + .setSubtitle(displayAuthor) var contentUri = this.getContentUri() metadataBuilder.setMediaUri(contentUri) diff --git a/android/app/src/main/java/com/audiobookshelf/app/device/FolderScanner.kt b/android/app/src/main/java/com/audiobookshelf/app/device/FolderScanner.kt index 0ec78863..d58901f9 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/device/FolderScanner.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/device/FolderScanner.kt @@ -11,6 +11,9 @@ import com.arthenica.ffmpegkit.Level import com.audiobookshelf.app.data.* import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.launch class FolderScanner(var ctx: Context) { private val tag = "FolderScanner" @@ -69,7 +72,7 @@ class FolderScanner(var ctx: Context) { var localFiles = mutableListOf() var index = 1 var startOffset = 0.0 - var coverPath:String? = null + var coverContentUrl:String? = null var filesInFolder = it.search(false, DocumentFileType.FILE, arrayOf("audio/*", "image/*")) @@ -146,14 +149,14 @@ class FolderScanner(var ctx: Context) { if (existingLocalFile == null) { isNewOrUpdated = true } - if (existingMediaItem != null && existingMediaItem.coverPath == null) { + if (existingMediaItem != null && existingMediaItem.coverContentUrl == null) { // Existing media item did not have a cover - cover found on scan isNewOrUpdated = true } // First image file use as cover path - if (coverPath == null) { - coverPath = localFile.contentUrl + if (coverContentUrl == null) { + coverContentUrl = localFile.contentUrl } } } @@ -170,7 +173,7 @@ class FolderScanner(var ctx: Context) { else mediaItemsAdded++ Log.d(tag, "Found local media item named $itemFolderName with ${audioTracks.size} tracks and ${localFiles.size} local files") - var localMediaItem = LocalMediaItem(itemId, itemFolderName, localFolder.mediaType, localFolder.id, it.uri.toString(), it.getSimplePath(ctx), it.getAbsolutePath(ctx),audioTracks,localFiles,coverPath) + var localMediaItem = LocalMediaItem(itemId, itemFolderName, localFolder.mediaType, localFolder.id, it.uri.toString(), it.getSimplePath(ctx), it.getAbsolutePath(ctx),audioTracks,localFiles,coverContentUrl) mediaItems.add(localMediaItem) } } diff --git a/android/app/src/main/java/com/audiobookshelf/app/server/ApiHandler.kt b/android/app/src/main/java/com/audiobookshelf/app/server/ApiHandler.kt index 6f602662..cb71f6ac 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/server/ApiHandler.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/server/ApiHandler.kt @@ -109,11 +109,14 @@ class ApiHandler { } } - fun playLibraryItem(libraryItemId:String, cb: (PlaybackSession) -> Unit) { + fun playLibraryItem(libraryItemId:String, forceTranscode:Boolean, cb: (PlaybackSession) -> Unit) { val mapper = jacksonObjectMapper() var payload = JSObject() payload.put("mediaPlayer", "exo-player") - payload.put("forceDirectPlay", true) + + // Only if direct play fails do we force transcode + if (!forceTranscode) payload.put("forceDirectPlay", true) + else payload.put("forceTranscode", true) postRequest("/api/items/$libraryItemId/play", payload) { it.put("serverUrl", serverUrl) diff --git a/components/app/AudioPlayer.vue b/components/app/AudioPlayer.vue index 7877ab4d..7b38f251 100644 --- a/components/app/AudioPlayer.vue +++ b/components/app/AudioPlayer.vue @@ -1,5 +1,5 @@