diff --git a/android/app/src/main/java/com/audiobookshelf/app/data/DeviceClasses.kt b/android/app/src/main/java/com/audiobookshelf/app/data/DeviceClasses.kt index 76b8f4b4..9c242b42 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/data/DeviceClasses.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/data/DeviceClasses.kt @@ -1,12 +1,14 @@ package com.audiobookshelf.app.data import android.content.Context +import android.net.Uri import android.support.v4.media.MediaDescriptionCompat import android.util.Log import com.fasterxml.jackson.annotation.JsonIgnore import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonSubTypes import com.fasterxml.jackson.annotation.JsonTypeInfo +import java.io.File enum class LockOrientationSetting { NONE, PORTRAIT, LANDSCAPE @@ -57,6 +59,19 @@ data class LocalFile( var mimeType:String?, var size:Long ) { + @JsonIgnore + fun exists(ctx: Context): Boolean { + if (contentUrl.startsWith("content:")) { + return try { + ctx.contentResolver.openFileDescriptor(Uri.parse(contentUrl), "r")?.use { true } ?: false + } catch (e: Exception) { + Log.w("LocalFile", "Cannot access SAF file $contentUrl", e) + false + } + } + return File(absolutePath).exists() + } + @JsonIgnore fun isAudioFile():Boolean { if (mimeType == "application/octet-stream") return true @@ -218,4 +233,3 @@ data class DeviceData( } } } - diff --git a/android/app/src/main/java/com/audiobookshelf/app/data/FolderScanResult.kt b/android/app/src/main/java/com/audiobookshelf/app/data/FolderScanResult.kt deleted file mode 100644 index f4f706bf..00000000 --- a/android/app/src/main/java/com/audiobookshelf/app/data/FolderScanResult.kt +++ /dev/null @@ -1,15 +0,0 @@ -package com.audiobookshelf.app.data - -data class FolderScanResult( - var itemsAdded:Int, - var itemsUpdated:Int, - var itemsRemoved:Int, - var itemsUpToDate:Int, - val localFolder:LocalFolder, - val localLibraryItems:List, -) - -data class LocalLibraryItemScanResult( - val updated:Boolean, - val localLibraryItem:LocalLibraryItem, -) diff --git a/android/app/src/main/java/com/audiobookshelf/app/data/LocalLibraryItem.kt b/android/app/src/main/java/com/audiobookshelf/app/data/LocalLibraryItem.kt index 635c98f2..9251e7d9 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/data/LocalLibraryItem.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/data/LocalLibraryItem.kt @@ -80,7 +80,7 @@ class LocalLibraryItem( } @JsonIgnore - fun hasTracks(episode:PodcastEpisode?): Boolean { + fun hasTracks(ctx: Context, episode:PodcastEpisode?): Boolean { var audioTracks = media.getAudioTracks() as MutableList if (episode != null) { // Get podcast episode audio track episode.audioTrack?.let { at -> mutableListOf(at) }?.let { tracks -> audioTracks = tracks } @@ -91,15 +91,19 @@ class LocalLibraryItem( if (it.metadata === null) { return false } - // Check that file exists - val file = File(it.metadata!!.path) - if (!file.exists()) { + if (!trackExists(ctx, it.contentUrl, it.metadata!!.path)) { return false } } return true } + /** App-private files have paths; SAF files must be validated through their persisted URI. */ + @JsonIgnore + private fun trackExists(ctx: Context, contentUrl: String?, path: String): Boolean { + return LocalFile("", null, contentUrl ?: "", "", path, "", null, 0).exists(ctx) + } + @JsonIgnore fun getPlaybackSession(episode:PodcastEpisode?, deviceInfo:DeviceInfo):PlaybackSession { val localEpisodeId = episode?.id diff --git a/android/app/src/main/java/com/audiobookshelf/app/device/FolderScanner.kt b/android/app/src/main/java/com/audiobookshelf/app/device/FolderScanner.kt index 951b6a3b..c53b9b8b 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/device/FolderScanner.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/device/FolderScanner.kt @@ -7,570 +7,299 @@ import androidx.documentfile.provider.DocumentFile import com.anggrayudi.storage.file.* import com.audiobookshelf.app.data.* import com.audiobookshelf.app.models.DownloadItem -import com.fasterxml.jackson.core.json.JsonReadFeature -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.audiobookshelf.app.models.DownloadItemPart import java.io.File -class FolderScanner(var ctx: Context) { +/** Creates local-library records from the completed download manifest, not a recursive rescan. */ +class FolderScanner(private val ctx: Context) { private val tag = "FolderScanner" - private var jacksonMapper = - jacksonObjectMapper() - .enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature()) data class DownloadItemScanResult( val localLibraryItem: LocalLibraryItem, var localMediaProgress: LocalMediaProgress? ) - private fun getLocalLibraryItemId(mediaItemId: String): String { - return "local_" + DeviceManager.getBase64Id(mediaItemId) - } + private fun localLibraryItemId(mediaItemId: String) = "local_${DeviceManager.getBase64Id(mediaItemId)}" - private fun scanInternalDownloadItem( - downloadItem: DownloadItem, - cb: (DownloadItemScanResult?) -> Unit - ) { - val localLibraryItemId = "local_${downloadItem.libraryItemId}" - - var localEpisodeId: String? = null - var localLibraryItem: LocalLibraryItem? - if (downloadItem.mediaType == "book") { - localLibraryItem = - LocalLibraryItem( - localLibraryItemId, - downloadItem.localFolder.id, - downloadItem.itemFolderPath, - downloadItem.itemFolderPath, - "", - false, - downloadItem.mediaType, - downloadItem.media.getLocalCopy(), - mutableListOf(), - null, - null, - true, - downloadItem.serverConnectionConfigId, - downloadItem.serverAddress, - downloadItem.serverUserId, - downloadItem.libraryItemId - ) - } else { - // Lookup or create podcast local library item - localLibraryItem = DeviceManager.dbManager.getLocalLibraryItem(localLibraryItemId) - if (localLibraryItem == null) { - Log.d( - tag, - "[FolderScanner] Podcast local library item not created yet for ${downloadItem.media.metadata.title}" - ) - localLibraryItem = - LocalLibraryItem( - localLibraryItemId, - downloadItem.localFolder.id, - downloadItem.itemFolderPath, - downloadItem.itemFolderPath, - "", - false, - downloadItem.mediaType, - downloadItem.media.getLocalCopy(), - mutableListOf(), - null, - null, - true, - downloadItem.serverConnectionConfigId, - downloadItem.serverAddress, - downloadItem.serverUserId, - downloadItem.libraryItemId - ) - } + private fun createLocalFile(part: DownloadItemPart, externalFile: DocumentFile? = null): LocalFile? { + if (part.isInternalStorage) { + val file = File(part.finalDestinationPath) + if (!file.exists()) return null + return LocalFile( + DeviceManager.getBase64Id(file.name), + file.name, + Uri.fromFile(file).toString(), + file.getBasePath(ctx), + file.absolutePath, + file.getSimplePath(ctx), + file.mimeType, + file.length() + ) } - val audioTracks: MutableList = mutableListOf() - var foundEBookFile = false - - downloadItem.downloadItemParts.forEach { downloadItemPart -> - Log.d( - tag, - "Scan internal storage item with finalDestinationUri=${downloadItemPart.finalDestinationUri}" + part.completedDestinationUri?.let { contentUrl -> + val uri = Uri.parse(contentUrl) + val size = + try { + ctx.contentResolver.openFileDescriptor(uri, "r")?.use { descriptor -> + descriptor.statSize.coerceAtLeast(0L) + } ?: 0L + } catch (e: Exception) { + Log.e(tag, "Could not open completed SAF file: $contentUrl", e) + return null + } + // Android 10 DownloadsProvider may not reconstruct a DocumentFile for an audio URI even + // though the URI remains readable. Keep the URI as the authoritative local-file location. + return LocalFile( + DeviceManager.getBase64Id(contentUrl), + part.filename, + contentUrl, + part.localFolderName, + part.finalDestinationPath, + part.finalDestinationPath, + mimeTypeFor(part), + size ) + } - val file = File(downloadItemPart.finalDestinationPath) - Log.d(tag, "Scan internal storage item created file ${file.name}") + // Do not reconstruct a DocumentFile from an absolute path: on Android 10 that becomes a + // file:// URI, which DocumentsContract rejects. The caller resolves this from the persisted + // SAF tree grant instead. + val document = externalFile + if (document == null || !document.exists()) { + Log.e(tag, "Could not resolve downloaded SAF file: ${part.finalDestinationPath}") + return null + } + return LocalFile( + DeviceManager.getBase64Id(document.id), + document.name, + document.uri.toString(), + document.getBasePath(ctx), + document.getAbsolutePath(ctx), + document.getSimplePath(ctx), + document.mimeType, + document.length() + ) + } - if (file == null) { - Log.e( - tag, - "scanInternalDownloadItem: Null docFile for path ${downloadItemPart.finalDestinationPath}" - ) - } else { - if (downloadItemPart.audioTrack != null) { - val audioTrackFromServer = downloadItemPart.audioTrack - Log.d( - tag, - "scanInternalDownloadItem: Audio Track from Server index = ${audioTrackFromServer.index}" + private fun newLocalLibraryItem( + id: String, + downloadItem: DownloadItem, + basePath: String, + absolutePath: String, + contentUrl: String + ) = + LocalLibraryItem( + id, + downloadItem.localFolder.id, + basePath, + absolutePath, + contentUrl, + false, + downloadItem.mediaType, + downloadItem.media.getLocalCopy(), + mutableListOf(), + null, + null, + true, + downloadItem.serverConnectionConfigId, + downloadItem.serverAddress, + downloadItem.serverUserId, + downloadItem.libraryItemId ) - val localFileId = DeviceManager.getBase64Id(file.name) - Log.d(tag, "Scan internal file localFileId=$localFileId") - val localFile = - LocalFile( - localFileId, - file.name, - downloadItemPart.finalDestinationUri.toString(), - file.getBasePath(ctx), - file.absolutePath, - file.getSimplePath(ctx), - file.mimeType, - file.length() - ) - localLibraryItem.localFiles.add(localFile) + private fun scanParts( + item: DownloadItem, + localItem: LocalLibraryItem, + externalFolder: DocumentFile? = null, + callback: (DownloadItemScanResult?) -> Unit + ) { + val tracks = mutableListOf() + var foundEbook = false + var localEpisodeId: String? = null - val trackFileMetadata = + item.downloadItemParts.forEach { part -> + val externalFile = + if (part.isInternalStorage) { + null + } else { + part.completedDestinationUri + ?.let { DocumentFileCompat.fromUri(ctx, Uri.parse(it)) } + ?: resolveExternalFile(externalFolder, part) + } + Log.d(tag, "Resolve part ${part.filename}: externalFile=${externalFile?.uri}") + val localFile = createLocalFile(part, externalFile) ?: return@forEach + when { + part.audioTrack != null -> { + val serverTrack = part.audioTrack + localItem.localFiles.removeAll { it.id == localFile.id } + localItem.localFiles.add(localFile) + val metadata = FileMetadata( - file.name, - file.extension, - file.absolutePath, - file.getBasePath(ctx), - file.length() + localFile.filename ?: "", + File(localFile.filename ?: "").extension, + localFile.absolutePath, + localFile.basePath, + localFile.size ) - // Create new audio track val track = AudioTrack( - audioTrackFromServer.index, - audioTrackFromServer.startOffset, - audioTrackFromServer.duration, + serverTrack.index, + serverTrack.startOffset, + serverTrack.duration, localFile.filename ?: "", localFile.contentUrl, localFile.mimeType ?: "", - trackFileMetadata, + metadata, true, - localFileId, - audioTrackFromServer.index + localFile.id, + serverTrack.index ) - audioTracks.add(track) - - Log.d( - tag, - "scanInternalDownloadItem: Created Audio Track with index ${track.index} from local file ${localFile.absolutePath}" - ) - - // Add podcast episodes to library - downloadItemPart.episode?.let { podcastEpisode -> - val podcast = localLibraryItem.media as Podcast - val newEpisode = podcast.addEpisode(track, podcastEpisode) - localEpisodeId = newEpisode.id - Log.d( - tag, - "scanInternalDownloadItem: Added episode to podcast ${podcastEpisode.title} ${track.title} | Track index: ${podcastEpisode.audioTrack?.index}" - ) + tracks.add(track) + Log.d(tag, "Added local audio track ${track.contentUrl} (${track.metadata?.path})") + part.episode?.let { episode -> + val podcast = localItem.media as Podcast + localEpisodeId = podcast.addEpisode(track, episode).id } - } else if (downloadItemPart.ebookFile != null) { - foundEBookFile = true - Log.d(tag, "scanInternalDownloadItem: Ebook file found with mimetype=${file.mimeType}") - val localFileId = DeviceManager.getBase64Id(file.name) - val localFile = - LocalFile( - localFileId, - file.name, - Uri.fromFile(file).toString(), - file.getBasePath(ctx), - file.absolutePath, - file.getSimplePath(ctx), - file.mimeType, - file.length() - ) - localLibraryItem.localFiles.add(localFile) - - val ebookFile = + } + part.ebookFile != null -> { + foundEbook = true + localItem.localFiles.removeAll { it.id == localFile.id } + localItem.localFiles.add(localFile) + (localItem.media as Book).ebookFile = EBookFile( - downloadItemPart.ebookFile.ino, - downloadItemPart.ebookFile.metadata, - downloadItemPart.ebookFile.ebookFormat, + part.ebookFile.ino, + part.ebookFile.metadata, + part.ebookFile.ebookFormat, true, - localFileId, + localFile.id, localFile.contentUrl ) - (localLibraryItem.media as Book).ebookFile = ebookFile - Log.d(tag, "scanInternalDownloadItem: Ebook file added to lli ${localFile.contentUrl}") - } else { - val localFileId = DeviceManager.getBase64Id(file.name) - val localFile = - LocalFile( - localFileId, - file.name, - Uri.fromFile(file).toString(), - file.getBasePath(ctx), - file.absolutePath, - file.getSimplePath(ctx), - file.mimeType, - file.length() - ) - - localLibraryItem.coverAbsolutePath = localFile.absolutePath - localLibraryItem.coverContentUrl = localFile.contentUrl - localLibraryItem.localFiles.add(localFile) + } + else -> { + localItem.coverAbsolutePath = localFile.absolutePath + localItem.coverContentUrl = localFile.contentUrl + localItem.localFiles.removeAll { it.id == localFile.id } + localItem.localFiles.add(localFile) } } } - if (audioTracks.isEmpty() && !foundEBookFile) { - Log.d( - tag, - "scanDownloadItem did not find any audio tracks or ebook file in folder for ${downloadItem.itemFolderPath}" - ) - return cb(null) + if (tracks.isEmpty() && !foundEbook) { + callback(null) + return } - - // For books sort audio tracks then set - if (downloadItem.mediaType == "book") { - audioTracks.sortBy { it.index } - - var indexCheck = 1 - var startOffset = 0.0 - audioTracks.forEach { audioTrack -> - if (audioTrack.index != indexCheck || audioTrack.startOffset != startOffset) { - audioTrack.index = indexCheck - audioTrack.startOffset = startOffset - } - indexCheck++ - startOffset += audioTrack.duration + if (item.mediaType == "book") { + tracks.sortBy { it.index } + var expectedIndex = 1 + var offset = 0.0 + tracks.forEach { track -> + track.index = expectedIndex++ + track.startOffset = offset + offset += track.duration } - - localLibraryItem.media.setAudioTracks(audioTracks) + localItem.media.setAudioTracks(tracks) } - val downloadItemScanResult = DownloadItemScanResult(localLibraryItem, null) - - // If library item had media progress then make local media progress and save - downloadItem.userMediaProgress?.let { mediaProgress -> - val localMediaProgressId = - if (downloadItem.episodeId.isNullOrEmpty()) localLibraryItemId - else "$localLibraryItemId-$localEpisodeId" - val newLocalMediaProgress = + val result = DownloadItemScanResult(localItem, null) + item.userMediaProgress?.let { progress -> + val progressId = if (item.episodeId.isNullOrEmpty()) localItem.id else "${localItem.id}-$localEpisodeId" + result.localMediaProgress = LocalMediaProgress( - id = localMediaProgressId, - localLibraryItemId = localLibraryItemId, - localEpisodeId = localEpisodeId, - duration = mediaProgress.duration, - progress = mediaProgress.progress, - currentTime = mediaProgress.currentTime, - isFinished = mediaProgress.isFinished, - ebookLocation = mediaProgress.ebookLocation, - ebookProgress = mediaProgress.ebookProgress, - lastUpdate = mediaProgress.lastUpdate, - startedAt = mediaProgress.startedAt, - finishedAt = mediaProgress.finishedAt, - serverConnectionConfigId = downloadItem.serverConnectionConfigId, - serverAddress = downloadItem.serverAddress, - serverUserId = downloadItem.serverUserId, - libraryItemId = downloadItem.libraryItemId, - episodeId = downloadItem.episodeId + progressId, + localItem.id, + localEpisodeId, + progress.duration, + progress.progress, + progress.currentTime, + progress.isFinished, + progress.ebookLocation, + progress.ebookProgress, + progress.lastUpdate, + progress.startedAt, + progress.finishedAt, + item.serverConnectionConfigId, + item.serverAddress, + item.serverUserId, + item.libraryItemId, + item.episodeId ) - Log.d( - tag, - "scanLibraryItemFolder: Saving local media progress ${newLocalMediaProgress.id} at progress ${newLocalMediaProgress.progress}" - ) - DeviceManager.dbManager.saveLocalMediaProgress(newLocalMediaProgress) - - downloadItemScanResult.localMediaProgress = newLocalMediaProgress + DeviceManager.dbManager.saveLocalMediaProgress(result.localMediaProgress!!) } - - DeviceManager.dbManager.saveLocalLibraryItem(localLibraryItem) - - cb(downloadItemScanResult) + DeviceManager.dbManager.saveLocalLibraryItem(localItem) + callback(result) } - // Scan item after download and create local library item - fun scanDownloadItem(downloadItem: DownloadItem, cb: (DownloadItemScanResult?) -> Unit) { - // If downloading to internal storage handle separately - if (downloadItem.isInternalStorage) { - scanInternalDownloadItem(downloadItem, cb) + private fun findFolderByPath(root: DocumentFile, subPath: String): DocumentFile? { + if (subPath.isBlank()) return root + var current = root + subPath.split('/').filter { it.isNotBlank() }.forEach { segment -> + if (segment == "." || segment == "..") return null + current = current.findFile(segment) ?: return null + } + return current + } + + /** + * DownloadsProvider on Android 10 may expose an audio document without its extension through + * DocumentFile.findFile(). Match the manifest first, then match the provider-normalized base + * filename. MIME type and server-reported size are unreliable for Opus on this platform. + */ + private fun resolveExternalFile(folder: DocumentFile?, part: DownloadItemPart): DocumentFile? { + if (folder == null) return null + folder.findFile(part.filename)?.let { return it } + val expectedBaseName = part.filename.substringBeforeLast('.') + return folder.listFiles().firstOrNull { document -> + document.name == part.filename || + document.fullName == part.filename || + (part.audioTrack != null && document.isFile && + (document.name ?: "").substringBeforeLast('.') == expectedBaseName) || + (part.audioTrack != null && document.isFile && + document.fullName.substringBeforeLast('.') == expectedBaseName) + } + } + + private fun mimeTypeFor(part: DownloadItemPart): String? { + return part.audioTrack?.mimeType + ?: when (part.ebookFile?.ebookFormat?.lowercase()) { + "epub" -> "application/epub+zip" + "pdf" -> "application/pdf" + else -> "image/jpeg" + } + } + + fun scanDownloadItem(item: DownloadItem, callback: (DownloadItemScanResult?) -> Unit) { + if (item.isInternalStorage) { + val id = "local_${item.libraryItemId}" + val localItem = + DeviceManager.dbManager.getLocalLibraryItem(id) + ?: newLocalLibraryItem(id, item, item.itemFolderPath, item.itemFolderPath, "") + scanParts(item, localItem, callback = callback) return } - val folderDf = DocumentFileCompat.fromUri(ctx, Uri.parse(downloadItem.localFolder.contentUrl)) - val foldersFound = folderDf?.search(true, DocumentFileType.FOLDER) ?: mutableListOf() - - var itemFolderId = "" - var itemFolderUrl = "" - var itemFolderBasePath = "" - var itemFolderAbsolutePath = "" - foldersFound.forEach { - // e.g. absolute path is "storage/emulated/0/Audiobooks/Orson Scott Card/Enders Game" - // and itemSubfolder is "Orson Scott Card/Enders Game" - if (it.getAbsolutePath(ctx).endsWith(downloadItem.itemSubfolder)) { - itemFolderId = it.id - itemFolderUrl = it.uri.toString() - itemFolderBasePath = it.getBasePath(ctx) - itemFolderAbsolutePath = it.getAbsolutePath(ctx) - } + val root = DocumentFileCompat.fromUri(ctx, Uri.parse(item.localFolder.contentUrl)) + if (root == null) { + Log.e(tag, "Invalid SAF root: ${item.localFolder.contentUrl}") + callback(null) + return } - - if (itemFolderUrl == "") { - Log.d(tag, "scanDownloadItem failed to find media folder") - return cb(null) + val itemFolder = findFolderByPath(root, item.itemSubfolder) + if (itemFolder == null) { + Log.e(tag, "SAF item folder not found: ${item.itemSubfolder}") + callback(null) + return } - val df: DocumentFile? = DocumentFileCompat.fromUri(ctx, Uri.parse(itemFolderUrl)) - - if (df == null) { - Log.e(tag, "Folder Doc File Invalid ${downloadItem.itemFolderPath}") - return cb(null) - } - - val localLibraryItemId = getLocalLibraryItemId(itemFolderId) - Log.d( - tag, - "scanDownloadItem starting for ${downloadItem.itemFolderPath} | ${df.uri} | Item Folder Id:$itemFolderId | LLI Id:$localLibraryItemId" - ) - - // Search for files in media item folder - // m4b files showing as mimeType application/octet-stream on Android 10 and earlier see #154 - val filesFound = - df.search( - false, - DocumentFileType.FILE, - arrayOf("audio/*", "image/*", "video/mp4", "application/*") - ) - Log.d(tag, "scanDownloadItem ${filesFound.size} files found in ${downloadItem.itemFolderPath}") - - var localEpisodeId: String? = null - var localLibraryItem: LocalLibraryItem? - if (downloadItem.mediaType == "book") { - localLibraryItem = - LocalLibraryItem( - localLibraryItemId, - downloadItem.localFolder.id, - itemFolderBasePath, - itemFolderAbsolutePath, - itemFolderUrl, - false, - downloadItem.mediaType, - downloadItem.media.getLocalCopy(), - mutableListOf(), - null, - null, - true, - downloadItem.serverConnectionConfigId, - downloadItem.serverAddress, - downloadItem.serverUserId, - downloadItem.libraryItemId - ) - } else { - // Lookup or create podcast local library item - localLibraryItem = DeviceManager.dbManager.getLocalLibraryItem(localLibraryItemId) - if (localLibraryItem == null) { - Log.d( - tag, - "[FolderScanner] Podcast local library item not created yet for ${downloadItem.media.metadata.title}" - ) - localLibraryItem = - LocalLibraryItem( - localLibraryItemId, - downloadItem.localFolder.id, - itemFolderBasePath, - itemFolderAbsolutePath, - itemFolderUrl, - false, - downloadItem.mediaType, - downloadItem.media.getLocalCopy(), - mutableListOf(), - null, - null, - true, - downloadItem.serverConnectionConfigId, - downloadItem.serverAddress, - downloadItem.serverUserId, - downloadItem.libraryItemId - ) - } - } - - val audioTracks: MutableList = mutableListOf() - var foundEBookFile = false - - filesFound.forEach { docFile -> - val itemPart = - downloadItem.downloadItemParts.find { itemPart -> itemPart.filename == docFile.name } - if (itemPart == null) { - if (downloadItem.mediaType == "book" - ) { // for books every download item should be a file found - Log.e( - tag, - "scanDownloadItem: Item part not found for doc file ${docFile.name} | ${docFile.getAbsolutePath(ctx)} | ${docFile.uri}" - ) - } - } else if (itemPart.audioTrack != null) { // Is audio track - val audioTrackFromServer = itemPart.audioTrack - Log.d( - tag, - "scanDownloadItem: Audio Track from Server index = ${audioTrackFromServer.index}" - ) - - val localFileId = DeviceManager.getBase64Id(docFile.id) - val localFile = - LocalFile( - localFileId, - docFile.name, - docFile.uri.toString(), - docFile.getBasePath(ctx), - docFile.getAbsolutePath(ctx), - docFile.getSimplePath(ctx), - docFile.mimeType, - docFile.length() - ) - localLibraryItem.localFiles.add(localFile) - - // Create new audio track - val trackFileMetadata = - FileMetadata( - docFile.name ?: "", - docFile.extension ?: "", - docFile.getAbsolutePath(ctx), - docFile.getBasePath(ctx), - docFile.length() - ) - val track = - AudioTrack( - audioTrackFromServer.index, - audioTrackFromServer.startOffset, - audioTrackFromServer.duration, - localFile.filename ?: "", - localFile.contentUrl, - localFile.mimeType ?: "", - trackFileMetadata, - true, - localFileId, - audioTrackFromServer.index - ) - audioTracks.add(track) - - Log.d( - tag, - "scanDownloadItem: Created Audio Track with index ${track.index} from local file ${localFile.absolutePath}" - ) - - // Add podcast episodes to library - itemPart.episode?.let { podcastEpisode -> - val podcast = localLibraryItem.media as Podcast - val newEpisode = podcast.addEpisode(track, podcastEpisode) - localEpisodeId = newEpisode.id - Log.d( - tag, - "scanDownloadItem: Added episode to podcast ${podcastEpisode.title} ${track.title} | Track index: ${podcastEpisode.audioTrack?.index}" - ) - } - } else if (itemPart.ebookFile != null) { // Ebook - foundEBookFile = true - Log.d(tag, "scanDownloadItem: Ebook file found with mimetype=${docFile.mimeType}") - val localFileId = DeviceManager.getBase64Id(docFile.id) - val localFile = - LocalFile( - localFileId, - docFile.name, - docFile.uri.toString(), - docFile.getBasePath(ctx), - docFile.getAbsolutePath(ctx), - docFile.getSimplePath(ctx), - docFile.mimeType, - docFile.length() - ) - localLibraryItem.localFiles.add(localFile) - - val ebookFile = - EBookFile( - itemPart.ebookFile.ino, - itemPart.ebookFile.metadata, - itemPart.ebookFile.ebookFormat, - true, - localFileId, - localFile.contentUrl - ) - (localLibraryItem.media as Book).ebookFile = ebookFile - Log.d(tag, "scanDownloadItem: Ebook file added to lli ${localFile.contentUrl}") - } else { // Cover image - val localFileId = DeviceManager.getBase64Id(docFile.id) - val localFile = - LocalFile( - localFileId, - docFile.name, - docFile.uri.toString(), - docFile.getBasePath(ctx), - docFile.getAbsolutePath(ctx), - docFile.getSimplePath(ctx), - docFile.mimeType, - docFile.length() - ) - - localLibraryItem.coverAbsolutePath = localFile.absolutePath - localLibraryItem.coverContentUrl = localFile.contentUrl - localLibraryItem.localFiles.add(localFile) - } - } - - if (audioTracks.isEmpty() && !foundEBookFile) { - Log.d( - tag, - "scanDownloadItem did not find any audio tracks or ebook file in folder for ${downloadItem.itemFolderPath}" - ) - return cb(null) - } - - // For books sort audio tracks then set - if (downloadItem.mediaType == "book") { - audioTracks.sortBy { it.index } - - var indexCheck = 1 - var startOffset = 0.0 - audioTracks.forEach { audioTrack -> - if (audioTrack.index != indexCheck || audioTrack.startOffset != startOffset) { - audioTrack.index = indexCheck - audioTrack.startOffset = startOffset - } - indexCheck++ - startOffset += audioTrack.duration - } - - localLibraryItem.media.setAudioTracks(audioTracks) - } - - val downloadItemScanResult = DownloadItemScanResult(localLibraryItem, null) - - // If library item had media progress then make local media progress and save - downloadItem.userMediaProgress?.let { mediaProgress -> - val localMediaProgressId = - if (downloadItem.episodeId.isNullOrEmpty()) localLibraryItemId - else "$localLibraryItemId-$localEpisodeId" - val newLocalMediaProgress = - LocalMediaProgress( - id = localMediaProgressId, - localLibraryItemId = localLibraryItemId, - localEpisodeId = localEpisodeId, - duration = mediaProgress.duration, - progress = mediaProgress.progress, - currentTime = mediaProgress.currentTime, - isFinished = mediaProgress.isFinished, - ebookLocation = mediaProgress.ebookLocation, - ebookProgress = mediaProgress.ebookProgress, - lastUpdate = mediaProgress.lastUpdate, - startedAt = mediaProgress.startedAt, - finishedAt = mediaProgress.finishedAt, - serverConnectionConfigId = downloadItem.serverConnectionConfigId, - serverAddress = downloadItem.serverAddress, - serverUserId = downloadItem.serverUserId, - libraryItemId = downloadItem.libraryItemId, - episodeId = downloadItem.episodeId - ) - Log.d( - tag, - "scanLibraryItemFolder: Saving local media progress ${newLocalMediaProgress.id} at progress ${newLocalMediaProgress.progress}" - ) - - DeviceManager.dbManager.saveLocalMediaProgress(newLocalMediaProgress) - - downloadItemScanResult.localMediaProgress = newLocalMediaProgress - } - - DeviceManager.dbManager.saveLocalLibraryItem(localLibraryItem) - - cb(downloadItemScanResult) + val id = localLibraryItemId(itemFolder.id) + val localItem = + DeviceManager.dbManager.getLocalLibraryItem(id) + ?: newLocalLibraryItem( + id, + item, + itemFolder.getBasePath(ctx), + itemFolder.getAbsolutePath(ctx), + itemFolder.uri.toString() + ) + scanParts(item, localItem, itemFolder, callback) } } diff --git a/android/app/src/main/java/com/audiobookshelf/app/managers/DbManager.kt b/android/app/src/main/java/com/audiobookshelf/app/managers/DbManager.kt index b2f8293e..d02f2d3c 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/managers/DbManager.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/managers/DbManager.kt @@ -122,6 +122,21 @@ class DbManager { return downloadItems } + /** + * The downloader now persists app-owned staging paths instead of DownloadManager state. Old + * queue entries cannot be resumed safely, but completed local media lives in other books and is + * deliberately left alone. + */ + fun clearLegacyDownloadQueueOnce() { + val metadata = Paper.book("downloadQueueMetadata") + val architectureVersion = metadata.read("architectureVersion") ?: 0 + if (architectureVersion >= 2) return + + Paper.book("downloadItems").destroy() + metadata.write("architectureVersion", 2) + Log.i(tag, "Cleared legacy persisted download queue for architecture v2") + } + fun saveLocalMediaProgress(mediaProgress: LocalMediaProgress) { Paper.book("localMediaProgress").write(mediaProgress.id, mediaProgress) } @@ -148,7 +163,7 @@ class DbManager { } // Make sure all local file ids still exist - fun cleanLocalLibraryItems() { + fun cleanLocalLibraryItems(context: Context) { val localLibraryItems = getLocalLibraryItems() localLibraryItems.forEach { lli -> @@ -157,15 +172,15 @@ class DbManager { // Check local files lli.localFiles = lli.localFiles.filter { localFile -> - val file = File(localFile.absolutePath) - if (!file.exists()) { + val exists = localFile.exists(context) + if (!exists) { Log.d( tag, "cleanLocalLibraryItems: Local file ${localFile.absolutePath} was removed from library item ${lli.media.metadata.title}" ) hasUpdates = true } - file.exists() + exists } as MutableList @@ -203,9 +218,10 @@ class DbManager { // Check cover still there lli.coverAbsolutePath?.let { - val coverFile = File(it) - - if (!coverFile.exists()) { + val coverExists = lli.localFiles.any { localFile -> + localFile.absolutePath == it && localFile.exists(context) + } + if (!coverExists) { Log.d( tag, "cleanLocalLibraryItems: Cover $it was removed from library item ${lli.media.metadata.title}" diff --git a/android/app/src/main/java/com/audiobookshelf/app/managers/DownloadItemManager.kt b/android/app/src/main/java/com/audiobookshelf/app/managers/DownloadItemManager.kt index b57cc9f6..b4e304b3 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/managers/DownloadItemManager.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/managers/DownloadItemManager.kt @@ -1,15 +1,8 @@ package com.audiobookshelf.app.managers -import android.app.DownloadManager import android.net.Uri import android.util.Log import androidx.documentfile.provider.DocumentFile -import com.anggrayudi.storage.callback.FileCallback -import com.anggrayudi.storage.file.DocumentFileCompat -import com.anggrayudi.storage.file.MimeType -import com.anggrayudi.storage.file.getAbsolutePath -import com.anggrayudi.storage.file.moveFileTo -import com.anggrayudi.storage.media.FileDescription import com.audiobookshelf.app.MainActivity import com.audiobookshelf.app.device.DeviceManager import com.audiobookshelf.app.device.FolderScanner @@ -19,36 +12,32 @@ import com.fasterxml.jackson.core.json.JsonReadFeature import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.getcapacitor.JSObject import java.io.File -import java.io.FileOutputStream -import java.util.* +import java.io.FileInputStream +import java.util.concurrent.ConcurrentHashMap +import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.launch +import okhttp3.Call -/** Manages download items and their parts. */ +/** Owns the Android download queue and writes all bytes to app-owned staging files. */ class DownloadItemManager( - var downloadManager: DownloadManager, - private var folderScanner: FolderScanner, - var mainActivity: MainActivity, - private var clientEventEmitter: DownloadEventEmitter + private val folderScanner: FolderScanner, + private val mainActivity: MainActivity, + private val clientEventEmitter: DownloadEventEmitter ) { - val tag = "DownloadItemManager" + private val tag = "DownloadItemManager" private val maxSimultaneousDownloads = 3 - private var jacksonMapper = - jacksonObjectMapper() - .enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature()) + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO) + private val activeCalls = ConcurrentHashMap() + private var watcherRunning = false + private val jacksonMapper = + jacksonObjectMapper().enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature()) - enum class DownloadCheckStatus { - InProgress, - Successful, - Failed - } - - var downloadItemQueue: MutableList = - mutableListOf() // All pending and downloading items - var currentDownloadItemParts: MutableList = - mutableListOf() // Item parts currently being downloaded + var downloadItemQueue: MutableList = mutableListOf() + var currentDownloadItemParts: MutableList = mutableListOf() interface DownloadEventEmitter { fun onDownloadItem(downloadItem: DownloadItem) @@ -61,323 +50,223 @@ class DownloadItemManager( fun onComplete(failed: Boolean) } - companion object { - var isDownloading: Boolean = false + init { + DeviceManager.dbManager.clearLegacyDownloadQueueOnce() } - /** Adds a download item to the queue and starts processing the queue. */ + @Synchronized fun addDownloadItem(downloadItem: DownloadItem) { DeviceManager.dbManager.saveDownloadItem(downloadItem) - Log.i(tag, "Add download item ${downloadItem.media.metadata.title}") - downloadItemQueue.add(downloadItem) clientEventEmitter.onDownloadItem(downloadItem) checkUpdateDownloadQueue() } - /** Checks and updates the download queue. */ + @Synchronized private fun checkUpdateDownloadQueue() { - for (downloadItem in downloadItemQueue) { - val numPartsToGet = maxSimultaneousDownloads - currentDownloadItemParts.size - val nextDownloadItemParts = downloadItem.getNextDownloadItemParts(numPartsToGet) - Log.d( - tag, - "checkUpdateDownloadQueue: numPartsToGet=$numPartsToGet, nextDownloadItemParts=${nextDownloadItemParts.size}" - ) - - if (nextDownloadItemParts.isNotEmpty()) { - processDownloadItemParts(nextDownloadItemParts) - } - - if (currentDownloadItemParts.size >= maxSimultaneousDownloads) { - break - } + for (downloadItem in downloadItemQueue.toList()) { + val availableSlots = maxSimultaneousDownloads - currentDownloadItemParts.size + if (availableSlots <= 0) break + downloadItem.getNextDownloadItemParts(availableSlots).forEach(::startDownload) } - if (currentDownloadItemParts.isNotEmpty()) startWatchingDownloads() } - /** Processes the download item parts. */ - private fun processDownloadItemParts(nextDownloadItemParts: List) { - nextDownloadItemParts.forEach { - if (it.isInternalStorage) { - startInternalDownload(it) - } else { - startExternalDownload(it) - } - } - } - - /** Starts an internal download. */ - private fun startInternalDownload(downloadItemPart: DownloadItemPart) { - val file = File(downloadItemPart.finalDestinationPath) - file.parentFile?.mkdirs() - - val fileOutputStream = FileOutputStream(downloadItemPart.finalDestinationPath) - val internalProgressCallback = + private fun startDownload(part: DownloadItemPart) { + val stagingFile = File(part.destinationPath) + stagingFile.parentFile?.mkdirs() + part.downloadId = APP_MANAGED_DOWNLOAD_ID + part.lastUpdateTime = System.currentTimeMillis() + currentDownloadItemParts.add(part) + val callback = object : InternalProgressCallback { override fun onProgress(totalBytesWritten: Long, progress: Long) { - downloadItemPart.bytesDownloaded = totalBytesWritten - downloadItemPart.progress = progress + synchronized(this@DownloadItemManager) { + part.bytesDownloaded = totalBytesWritten + part.progress = progress + part.lastUpdateTime = System.currentTimeMillis() + } } override fun onComplete(failed: Boolean) { - downloadItemPart.failed = failed - downloadItemPart.completed = true - } - } - - Log.d( - tag, - "Start internal download to destination path ${downloadItemPart.finalDestinationPath} from ${downloadItemPart.serverUrl}" - ) - InternalDownloadManager(fileOutputStream, internalProgressCallback) - .download(downloadItemPart.serverUrl) - downloadItemPart.downloadId = 1 - currentDownloadItemParts.add(downloadItemPart) - } - - /** Starts an external download. */ - private fun startExternalDownload(downloadItemPart: DownloadItemPart) { - val dlRequest = downloadItemPart.getDownloadRequest() - val downloadId = downloadManager.enqueue(dlRequest) - downloadItemPart.downloadId = downloadId - Log.d(tag, "checkUpdateDownloadQueue: Starting download item part, downloadId=$downloadId") - currentDownloadItemParts.add(downloadItemPart) - } - - /** Starts watching the downloads. */ - private fun startWatchingDownloads() { - if (isDownloading) return // Already watching - - GlobalScope.launch(Dispatchers.IO) { - Log.d(tag, "Starting watching downloads") - isDownloading = true - - while (currentDownloadItemParts.isNotEmpty()) { - val itemParts = currentDownloadItemParts.filter { !it.isMoving } - for (downloadItemPart in itemParts) { - if (downloadItemPart.isInternalStorage) { - handleInternalDownloadPart(downloadItemPart) - } else { - handleExternalDownloadPart(downloadItemPart) - } - } - - delay(500) - - if (currentDownloadItemParts.size < maxSimultaneousDownloads) { - checkUpdateDownloadQueue() - } - } - - Log.d(tag, "Finished watching downloads") - isDownloading = false - } - } - - /** Handles an internal download part. */ - private fun handleInternalDownloadPart(downloadItemPart: DownloadItemPart) { - clientEventEmitter.onDownloadItemPartUpdate(downloadItemPart) - - if (downloadItemPart.completed) { - val downloadItem = downloadItemQueue.find { it.id == downloadItemPart.downloadItemId } - downloadItem?.let { checkDownloadItemFinished(it) } - currentDownloadItemParts.remove(downloadItemPart) - } - } - - /** Handles an external download part. */ - private fun handleExternalDownloadPart(downloadItemPart: DownloadItemPart) { - val downloadCheckStatus = checkDownloadItemPart(downloadItemPart) - clientEventEmitter.onDownloadItemPartUpdate(downloadItemPart) - - // Will move to final destination, remove current item parts, and check if download item is - // finished - handleDownloadItemPartCheck(downloadCheckStatus, downloadItemPart) - } - - /** Checks the status of a download item part. */ - private fun checkDownloadItemPart(downloadItemPart: DownloadItemPart): DownloadCheckStatus { - val downloadId = downloadItemPart.downloadId ?: return DownloadCheckStatus.Failed - - val query = DownloadManager.Query().setFilterById(downloadId) - downloadManager.query(query).use { - if (it.moveToFirst()) { - val bytesColumnIndex = it.getColumnIndex(DownloadManager.COLUMN_TOTAL_SIZE_BYTES) - val statusColumnIndex = it.getColumnIndex(DownloadManager.COLUMN_STATUS) - val bytesDownloadedColumnIndex = - it.getColumnIndex(DownloadManager.COLUMN_BYTES_DOWNLOADED_SO_FAR) - - val totalBytes = if (bytesColumnIndex >= 0) it.getInt(bytesColumnIndex) else 0 - val downloadStatus = if (statusColumnIndex >= 0) it.getInt(statusColumnIndex) else 0 - val bytesDownloadedSoFar = - if (bytesDownloadedColumnIndex >= 0) it.getLong(bytesDownloadedColumnIndex) else 0 - Log.d( - tag, - "checkDownloads Download ${downloadItemPart.filename} bytes $totalBytes | bytes dled $bytesDownloadedSoFar | downloadStatus $downloadStatus" - ) - - return when (downloadStatus) { - DownloadManager.STATUS_SUCCESSFUL -> { - Log.d(tag, "checkDownloads Download ${downloadItemPart.filename} Successful") - downloadItemPart.completed = true - downloadItemPart.progress = 1 - downloadItemPart.bytesDownloaded = bytesDownloadedSoFar - - DownloadCheckStatus.Successful - } - DownloadManager.STATUS_FAILED -> { - Log.d(tag, "checkDownloads Download ${downloadItemPart.filename} Failed") - downloadItemPart.completed = true - downloadItemPart.failed = true - - DownloadCheckStatus.Failed - } - else -> { - val percentProgress = - if (totalBytes > 0) ((bytesDownloadedSoFar * 100L) / totalBytes) else 0 - Log.d( - tag, - "checkDownloads Download ${downloadItemPart.filename} Progress = $percentProgress%" - ) - downloadItemPart.progress = percentProgress - downloadItemPart.bytesDownloaded = bytesDownloadedSoFar - - DownloadCheckStatus.InProgress - } - } - } else { - Log.d(tag, "Download ${downloadItemPart.filename} not found in dlmanager") - downloadItemPart.completed = true - downloadItemPart.failed = true - return DownloadCheckStatus.Failed - } - } - } - - /** Handles the result of a download item part check. */ - private fun handleDownloadItemPartCheck( - downloadCheckStatus: DownloadCheckStatus, - downloadItemPart: DownloadItemPart - ) { - val downloadItem = downloadItemQueue.find { it.id == downloadItemPart.downloadItemId } - if (downloadItem == null) { - Log.e( - tag, - "Download item part finished but download item not found ${downloadItemPart.filename}" - ) - currentDownloadItemParts.remove(downloadItemPart) - } else if (downloadCheckStatus == DownloadCheckStatus.Successful) { - moveDownloadedFile(downloadItem, downloadItemPart) - } else if (downloadCheckStatus != DownloadCheckStatus.InProgress) { - checkDownloadItemFinished(downloadItem) - currentDownloadItemParts.remove(downloadItemPart) - } - } - - /** Moves the downloaded file to its final destination. */ - private fun moveDownloadedFile(downloadItem: DownloadItem, downloadItemPart: DownloadItemPart) { - val file = DocumentFileCompat.fromUri(mainActivity, downloadItemPart.destinationUri) - Log.d(tag, "DOWNLOAD: DESTINATION URI ${downloadItemPart.destinationUri}") - - val fcb = - object : FileCallback() { - override fun onPrepare() { - Log.d(tag, "DOWNLOAD: PREPARING MOVE FILE") - } - - override fun onFailed(errorCode: ErrorCode) { - Log.e(tag, "DOWNLOAD: FAILED TO MOVE FILE $errorCode") - downloadItemPart.failed = true - downloadItemPart.isMoving = false - file?.delete() - checkDownloadItemFinished(downloadItem) - currentDownloadItemParts.remove(downloadItemPart) - } - - override fun onCompleted(result: Any) { - Log.d(tag, "DOWNLOAD: FILE MOVE COMPLETED") - val resultDocFile = result as DocumentFile - Log.d( - tag, - "DOWNLOAD: COMPLETED FILE INFO (name=${resultDocFile.name}) ${resultDocFile.getAbsolutePath(mainActivity)}" - ) - - // Rename to fix appended .mp3 on m4b/m4a files - // REF: https://github.com/anggrayudi/SimpleStorage/issues/94 - val docNameLowerCase = resultDocFile.name?.lowercase(Locale.getDefault()) ?: "" - if (docNameLowerCase.endsWith(".m4b.mp3") || docNameLowerCase.endsWith(".m4a.mp3") - ) { - resultDocFile.renameTo(downloadItemPart.filename) + synchronized(this@DownloadItemManager) { + part.failed = failed + part.completed = true + part.lastUpdateTime = System.currentTimeMillis() + activeCalls.remove(part.id) } - - downloadItemPart.moved = true - downloadItemPart.isMoving = false - checkDownloadItemFinished(downloadItem) - currentDownloadItemParts.remove(downloadItemPart) } } + activeCalls[part.id] = InternalDownloadManager(stagingFile, part.fileSize, callback).download(part.serverUrl) + } - val localFolderFile = - DocumentFileCompat.fromUri(mainActivity, Uri.parse(downloadItemPart.localFolderUrl)) - if (localFolderFile == null) { - // Failed - downloadItemPart.failed = true - Log.e(tag, "Local Folder File from uri is null") - checkDownloadItemFinished(downloadItem) - currentDownloadItemParts.remove(downloadItemPart) - } else { - downloadItemPart.isMoving = true - val mimetype = if (downloadItemPart.audioTrack != null) MimeType.AUDIO else MimeType.IMAGE - val fileDescription = - FileDescription( - downloadItemPart.filename, - downloadItemPart.finalDestinationSubfolder, - mimetype - ) - file?.moveFileTo(mainActivity, localFolderFile, fileDescription, fcb) + @Synchronized + private fun startWatchingDownloads() { + if (watcherRunning) return + watcherRunning = true + scope.launch { + while (true) { + val activeParts = synchronized(this@DownloadItemManager) { currentDownloadItemParts.toList() } + if (activeParts.isEmpty()) break + activeParts.forEach(::handlePartUpdate) + delay(WATCH_INTERVAL_MS) + synchronized(this@DownloadItemManager) { checkUpdateDownloadQueue() } + } + synchronized(this@DownloadItemManager) { watcherRunning = false } } } - /** Checks if a download item is finished and processes it. */ + private fun handlePartUpdate(part: DownloadItemPart) { + clientEventEmitter.onDownloadItemPartUpdate(part) + if (!part.completed) { + val lastUpdate = part.lastUpdateTime ?: return + if (System.currentTimeMillis() - lastUpdate > STALL_TIMEOUT_MS) { + Log.e(tag, "Download stalled: ${part.filename}") + activeCalls.remove(part.id)?.cancel() + synchronized(this) { + part.failed = true + part.completed = true + } + } + return + } + + val item = synchronized(this) { downloadItemQueue.find { it.id == part.downloadItemId } } + if (item == null) { + removeActivePart(part) + return + } + if (part.failed) { + removeActivePart(part) + return + } + if (part.isInternalStorage) finalizeInternalFile(item, part) else moveDownloadedFile(item, part) + } + + private fun finalizeInternalFile(item: DownloadItem, part: DownloadItemPart) { + if (part.moved || part.isMoving) return + part.isMoving = true + val stagingFile = File(part.destinationPath) + val finalFile = File(part.finalDestinationPath) + finalFile.parentFile?.mkdirs() + if (finalFile.exists() && !finalFile.delete()) { + failFinalization(item, part, "Could not replace existing internal file") + return + } + if (!stagingFile.renameTo(finalFile)) { + failFinalization(item, part, "Could not finalize internal staging file") + return + } + part.moved = true + part.isMoving = false + removeActivePart(part) + checkDownloadItemFinished(item) + } + + private fun moveDownloadedFile(item: DownloadItem, part: DownloadItemPart) { + if (part.moved || part.isMoving) return + val destinationRoot = DocumentFile.fromTreeUri(mainActivity, Uri.parse(part.localFolderUrl)) + if (destinationRoot == null) { + failFinalization(item, part, "Could not resolve SAF destination") + return + } + part.isMoving = true + scope.launch { + try { + val destinationFolder = getOrCreateFolder(destinationRoot, part.finalDestinationSubfolder) + ?: throw IllegalStateException("Could not create SAF destination folder") + destinationFolder.findFile(part.filename)?.let { existing -> + if (!existing.delete()) throw IllegalStateException("Could not replace ${part.filename}") + } + val destinationFile = destinationFolder.createFile(mimeTypeFor(part), part.filename) + ?: throw IllegalStateException("Could not create ${part.filename}") + val stagingFile = File(part.destinationPath) + FileInputStream(stagingFile).use { input -> + mainActivity.contentResolver.openOutputStream(destinationFile.uri, "w")?.use { output -> + input.copyTo(output) + } ?: throw IllegalStateException("Could not open SAF output stream") + } + if (destinationFile.length() != stagingFile.length()) { + destinationFile.delete() + throw IllegalStateException("SAF copy size mismatch for ${part.filename}") + } + stagingFile.delete() + part.completedDestinationUri = destinationFile.uri.toString() + part.moved = true + part.isMoving = false + removeActivePart(part) + checkDownloadItemFinished(item) + } catch (e: Exception) { + failFinalization(item, part, "SAF copy failed: ${e.message}") + } + } + } + + private fun getOrCreateFolder(root: DocumentFile, relativePath: String): DocumentFile? { + var current = root + relativePath.split('/').filter { it.isNotBlank() }.forEach { segment -> + if (segment == "." || segment == "..") return null + current = current.findFile(segment) ?: current.createDirectory(segment) ?: return null + } + return current + } + + private fun mimeTypeFor(part: DownloadItemPart): String { + return part.audioTrack?.mimeType + ?: when (part.ebookFile?.ebookFormat?.lowercase()) { + "epub" -> "application/epub+zip" + "pdf" -> "application/pdf" + else -> "image/jpeg" + } + } + + private fun failFinalization(item: DownloadItem, part: DownloadItemPart, message: String) { + Log.e(tag, message) + part.failed = true + part.isMoving = false + part.completed = true + removeActivePart(part) + } + + @Synchronized + private fun removeActivePart(part: DownloadItemPart) { + activeCalls.remove(part.id) + currentDownloadItemParts.remove(part) + } + private fun checkDownloadItemFinished(downloadItem: DownloadItem) { - if (downloadItem.isDownloadFinished) { - Log.i(tag, "Download Item finished ${downloadItem.media.metadata.title}") - - GlobalScope.launch(Dispatchers.IO) { - folderScanner.scanDownloadItem(downloadItem) { downloadItemScanResult -> - Log.d( - tag, - "Item download complete ${downloadItem.itemTitle} | local library item id: ${downloadItemScanResult?.localLibraryItem?.id}" - ) - - val jsobj = - JSObject().apply { - put("libraryItemId", downloadItem.id) - put("localFolderId", downloadItem.localFolder.id) - - downloadItemScanResult?.localLibraryItem?.let { localLibraryItem -> - put( - "localLibraryItem", - JSObject(jacksonMapper.writeValueAsString(localLibraryItem)) - ) - } - downloadItemScanResult?.localMediaProgress?.let { localMediaProgress -> - put( - "localMediaProgress", - JSObject(jacksonMapper.writeValueAsString(localMediaProgress)) - ) - } + if (!downloadItem.isDownloadFinished) return + scope.launch { + folderScanner.scanDownloadItem(downloadItem) { scanResult -> + val event = + JSObject().apply { + put("libraryItemId", downloadItem.id) + put("localFolderId", downloadItem.localFolder.id) + scanResult?.localLibraryItem?.let { + put("localLibraryItem", JSObject(jacksonMapper.writeValueAsString(it))) } - - launch(Dispatchers.Main) { - clientEventEmitter.onDownloadItemComplete(jsobj) - downloadItemQueue.remove(downloadItem) - DeviceManager.dbManager.removeDownloadItem(downloadItem.id) - } + scanResult?.localMediaProgress?.let { + put("localMediaProgress", JSObject(jacksonMapper.writeValueAsString(it))) + } + } + clientEventEmitter.onDownloadItemComplete(event) + synchronized(this@DownloadItemManager) { + downloadItemQueue.remove(downloadItem) + DeviceManager.dbManager.removeDownloadItem(downloadItem.id) } } } } + + fun destroy() { + activeCalls.values.forEach(Call::cancel) + activeCalls.clear() + scope.cancel() + } + + private companion object { + const val APP_MANAGED_DOWNLOAD_ID = -1L + const val WATCH_INTERVAL_MS = 500L + const val STALL_TIMEOUT_MS = 60_000L + } } diff --git a/android/app/src/main/java/com/audiobookshelf/app/managers/InternalDownloadManager.kt b/android/app/src/main/java/com/audiobookshelf/app/managers/InternalDownloadManager.kt index 3f1f9348..128d41a6 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/managers/InternalDownloadManager.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/managers/InternalDownloadManager.kt @@ -1,114 +1,113 @@ package com.audiobookshelf.app.managers import android.util.Log -import java.io.* +import java.io.File +import java.io.FileOutputStream +import java.io.IOException import java.util.concurrent.TimeUnit -import okhttp3.* +import okhttp3.Call +import okhttp3.Callback +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response -/** - * Manages the internal download process. - * - * @property outputStream The output stream to write the downloaded data. - * @property progressCallback The callback to report download progress. - */ +/** Streams a download into an app-owned staging file. */ class InternalDownloadManager( - private val outputStream: FileOutputStream, + private val destinationFile: File, + private val expectedSize: Long, private val progressCallback: DownloadItemManager.InternalProgressCallback -) : AutoCloseable { - +) { private val tag = "InternalDownloadManager" - private val client: OkHttpClient = - OkHttpClient.Builder().connectTimeout(30, TimeUnit.SECONDS).build() - private val writer = BinaryFileWriter(outputStream, progressCallback) + private val client = + OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .writeTimeout(60, TimeUnit.SECONDS) + .build() /** - * Downloads a file from the given URL. - * - * @param url The URL to download the file from. - * @throws IOException If an I/O error occurs. + * Returns the active call so the queue can cancel a stalled transfer. A partial staging file is + * retained only when the server proves that it honoured a subsequent range request. */ - @Throws(IOException::class) - fun download(url: String) { - val request: Request = Request.Builder().url(url).addHeader("Accept-Encoding", "identity").build() - client.newCall(request) - .enqueue( - object : Callback { - override fun onFailure(call: Call, e: IOException) { - Log.e(tag, "Download URL $url FAILED", e) - progressCallback.onComplete(true) - } + fun download(url: String): Call { + destinationFile.parentFile?.mkdirs() + val existingBytes = destinationFile.takeIf { it.exists() }?.length() ?: 0L + val request = + Request.Builder() + .url(url) + .addHeader("Accept-Encoding", "identity") + .apply { + if (existingBytes > 0L) header("Range", "bytes=$existingBytes-") + } + .build() + val call = client.newCall(request) + call.enqueue( + object : Callback { + override fun onFailure(call: Call, e: IOException) { + Log.e(tag, "Download URL failed", e) + progressCallback.onComplete(true) + } - override fun onResponse(call: Call, response: Response) { - response.body?.let { responseBody -> - val length: Long = response.header("Content-Length")?.toLongOrNull() ?: 0L - writer.write(responseBody.byteStream(), length) + override fun onResponse(call: Call, response: Response) { + response.use { + try { + val append = existingBytes > 0L && response.code == 206 && hasExpectedRange(response, existingBytes) + if (existingBytes > 0L && !append && response.code != 200) { + Log.e(tag, "Invalid resume response ${response.code} for offset $existingBytes") + progressCallback.onComplete(true) + return + } + if (!response.isSuccessful || response.body == null) { + Log.e(tag, "Download HTTP failure ${response.code}") + progressCallback.onComplete(true) + return + } + + val startingBytes = if (append) existingBytes else 0L + val responseLength = response.body!!.contentLength() + val totalLength = + if (expectedSize > 0L) expectedSize + else if (responseLength >= 0L) startingBytes + responseLength + else 0L + + FileOutputStream(destinationFile, append).use { output -> + response.body!!.byteStream().use { input -> + val buffer = ByteArray(CHUNK_SIZE) + var totalBytes = startingBytes + while (true) { + val read = input.read(buffer) + if (read < 0) break + output.write(buffer, 0, read) + totalBytes += read + val progress = if (totalLength > 0L) (totalBytes * 100L) / totalLength else 0L + progressCallback.onProgress(totalBytes, progress.coerceAtMost(100L)) } - ?: run { - Log.e(tag, "Response doesn't contain a file") - progressCallback.onComplete(true) - } } } - ) + + if (expectedSize > 0L && destinationFile.length() != expectedSize) { + Log.e(tag, "Downloaded size ${destinationFile.length()} did not match $expectedSize") + progressCallback.onComplete(true) + } else { + progressCallback.onComplete(false) + } + } catch (e: IOException) { + Log.e(tag, "Could not write staging file", e) + progressCallback.onComplete(true) + } + } + } + } + ) + return call } - /** - * Closes the download manager and releases resources. - * - * @throws Exception If an error occurs during closing. - */ - @Throws(Exception::class) - override fun close() { - writer.close() - } -} - -/** - * Writes binary data to an output stream. - * - * @property outputStream The output stream to write the data to. - * @property progressCallback The callback to report write progress. - */ -class BinaryFileWriter( - private val outputStream: OutputStream, - private val progressCallback: DownloadItemManager.InternalProgressCallback -) : AutoCloseable { - - /** - * Writes data from the input stream to the output stream. - * - * @param inputStream The input stream to read the data from. - * @param length The total length of the data to be written. - * @return The total number of bytes written. - * @throws IOException If an I/O error occurs. - */ - @Throws(IOException::class) - fun write(inputStream: InputStream, length: Long): Long { - BufferedInputStream(inputStream).use { input -> - val dataBuffer = ByteArray(CHUNK_SIZE) - var totalBytes: Long = 0 - var readBytes: Int - while (input.read(dataBuffer).also { readBytes = it } != -1) { - totalBytes += readBytes - outputStream.write(dataBuffer, 0, readBytes) - progressCallback.onProgress(totalBytes, (totalBytes * 100L) / length) - } - progressCallback.onComplete(false) - return totalBytes - } - } - - /** - * Closes the writer and releases resources. - * - * @throws IOException If an error occurs during closing. - */ - @Throws(IOException::class) - override fun close() { - outputStream.close() - } - - companion object { - private const val CHUNK_SIZE = 8192 // Increased chunk size for better performance + private fun hasExpectedRange(response: Response, offset: Long): Boolean { + val range = response.header("Content-Range") ?: return false + return range.startsWith("bytes $offset-") + } + + private companion object { + const val CHUNK_SIZE = 8 * 1024 } } diff --git a/android/app/src/main/java/com/audiobookshelf/app/models/DownloadItem.kt b/android/app/src/main/java/com/audiobookshelf/app/models/DownloadItem.kt index 918744eb..95ec320b 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/models/DownloadItem.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/models/DownloadItem.kt @@ -25,7 +25,7 @@ data class DownloadItem( val isInternalStorage get() = localFolder.id.startsWith("internal-") @get:JsonIgnore - val isDownloadFinished get() = !downloadItemParts.any { !it.completed || it.isMoving } + val isDownloadFinished get() = !downloadItemParts.any { !it.completed || it.isMoving || it.failed } @JsonIgnore fun getNextDownloadItemParts(limit:Int): MutableList { diff --git a/android/app/src/main/java/com/audiobookshelf/app/models/DownloadItemPart.kt b/android/app/src/main/java/com/audiobookshelf/app/models/DownloadItemPart.kt index fd65271e..a60a24cd 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/models/DownloadItemPart.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/models/DownloadItemPart.kt @@ -1,6 +1,5 @@ package com.audiobookshelf.app.models -import android.app.DownloadManager import android.net.Uri import android.util.Log import com.audiobookshelf.app.data.AudioTrack @@ -16,6 +15,8 @@ data class DownloadItemPart( val downloadItemId: String, val filename: String, val fileSize: Long, + /** App-owned staging location. This is intentionally a String so it survives process storage. */ + val destinationPath: String, val finalDestinationPath:String, val serverPath: String, val localFolderName: String, @@ -31,8 +32,11 @@ data class DownloadItemPart( @JsonIgnore val uri: Uri, @JsonIgnore val destinationUri: Uri, @JsonIgnore val finalDestinationUri: Uri, + /** Final SAF document returned by the provider after a successful move. */ + @JsonIgnore var completedDestinationUri: String?, val finalDestinationSubfolder: String, var downloadId: Long?, + var lastUpdateTime: Long?, var progress: Long, var bytesDownloaded: Long ) { @@ -53,6 +57,7 @@ data class DownloadItemPart( downloadItemId, filename = filename, fileSize = fileSize, + destinationPath = destinationFile.absolutePath, finalDestinationPath = finalDestinationFile.absolutePath, serverPath = serverPath, localFolderName = localFolder.name, @@ -68,8 +73,10 @@ data class DownloadItemPart( uri = downloadUri, destinationUri = destinationUri, finalDestinationUri = finalDestinationUri, + completedDestinationUri = null, finalDestinationSubfolder = subfolder, downloadId = null, + lastUpdateTime = null, progress = 0, bytesDownloaded = 0 ) @@ -82,13 +89,4 @@ data class DownloadItemPart( @get:JsonIgnore val serverUrl get() = uri.toString() - @JsonIgnore - fun getDownloadRequest(): DownloadManager.Request { - val dlRequest = DownloadManager.Request(uri) - dlRequest.setTitle(filename) - dlRequest.setDescription("Downloading to $localFolderName with filename $filename") - dlRequest.setNotificationVisibility(DownloadManager.Request.VISIBILITY_VISIBLE) - dlRequest.setDestinationUri(destinationUri) - return dlRequest - } } diff --git a/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsAudioPlayer.kt b/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsAudioPlayer.kt index 41d7cbf7..7e036f79 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsAudioPlayer.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsAudioPlayer.kt @@ -229,7 +229,7 @@ class AbsAudioPlayer : Plugin() { return call.resolve(JSObject("{\"error\":\"Podcast episode not found\"}")) } } - if (!it.hasTracks(episode)) { + if (!it.hasTracks(mainActivity, episode)) { return call.resolve(JSObject("{\"error\":\"No audio files found on device. Download book again to fix.\"}")) } diff --git a/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsDatabase.kt b/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsDatabase.kt index 473e69ce..0964d470 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsDatabase.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsDatabase.kt @@ -40,7 +40,7 @@ class AbsDatabase : Plugin() { secureStorage = SecureStorage(mainActivity) DeviceManager.dbManager.cleanLocalMediaProgress() - DeviceManager.dbManager.cleanLocalLibraryItems() + DeviceManager.dbManager.cleanLocalLibraryItems(mainActivity) DeviceManager.dbManager.cleanLogs() } diff --git a/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsDownloader.kt b/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsDownloader.kt index d3e42029..3dde53d2 100644 --- a/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsDownloader.kt +++ b/android/app/src/main/java/com/audiobookshelf/app/plugins/AbsDownloader.kt @@ -1,7 +1,5 @@ package com.audiobookshelf.app.plugins -import android.app.DownloadManager -import android.content.Context import android.os.Environment import android.util.Log import com.audiobookshelf.app.MainActivity @@ -27,7 +25,6 @@ class AbsDownloader : Plugin() { private var jacksonMapper = jacksonObjectMapper().enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature()) lateinit var mainActivity: MainActivity - lateinit var downloadManager: DownloadManager lateinit var apiHandler: ApiHandler lateinit var folderScanner: FolderScanner lateinit var downloadItemManager: DownloadItemManager @@ -46,10 +43,14 @@ class AbsDownloader : Plugin() { override fun load() { mainActivity = (activity as MainActivity) - downloadManager = activity.getSystemService(Context.DOWNLOAD_SERVICE) as DownloadManager folderScanner = FolderScanner(mainActivity) apiHandler = ApiHandler(mainActivity) - downloadItemManager = DownloadItemManager(downloadManager, folderScanner, mainActivity, clientEventEmitter) + downloadItemManager = DownloadItemManager(folderScanner, mainActivity, clientEventEmitter) + } + + override fun handleOnDestroy() { + if (::downloadItemManager.isInitialized) downloadItemManager.destroy() + super.handleOnDestroy() } @PluginMethod @@ -132,7 +133,15 @@ class AbsDownloader : Plugin() { private fun startLibraryItemDownload(libraryItem: LibraryItem, localFolder: LocalFolder, episode:PodcastEpisode?) { val isInternal = localFolder.id.startsWith("internal-") - val tempFolderPath = if (isInternal) "${mainActivity.filesDir}/downloads/${libraryItem.id}" else mainActivity.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS) + val finalInternalFolderPath = "${mainActivity.filesDir}/downloads/${libraryItem.id}" + // Keep internal staging on the same filesystem as its final file so finalization is a rename. + // External-folder downloads can use app external storage and are moved through SAF afterwards. + val tempFolderPath = + if (isInternal) { + "${mainActivity.filesDir}/download-staging/${libraryItem.id}" + } else { + "${mainActivity.getExternalFilesDir(Environment.DIRECTORY_DOWNLOADS) ?: mainActivity.filesDir}/download-staging/${libraryItem.id}" + } Log.d(tag, "downloadCacheDirectory=$tempFolderPath") @@ -143,7 +152,7 @@ class AbsDownloader : Plugin() { val tracks = libraryItem.media.getAudioTracks() Log.d(tag, "Starting library item download with ${tracks.size} tracks") val itemSubfolder = "$bookAuthor/$bookTitle" - val itemFolderPath = if (isInternal) "$tempFolderPath" else "${localFolder.absolutePath}/$itemSubfolder" + val itemFolderPath = if (isInternal) finalInternalFolderPath else "${localFolder.absolutePath}/$itemSubfolder" val downloadItem = DownloadItem(libraryItem.id, libraryItem.id, null, libraryItem.userMediaProgress,DeviceManager.serverConnectionConfig?.id ?: "", DeviceManager.serverAddress, DeviceManager.serverUserId, libraryItem.mediaType, itemFolderPath, localFolder, bookTitle, itemSubfolder, libraryItem.media, mutableListOf()) val book = libraryItem.media as Book @@ -152,12 +161,7 @@ class AbsDownloader : Plugin() { val serverPath = "/api/items/${libraryItem.id}/file/${ebookFile.ino}/download" val destinationFilename = getFilenameFromRelPath(ebookFile.metadata?.relPath ?: "") val finalDestinationFile = File("$itemFolderPath/$destinationFilename") - val destinationFile = File("$tempFolderPath/$destinationFilename") - - if (destinationFile.exists()) { - Log.d(tag, "TEMP ebook file already exists, removing it from ${destinationFile.absolutePath}") - destinationFile.delete() - } + val destinationFile = File("$tempFolderPath/$destinationFilename.part") if (finalDestinationFile.exists()) { Log.d(tag, "ebook file already exists, removing it from ${finalDestinationFile.absolutePath}") @@ -181,12 +185,7 @@ class AbsDownloader : Plugin() { Log.d(tag, "Audio File Server Path $serverPath | AF RelPath ${audioTrack.relPath} | LocalFolder Path ${localFolder.absolutePath} | DestName $destinationFilename") val finalDestinationFile = File("$itemFolderPath/$destinationFilename") - val destinationFile = File("$tempFolderPath/$destinationFilename") - - if (destinationFile.exists()) { - Log.d(tag, "TEMP Audio file already exists, removing it from ${destinationFile.absolutePath}") - destinationFile.delete() - } + val destinationFile = File("$tempFolderPath/$destinationFilename.part") if (finalDestinationFile.exists()) { Log.d(tag, "Audio file already exists, removing it from ${finalDestinationFile.absolutePath}") @@ -205,14 +204,9 @@ class AbsDownloader : Plugin() { val serverPath = "/api/items/${libraryItem.id}/cover" val destinationFilename = "cover-${libraryItem.id}.jpg" - val destinationFile = File("$tempFolderPath/$destinationFilename") + val destinationFile = File("$tempFolderPath/$destinationFilename.part") val finalDestinationFile = File("$itemFolderPath/$destinationFilename") - if (destinationFile.exists()) { - Log.d(tag, "TEMP Audio file already exists, removing it from ${destinationFile.absolutePath}") - destinationFile.delete() - } - if (finalDestinationFile.exists()) { Log.d(tag, "Cover already exists, removing it from ${finalDestinationFile.absolutePath}") finalDestinationFile.delete() @@ -233,7 +227,7 @@ class AbsDownloader : Plugin() { val fileSize = audioTrack?.metadata?.size ?: 0 Log.d(tag, "Starting podcast episode download") - val itemFolderPath = if (isInternal) "$tempFolderPath" else "${localFolder.absolutePath}/$podcastTitle" + val itemFolderPath = if (isInternal) finalInternalFolderPath else "${localFolder.absolutePath}/$podcastTitle" val downloadItemId = "${libraryItem.id}-${episode?.id}" val downloadItem = DownloadItem(downloadItemId, libraryItem.id, episode?.id, libraryItem.userMediaProgress, DeviceManager.serverConnectionConfig?.id ?: "", DeviceManager.serverAddress, DeviceManager.serverUserId, libraryItem.mediaType, itemFolderPath, localFolder, podcastTitle, podcastTitle, libraryItem.media, mutableListOf()) @@ -241,7 +235,7 @@ class AbsDownloader : Plugin() { var destinationFilename = getFilenameFromRelPath(audioTrack?.relPath ?: "") Log.d(tag, "Audio File Server Path $serverPath | AF RelPath ${audioTrack?.relPath} | LocalFolder Path ${localFolder.absolutePath} | DestName $destinationFilename") - var destinationFile = File("$tempFolderPath/$destinationFilename") + var destinationFile = File("$tempFolderPath/$destinationFilename.part") var finalDestinationFile = File("$itemFolderPath/$destinationFilename") if (finalDestinationFile.exists()) { Log.d(tag, "Audio file already exists, removing it from ${finalDestinationFile.absolutePath}") @@ -258,7 +252,7 @@ class AbsDownloader : Plugin() { serverPath = "/api/items/${libraryItem.id}/cover" destinationFilename = "cover.jpg" - destinationFile = File("$tempFolderPath/$destinationFilename") + destinationFile = File("$tempFolderPath/$destinationFilename.part") finalDestinationFile = File("$itemFolderPath/$destinationFilename") if (finalDestinationFile.exists()) {