Media folder management page, android media folder scanner

android/app/src/main/java/com/audiobookshelf/app/StorageManager.kt

@@ -65,8 +65,9 @@ class StorageManager : Plugin() {
         var absolutePath = folder.getAbsolutePath(activity)
         var storageType = folder.getStorageType(activity)
         var simplePath = folder.getSimplePath(activity)
+        var folderId = android.util.Base64.encodeToString(folder.id.toByteArray(), android.util.Base64.DEFAULT)
 
-        var localFolder = LocalFolder(folder.id, folder.name, folder.uri.toString(),absolutePath, simplePath, storageType.toString(), mediaType)
+        var localFolder = LocalFolder(folderId, folder.name, folder.uri.toString(),absolutePath, simplePath, storageType.toString(), mediaType)
 
         DeviceManager.dbManager.saveLocalFolder(localFolder)
         call.resolve(JSObject(jacksonObjectMapper().writeValueAsString(localFolder)))
@@ -127,13 +128,15 @@ class StorageManager : Plugin() {
   }
 
   @PluginMethod
-  fun searchFolder(call: PluginCall) {
+  fun scanFolder(call: PluginCall) {
     var folderId =  call.data.getString("folderId", "").toString()
-    var folder: LocalFolder? = DeviceManager.dbManager.loadLocalFolder(folderId)
+    var forceAudioProbe = call.data.getBoolean("forceAudioProbe")
+    Log.d(TAG, "Scan Folder $folderId | Force Audio Probe $forceAudioProbe")
+
+    var folder: LocalFolder? = DeviceManager.dbManager.getLocalFolder(folderId)
     folder?.let {
-      Log.d(TAG, "Searching folder ${it.contentUrl}")
       var folderScanner = FolderScanner(context)
-      var folderScanResult = folderScanner.scanForMediaItems(it.contentUrl, it.mediaType)
+      var folderScanResult = folderScanner.scanForMediaItems(it, forceAudioProbe)
       if (folderScanResult == null) {
         Log.d(TAG, "NO Scan DATA")
         return call.resolve(JSObject())
@@ -141,65 +144,15 @@ class StorageManager : Plugin() {
         Log.d(TAG, "Scan DATA ${jacksonObjectMapper().writeValueAsString(folderScanResult)}")
         return call.resolve(JSObject(jacksonObjectMapper().writeValueAsString(folderScanResult)))
       }
-    }
-    Log.d(TAG, "Folder not found $folderId")
-    call.resolve(JSObject())
-
-//
-//    var df: DocumentFile? = DocumentFileCompat.fromUri(context, Uri.parse(folderUrl))
-//
-//    if (df == null) {
-//      Log.e(TAG, "Folder Doc File Invalid $folderUrl")
-//      var jsobj = JSObject()
-//      jsobj.put("folders", JSArray())
-//      jsobj.put("files", JSArray())
-//      call.resolve(jsobj)
-//      return
-//    }
-//
-//    Log.d(TAG, "Folder as DF ${df.isDirectory} | ${df.getSimplePath(context)} | ${df.getBasePath(context)} | ${df.name}")
-//
-//    var mediaFolders = mutableListOf<MediaFolder>()
-//    var foldersFound = df.search(false, DocumentFileType.FOLDER)
-//
-//    foldersFound.forEach {
-//      Log.d(TAG, "Iterating over Folder Found ${it.name} | ${it.getSimplePath(context)} | URI: ${it.uri}")
-//      var folderName = it.name ?: ""
-//      var mediaFiles = mutableListOf<MediaFile>()
-//
-//      var filesInFolder = it.search(false, DocumentFileType.FILE, arrayOf("audio/*", "image/*"))
-//      filesInFolder.forEach { it2 ->
-//        var mimeType = it2?.mimeType ?: ""
-//        var filename = it2?.name ?: ""
-//        var isAudio = mimeType.startsWith("audio")
-//        Log.d(TAG, "Found $mimeType file $filename in folder $folderName")
-//        var imageFile = MediaFile(it2.uri, filename, it2.getSimplePath(context), it2.length(), mimeType, isAudio)
-//        mediaFiles.add(imageFile)
-//      }
-//      if (mediaFiles.size > 0) {
-//        mediaFolders.add(MediaFolder(it.uri, folderName, it.getSimplePath(context), mediaFiles))
-//      }
-//    }
-//
-//    // Files in root dir
-//    var rootMediaFiles = mutableListOf<MediaFile>()
-//    var mediaFilesFound:List<DocumentFile> = df.search(false, DocumentFileType.FILE, arrayOf("audio/*", "image/*"))
-//    mediaFilesFound.forEach {
-//      Log.d(TAG, "Folder Root File Found ${it.name} | ${it.getSimplePath(context)} | URI: ${it.uri} | ${it.mimeType}")
-//      var mimeType = it?.mimeType ?: ""
-//      var filename = it?.name ?: ""
-//      var isAudio = mimeType.startsWith("audio")
-//      Log.d(TAG, "Found $mimeType file $filename in root folder")
-//      var imageFile = MediaFile(it.uri, filename, it.getSimplePath(context), it.length(), mimeType, isAudio)
-//      rootMediaFiles.add(imageFile)
-//    }
-//
-//    var jsobj = JSObject()
-//    jsobj.put("folders", mediaFolders.map{ it.toJSObject() })
-//    jsobj.put("files", rootMediaFiles.map{ it.toJSObject() })
-//    call.resolve(jsobj)
+    } ?: call.resolve(JSObject())
   }
 
+  @PluginMethod
+  fun removeFolder(call: PluginCall) {
+    var folderId = call.data.getString("folderId", "").toString()
+    DeviceManager.dbManager.removeLocalFolder(folderId)
+    call.resolve()
+  }
 
   @PluginMethod
   fun delete(call: PluginCall) {

android/app/src/main/java/com/audiobookshelf/app/data/DataClasses.kt

@@ -151,5 +151,6 @@ data class AudioTrack(
   var contentUrl:String,
   var mimeType:String,
   var isLocal:Boolean,
+  var localFileId:String?,
   var audioProbeResult:AudioProbeResult?
 )

android/app/src/main/java/com/audiobookshelf/app/data/DbManager.kt

@@ -8,6 +8,9 @@ import com.getcapacitor.PluginCall
 import com.getcapacitor.PluginMethod
 import com.getcapacitor.annotation.CapacitorPlugin
 import io.paperdb.Paper
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.GlobalScope
+import kotlinx.coroutines.launch
 import org.json.JSONObject
 
 @CapacitorPlugin(name = "DbManager")
@@ -33,13 +36,26 @@ class DbManager : Plugin() {
     return localMediaItems
   }
 
+  fun getLocalMediaItemsInFolder(folderId:String):List<LocalMediaItem> {
+    var localMediaItems = loadLocalMediaItems()
+    return localMediaItems.filter {
+      it.folderId == folderId
+    }
+  }
+
   fun loadLocalMediaItem(localMediaItemId:String):LocalMediaItem? {
     return Paper.book("localMediaItems").read(localMediaItemId)
   }
 
+  fun removeLocalMediaItem(localMediaItemId:String) {
+    Paper.book("localMediaItems").delete(localMediaItemId)
+  }
+
   fun saveLocalMediaItems(localMediaItems:List<LocalMediaItem>) {
-    localMediaItems.map {
-      Paper.book("localMediaItems").write(it.id, it)
+    GlobalScope.launch(Dispatchers.IO) {
+      localMediaItems.map {
+        Paper.book("localMediaItems").write(it.id, it)
+      }
     }
   }
 
@@ -47,7 +63,7 @@ class DbManager : Plugin() {
     Paper.book("localFolders").write(localFolder.id,localFolder)
   }
 
-  fun loadLocalFolder(folderId:String):LocalFolder? {
+  fun getLocalFolder(folderId:String):LocalFolder? {
     return Paper.book("localFolders").read(folderId)
   }
 
@@ -62,6 +78,14 @@ class DbManager : Plugin() {
     return localFolders
   }
 
+  fun removeLocalFolder(folderId:String) {
+    var localMediaItems = getLocalMediaItemsInFolder(folderId)
+    localMediaItems.forEach {
+      Paper.book("localMediaItems").delete(it.id)
+    }
+    Paper.book("localFolders").delete(folderId)
+  }
+
   fun saveObject(db:String, key:String, value:JSONObject) {
     Log.d(tag, "Saving Object $key ${value.toString()}")
     Paper.book(db).write(key, value)
@@ -78,13 +102,16 @@ class DbManager : Plugin() {
     var db = call.getString("db", "").toString()
     var key = call.getString("key", "").toString()
     var value = call.getObject("value")
-    if (db == "" || key == "" || value == null) {
-      Log.d(tag, "saveFromWebview Invalid key/value")
-    } else {
-      var json = value as JSONObject
-      saveObject(db, key, json)
+
+    GlobalScope.launch(Dispatchers.IO) {
+      if (db == "" || key == "" || value == null) {
+        Log.d(tag, "saveFromWebview Invalid key/value")
+      } else {
+        var json = value as JSONObject
+        saveObject(db, key, json)
+      }
+      call.resolve()
     }
-    call.resolve()
   }
 
   @PluginMethod
@@ -102,24 +129,36 @@ class DbManager : Plugin() {
   }
 
   @PluginMethod
-  fun localFoldersFromWebView(call:PluginCall) {
-    var folders = getAllLocalFolders()
-    var folderObjArray = jacksonObjectMapper().writeValueAsString(folders)
-    var jsobj = JSObject()
-    jsobj.put("folders", folderObjArray)
-    call.resolve(jsobj)
+  fun getLocalFolders_WV(call:PluginCall) {
+    GlobalScope.launch(Dispatchers.IO) {
+      var folders = getAllLocalFolders()
+      var folderObjArray = jacksonObjectMapper().writeValueAsString(folders)
+      var jsobj = JSObject()
+      jsobj.put("folders", folderObjArray)
+      call.resolve(jsobj)
+    }
   }
 
   @PluginMethod
-  fun loadMediaItemsInFolder(call:PluginCall) {
+  fun getLocalFolder_WV(call:PluginCall) {
     var folderId = call.getString("folderId", "").toString()
-    var localMediaItems = loadLocalMediaItems().filter {
-      it.folderId == folderId
+    GlobalScope.launch(Dispatchers.IO) {
+      getLocalFolder(folderId)?.let {
+        var folderObj = jacksonObjectMapper().writeValueAsString(it)
+        call.resolve(JSObject(folderObj))
+      } ?: call.resolve()
     }
+  }
 
-    var mediaItemsArray = jacksonObjectMapper().writeValueAsString(localMediaItems)
-    var jsobj = JSObject()
-    jsobj.put("localMediaItems", mediaItemsArray)
-    call.resolve(jsobj)
+  @PluginMethod
+  fun getLocalMediaItemsInFolder_WV(call:PluginCall) {
+    var folderId = call.getString("folderId", "").toString()
+    GlobalScope.launch(Dispatchers.IO) {
+      var localMediaItems = getLocalMediaItemsInFolder(folderId)
+      var mediaItemsArray = jacksonObjectMapper().writeValueAsString(localMediaItems)
+      var jsobj = JSObject()
+      jsobj.put("localMediaItems", mediaItemsArray)
+      call.resolve(jsobj)
+    }
   }
 }

android/app/src/main/java/com/audiobookshelf/app/data/FolderScanResult.kt

@@ -1,9 +1,10 @@
 package com.audiobookshelf.app.data
 
 data class FolderScanResult(
-  val name:String?,
-  val absolutePath:String,
-  val mediaType:String,
-  val contentUrl:String,
-  val localMediaItems:MutableList<LocalMediaItem>,
+  var itemsAdded:Int,
+  var itemsUpdated:Int,
+  var itemsRemoved:Int,
+  var itemsUpToDate:Int,
+  val localFolder:LocalFolder,
+  val localMediaItems:List<LocalMediaItem>,
 )

android/app/src/main/java/com/audiobookshelf/app/device/DeviceManager.kt

@@ -1,6 +1,7 @@
 package com.audiobookshelf.app.device
 
 import android.util.Log
+import com.anggrayudi.storage.file.id
 import com.audiobookshelf.app.data.DbManager
 import com.audiobookshelf.app.data.DeviceData
 import com.audiobookshelf.app.data.ServerConfig
@@ -17,4 +18,8 @@ object DeviceManager {
   init {
     Log.d(tag, "Device Manager Singleton invoked")
   }
+
+  fun getBase64Id(id:String):String {
+    return android.util.Base64.encodeToString(id.toByteArray(), android.util.Base64.DEFAULT)
+  }
 }

android/app/src/main/java/com/audiobookshelf/app/device/FolderScanner.kt

@@ -5,7 +5,9 @@ import android.net.Uri
 import android.util.Log
 import androidx.documentfile.provider.DocumentFile
 import com.anggrayudi.storage.file.*
+import com.arthenica.ffmpegkit.FFmpegKitConfig
 import com.arthenica.ffmpegkit.FFprobeKit
+import com.arthenica.ffmpegkit.Level
 import com.audiobookshelf.app.data.*
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
@@ -13,25 +15,55 @@ import com.fasterxml.jackson.module.kotlin.readValue
 class FolderScanner(var ctx: Context) {
   private val tag = "FolderScanner"
 
-   fun scanForMediaItems(folderUrl: String, mediaType:String):FolderScanResult? {
-     var df: DocumentFile? = DocumentFileCompat.fromUri(ctx, Uri.parse(folderUrl))
+  // TODO: CLEAN this monster! Divide into bite-size methods
+   fun scanForMediaItems(localFolder:LocalFolder, forceAudioProbe:Boolean):FolderScanResult? {
+    FFmpegKitConfig.enableLogCallback { log ->
+      if (log.level != Level.AV_LOG_STDERR) { // STDERR is filled with junk
+        Log.d(tag, "FFmpeg-Kit Log: (${log.level}) ${log.message}")
+      }
+    }
+
+     var df: DocumentFile? = DocumentFileCompat.fromUri(ctx, Uri.parse(localFolder.contentUrl))
 
      if (df == null) {
-       Log.e(tag, "Folder Doc File Invalid $folderUrl")
+       Log.e(tag, "Folder Doc File Invalid $localFolder.contentUrl")
        return null
      }
-     var folderName = df.name ?: ""
-     var folderPath = df.getAbsolutePath(ctx)
-     var folderUrl = df.uri.toString()
-     var folderId = df.id
 
+     var mediaItemsUpdated = 0
+     var mediaItemsAdded = 0
+     var mediaItemsRemoved = 0
+     var mediaItemsUpToDate = 0
+
+      // Search for files in media item folder
      var foldersFound = df.search(false, DocumentFileType.FOLDER)
 
+      // Match folders found with media items already saved in db
+     var existingMediaItems = DeviceManager.dbManager.getLocalMediaItemsInFolder(localFolder.id)
+
+     // Remove existing items no longer there
+     existingMediaItems = existingMediaItems.filter { lmi ->
+       var fileFound = foldersFound.find { f -> lmi.id == DeviceManager.getBase64Id(f.id)  }
+       if (fileFound == null) {
+         Log.d(tag, "Existing media item is no longer in file system ${lmi.name}")
+         DeviceManager.dbManager.removeLocalMediaItem(lmi.id)
+         mediaItemsRemoved++
+       }
+       fileFound != null
+     }
+
      var mediaItems = mutableListOf<LocalMediaItem>()
 
      foldersFound.forEach {
        Log.d(tag, "Iterating over Folder Found ${it.name} | ${it.getSimplePath(ctx)} | URI: ${it.uri}")
+
        var itemFolderName = it.name ?: ""
+       var itemId = DeviceManager.getBase64Id(it.id)
+
+       var existingMediaItem = existingMediaItems.find { emi -> emi.id == itemId }
+       var existingLocalFiles = existingMediaItem?.localFiles ?: mutableListOf()
+       var existingAudioTracks = existingMediaItem?.audioTracks ?: mutableListOf()
+       var isNewOrUpdated = existingMediaItem == null
 
        var audioTracks = mutableListOf<AudioTrack>()
        var localFiles = mutableListOf<LocalFile>()
@@ -40,53 +72,119 @@ class FolderScanner(var ctx: Context) {
        var coverPath:String? = null
 
        var filesInFolder = it.search(false, DocumentFileType.FILE, arrayOf("audio/*", "image/*"))
-       filesInFolder.forEach { it2 ->
-         var mimeType = it2?.mimeType ?: ""
-         var filename = it2?.name ?: ""
+
+       var existingLocalFilesRemoved = existingLocalFiles.filter { elf ->
+         filesInFolder.find { fif -> DeviceManager.getBase64Id(fif.id) == elf.id } == null // File was not found in media item folder
+       }
+       if (existingLocalFilesRemoved.isNotEmpty()) {
+         Log.d(tag, "${existingLocalFilesRemoved.size} Local files were removed from local media item ${existingMediaItem?.name}")
+         isNewOrUpdated = true
+       }
+
+       filesInFolder.forEach { file ->
+         var mimeType = file?.mimeType ?: ""
+         var filename = file?.name ?: ""
          var isAudio = mimeType.startsWith("audio")
          Log.d(tag, "Found $mimeType file $filename in folder $itemFolderName")
 
-         var localFile = LocalFile(it2.id,it2.name,it2.uri.toString(),it2.getAbsolutePath(ctx),it2.getSimplePath(ctx),it2.mimeType,it2.length())
+         var localFileId = DeviceManager.getBase64Id(file.id)
+
+         var localFile = LocalFile(localFileId,filename,file.uri.toString(),file.getAbsolutePath(ctx),file.getSimplePath(ctx),mimeType,file.length())
          localFiles.add(localFile)
 
-         Log.d(tag, "File attributes Id:${it2.id}|ContentUrl:${localFile.contentUrl}|isDownloadsDocument:${it2.isDownloadsDocument}")
+         Log.d(tag, "File attributes Id:${localFileId}|ContentUrl:${localFile.contentUrl}|isDownloadsDocument:${file.isDownloadsDocument}")
 
          if (isAudio) {
-           Log.d(tag, "Scanning Audio File Path ${localFile.absolutePath}")
+           var audioTrackToAdd:AudioTrack? = null
 
-           // TODO: Make asynchronous
-           var session = FFprobeKit.execute("-i \"${localFile.absolutePath}\" -print_format json -show_format -show_streams -select_streams a -show_chapters -loglevel quiet")
-           var sessionData = session.output
-           Log.d(tag, "AFTER FFPROBE STRING $sessionData")
+           var existingAudioTrack = existingAudioTracks.find { eat -> eat.localFileId == localFileId }
+           if (existingAudioTrack != null) { // Update existing audio track
+             if (existingAudioTrack.index != index) {
+               Log.d(tag, "Updating Audio track index from ${existingAudioTrack.index} to $index")
+               existingAudioTrack.index = index
+               isNewOrUpdated = true
+             }
+             if (existingAudioTrack.startOffset != startOffset) {
+               Log.d(tag, "Updating Audio track startOffset ${existingAudioTrack.startOffset} to $startOffset")
+               existingAudioTrack.startOffset = startOffset
+               isNewOrUpdated = true
+             }
+           }
 
-           val mapper = jacksonObjectMapper()
-           val audioProbeResult = mapper.readValue<AudioProbeResult>(sessionData)
-           Log.d(tag, "Probe Result DATA ${audioProbeResult.duration} | ${audioProbeResult.size} | ${audioProbeResult.title} | ${audioProbeResult.artist}")
+           if (existingAudioTrack == null || forceAudioProbe) {
+             Log.d(tag, "Scanning Audio File Path ${localFile.absolutePath}")
 
-           var track = AudioTrack(index, startOffset, audioProbeResult.duration, filename, localFile.contentUrl, mimeType, true, audioProbeResult)
-           audioTracks.add(track)
-           startOffset += audioProbeResult.duration
+             // TODO: Make asynchronous
+             var session = FFprobeKit.execute("-i \"${localFile.absolutePath}\" -print_format json -show_format -show_streams -select_streams a -show_chapters -loglevel quiet")
+
+             val audioProbeResult = jacksonObjectMapper().readValue<AudioProbeResult>(session.output)
+             Log.d(tag, "Probe Result DATA ${audioProbeResult.duration} | ${audioProbeResult.size} | ${audioProbeResult.title} | ${audioProbeResult.artist}")
+
+             if (existingAudioTrack != null) {
+               // Update audio probe data on existing audio track
+               existingAudioTrack.audioProbeResult = audioProbeResult
+               audioTrackToAdd = existingAudioTrack
+             } else {
+               // Create new audio track
+               var track = AudioTrack(index, startOffset, audioProbeResult.duration, filename, localFile.contentUrl, mimeType, true, localFileId, audioProbeResult)
+               audioTrackToAdd = track
+             }
+
+             startOffset += audioProbeResult.duration
+             index++
+             isNewOrUpdated = true
+           } else {
+             audioTrackToAdd = existingAudioTrack
+           }
+
+           startOffset += audioTrackToAdd.duration
+           index++
+           audioTracks.add(audioTrackToAdd)
          } else {
+           var existingLocalFile = existingLocalFiles.find { elf -> elf.id == localFileId }
+
+           if (existingLocalFile == null) {
+             isNewOrUpdated = true
+           }
+           if (existingMediaItem != null && existingMediaItem.coverPath == null) {
+             // Existing media item did not have a cover - cover found on scan
+             isNewOrUpdated = true
+           }
+
            // First image file use as cover path
            if (coverPath == null) {
              coverPath = localFile.absolutePath
            }
          }
        }
-       if (audioTracks.size > 0) {
-        Log.d(tag, "Found local media item named $itemFolderName with ${audioTracks.size} tracks")
-         var localMediaItem = LocalMediaItem(it.id, itemFolderName, mediaType, folderId, it.uri.toString(), it.getSimplePath(ctx), it.getAbsolutePath(ctx),audioTracks,localFiles,coverPath)
+
+       if (existingMediaItem != null && audioTracks.isEmpty()) {
+         Log.d(tag, "Local media item ${existingMediaItem.name} no longer has audio tracks - removing item")
+         DeviceManager.dbManager.removeLocalMediaItem(existingMediaItem.id)
+         mediaItemsRemoved++
+       } else if (existingMediaItem != null && !isNewOrUpdated) {
+         Log.d(tag, "Local media item ${existingMediaItem.name} has no updates")
+         mediaItemsUpToDate++
+       } else if (audioTracks.isNotEmpty()) {
+         if (existingMediaItem != null) mediaItemsUpdated++
+         else mediaItemsAdded++
+
+        Log.d(tag, "Found local media item named $itemFolderName with ${audioTracks.size} tracks and ${localFiles.size} local files")
+         var localMediaItem = LocalMediaItem(itemId, itemFolderName, localFolder.mediaType, localFolder.id, it.uri.toString(), it.getSimplePath(ctx), it.getAbsolutePath(ctx),audioTracks,localFiles,coverPath)
          mediaItems.add(localMediaItem)
        }
      }
 
-     return if (mediaItems.size > 0) {
-       Log.d(tag, "Found ${mediaItems.size} Media Items")
+     Log.d(tag, "Folder $${localFolder.name} scan Results: $mediaItemsAdded Added | $mediaItemsUpdated Updated | $mediaItemsRemoved Removed | $mediaItemsUpToDate Up-to-date")
+
+     return if (mediaItems.isNotEmpty()) {
        DeviceManager.dbManager.saveLocalMediaItems(mediaItems)
-       FolderScanResult(folderName, folderPath, mediaType, folderUrl, mediaItems)
+
+       var folderMediaItems = DeviceManager.dbManager.getLocalMediaItemsInFolder(localFolder.id) // Get all local media items
+       FolderScanResult(mediaItemsAdded, mediaItemsUpdated, mediaItemsRemoved, mediaItemsUpToDate, localFolder, folderMediaItems)
      } else {
-       Log.d(tag, "No Media Items Found")
-       null
+       Log.d(tag, "No Media Items to save")
+       FolderScanResult(mediaItemsAdded, mediaItemsUpdated, mediaItemsRemoved, mediaItemsUpToDate, localFolder, mutableListOf())
      }
    }
 }