Fix refresh feed when book is deleted and belonged to a series/collection

Book query optimizations

server/managers/RssFeedManager.js

@@ -98,11 +98,22 @@ class RssFeedManager {
             podcastId: feed.entity.mediaId
           },
           attributes: ['id', 'updatedAt'],
-          order: [['createdAt', 'DESC']]
+          order: [['updatedAt', 'DESC']]
         })
+
         if (mostRecentPodcastEpisode && mostRecentPodcastEpisode.updatedAt > newEntityUpdatedAt) {
           newEntityUpdatedAt = mostRecentPodcastEpisode.updatedAt
         }
+      } else {
+        const book = await Database.bookModel.findOne({
+          where: {
+            id: feed.entity.mediaId
+          },
+          attributes: ['id', 'updatedAt']
+        })
+        if (book && book.updatedAt > newEntityUpdatedAt) {
+          newEntityUpdatedAt = book.updatedAt
+        }
       }
 
       return newEntityUpdatedAt > feed.entityUpdatedAt
@@ -111,7 +122,7 @@ class RssFeedManager {
         attributes: ['id', 'updatedAt'],
         include: {
           model: Database.bookModel,
-          attributes: ['id'],
+          attributes: ['id', 'audioFiles', 'updatedAt'],
           through: {
             attributes: []
           },
@@ -122,13 +133,16 @@ class RssFeedManager {
         }
       })
 
+      const totalBookTracks = feed.entity.books.reduce((total, book) => total + book.includedAudioFiles.length, 0)
+      if (feed.feedEpisodes.length !== totalBookTracks) {
+        return true
+      }
+
       let newEntityUpdatedAt = feed.entity.updatedAt
 
       const mostRecentItemUpdatedAt = feed.entity.books.reduce((mostRecent, book) => {
-        if (book.libraryItem.updatedAt > mostRecent) {
+        let updatedAt = book.libraryItem.updatedAt > book.updatedAt ? book.libraryItem.updatedAt : book.updatedAt
-          return book.libraryItem.updatedAt
+        return updatedAt > mostRecent ? updatedAt : mostRecent
-        }
-        return mostRecent
       }, 0)
 
       if (mostRecentItemUpdatedAt > newEntityUpdatedAt) {
@@ -151,6 +165,9 @@ class RssFeedManager {
     let feed = await Database.feedModel.findOne({
       where: {
         slug: req.params.slug
+      },
+      include: {
+        model: Database.feedEpisodeModel
       }
     })
     if (!feed) {
@@ -163,8 +180,6 @@ class RssFeedManager {
     if (feedRequiresUpdate) {
       Logger.info(`[RssFeedManager] Feed "${feed.title}" requires update - updating feed`)
       feed = await feed.updateFeedForEntity()
-    } else {
-      feed.feedEpisodes = await feed.getFeedEpisodes()
     }
 
     const xml = feed.buildXml(req.originalHostPrefix)

server/migrations/changelog.md

@@ -12,3 +12,4 @@ Please add a record of every database migration that you create to this file. Th
 | v2.17.4        | v2.17.4-use-subfolder-for-oidc-redirect-uris | Save subfolder to OIDC redirect URIs to support existing installations                                        |
 | v2.17.5        | v2.17.5-remove-host-from-feed-urls           | removes the host (serverAddress) from URL columns in the feeds and feedEpisodes tables                        |
 | v2.17.6        | v2.17.6-share-add-isdownloadable             | Adds the isDownloadable column to the mediaItemShares table                                                   |
+| v2.17.7        | v2.17.7-add-indices                          | Adds indices to the libraryItems and books tables to reduce query times                                       |

server/migrations/v2.17.7-add-indices.js

@@ -0,0 +1,83 @@
+/**
+ * @typedef MigrationContext
+ * @property {import('sequelize').QueryInterface} queryInterface - a suquelize QueryInterface object.
+ * @property {import('../Logger')} logger - a Logger object.
+ *
+ * @typedef MigrationOptions
+ * @property {MigrationContext} context - an object containing the migration context.
+ */
+
+const migrationVersion = '2.17.7'
+const migrationName = `${migrationVersion}-add-indices`
+const loggerPrefix = `[${migrationVersion} migration]`
+
+/**
+ * This upward migration adds some indices to the libraryItems and books tables to improve query performance
+ *
+ * @param {MigrationOptions} options - an object containing the migration context.
+ * @returns {Promise<void>} - A promise that resolves when the migration is complete.
+ */
+async function up({ context: { queryInterface, logger } }) {
+  // Upwards migration script
+  logger.info(`${loggerPrefix} UPGRADE BEGIN: ${migrationName}`)
+
+  await addIndex(queryInterface, logger, 'libraryItems', ['libraryId', 'mediaType', 'size'])
+  await addIndex(queryInterface, logger, 'books', ['duration'])
+
+  logger.info(`${loggerPrefix} UPGRADE END: ${migrationName}`)
+}
+
+/**
+ * This downward migration script removes the indices added in the upward migration script
+ *
+ * @param {MigrationOptions} options - an object containing the migration context.
+ * @returns {Promise<void>} - A promise that resolves when the migration is complete.
+ */
+async function down({ context: { queryInterface, logger } }) {
+  // Downward migration script
+  logger.info(`${loggerPrefix} DOWNGRADE BEGIN: ${migrationName}`)
+
+  await removeIndex(queryInterface, logger, 'libraryItems', ['libraryId', 'mediaType', 'size'])
+  await removeIndex(queryInterface, logger, 'books', ['duration'])
+
+  logger.info(`${loggerPrefix} DOWNGRADE END: ${migrationName}`)
+}
+
+/**
+ * Utility function to add an index to a table. If the index already exists, it logs a message and continues.
+ *
+ * @param {import('sequelize').QueryInterface} queryInterface
+ * @param {import ('../Logger')} logger
+ * @param {string} tableName
+ * @param {string[]} columns
+ */
+async function addIndex(queryInterface, logger, tableName, columns) {
+  try {
+    logger.info(`${loggerPrefix} adding index [${columns.join(', ')}] to table "${tableName}"`)
+    await queryInterface.addIndex(tableName, columns)
+    logger.info(`${loggerPrefix} added index [${columns.join(', ')}] to table "${tableName}"`)
+  } catch (error) {
+    if (error.name === 'SequelizeDatabaseError' && error.message.includes('already exists')) {
+      logger.info(`${loggerPrefix} index [${columns.join(', ')}] for table "${tableName}" already exists`)
+    } else {
+      throw error
+    }
+  }
+}
+
+/**
+ * Utility function to remove an index from a table.
+ * Sequelize implemets it using DROP INDEX IF EXISTS, so it won't throw an error if the index doesn't exist.
+ *
+ * @param {import('sequelize').QueryInterface} queryInterface
+ * @param {import ('../Logger')} logger
+ * @param {string} tableName
+ * @param {string[]} columns
+ */
+async function removeIndex(queryInterface, logger, tableName, columns) {
+  logger.info(`${loggerPrefix} removing index [${columns.join(', ')}] from table "${tableName}"`)
+  await queryInterface.removeIndex(tableName, columns)
+  logger.info(`${loggerPrefix} removed index [${columns.join(', ')}] from table "${tableName}"`)
+}
+
+module.exports = { up, down }

server/models/Book.js

@@ -321,10 +321,10 @@ class Book extends Model {
           // },
           {
             fields: ['publishedYear']
+          },
+          {
+            fields: ['duration']
           }
-          // {
-          //   fields: ['duration']
-          // }
         ]
       }
     )

server/models/Feed.js

@@ -107,6 +107,9 @@ class Feed extends Model {
       entityUpdatedAt = libraryItem.media.podcastEpisodes.reduce((mostRecent, episode) => {
         return episode.updatedAt > mostRecent ? episode.updatedAt : mostRecent
       }, entityUpdatedAt)
+    } else if (libraryItem.media.updatedAt > entityUpdatedAt) {
+      // Book feeds will use Book.updatedAt if more recent
+      entityUpdatedAt = libraryItem.media.updatedAt
     }
 
     const feedObj = {
@@ -187,7 +190,8 @@ class Feed extends Model {
     const booksWithTracks = collectionExpanded.books.filter((book) => book.includedAudioFiles.length)
 
     const entityUpdatedAt = booksWithTracks.reduce((mostRecent, book) => {
-      return book.libraryItem.updatedAt > mostRecent ? book.libraryItem.updatedAt : mostRecent
+      const updatedAt = book.libraryItem.updatedAt > book.updatedAt ? book.libraryItem.updatedAt : book.updatedAt
+      return updatedAt > mostRecent ? updatedAt : mostRecent
     }, collectionExpanded.updatedAt)
 
     const firstBookWithCover = booksWithTracks.find((book) => book.coverPath)
@@ -275,7 +279,8 @@ class Feed extends Model {
   static getFeedObjForSeries(userId, seriesExpanded, slug, serverAddress, feedOptions = null) {
     const booksWithTracks = seriesExpanded.books.filter((book) => book.includedAudioFiles.length)
     const entityUpdatedAt = booksWithTracks.reduce((mostRecent, book) => {
-      return book.libraryItem.updatedAt > mostRecent ? book.libraryItem.updatedAt : mostRecent
+      const updatedAt = book.libraryItem.updatedAt > book.updatedAt ? book.libraryItem.updatedAt : book.updatedAt
+      return updatedAt > mostRecent ? updatedAt : mostRecent
     }, seriesExpanded.updatedAt)
 
     const firstBookWithCover = booksWithTracks.find((book) => book.coverPath)
@@ -516,17 +521,24 @@ class Feed extends Model {
     try {
       const updatedFeed = await this.update(feedObj, { transaction })
 
-      // Remove existing feed episodes
+      const existingFeedEpisodeIds = this.feedEpisodes.map((ep) => ep.id)
-      await feedEpisodeModel.destroy({
-        where: {
-          feedId: this.id
-        },
-        transaction
-      })
 
       // Create new feed episodes
       updatedFeed.feedEpisodes = await feedEpisodeCreateFunc(feedEpisodeCreateFuncEntity, updatedFeed, this.slug, transaction)
 
+      const newFeedEpisodeIds = updatedFeed.feedEpisodes.map((ep) => ep.id)
+      const feedEpisodeIdsToRemove = existingFeedEpisodeIds.filter((epid) => !newFeedEpisodeIds.includes(epid))
+
+      if (feedEpisodeIdsToRemove.length) {
+        Logger.info(`[Feed] Removing ${feedEpisodeIdsToRemove.length} episodes from feed ${this.id}`)
+        await feedEpisodeModel.destroy({
+          where: {
+            id: feedEpisodeIdsToRemove
+          },
+          transaction
+        })
+      }
+
       await transaction.commit()
 
       return updatedFeed

server/models/FeedEpisode.js

@@ -53,9 +53,10 @@ class FeedEpisode extends Model {
    * @param {import('./Feed')} feed
    * @param {string} slug
    * @param {import('./PodcastEpisode')} episode
+   * @param {string} [existingEpisodeId]
    */
-  static getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode) {
+  static getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode, existingEpisodeId = null) {
-    const episodeId = uuidv4()
+    const episodeId = existingEpisodeId || uuidv4()
     return {
       id: episodeId,
       title: episode.title,
@@ -94,11 +95,18 @@ class FeedEpisode extends Model {
       libraryItemExpanded.media.podcastEpisodes.sort((a, b) => new Date(a.pubDate) - new Date(b.pubDate))
     }
 
+    let numExisting = 0
     for (const episode of libraryItemExpanded.media.podcastEpisodes) {
-      feedEpisodeObjs.push(this.getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode))
+      // Check for existing episode by filepath
+      const existingEpisode = feed.feedEpisodes?.find((feedEpisode) => {
+        return feedEpisode.filePath === episode.audioFile.metadata.path
+      })
+      numExisting = existingEpisode ? numExisting + 1 : numExisting
+
+      feedEpisodeObjs.push(this.getFeedEpisodeObjFromPodcastEpisode(libraryItemExpanded, feed, slug, episode, existingEpisode?.id))
     }
-    Logger.info(`[FeedEpisode] Creating ${feedEpisodeObjs.length} episodes for feed ${feed.id}`)
+    Logger.info(`[FeedEpisode] Upserting ${feedEpisodeObjs.length} episodes for feed ${feed.id} (${numExisting} existing)`)
-    return this.bulkCreate(feedEpisodeObjs, { transaction })
+    return this.bulkCreate(feedEpisodeObjs, { transaction, updateOnDuplicate: ['title', 'author', 'description', 'siteURL', 'enclosureURL', 'enclosureType', 'enclosureSize', 'pubDate', 'season', 'episode', 'episodeType', 'duration', 'filePath', 'explicit'] })
   }
 
   /**
@@ -127,11 +135,12 @@ class FeedEpisode extends Model {
    * @param {string} slug
    * @param {import('./Book').AudioFileObject} audioTrack
    * @param {boolean} useChapterTitles
+   * @param {string} [existingEpisodeId]
    */
-  static getFeedEpisodeObjFromAudiobookTrack(book, pubDateStart, feed, slug, audioTrack, useChapterTitles) {
+  static getFeedEpisodeObjFromAudiobookTrack(book, pubDateStart, feed, slug, audioTrack, useChapterTitles, existingEpisodeId = null) {
     // Example: <pubDate>Fri, 04 Feb 2015 00:00:00 GMT</pubDate>
     let timeOffset = isNaN(audioTrack.index) ? 0 : Number(audioTrack.index) * 1000 // Offset pubdate to ensure correct order
-    let episodeId = uuidv4()
+    let episodeId = existingEpisodeId || uuidv4()
 
     // e.g. Track 1 will have a pub date before Track 2
     const audiobookPubDate = date.format(new Date(pubDateStart.valueOf() + timeOffset), 'ddd, DD MMM YYYY HH:mm:ss [GMT]')
@@ -179,11 +188,18 @@ class FeedEpisode extends Model {
     const useChapterTitles = this.checkUseChapterTitlesForEpisodes(libraryItemExpanded.media)
 
     const feedEpisodeObjs = []
+    let numExisting = 0
     for (const track of libraryItemExpanded.media.trackList) {
-      feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(libraryItemExpanded.media, libraryItemExpanded.createdAt, feed, slug, track, useChapterTitles))
+      // Check for existing episode by filepath
+      const existingEpisode = feed.feedEpisodes?.find((episode) => {
+        return episode.filePath === track.metadata.path
+      })
+      numExisting = existingEpisode ? numExisting + 1 : numExisting
+
+      feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(libraryItemExpanded.media, libraryItemExpanded.createdAt, feed, slug, track, useChapterTitles, existingEpisode?.id))
     }
-    Logger.info(`[FeedEpisode] Creating ${feedEpisodeObjs.length} episodes for feed ${feed.id}`)
+    Logger.info(`[FeedEpisode] Upserting ${feedEpisodeObjs.length} episodes for feed ${feed.id} (${numExisting} existing)`)
-    return this.bulkCreate(feedEpisodeObjs, { transaction })
+    return this.bulkCreate(feedEpisodeObjs, { transaction, updateOnDuplicate: ['title', 'author', 'description', 'siteURL', 'enclosureURL', 'enclosureType', 'enclosureSize', 'pubDate', 'season', 'episode', 'episodeType', 'duration', 'filePath', 'explicit'] })
   }
 
   /**
@@ -200,14 +216,21 @@ class FeedEpisode extends Model {
     }).libraryItem.createdAt
 
     const feedEpisodeObjs = []
+    let numExisting = 0
     for (const book of books) {
       const useChapterTitles = this.checkUseChapterTitlesForEpisodes(book)
       for (const track of book.trackList) {
-        feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(book, earliestLibraryItemCreatedAt, feed, slug, track, useChapterTitles))
+        // Check for existing episode by filepath
+        const existingEpisode = feed.feedEpisodes?.find((episode) => {
+          return episode.filePath === track.metadata.path
+        })
+        numExisting = existingEpisode ? numExisting + 1 : numExisting
+
+        feedEpisodeObjs.push(this.getFeedEpisodeObjFromAudiobookTrack(book, earliestLibraryItemCreatedAt, feed, slug, track, useChapterTitles, existingEpisode?.id))
       }
     }
-    Logger.info(`[FeedEpisode] Creating ${feedEpisodeObjs.length} episodes for feed ${feed.id}`)
+    Logger.info(`[FeedEpisode] Upserting ${feedEpisodeObjs.length} episodes for feed ${feed.id} (${numExisting} existing)`)
-    return this.bulkCreate(feedEpisodeObjs, { transaction })
+    return this.bulkCreate(feedEpisodeObjs, { transaction, updateOnDuplicate: ['title', 'author', 'description', 'siteURL', 'enclosureURL', 'enclosureType', 'enclosureSize', 'pubDate', 'season', 'episode', 'episodeType', 'duration', 'filePath', 'explicit'] })
   }
 
   /**

server/models/LibraryItem.js

@@ -1061,6 +1061,9 @@ class LibraryItem extends Model {
           {
             fields: ['libraryId', 'mediaType']
           },
+          {
+            fields: ['libraryId', 'mediaType', 'size']
+          },
           {
             fields: ['libraryId', 'mediaId', 'mediaType']
           },