diff --git a/app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackViewModel.kt b/app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackViewModel.kt
index fa71dc7f..1e56ce08 100644
--- a/app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackViewModel.kt
+++ b/app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackViewModel.kt
@@ -325,12 +325,15 @@ class PlaybackViewModel
             withContext(Dispatchers.IO) {
                 Timber.i("Playing ${item.id}")
 
-                // Starting playback, so want to invalidate the last played timestamp for this item
-                datePlayedService.invalidate(item)
                 // New item, so we can clear the media segment tracker & subtitle cues
-                autoSkippedSegments.clear()
+                resetSegmentState()
                 this@PlaybackViewModel.subtitleCues.setValueOnMain(listOf())
 
+                viewModelScope.launchIO {
+                    // Starting playback, so want to invalidate the last played timestamp for this item
+                    datePlayedService.invalidate(item)
+                }
+
                 if (item.type !in supportItemKinds) {
                     showToast(
                         context,
@@ -457,7 +460,7 @@ class PlaybackViewModel
                     player.prepare()
                     player.play()
                 }
-                listenForSegments()
+                listenForSegments(item.id)
                 return@withContext true
             }
 
@@ -809,10 +812,19 @@ class PlaybackViewModel
 
         private var segmentJob: Job? = null
 
+        /**
+         * Cancels listening for segments and clears current segment state
+         */
+        private suspend fun resetSegmentState() {
+            segmentJob?.cancel()
+            autoSkippedSegments.clear()
+            currentSegment.setValueOnMain(null)
+        }
+
         /**
          * This sets up a coroutine to periodically check whether the current playback progress is within a media segment (intro, outro, etc)
          */
-        private fun listenForSegments() {
+        private fun listenForSegments(itemId: UUID) {
             segmentJob?.cancel()
             segmentJob =
                 viewModelScope.launchIO {
@@ -828,7 +840,10 @@ class PlaybackViewModel
                                     .firstOrNull {
                                         it.type != MediaSegmentType.UNKNOWN && currentTicks >= it.startTicks && currentTicks < it.endTicks
                                     }
-                            if (currentSegment != null && autoSkippedSegments.add(currentSegment.id)) {
+                            if (currentSegment != null &&
+                                currentSegment.itemId == this@PlaybackViewModel.itemId &&
+                                autoSkippedSegments.add(currentSegment.id)
+                            ) {
                                 Timber.d(
                                     "Found media segment for %s: %s, %s",
                                     currentSegment.itemId,