Add extra debug info to overlay (#258)

Adds some extra info to the playback debug info overlay such as
ExoPlayer decoder names and transcode info

app/src/main/java/com/github/damontecres/wholphin/ui/playback/CurrentPlayback.kt

@@ -5,6 +5,7 @@ import com.github.damontecres.wholphin.preferences.PlayerBackend
 import com.github.damontecres.wholphin.util.TrackSupport
 import org.jellyfin.sdk.model.api.MediaSourceInfo
 import org.jellyfin.sdk.model.api.PlayMethod
+import org.jellyfin.sdk.model.api.TranscodingInfo
 
 data class CurrentPlayback(
     val item: BaseItem,
@@ -14,4 +15,7 @@ data class CurrentPlayback(
     val playSessionId: String?,
     val liveStreamId: String?,
     val mediaSourceInfo: MediaSourceInfo,
+    val videoDecoder: String? = null,
+    val audioDecoder: String? = null,
+    val transcodeInfo: TranscodingInfo? = null,
 )

app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackDebugOverlay.kt

@@ -0,0 +1,111 @@
+package com.github.damontecres.wholphin.ui.playback
+
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.padding
+import androidx.compose.runtime.Composable
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.unit.dp
+import androidx.tv.material3.MaterialTheme
+import androidx.tv.material3.ProvideTextStyle
+import androidx.tv.material3.Text
+import com.github.damontecres.wholphin.ui.byteRateSuffixes
+import com.github.damontecres.wholphin.ui.formatBytes
+import com.github.damontecres.wholphin.ui.letNotEmpty
+import org.jellyfin.sdk.model.api.TranscodingInfo
+
+@Composable
+fun PlaybackDebugOverlay(
+    currentPlayback: CurrentPlayback?,
+    modifier: Modifier = Modifier,
+) {
+    Column(
+        verticalArrangement = Arrangement.spacedBy(8.dp),
+        modifier = modifier,
+    ) {
+        Row(
+            horizontalArrangement = Arrangement.spacedBy(48.dp),
+            modifier = Modifier.padding(start = 8.dp, top = 8.dp),
+        ) {
+            ProvideTextStyle(MaterialTheme.typography.bodySmall.copy(color = MaterialTheme.colorScheme.onSurface)) {
+                SimpleTable(
+                    listOf(
+                        "Backend:" to currentPlayback?.backend?.toString(),
+                        "Play method:" to currentPlayback?.playMethod?.serialName,
+                        "Video Decoder:" to currentPlayback?.videoDecoder,
+                        "Audio Decoder:" to currentPlayback?.audioDecoder,
+                    ),
+                )
+                currentPlayback?.transcodeInfo?.let {
+                    TranscodeInfo(it, Modifier)
+                }
+            }
+        }
+        currentPlayback?.tracks?.letNotEmpty {
+            PlaybackTrackInfo(
+                trackSupport = it,
+            )
+        }
+    }
+}
+
+@Composable
+fun TranscodeInfo(
+    info: TranscodingInfo,
+    modifier: Modifier = Modifier,
+) {
+    Row(
+        horizontalArrangement = Arrangement.spacedBy(16.dp),
+        modifier = modifier,
+    ) {
+        SimpleTable(
+            listOf(
+                "Reason:" to info.transcodeReasons.joinToString(", "),
+                "HW Accel:" to info.hardwareAccelerationType?.toString(),
+                "Container:" to info.container,
+                "Bitrate:" to info.bitrate?.let { formatBytes(it, byteRateSuffixes) },
+            ),
+        )
+        SimpleTable(
+            listOf(
+                "Video:" to "${info.videoCodec}, ${info.width}x${info.height}",
+                "Video Direct:" to info.isVideoDirect.toString(),
+                "Audio:" to "${info.audioCodec}, ch=${info.audioChannels}",
+                "Audio Direct:" to info.isAudioDirect.toString(),
+            ),
+        )
+    }
+}
+
+@Composable
+fun SimpleTable(
+    rows: List<Pair<String, String?>>,
+    modifier: Modifier = Modifier,
+) {
+    Row(
+        horizontalArrangement = Arrangement.spacedBy(4.dp),
+        modifier = modifier,
+    ) {
+        Column(
+            verticalArrangement = Arrangement.spacedBy(4.dp),
+            modifier = Modifier,
+        ) {
+            rows.forEach {
+                Text(
+                    text = it.first,
+                )
+            }
+        }
+        Column(
+            verticalArrangement = Arrangement.spacedBy(4.dp),
+            modifier = Modifier,
+        ) {
+            rows.forEach {
+                Text(
+                    text = it.second.toString(),
+                )
+            }
+        }
+    }
+}

app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackOverlay.kt

@@ -60,7 +60,6 @@ import com.github.damontecres.wholphin.ui.cards.ChapterCard
 import com.github.damontecres.wholphin.ui.cards.SeasonCard
 import com.github.damontecres.wholphin.ui.ifElse
 import com.github.damontecres.wholphin.ui.isNotNullOrBlank
-import com.github.damontecres.wholphin.ui.letNotEmpty
 import com.github.damontecres.wholphin.ui.tryRequestFocus
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.isActive
@@ -416,32 +415,14 @@ fun PlaybackOverlay(
                 Modifier
                     .align(Alignment.TopStart),
         ) {
-            Column(
-                verticalArrangement = Arrangement.spacedBy(8.dp),
+            PlaybackDebugOverlay(
+                currentPlayback = currentPlayback,
                 modifier =
                     Modifier
                         .align(Alignment.TopStart)
-                        .padding(16.dp)
+                        .padding(8.dp)
                         .background(AppColors.TransparentBlack50),
-            ) {
-                Text(
-                    text = "Backend: ${currentPlayback?.backend}",
-                    style = MaterialTheme.typography.bodySmall,
-                    color = MaterialTheme.colorScheme.onSurface,
-                    modifier = Modifier.padding(start = 8.dp),
             )
-                Text(
-                    text = "Play method: ${currentPlayback?.playMethod?.serialName}",
-                    style = MaterialTheme.typography.bodySmall,
-                    color = MaterialTheme.colorScheme.onSurface,
-                    modifier = Modifier.padding(start = 8.dp),
-                )
-                currentPlayback?.tracks?.letNotEmpty {
-                    PlaybackTrackInfo(
-                        trackSupport = it,
-                    )
-                }
-            }
         }
     }
 }

app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackPage.kt

@@ -23,6 +23,7 @@ import androidx.compose.foundation.layout.widthIn
 import androidx.compose.foundation.shape.RoundedCornerShape
 import androidx.compose.runtime.Composable
 import androidx.compose.runtime.LaunchedEffect
+import androidx.compose.runtime.collectAsState
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.livedata.observeAsState
 import androidx.compose.runtime.mutableFloatStateOf
@@ -122,7 +123,7 @@ fun PlaybackPage(
             val player = viewModel.player
             val mediaInfo by viewModel.currentMediaInfo.observeAsState()
 
-            val currentPlayback by viewModel.currentPlayback.observeAsState(null)
+            val currentPlayback by viewModel.currentPlayback.collectAsState()
             val currentItemPlayback by viewModel.currentItemPlayback.observeAsState(
                 ItemPlayback(
                     userId = -1,

app/src/main/java/com/github/damontecres/wholphin/ui/playback/PlaybackViewModel.kt

@@ -9,6 +9,7 @@ import androidx.lifecycle.MutableLiveData
 import androidx.lifecycle.ViewModel
 import androidx.lifecycle.viewModelScope
 import androidx.media3.common.C
+import androidx.media3.common.Format
 import androidx.media3.common.MediaItem
 import androidx.media3.common.PlaybackException
 import androidx.media3.common.Player
@@ -16,6 +17,10 @@ import androidx.media3.common.Tracks
 import androidx.media3.common.text.Cue
 import androidx.media3.common.text.CueGroup
 import androidx.media3.common.util.UnstableApi
+import androidx.media3.exoplayer.DecoderCounters
+import androidx.media3.exoplayer.DecoderReuseEvaluation
+import androidx.media3.exoplayer.ExoPlayer
+import androidx.media3.exoplayer.analytics.AnalyticsListener
 import com.github.damontecres.wholphin.data.ItemPlaybackDao
 import com.github.damontecres.wholphin.data.ItemPlaybackRepository
 import com.github.damontecres.wholphin.data.ServerRepository
@@ -54,11 +59,15 @@ import com.github.damontecres.wholphin.util.subtitleMimeTypes
 import com.github.damontecres.wholphin.util.supportItemKinds
 import dagger.hilt.android.lifecycle.HiltViewModel
 import dagger.hilt.android.qualifiers.ApplicationContext
+import kotlinx.coroutines.CancellationException
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.Job
 import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.collectLatest
 import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.onEach
+import kotlinx.coroutines.flow.update
 import kotlinx.coroutines.isActive
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Mutex
@@ -67,10 +76,12 @@ import kotlinx.coroutines.withContext
 import org.jellyfin.sdk.api.client.ApiClient
 import org.jellyfin.sdk.api.client.extensions.mediaInfoApi
 import org.jellyfin.sdk.api.client.extensions.mediaSegmentsApi
+import org.jellyfin.sdk.api.client.extensions.sessionApi
 import org.jellyfin.sdk.api.client.extensions.trickplayApi
 import org.jellyfin.sdk.api.client.extensions.userLibraryApi
 import org.jellyfin.sdk.api.client.extensions.videosApi
 import org.jellyfin.sdk.api.sockets.subscribe
+import org.jellyfin.sdk.model.DeviceInfo
 import org.jellyfin.sdk.model.api.BaseItemKind
 import org.jellyfin.sdk.model.api.DeviceProfile
 import org.jellyfin.sdk.model.api.MediaSegmentDto
@@ -88,6 +99,7 @@ import java.util.Date
 import java.util.UUID
 import javax.inject.Inject
 import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.Duration.Companion.seconds
 
 /**
  * This [ViewModel] is responsible for playing media including moving through playlists (including next up episodes)
@@ -106,8 +118,10 @@ class PlaybackViewModel
         private val itemPlaybackRepository: ItemPlaybackRepository,
         private val playerFactory: PlayerFactory,
         private val datePlayedService: DatePlayedService,
+        private val deviceInfo: DeviceInfo,
     ) : ViewModel(),
-        Player.Listener {
+        Player.Listener,
+        AnalyticsListener {
         val player by lazy {
             playerFactory.createVideoPlayer()
         }
@@ -116,7 +130,7 @@ class PlaybackViewModel
         val loading = MutableLiveData<LoadingState>(LoadingState.Loading)
 
         val currentMediaInfo = MutableLiveData<CurrentMediaInfo>(CurrentMediaInfo.EMPTY)
-        val currentPlayback = MutableLiveData<CurrentPlayback?>(null)
+        val currentPlayback = MutableStateFlow<CurrentPlayback?>(null)
         val currentItemPlayback = MutableLiveData<ItemPlayback>()
         val currentSegment = EqualityMutableLiveData<MediaSegmentDto?>(null)
         private val autoSkippedSegments = mutableSetOf<UUID>()
@@ -138,7 +152,9 @@ class PlaybackViewModel
 
         init {
             player.addListener(this)
+            (player as? ExoPlayer)?.addAnalyticsListener(this)
             addCloseable { player.removeListener(this@PlaybackViewModel) }
+            addCloseable { (player as? ExoPlayer)?.removeAnalyticsListener(this@PlaybackViewModel) }
             addCloseable {
                 this@PlaybackViewModel.activityListener?.let {
                     it.release()
@@ -147,6 +163,7 @@ class PlaybackViewModel
             }
             addCloseable { player.release() }
             subscribe()
+            listenForTranscodeReason()
         }
 
         /**
@@ -401,10 +418,12 @@ class PlaybackViewModel
                             }
                         }
                         withContext(Dispatchers.Main) {
-                            this@PlaybackViewModel.currentPlayback.value =
-                                currentPlayback.copy(
+                            this@PlaybackViewModel.currentPlayback.update {
+                                (it ?: currentPlayback).copy(
                                     tracks = checkForSupport(player.currentTracks),
                                 )
+                            }
+
                             this@PlaybackViewModel.currentItemPlayback.value = itemPlayback
                         }
 
@@ -559,7 +578,7 @@ class PlaybackViewModel
                     this@PlaybackViewModel.activityListener = activityListener
 
                     loading.value = LoadingState.Success
-                    this@PlaybackViewModel.currentPlayback.value = playback
+                    this@PlaybackViewModel.currentPlayback.update { playback }
                     this@PlaybackViewModel.currentItemPlayback.value = itemPlayback
                     player.setMediaItem(
                         mediaItem,
@@ -718,6 +737,39 @@ class PlaybackViewModel
                 }
         }
 
+        private fun listenForTranscodeReason() {
+            viewModelScope.launchIO {
+                currentPlayback.collectLatest {
+                    if (it != null) {
+                        try {
+                            var transcodeInfo = it.transcodeInfo
+                            while (isActive && it.playMethod == PlayMethod.TRANSCODE && transcodeInfo == null) {
+                                delay(2.seconds)
+                                transcodeInfo =
+                                    api.sessionApi
+                                        .getSessions(deviceId = deviceInfo.id)
+                                        .content
+                                        .firstOrNull()
+                                        ?.transcodingInfo
+                                if (transcodeInfo == null) delay(3.seconds)
+                            }
+                            Timber.v("transcodeInfo=$transcodeInfo")
+                            currentPlayback.update { current ->
+                                current?.copy(transcodeInfo = transcodeInfo)
+                            }
+                        } catch (ex: Exception) {
+                            if (ex !is CancellationException) {
+                                Timber.w(ex, "Exception trying to get session info")
+                                currentPlayback.update { current ->
+                                    current?.copy(transcodeInfo = null)
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
         private var lastInteractionDate: Date = Date()
 
         /**
@@ -789,11 +841,12 @@ class PlaybackViewModel
         }
 
         override fun onTracksChanged(tracks: Tracks) {
-            currentPlayback.value =
-                currentPlayback.value?.copy(
+            currentPlayback.update {
+                it?.copy(
                     tracks = checkForSupport(tracks),
                 )
             }
+        }
 
         override fun onCues(cueGroup: CueGroup) {
             subtitleCues.value = cueGroup.cues
@@ -889,4 +942,66 @@ class PlaybackViewModel
                     currentMediaInfo.setValueOnMain(newMediaInfo)
                 }
             }
+
+        override fun onVideoDecoderInitialized(
+            eventTime: AnalyticsListener.EventTime,
+            decoderName: String,
+            initializedTimestampMs: Long,
+            initializationDurationMs: Long,
+        ) {
+            Timber.v("onVideoDecoderInitialized: decoder=$decoderName")
+            currentPlayback.update { it?.copy(videoDecoder = decoderName) }
+        }
+
+        override fun onVideoDisabled(
+            eventTime: AnalyticsListener.EventTime,
+            decoderCounters: DecoderCounters,
+        ) {
+            Timber.d("onVideoDisabled")
+            currentPlayback.update { it?.copy(videoDecoder = null) }
+        }
+
+        override fun onVideoInputFormatChanged(
+            eventTime: AnalyticsListener.EventTime,
+            format: Format,
+            decoderReuseEvaluation: DecoderReuseEvaluation?,
+        ) {
+            decoderReuseEvaluation?.let { decoder ->
+                if (decoder.result != DecoderReuseEvaluation.REUSE_RESULT_NO) {
+                    Timber.d("onVideoInputFormatChanged: decoder=${decoder.decoderName}")
+                    currentPlayback.update { it?.copy(videoDecoder = decoder.decoderName) }
+                }
+            }
+        }
+
+        override fun onAudioDecoderInitialized(
+            eventTime: AnalyticsListener.EventTime,
+            decoderName: String,
+            initializedTimestampMs: Long,
+            initializationDurationMs: Long,
+        ) {
+            Timber.d("decoder: onAudioDecoderInitialized: decoder=$decoderName")
+            currentPlayback.update { it?.copy(audioDecoder = decoderName) }
+        }
+
+        override fun onAudioInputFormatChanged(
+            eventTime: AnalyticsListener.EventTime,
+            format: Format,
+            decoderReuseEvaluation: DecoderReuseEvaluation?,
+        ) {
+            decoderReuseEvaluation?.let { decoder ->
+                if (decoder.result != DecoderReuseEvaluation.REUSE_RESULT_NO) {
+                    Timber.d("decoder: onAudioInputFormatChanged: decoder=${decoder.decoderName}")
+                    currentPlayback.update { it?.copy(audioDecoder = decoder.decoderName) }
+                }
+            }
+        }
+
+        override fun onAudioDisabled(
+            eventTime: AnalyticsListener.EventTime,
+            decoderCounters: DecoderCounters,
+        ) {
+            Timber.d("decoder: onAudioDisabled")
+            currentPlayback.update { it?.copy(audioDecoder = null) }
+        }
     }