Support for trickplay images

app/src/main/java/com/github/damontecres/dolphin/ui/CoilTrickplayTransformation.kt

@@ -0,0 +1,33 @@
+package com.github.damontecres.dolphin.ui
+
+import android.graphics.Bitmap
+import androidx.core.graphics.scale
+import coil3.size.Size
+import coil3.size.pxOrElse
+import coil3.transform.Transformation
+
+class CoilTrickplayTransformation(
+    val targetWidth: Int,
+    val targetHeight: Int,
+    val numRows: Int,
+    val numColumns: Int,
+    val imageIndex: Int,
+    val index: Int,
+) : Transformation() {
+    private val x: Int = imageIndex % numColumns
+    private val y: Int = imageIndex / numRows
+
+    override val cacheKey: String
+        get() = "CoilTrickplayTransformation_$index,$x,$y"
+
+    override suspend fun transform(
+        input: Bitmap,
+        size: Size,
+    ): Bitmap {
+        val width = input.width / numColumns
+        val height = input.height / numRows
+        return Bitmap
+            .createBitmap(input, x * width, y * height, width, height)
+            .scale(size.width.pxOrElse { targetWidth }, size.height.pxOrElse { targetHeight })
+    }
+}

app/src/main/java/com/github/damontecres/dolphin/ui/UiConstants.kt

@@ -26,6 +26,7 @@ val DefaultItemFields =
         ItemFields.SEASON_USER_DATA,
         ItemFields.CHILD_COUNT,
         ItemFields.OVERVIEW,
+        ItemFields.TRICKPLAY,
     )
 
 @Preview(

app/src/main/java/com/github/damontecres/dolphin/ui/detail/SeriesDetails.kt

@@ -106,6 +106,7 @@ class SeriesViewModel
                                 ItemFields.MEDIA_STREAMS,
                                 ItemFields.OVERVIEW,
                                 ItemFields.CUSTOM_RATING,
+                                ItemFields.TRICKPLAY,
                             ),
                     )
                 val pager = ItemPager(api, request, viewModelScope)

app/src/main/java/com/github/damontecres/dolphin/ui/playback/PlaybackContent.kt

@@ -73,6 +73,7 @@ fun PlaybackContent(
     val duration by viewModel.duration.observeAsState(null)
     val audioStreams by viewModel.audioStreams.observeAsState(listOf())
     val subtitleStreams by viewModel.subtitleStreams.observeAsState(listOf())
+    val trickplay by viewModel.trickplay.observeAsState(null)
 
     if (stream == null) {
         // TODO loading
@@ -228,6 +229,8 @@ fun PlaybackContent(
                         subtitleIndex = null,
                         audioIndex = null,
                         audioStreams = audioStreams,
+                        trickplayInfo = trickplay,
+                        trickplayUrlFor = viewModel::getTrickplayUrl,
                     )
                 }
             }

app/src/main/java/com/github/damontecres/dolphin/ui/playback/PlaybackOverlay.kt

@@ -1,21 +1,26 @@
 package com.github.damontecres.dolphin.ui.playback
 
+import androidx.compose.animation.AnimatedVisibility
 import androidx.compose.foundation.interaction.MutableInteractionSource
 import androidx.compose.foundation.interaction.collectIsFocusedAsState
 import androidx.compose.foundation.layout.Box
 import androidx.compose.foundation.layout.Column
 import androidx.compose.foundation.layout.fillMaxWidth
 import androidx.compose.runtime.Composable
+import androidx.compose.runtime.LaunchedEffect
 import androidx.compose.runtime.getValue
 import androidx.compose.runtime.mutableFloatStateOf
-import androidx.compose.runtime.mutableStateOf
 import androidx.compose.runtime.remember
 import androidx.compose.runtime.setValue
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
 import androidx.compose.ui.layout.ContentScale
+import androidx.compose.ui.platform.LocalDensity
+import androidx.compose.ui.unit.dp
 import androidx.media3.common.Player
 import androidx.tv.material3.Text
+import org.jellyfin.sdk.model.api.TrickplayInfo
+import timber.log.Timber
 
 @Composable
 fun PlaybackOverlay(
@@ -38,12 +43,14 @@ fun PlaybackOverlay(
     audioStreams: List<AudioStream>,
     modifier: Modifier = Modifier,
     subtitle: String? = null,
+    trickplayInfo: TrickplayInfo? = null,
+    trickplayUrlFor: (Int) -> String? = { null },
     seekBarInteractionSource: MutableInteractionSource = remember { MutableInteractionSource() },
 ) {
     // Will be used for preview/trick play images
-    var seekProgress by remember { mutableFloatStateOf(-1f) }
+    var seekProgressPercent by remember { mutableFloatStateOf(-1f) }
+    val seekProgressMs = seekProgressPercent * playerControls.duration
     val seekBarFocused by seekBarInteractionSource.collectIsFocusedAsState()
-    var seekBarDragging by remember { mutableStateOf(false) }
 
     Box(
         modifier = modifier,
@@ -60,6 +67,45 @@ fun PlaybackOverlay(
                     text = it,
                 )
             }
+            if (seekBarInteractionSource.collectIsFocusedAsState().value) {
+                LaunchedEffect(Unit) {
+                    seekProgressPercent =
+                        (playerControls.currentPosition.toFloat() / playerControls.duration)
+                }
+            }
+            if (trickplayInfo != null) {
+                AnimatedVisibility(seekProgressPercent >= 0 && seekBarFocused) {
+                    val tilesPerImage = trickplayInfo.tileWidth * trickplayInfo.tileHeight
+                    val index =
+                        (seekProgressMs / trickplayInfo.interval).toInt() / tilesPerImage
+
+                    val yOffsetDp = 180.dp + 160.dp
+                    val heightPx = with(LocalDensity.current) { yOffsetDp.toPx().toInt() }
+
+                    val imageUrl = trickplayUrlFor(index)
+                    Timber.v("Trickplay imageUrl: $imageUrl")
+
+                    if (imageUrl != null) {
+                        SeekPreviewImage(
+                            modifier =
+                                Modifier
+//                                    .align(Alignment.TopStart)
+                                    .offsetByPercent(
+                                        xPercentage = seekProgressPercent.coerceIn(0f, 1f),
+//                                        yOffset = heightPx,
+//                                yPercentage = 1 - controlHeight,
+                                    ),
+                            previewImageUrl = imageUrl,
+                            duration = playerControls.duration,
+                            seekProgress = seekProgressPercent,
+                            videoWidth = trickplayInfo.width,
+                            videoHeight = trickplayInfo.height,
+                            trickPlayInfo = trickplayInfo,
+                        )
+                    }
+                }
+            }
+
             PlaybackControls(
                 modifier = Modifier.fillMaxWidth(),
                 subtitleStreams = subtitleStreams,
@@ -68,7 +114,7 @@ fun PlaybackOverlay(
                 controllerViewState = controllerViewState,
                 showDebugInfo = showDebugInfo,
                 onSeekProgress = {
-                    seekProgress = it
+                    seekProgressPercent = it
                     onSeekBarChange(it)
                 },
                 showPlay = showPlay,

app/src/main/java/com/github/damontecres/dolphin/ui/playback/PlaybackViewModel.kt

@@ -19,13 +19,17 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
 import org.jellyfin.sdk.api.client.ApiClient
 import org.jellyfin.sdk.api.client.extensions.mediaInfoApi
+import org.jellyfin.sdk.api.client.extensions.trickplayApi
 import org.jellyfin.sdk.api.client.extensions.userLibraryApi
 import org.jellyfin.sdk.api.client.extensions.videosApi
+import org.jellyfin.sdk.model.api.BaseItemDto
 import org.jellyfin.sdk.model.api.BaseItemKind
 import org.jellyfin.sdk.model.api.DeviceProfile
 import org.jellyfin.sdk.model.api.MediaStreamType
 import org.jellyfin.sdk.model.api.PlaybackInfoDto
+import org.jellyfin.sdk.model.api.TrickplayInfo
 import org.jellyfin.sdk.model.extensions.ticks
+import org.jellyfin.sdk.model.serializer.toUUIDOrNull
 import timber.log.Timber
 import java.util.UUID
 import javax.inject.Inject
@@ -74,8 +78,10 @@ class PlaybackViewModel
         val audioStreams = MutableLiveData<List<AudioStream>>(listOf())
         val subtitleStreams = MutableLiveData<List<SubtitleStream>>(listOf())
         val currentPlayback = MutableLiveData<CurrentPlayback?>(null)
+        val trickplay = MutableLiveData<TrickplayInfo?>(null)
 
         private lateinit var deviceProfile: DeviceProfile
+        private lateinit var dto: BaseItemDto
 
         init {
             addCloseable { player.release() }
@@ -91,6 +97,7 @@ class PlaybackViewModel
             val item = destination.item
             viewModelScope.launch(Dispatchers.IO) {
                 val base = item?.data ?: api.userLibraryApi.getItem(itemId).content
+                dto = base
 
                 val title = base.name
                 val subtitle =
@@ -170,7 +177,7 @@ class PlaybackViewModel
             }
         }
 
-        fun changeStreams(
+        private suspend fun changeStreams(
             itemId: UUID,
             audioIndex: Int?,
             subtitleIndex: Int?,
@@ -185,72 +192,111 @@ class PlaybackViewModel
                 Timber.i("No change in playback for changeStreams")
                 return
             }
-            viewModelScope.launch(Dispatchers.IO) {
+            val response =
-                val response =
+                api.mediaInfoApi
-                    api.mediaInfoApi
+                    .getPostedPlaybackInfo(
-                        .getPostedPlaybackInfo(
+                        itemId,
-                            itemId,
+                        // TODO device profile, etc
-                            // TODO device profile, etc
+                        PlaybackInfoDto(
-                            PlaybackInfoDto(
+                            startTimeTicks = null,
-                                startTimeTicks = null,
+                            deviceProfile = deviceProfile,
-                                deviceProfile = deviceProfile,
+                            enableDirectPlay = true,
-                                enableDirectPlay = true,
+                            enableDirectStream = true,
-                                enableDirectStream = true,
+                            maxAudioChannels = null,
-                                maxAudioChannels = null,
+                            audioStreamIndex = audioIndex,
-                                audioStreamIndex = audioIndex,
+                            subtitleStreamIndex = subtitleIndex,
-                                subtitleStreamIndex = subtitleIndex,
+                            allowVideoStreamCopy = true,
-                                allowVideoStreamCopy = true,
+                            allowAudioStreamCopy = true,
-                                allowAudioStreamCopy = true,
+                            autoOpenLiveStream = true,
-                                autoOpenLiveStream = true,
+                            mediaSourceId = null,
-                                mediaSourceId = null,
+                        ),
-                            ),
+                    ).content
-                        ).content
+            val source = response.mediaSources.firstOrNull()
-                val source = response.mediaSources.firstOrNull()
+            source?.let { source ->
-                source?.let { source ->
+                val mediaUrl =
-                    val mediaUrl =
+                    if (source.supportsDirectPlay) {
-                        if (source.supportsDirectPlay) {
+                        api.videosApi.getVideoStreamUrl(
-                            api.videosApi.getVideoStreamUrl(
+                            itemId = itemId,
-                                itemId = itemId,
+                            mediaSourceId = source.id,
-                                mediaSourceId = source.id,
+                            static = true,
-                                static = true,
+                            tag = source.eTag,
-                                tag = source.eTag,
-                            )
-                        } else if (source.supportsDirectStream) {
-                            api.createUrl(source.transcodingUrl!!)
-                        } else {
-                            api.createUrl(source.transcodingUrl!!)
-                        }
-                    val transcodeType =
-                        when {
-                            source.supportsDirectPlay -> TranscodeType.DIRECT_PLAY
-                            source.supportsDirectStream -> TranscodeType.DIRECT_STREAM
-                            source.supportsTranscoding -> TranscodeType.TRANSCODE
-                            else -> throw Exception("No supported playback method")
-                        }
-                    val decision = StreamDecision(itemId, transcodeType, mediaUrl)
-                    Timber.v("Playback decision: $decision")
-
-                    withContext(Dispatchers.Main) {
-                        duration.value = source.runTimeTicks?.ticks
-                        stream.value = decision
-                        currentPlayback.value = CurrentPlayback(itemId, audioIndex, subtitleIndex)
-                        player.setMediaItem(
-                            decision.mediaItem,
-                            positionMs,
                         )
+                    } else if (source.supportsDirectStream) {
+                        api.createUrl(source.transcodingUrl!!)
+                    } else {
+                        api.createUrl(source.transcodingUrl!!)
                     }
+                val transcodeType =
+                    when {
+                        source.supportsDirectPlay -> TranscodeType.DIRECT_PLAY
+                        source.supportsDirectStream -> TranscodeType.DIRECT_STREAM
+                        source.supportsTranscoding -> TranscodeType.TRANSCODE
+                        else -> throw Exception("No supported playback method")
+                    }
+                val decision = StreamDecision(itemId, transcodeType, mediaUrl)
+                Timber.v("Playback decision: $decision")
+
+                withContext(Dispatchers.Main) {
+                    duration.value = source.runTimeTicks?.ticks
+                    stream.value = decision
+                    currentPlayback.value =
+                        CurrentPlayback(
+                            itemId,
+                            audioIndex,
+                            subtitleIndex,
+                            source.id?.toUUIDOrNull(),
+                        )
+                    player.setMediaItem(
+                        decision.mediaItem,
+                        positionMs,
+                    )
+                }
+                val trickPlayInfo =
+                    dto.trickplay
+                        ?.get(source.id)
+                        ?.values
+                        ?.firstOrNull()
+                Timber.v("Trickplay info: $trickPlayInfo")
+                withContext(Dispatchers.Main) {
+                    trickplay.value = trickPlayInfo
                 }
             }
         }
 
         fun changeAudioStream(index: Int) {
             val itemId = currentPlayback.value?.itemId ?: return
-            changeStreams(itemId, index, currentPlayback.value?.subtitleIndex, player.currentPosition)
+            viewModelScope.launch {
+                changeStreams(
+                    itemId,
+                    index,
+                    currentPlayback.value?.subtitleIndex,
+                    player.currentPosition,
+                )
+            }
         }
 
         fun changeSubtitleStream(index: Int?) {
             val itemId = currentPlayback.value?.itemId ?: return
-            changeStreams(itemId, currentPlayback.value?.audioIndex, index, player.currentPosition)
+            viewModelScope.launch {
+                changeStreams(
+                    itemId,
+                    currentPlayback.value?.audioIndex,
+                    index,
+                    player.currentPosition,
+                )
+            }
+        }
+
+        fun getTrickplayUrl(index: Int): String? {
+            val itemId = dto.id
+            val mediaSourceId = currentPlayback.value?.mediaSourceId
+            val trickPlayInfo = trickplay.value ?: return null
+            return api.trickplayApi.getTrickplayTileImageUrl(
+                itemId,
+                trickPlayInfo.width,
+                index,
+                mediaSourceId,
+            )
         }
     }
 
@@ -258,4 +304,5 @@ data class CurrentPlayback(
     val itemId: UUID,
     val audioIndex: Int?,
     val subtitleIndex: Int?,
+    val mediaSourceId: UUID?,
 )

app/src/main/java/com/github/damontecres/dolphin/ui/playback/SeekPreviewImage.kt

@@ -0,0 +1,126 @@
+package com.github.damontecres.dolphin.ui.playback
+
+import androidx.compose.foundation.background
+import androidx.compose.foundation.border
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.height
+import androidx.compose.foundation.layout.width
+import androidx.compose.runtime.Composable
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.graphics.painter.Painter
+import androidx.compose.ui.layout.ContentScale
+import androidx.compose.ui.layout.layout
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.platform.LocalDensity
+import androidx.compose.ui.unit.dp
+import androidx.tv.material3.MaterialTheme
+import androidx.tv.material3.Text
+import coil3.compose.AsyncImage
+import coil3.request.CachePolicy
+import coil3.request.ImageRequest
+import coil3.request.transformations
+import com.github.damontecres.dolphin.ui.CoilTrickplayTransformation
+import com.github.damontecres.dolphin.ui.isNotNullOrBlank
+import org.jellyfin.sdk.model.api.TrickplayInfo
+import kotlin.time.Duration.Companion.seconds
+
+fun Modifier.offsetByPercent(
+    xPercentage: Float,
+    yOffset: Int,
+) = this.then(
+    Modifier.layout { measurable, constraints ->
+        val placeable = measurable.measure(constraints)
+        layout(placeable.width, placeable.height) {
+            placeable.placeRelative(
+                x =
+                    ((constraints.maxWidth * xPercentage).toInt() - placeable.width / 2)
+                        .coerceIn(0, constraints.maxWidth - placeable.width),
+                y = constraints.maxHeight - yOffset, // (constraints.maxHeight * yPercentage).toInt() - (placeable.height / 1.33f).toInt(),
+            )
+        }
+    },
+)
+
+fun Modifier.offsetByPercent(xPercentage: Float) =
+    this.then(
+        Modifier.layout { measurable, constraints ->
+            val placeable = measurable.measure(constraints)
+            layout(placeable.width, placeable.height) {
+                placeable.placeRelative(
+                    x =
+                        ((constraints.maxWidth * xPercentage).toInt() - placeable.width / 2)
+                            .coerceIn(0, constraints.maxWidth - placeable.width),
+                    y = 0,
+                )
+            }
+        },
+    )
+
+@Composable
+fun SeekPreviewImage(
+    previewImageUrl: String,
+    duration: Long,
+    seekProgress: Float,
+    videoWidth: Int?,
+    videoHeight: Int?,
+    trickPlayInfo: TrickplayInfo,
+    modifier: Modifier = Modifier,
+    placeHolder: Painter? = null,
+) {
+    val context = LocalContext.current
+
+    Column(
+        modifier = modifier,
+        verticalArrangement = Arrangement.spacedBy(8.dp),
+        horizontalAlignment = Alignment.CenterHorizontally,
+    ) {
+        if (previewImageUrl.isNotNullOrBlank() &&
+            videoWidth != null &&
+            videoHeight != null
+        ) {
+            val height = 160.dp
+            val width = height * (videoWidth.toFloat() / videoHeight)
+            val heightPx = with(LocalDensity.current) { height.toPx().toInt() }
+            val widthPx = with(LocalDensity.current) { width.toPx().toInt() }
+
+            val index = (duration * seekProgress / trickPlayInfo.interval).toInt() // Which tile
+            val numberOfTitlesPerImage = trickPlayInfo.tileHeight * trickPlayInfo.tileWidth
+            val imageIndex = index % numberOfTitlesPerImage
+
+            AsyncImage(
+                modifier =
+                    Modifier
+                        .width(width)
+                        .height(height)
+                        .background(Color.Black)
+                        .border(1.5.dp, color = MaterialTheme.colorScheme.border),
+                model =
+                    ImageRequest
+                        .Builder(context)
+                        .data(previewImageUrl)
+                        .memoryCachePolicy(CachePolicy.ENABLED)
+                        .transformations(
+                            CoilTrickplayTransformation(
+                                widthPx,
+                                heightPx,
+                                trickPlayInfo.tileHeight,
+                                trickPlayInfo.tileWidth,
+                                imageIndex,
+                                index,
+                            ),
+                        ).build(),
+                contentScale = ContentScale.None,
+                contentDescription = null,
+                placeholder = placeHolder,
+            )
+        }
+        Text(
+            text = (seekProgress * duration / 1000).toLong().seconds.toString(),
+            color = MaterialTheme.colorScheme.onSurface,
+            style = MaterialTheme.typography.labelLarge,
+        )
+    }
+}