Add optional AV1 software decoding via dav1d for ExoPlayer (#721)

## Description
Adds the [`dav1d` decoder
module](https://github.com/androidx/media/tree/release/libraries/decoder_av1)
for ExoPlayer as an option. It can be enabled in advanced settings.

Performance will vary between devices.

### Related issues
Closes #103

### Dev notes

As with the ffmpeg module, this one is not required to build the entire
app so it's easier for developers to get started.

This also introduces a customized `DefaultRenderersFactory` for
ExoPlayer, but its largely copied from the super class's implementation.

.github/actions/native-build/action.yml

@@ -15,6 +15,25 @@ runs:
         path: |
           app/libs
         key: ${{ runner.os }}-ffmpeg-${{ hashFiles('**/libs.versions.toml', '**/scripts/ffmpeg/build_ffmpeg_decoder.sh') }}
+    - name: Load libmpv module cache
+      id: cache_libmpv_module
+      if: ${{ inputs.cache }}
+      uses: actions/cache/restore@v4
+      with:
+        path: |
+          app/src/main/libs
+        key: ${{ runner.os }}-libmpv-${{ hashFiles('**/scripts/mpv/*.sh', '**/scripts/mpv/include/*', '**/scripts/mpv/scripts/*', 'app/src/main/jni/*') }}
+    - name: Install dependencies
+      if: steps.cache_ffmpeg_module.outputs.cache-hit != 'true' || steps.cache_libmpv_module.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        pip install jsonschema jinja2
+        sudo apt update
+        sudo apt install -y build-essential autoconf pkg-config libtool ninja-build unzip wget meson nasm
+
+
+# ffmpeg
     - name: Build ffmpeg decoder
       id: ffmpeg-decoder
       if: steps.cache_ffmpeg_module.outputs.cache-hit != 'true'
@@ -30,23 +49,7 @@ runs:
           app/libs
         key: ${{ steps.cache_ffmpeg_module.outputs.cache-primary-key }}
 
-    - name: Load libmpv module cache
+# libmpv
-      id: cache_libmpv_module
-      if: ${{ inputs.cache }}
-      uses: actions/cache/restore@v4
-      with:
-        path: |
-          app/src/main/libs
-        key: ${{ runner.os }}-libmpv-${{ hashFiles('**/scripts/mpv/*.sh', '**/scripts/mpv/include/*', '**/scripts/mpv/scripts/*', 'app/src/main/jni/*') }}
-
-    - name: Install dependencies
-      if: steps.cache_libmpv_module.outputs.cache-hit != 'true'
-      shell: bash
-      run: |
-        python -m pip install --upgrade pip
-        pip install jsonschema jinja2
-        sudo apt update
-        sudo apt install -y build-essential autoconf pkg-config libtool ninja-build unzip wget meson
     - name: Get libmpv dependencies
       if: steps.cache_libmpv_module.outputs.cache-hit != 'true'
       shell: bash

app/build.gradle.kts

@@ -21,6 +21,7 @@ plugins {
 val isCI = if (System.getenv("CI") != null) System.getenv("CI").toBoolean() else false
 val shouldSign = isCI && System.getenv("KEY_ALIAS") != null
 val ffmpegModuleExists = project.file("libs/lib-decoder-ffmpeg-release.aar").exists()
+val av1ModuleExists = project.file("libs/lib-decoder-av1-release.aar").exists()
 
 val gitTags =
     providers
@@ -291,6 +292,9 @@ dependencies {
     if (ffmpegModuleExists || isCI) {
         implementation(files("libs/lib-decoder-ffmpeg-release.aar"))
     }
+    if (av1ModuleExists || isCI) {
+        implementation(files("libs/lib-decoder-av1-release.aar"))
+    }
 
     testImplementation(libs.mockk.android)
     testImplementation(libs.mockk.agent)

app/src/main/java/com/github/damontecres/wholphin/preferences/AppPreference.kt

@@ -425,6 +425,18 @@ sealed interface AppPreference<Pref, T> {
                 summary = R.string.force_dovi_profile_7_summary,
             )
 
+        val DecodeAv1 =
+            AppSwitchPreference<AppPreferences>(
+                title = R.string.software_decoding_av1,
+                defaultValue = true,
+                getter = { it.playbackPreferences.overrides.decodeAv1 },
+                setter = { prefs, value ->
+                    prefs.updatePlaybackOverrides { decodeAv1 = value }
+                },
+                summaryOn = R.string.enabled,
+                summaryOff = R.string.disabled,
+            )
+
         val RememberSelectedTab =
             AppSwitchPreference<AppPreferences>(
                 title = R.string.remember_selected_tab,
@@ -1004,6 +1016,7 @@ val advancedPreferences =
                                 AppPreference.DirectPlayAss,
                                 AppPreference.DirectPlayPgs,
                                 AppPreference.DirectPlayDoviProfile7,
+                                AppPreference.DecodeAv1,
                             ),
                         ),
                         ConditionalPreferences(

app/src/main/java/com/github/damontecres/wholphin/services/DeviceProfileService.kt

@@ -43,6 +43,7 @@ class DeviceProfileService
                             assDirectPlay = prefs.overrides.directPlayAss,
                             pgsDirectPlay = prefs.overrides.directPlayPgs,
                             dolbyVisionELDirectPlay = prefs.overrides.directPlayDolbyVisionEL,
+                            decodeAv1 = prefs.overrides.decodeAv1,
                             jellyfinTenEleven =
                                 serverVersion != null && serverVersion >= ServerVersion(10, 11, 0),
                         )
@@ -57,6 +58,7 @@ class DeviceProfileService
                                 assDirectPlay = newConfig.assDirectPlay,
                                 pgsDirectPlay = newConfig.pgsDirectPlay,
                                 dolbyVisionELDirectPlay = newConfig.dolbyVisionELDirectPlay,
+                                decodeAv1 = prefs.overrides.decodeAv1,
                                 jellyfinTenEleven = newConfig.jellyfinTenEleven,
                             )
                     }
@@ -75,5 +77,6 @@ data class DeviceProfileConfiguration(
     val assDirectPlay: Boolean,
     val pgsDirectPlay: Boolean,
     val dolbyVisionELDirectPlay: Boolean,
+    val decodeAv1: Boolean,
     val jellyfinTenEleven: Boolean,
 )

app/src/main/java/com/github/damontecres/wholphin/services/PlayerFactory.kt

@@ -3,12 +3,19 @@
 package com.github.damontecres.wholphin.services
 
 import android.content.Context
+import android.os.Build
+import android.os.Handler
 import androidx.annotation.OptIn
 import androidx.datastore.core.DataStore
+import androidx.media3.common.C
 import androidx.media3.common.Player
 import androidx.media3.common.util.UnstableApi
 import androidx.media3.exoplayer.DefaultRenderersFactory
 import androidx.media3.exoplayer.ExoPlayer
+import androidx.media3.exoplayer.Renderer
+import androidx.media3.exoplayer.mediacodec.MediaCodecSelector
+import androidx.media3.exoplayer.video.MediaCodecVideoRenderer
+import androidx.media3.exoplayer.video.VideoRendererEventListener
 import com.github.damontecres.wholphin.preferences.AppPreference
 import com.github.damontecres.wholphin.preferences.AppPreferences
 import com.github.damontecres.wholphin.preferences.MediaExtensionStatus
@@ -18,6 +25,7 @@ import dagger.hilt.android.qualifiers.ApplicationContext
 import kotlinx.coroutines.flow.firstOrNull
 import kotlinx.coroutines.runBlocking
 import timber.log.Timber
+import java.lang.reflect.Constructor
 import javax.inject.Inject
 import javax.inject.Singleton
 
@@ -61,8 +69,8 @@ class PlayerFactory
                     PlayerBackend.EXO_PLAYER,
                     PlayerBackend.UNRECOGNIZED,
                     -> {
-                        val extensions =
+                        val extensions = prefs?.overrides?.mediaExtensionsEnabled
-                            runBlocking { appPreferences.data.firstOrNull() }?.playbackPreferences?.overrides?.mediaExtensionsEnabled
+                        val decodeAv1 = prefs?.overrides?.decodeAv1 == true
                         Timber.v("extensions=$extensions")
                         val rendererMode =
                             when (extensions) {
@@ -74,7 +82,7 @@ class PlayerFactory
                         ExoPlayer
                             .Builder(context)
                             .setRenderersFactory(
-                                DefaultRenderersFactory(context)
+                                WholphinRenderersFactory(context, decodeAv1)
                                     .setEnableDecoderFallback(true)
                                     .setExtensionRendererMode(rendererMode),
                             ).build()
@@ -96,3 +104,73 @@ val Player.isReleased: Boolean
             else -> throw IllegalStateException("Unknown Player type: ${this::class.qualifiedName}")
         }
     }
+
+// Code is adapted from https://github.com/androidx/media/blob/release/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/DefaultRenderersFactory.java#L436
+class WholphinRenderersFactory(
+    context: Context,
+    private val av1Enabled: Boolean,
+) : DefaultRenderersFactory(context) {
+    override fun buildVideoRenderers(
+        context: Context,
+        extensionRendererMode: Int,
+        mediaCodecSelector: MediaCodecSelector,
+        enableDecoderFallback: Boolean,
+        eventHandler: Handler,
+        eventListener: VideoRendererEventListener,
+        allowedVideoJoiningTimeMs: Long,
+        out: ArrayList<Renderer>,
+    ) {
+        var videoRendererBuilder =
+            MediaCodecVideoRenderer
+                .Builder(context)
+                .setCodecAdapterFactory(codecAdapterFactory)
+                .setMediaCodecSelector(mediaCodecSelector)
+                .setAllowedJoiningTimeMs(allowedVideoJoiningTimeMs)
+                .setEnableDecoderFallback(enableDecoderFallback)
+                .setEventHandler(eventHandler)
+                .setEventListener(eventListener)
+                .setMaxDroppedFramesToNotify(MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY)
+                .experimentalSetParseAv1SampleDependencies(false)
+                .experimentalSetLateThresholdToDropDecoderInputUs(C.TIME_UNSET)
+        if (Build.VERSION.SDK_INT >= 34) {
+            videoRendererBuilder =
+                videoRendererBuilder.experimentalSetEnableMediaCodecBufferDecodeOnlyFlag(
+                    false,
+                )
+        }
+        out.add(videoRendererBuilder.build())
+
+        if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) {
+            return
+        }
+        var extensionRendererIndex = out.size
+        if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) {
+            extensionRendererIndex--
+        }
+
+        if (av1Enabled) {
+            try {
+                val clazz = Class.forName("androidx.media3.decoder.av1.Libdav1dVideoRenderer")
+                val constructor: Constructor<*> =
+                    clazz.getConstructor(
+                        Long::class.javaPrimitiveType,
+                        Handler::class.java,
+                        VideoRendererEventListener::class.java,
+                        Int::class.javaPrimitiveType,
+                    )
+                val renderer =
+                    constructor.newInstance(
+                        allowedVideoJoiningTimeMs,
+                        eventHandler,
+                        eventListener,
+                        MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY,
+                    ) as Renderer
+                out.add(extensionRendererIndex++, renderer)
+                Timber.i("Loaded Libdav1dVideoRenderer.")
+            } catch (e: Exception) {
+                // The extension is present, but instantiation failed.
+                throw java.lang.IllegalStateException("Error instantiating AV1 extension", e)
+            }
+        }
+    }
+}

app/src/main/java/com/github/damontecres/wholphin/util/profile/DeviceProfileUtils.kt

@@ -67,6 +67,7 @@ fun createDeviceProfile(
     assDirectPlay: Boolean,
     pgsDirectPlay: Boolean,
     dolbyVisionELDirectPlay: Boolean,
+    decodeAv1: Boolean,
     jellyfinTenEleven: Boolean,
 ) = buildDeviceProfile {
     val allowedAudioCodecs =
@@ -338,6 +339,7 @@ fun createDeviceProfile(
 
         conditions {
             when {
+                decodeAv1 -> ProfileConditionValue.VIDEO_PROFILE notEquals "none"
                 !supportsAV1 -> ProfileConditionValue.VIDEO_PROFILE equals "none"
                 !supportsAV1Main10 -> ProfileConditionValue.VIDEO_PROFILE notEquals "main 10"
                 else -> ProfileConditionValue.VIDEO_PROFILE notEquals "none"
@@ -382,6 +384,7 @@ fun createDeviceProfile(
     }
 
     // AV1
+    if (!decodeAv1) {
         codecProfile {
             type = CodecType.VIDEO
             codec = Codec.Video.AV1
@@ -391,6 +394,7 @@ fun createDeviceProfile(
                 ProfileConditionValue.HEIGHT lowerThanOrEquals maxResolutionAV1.height
             }
         }
+    }
 
     // VC1
     codecProfile {
@@ -410,6 +414,7 @@ fun createDeviceProfile(
         buildSet {
             if (jellyfinTenEleven) add("DOVIInvalid")
 
+            if (!decodeAv1) {
                 if (!supportsAV1DolbyVision) {
                     add(VideoRangeType.DOVI.serialName)
                     if (!supportsAV1HDR10) add(VideoRangeType.DOVI_WITH_HDR10.serialName)
@@ -422,6 +427,7 @@ fun createDeviceProfile(
                     if (!mediaTest.supportsAV1HDR10()) add(VideoRangeType.HDR10.serialName)
                 }
             }
+        }
 
     // TODO Use VideoRangeType enum with Jellyfin 10.11 based SDK
     val unsupportedRangeTypesHevc =

app/src/main/proto/WholphinDataStore.proto

@@ -46,6 +46,7 @@ message PlaybackOverrides{
   bool direct_play_pgs = 4;
   MediaExtensionStatus media_extensions_enabled = 5;
   bool direct_play_dolby_vision_e_l = 6;
+  bool decode_av1 = 7;
 }
 
 message PlaybackPreferences {

app/src/main/res/values/strings.xml

@@ -460,6 +460,7 @@
     <string name="upcoming_movies">Upcoming Movies</string>
     <string name="upcoming_tv">Upcoming TV Shows</string>
     <string name="request_4k">Request in 4K</string>
+    <string name="software_decoding_av1">AV1 software decoding</string>
 
     <string-array name="theme_song_volume">
         <item>Disabled</item>

scripts/ffmpeg/build_ffmpeg_decoder.sh

@@ -15,6 +15,7 @@ PROJECT_ROOT="$(realpath "${SCRIPT_DIR}/../../")"
 ANDROID_ABI=21
 ENABLED_DECODERS=(dca ac3 eac3 mlp truehd flac alac pcm_mulaw pcm_alaw mp3)
 FFMPEG_BRANCH="release/6.0"
+DAV1D_BRANCH="1.5.3"
 
 # Path configs
 DIR_PATH="$(pwd)"
@@ -22,6 +23,7 @@ TARGET_PATH="$PROJECT_ROOT/app/libs"
 MEDIA_PATH="$DIR_PATH/ffmpeg_decoder/media"
 FFMPEG_MODULE_PATH="$MEDIA_PATH/libraries/decoder_ffmpeg/src/main"
 FFMPEG_PATH="$DIR_PATH/ffmpeg_decoder/ffmpeg"
+AV1_MODULE_PATH="$MEDIA_PATH/libraries/decoder_av1/src/main"
 HOST="$(uname -s | tr '[:upper:]' '[:lower:]')"
 HOST_PLATFORM="$HOST-x86_64"
 
@@ -48,16 +50,39 @@ else
   git clone https://github.com/FFmpeg/FFmpeg --depth 1 --single-branch -b "$FFMPEG_BRANCH" ffmpeg
 fi
 
-ln -s "$FFMPEG_PATH" "${FFMPEG_MODULE_PATH}/jni/ffmpeg"
+[[ ! -d "${FFMPEG_MODULE_PATH}/jni/ffmpeg" ]] && ln -s "$FFMPEG_PATH" "${FFMPEG_MODULE_PATH}/jni/ffmpeg"
 
 pushd "${FFMPEG_MODULE_PATH}/jni" || exit
 
 ./build_ffmpeg.sh "${FFMPEG_MODULE_PATH}" "${NDK_PATH}" "${HOST_PLATFORM}" "${ANDROID_ABI}" "${ENABLED_DECODERS[@]}"
 
+# av1 module
+
+pushd "$AV1_MODULE_PATH/jni" || exit
+
+if [[ ! -d cpu_features ]]; then
+  git clone https://github.com/google/cpu_features --depth 1 --single-branch cpu_features
+fi
+
+pushd "$AV1_MODULE_PATH/jni" || exit
+
+if [[ -d dav1d ]]; then
+  pushd dav1d || exit
+  git checkout --force "$DAV1D_BRANCH"
+else
+  git clone https://code.videolan.org/videolan/dav1d --depth 1 --single-branch -b "$DAV1D_BRANCH" dav1d
+fi
+
+pushd "$AV1_MODULE_PATH/jni" || exit
+
+/usr/bin/env bash ./build_dav1d.sh "${AV1_MODULE_PATH}" "${NDK_PATH}" "${HOST_PLATFORM}"
+
+
 pushd "$MEDIA_PATH" || exit
-./gradlew :lib-decoder-ffmpeg:assemble
+./gradlew :lib-decoder-ffmpeg:assemble :lib-decoder-av1:assemble
 popd || exit
 
 popd || exit
 cp "$MEDIA_PATH/libraries/decoder_ffmpeg/buildout/outputs/aar/lib-decoder-ffmpeg-release.aar" "$TARGET_PATH/"
+cp "$MEDIA_PATH/libraries/decoder_av1/buildout/outputs/aar/lib-decoder-av1-release.aar" "$TARGET_PATH/"
 popd || exit