diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 15e541ac..b28ac69f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -48,14 +48,14 @@ constraintlayout = "2.1.4" # Core dependencies android-gradlePlugin = { module = "com.android.tools.build:gradle", version.ref = "agp" } -androidx-activity = "androidx.activity:activity:1.8.1" +androidx-activity = "androidx.activity:activity:1.8.2" androidx-core = "androidx.core:core-ktx:1.12.0" androidx-appcompat = "androidx.appcompat:appcompat:1.6.1" -androidx-exifinterface = "androidx.exifinterface:exifinterface:1.3.6" +androidx-exifinterface = "androidx.exifinterface:exifinterface:1.3.7" # Fragment 1.7.0 alpha and Transition 1.5.0 alpha are required for predictive back to work with Fragments and transitions androidx-fragment = "androidx.fragment:fragment-ktx:1.7.0-alpha07" androidx-transition = "androidx.transition:transition-ktx:1.5.0-alpha05" -androidx-activity-compose = "androidx.activity:activity-compose:1.8.1" +androidx-activity-compose = "androidx.activity:activity-compose:1.8.2" androidx-navigation-fragment = { module = "androidx.navigation:navigation-fragment", version.ref = "androidx-navigation" } androidx-navigation-compose = { module = "androidx.navigation:navigation-compose", version.ref = "androidx-navigation" } androidx-navigation-testing = { module = "androidx.navigation:navigation-testing", version.ref = "androidx-navigation" } @@ -125,6 +125,7 @@ androidx-work-runtime-ktx = "androidx.work:work-runtime-ktx:2.9.0" androidx-core-remoteviews = "androidx.core:core-remoteviews:1.0.0" androidx-glance-appwidget = "androidx.glance:glance-appwidget:1.0.0" androidx-glance-material3 = "androidx.glance:glance-material3:1.0.0" +androidx-graphics-core = { group = "androidx.graphics", name = "graphics-core", version = "1.0.0-beta01" } androidx-startup = 'androidx.startup:startup-runtime:1.1.1' androidx-window = { module = "androidx.window:window", version.ref = "androidx-window" } androidx-window-java = { module = "androidx.window:window-java", version.ref = "androidx-window" } diff --git a/samples/README.md b/samples/README.md index 9ebb9d31..77743da7 100644 --- a/samples/README.md +++ b/samples/README.md @@ -107,11 +107,13 @@ A sample showcasing how to handle calls with the Jetpack Telecom API - [TextSpan](user-interface/text/src/main/java/com/example/platform/ui/text/TextSpan.kt): buildSpannedString is useful for quickly building a rich text. - [UltraHDR Image Capture](camera/camera2/src/main/java/com/example/platform/camera/imagecapture/Camera2UltraHDRCapture.kt): -This sample demonstrates how to capture a 10-bit compressed Ultra HDR still image. +This sample demonstrates how to capture a 10-bit compressed still image and - [UltraHDR to HDR Video](media/ultrahdr/src/main/java/com/example/platform/media/ultrahdr/video/UltraHDRToHDRVideo.kt): -This sample demonstrates converting a series of Ultra HDR still images into an HDR video. +This sample demonstrates converting a series of UltraHDR images into a HDR +- [UltraHDR x OpenGLES SurfaceView](graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/opengl/UltraHDRWithOpenGL.kt): +This sample demonstrates displaying an UltraHDR image via and OpenGL Pipeline - [Visualizing an UltraHDR Gainmap](graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/display/VisualizingAnUltraHDRGainmap.kt): -This sample demonstrates visualizing the underlying gainmap of an Ultra HDR image. +This sample demonstrates visualizing the underlying gainmap of an UltraHDR - [WindowInsetsAnimation](user-interface/window-insets/src/main/java/com/example/platform/ui/insets/WindowInsetsAnimation.kt): Shows how to react to the on-screen keyboard (IME) changing visibility, and also controlling the IME's visibility. - [WindowManager](user-interface/windowmanager/src/main/java/com/example/platform/ui/windowmanager/demos/WindowDemosActivity.kt): diff --git a/samples/graphics/ultrahdr/build.gradle.kts b/samples/graphics/ultrahdr/build.gradle.kts index 0c544bab..effd355d 100644 --- a/samples/graphics/ultrahdr/build.gradle.kts +++ b/samples/graphics/ultrahdr/build.gradle.kts @@ -1,4 +1,3 @@ - /* * Copyright 2023 The Android Open Source Project * @@ -32,4 +31,7 @@ dependencies { // Fresco implementation(libs.fresco) implementation(libs.fresco.nativeimagetranscoder) + + // Graphics Core + implementation(libs.androidx.graphics.core) } \ No newline at end of file diff --git a/samples/graphics/ultrahdr/src/main/assets/shaders/fs_uhdr_tonemapper.frag b/samples/graphics/ultrahdr/src/main/assets/shaders/fs_uhdr_tonemapper.frag new file mode 100644 index 00000000..77596de2 --- /dev/null +++ b/samples/graphics/ultrahdr/src/main/assets/shaders/fs_uhdr_tonemapper.frag @@ -0,0 +1,111 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This fragment shader performs tone mapping by converting HDR image data into SDR format +// using a combination of transfer functions, Gainmap adjustments, exposure and gamma +// correction factors, and epsilon values for numerical stability. +precision mediump float; +varying vec2 vTextureCoord; + +uniform highp float srcTF[7]; +uniform highp mat3 gamutTransform; +uniform highp float destTF[7]; +uniform sampler2D base; +uniform sampler2D gainmap; +uniform mediump vec3 logRatioMin; +uniform mediump vec3 logRatioMax; +uniform mediump vec3 gainmapGamma; +uniform mediump vec3 epsilonSdr; +uniform mediump vec3 epsilonHdr; +uniform mediump float W; +uniform highp int gainmapIsAlpha; +uniform highp int singleChannel; +uniform highp int noGamma; + +highp float fromSrc(highp float x) { + highp float G = srcTF[0]; + highp float A = srcTF[1]; + highp float B = srcTF[2]; + highp float C = srcTF[3]; + highp float D = srcTF[4]; + highp float E = srcTF[5]; + highp float F = srcTF[6]; + highp float s = sign(x); + x = abs(x); + x = x < D ? C * x + F : pow(A * x + B, G) + E; + return s * x; +} + +highp float toDest(highp float x) { + highp float G = destTF[0]; + highp float A = destTF[1]; + highp float B = destTF[2]; + highp float C = destTF[3]; + highp float D = destTF[4]; + highp float E = destTF[5]; + highp float F = destTF[6]; + highp float s = sign(x); + x = abs(x); + x = x < D ? C * x + F : pow(A * x + B, G) + E; + return s * x; +} + +highp vec4 sampleBase(vec2 coord) { + vec4 color = texture2D(base, vTextureCoord); + color = vec4(color.xyz / max(color.w, 0.0001), color.w); + color.x = fromSrc(color.x); + color.y = fromSrc(color.y); + color.z = fromSrc(color.z); + color.xyz *= color.w; + return color; +} + +void main() { + vec4 S = sampleBase(vTextureCoord); + vec4 G = texture2D(gainmap, vTextureCoord); + vec3 H; + + if (gainmapIsAlpha == 1) { + G = vec4(G.w, G.w, G.w, 1.0); + mediump float L; + + if (noGamma == 1) { + L = mix(logRatioMin.x, logRatioMax.x, G.x); + } else { + L = mix(logRatioMin.x, logRatioMax.x, pow(G.x, gainmapGamma.x)); + } + + H = (S.xyz + epsilonSdr) * exp(L * W) - epsilonHdr; + } else { + mediump vec3 L; + if (noGamma == 1) { + L = mix(logRatioMin, logRatioMax, G.xyz); + } else { + L = mix(logRatioMin, logRatioMax, pow(G.xyz, gainmapGamma)); + } + + H = (S.xyz + epsilonSdr) * exp(L * W) - epsilonHdr; + } + + vec4 result = vec4(H.xyz / max(S.w, 0.0001), S.w); + result.rgb = (gamutTransform * result.rgb); + result.x = toDest(result.x); + result.y = toDest(result.y); + result.z = toDest(result.z); + result.xyz *= result.w; + + gl_FragColor = result; +} \ No newline at end of file diff --git a/samples/graphics/ultrahdr/src/main/assets/shaders/vs_uhdr_texture_sampling.vert b/samples/graphics/ultrahdr/src/main/assets/shaders/vs_uhdr_texture_sampling.vert new file mode 100644 index 00000000..757f0fcb --- /dev/null +++ b/samples/graphics/ultrahdr/src/main/assets/shaders/vs_uhdr_texture_sampling.vert @@ -0,0 +1,30 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// This vertex shader program transforms vertices from object space to clip space and +// prepares texture coordinates for fragment shader processing in a fragment shader. +// +// refer to fs_uhdr_tonemapper.frag +// +uniform mat4 uMVPMatrix; +attribute vec4 aPosition; +attribute vec2 aTextureCoord; +varying vec2 vTextureCoord; + +void main() { + gl_Position = uMVPMatrix * aPosition; + vTextureCoord = aTextureCoord; +} \ No newline at end of file diff --git a/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/common/ColorModeControls.kt b/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/common/ColorModeControls.kt index 9e9d7f99..4b99fc4d 100644 --- a/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/common/ColorModeControls.kt +++ b/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/common/ColorModeControls.kt @@ -106,7 +106,11 @@ class ColorModeControls : LinearLayout, WindowObserver { binding.ultrahdrColorModeCurrentMode.run { val mode = when (it.colorMode) { - ActivityInfo.COLOR_MODE_DEFAULT -> resources.getString(R.string.color_mode_sdr) + ActivityInfo.COLOR_MODE_DEFAULT -> String.format( + resources.getString(R.string.color_mode_sdr_with_ratio), + sdrHdrRatio, + ) + ActivityInfo.COLOR_MODE_HDR -> String.format( resources.getString(R.string.color_mode_hdr_with_ratio), sdrHdrRatio, @@ -114,7 +118,7 @@ class ColorModeControls : LinearLayout, WindowObserver { else -> resources.getString(R.string.color_mode_unknown) } - text = "Activity Color Mode: " + mode + text = mode } } } diff --git a/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/opengl/UltraHDRWithOpenGL.kt b/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/opengl/UltraHDRWithOpenGL.kt new file mode 100644 index 00000000..212bf6fb --- /dev/null +++ b/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/opengl/UltraHDRWithOpenGL.kt @@ -0,0 +1,267 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.platform.graphics.ultrahdr.opengl + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.ColorSpace +import android.graphics.Typeface +import android.hardware.DataSpace +import android.os.Build +import android.os.Bundle +import android.view.Display +import android.view.LayoutInflater +import android.view.View +import android.view.ViewGroup +import android.widget.RadioButton +import androidx.annotation.RequiresApi +import androidx.fragment.app.Fragment +import androidx.graphics.lowlatency.BufferInfo +import androidx.graphics.opengl.FrameBuffer +import androidx.graphics.opengl.GLFrameBufferRenderer +import androidx.graphics.opengl.GLRenderer +import androidx.graphics.opengl.egl.EGLManager +import androidx.graphics.surface.SurfaceControlCompat +import androidx.hardware.SyncFenceCompat +import com.example.platform.graphics.ultrahdr.R +import com.example.platform.graphics.ultrahdr.databinding.UltrahdrWithGraphicsBinding +import com.google.android.catalog.framework.annotations.Sample +import java.util.function.Consumer + +@Sample( + name = "UltraHDR x OpenGLES SurfaceView", + description = "This sample demonstrates displaying an UltraHDR image via and OpenGL Pipeline " + + "and control the SurfaceView's rendering brightness.", + documentation = "https://developer.android.com/guide/topics/media/hdr-image-format", + tags = ["UltraHDR"], +) +@RequiresApi(Build.VERSION_CODES.UPSIDE_DOWN_CAKE) +class UltraHDRWithOpenGL : Fragment(), + GLRenderer.EGLContextCallback, + GLFrameBufferRenderer.Callback { + /** + * ExtendedBrightnessValue enum to control + */ + private enum class ExtendedBrightnessValue(val value: Int) { + ZERO_PERCENT(0), + THIRTY_PERCENT(1), + SEVENTY_PERCENT(2), + ONE_HUNDRED_PERCENT(3); + + fun toFloatValue() = when (this) { + ZERO_PERCENT -> 0.0f + THIRTY_PERCENT -> 0.3f + SEVENTY_PERCENT -> 0.7f + ONE_HUNDRED_PERCENT -> 1.0f + } + + companion object { + fun fromInt(value: Int) = ExtendedBrightnessValue.values().first { it.value == value } + } + } + + /** + * Android ViewBinding. + */ + private var _binding: UltrahdrWithGraphicsBinding? = null + private val binding get() = _binding!! + + /** + * Is the device screen wide gamut. + */ + private var _isWideGamut: Boolean? = null + private val isWideGamut get() = _isWideGamut!! + + private lateinit var glRenderer: GLRenderer + private lateinit var glFrameBufferRenderer: GLFrameBufferRenderer + private lateinit var ultraHDRGLRenderer: UltraHDRWithOpenGLRenderer + + private var hdrSdrRatio = 1.0f + private var desiredRatio = 1.0f + + private lateinit var bitmap: Bitmap + private val updateHdrSdrRatio = Consumer { glFrameBufferRenderer.render() } + + override fun onCreateView( + inflater: LayoutInflater, + container: ViewGroup?, + savedInstanceState: Bundle?, + ): View { + _binding = UltrahdrWithGraphicsBinding.inflate(inflater, container, false) + _isWideGamut = requireContext().display?.isWideColorGamut == true + return binding.root + } + + override fun onViewCreated(view: View, savedInstanceState: Bundle?) { + super.onViewCreated(view, savedInstanceState) + val stream = context?.assets?.open(ULTRA_HDR_IMAGE) + bitmap = BitmapFactory.decodeStream(stream) + + // Initialize GL pipeline when surface is created. + initUltraHDRGlRenderer() + initGlRenderer() + initGlFrameRenderer() + + // The ColorModeControls Class contain the necessary function to change the activities + // ColorMode to HDR, which allows and UltraHDRs images gain map to be used to enhance the + // image. + binding.colorModeControls.setWindow(requireActivity().window) + binding.colorModeControls.binding.ultrahdrColorModeHdr.visibility = View.GONE + binding.colorModeControls.binding.ultrahdrColorModeSdr.visibility = View.GONE + binding.openglSurfaceViewExtendedBrightness.typeface = Typeface.DEFAULT_BOLD + binding.openglBrightnessGroup.setOnCheckedChangeListener { group, i -> + val selected = group.findViewById(i) + val index = group.indexOfChild(selected) + val ebv = ExtendedBrightnessValue.fromInt(index) + val ebValue = ebv.toFloatValue() + desiredRatio = maxOf(1.0f, ultraHDRGLRenderer.desiredHdrSdrRatio * ebValue) + + binding.openglSurfaceViewExtendedBrightness.text = String.format( + resources.getString(R.string.ultrahdr_with_opengl_surface_view_extended_brightness), + desiredRatio, + ) + + // TODO Android U doesn't react well when desiredRatio changes from anything other than + // 1.0f. So for now, we will force to go back to 1.0f (0%) and then unlock other + // options + updateAvailableBrightnessOptions(ebv) + glFrameBufferRenderer.render() + } + + // Set Initial Render brightness to 0% + binding.openglBrightness0.isChecked = true + } + + private fun updateAvailableBrightnessOptions(ebv: ExtendedBrightnessValue) { + when (ebv) { + ExtendedBrightnessValue.ZERO_PERCENT -> { + binding.openglBrightness30.isEnabled = true + binding.openglBrightness70.isEnabled = true + binding.openglBrightness100.isEnabled = true + } + + else -> { + binding.openglBrightness30.isEnabled = false + binding.openglBrightness70.isEnabled = false + binding.openglBrightness100.isEnabled = false + } + } + } + + private fun initUltraHDRGlRenderer() { + ultraHDRGLRenderer = UltraHDRWithOpenGLRenderer(requireContext(), bitmap) + desiredRatio = ultraHDRGLRenderer.desiredHdrSdrRatio + binding.imageContainer.setImageBitmap(bitmap) + } + + private fun initGlRenderer() { + glRenderer = GLRenderer().apply { + registerEGLContextCallback(this@UltraHDRWithOpenGL) + start() + } + } + + private fun initGlFrameRenderer() { + glFrameBufferRenderer = GLFrameBufferRenderer.Builder(binding.surfaceView, this) + .setGLRenderer(glRenderer) + .build() + } + + override fun onAttach(context: Context) { + requireActivity().display + ?.registerHdrSdrRatioChangedListener(Runnable::run, updateHdrSdrRatio) + super.onAttach(context) + } + + override fun onDetach() { + super.onDetach() + binding.colorModeControls.detach() + requireActivity().display + ?.unregisterHdrSdrRatioChangedListener(updateHdrSdrRatio) + } + + override fun onDestroyView() { + glFrameBufferRenderer.release(true) + super.onDestroyView() + } + + override fun onEGLContextCreated(eglManager: EGLManager) { + val colorSpaceName = when (isWideGamut) { + true -> ColorSpace.Named.DISPLAY_P3 + false -> ColorSpace.Named.SRGB + } + + ultraHDRGLRenderer.onContextCreated(ColorSpace.get(colorSpaceName)) + } + + override fun onEGLContextDestroyed(eglManager: EGLManager) { + ultraHDRGLRenderer.onContextDestroyed() + } + + override fun onDrawFrame( + eglManager: EGLManager, + width: Int, + height: Int, + bufferInfo: BufferInfo, + transform: FloatArray, + ) { + hdrSdrRatio = requireActivity().display?.hdrSdrRatio ?: 1.0f + ultraHDRGLRenderer.onDrawFrame( + width, + height, + hdrSdrRatio, + bufferInfo.width, + bufferInfo.height, + transform, + ) + } + + override fun onDrawComplete( + targetSurfaceControl: SurfaceControlCompat, + transaction: SurfaceControlCompat.Transaction, + frameBuffer: FrameBuffer, + syncFence: SyncFenceCompat?, + ) { + transaction.setDataSpace(targetSurfaceControl, if (isWideGamut) P3_XRB else SRGB_XRB) + transaction.setExtendedRangeBrightness(targetSurfaceControl, hdrSdrRatio, desiredRatio) + } + + companion object { + /** + * Sample UltraHDR images paths + */ + private const val ULTRA_HDR_IMAGE = "gainmaps/lamps.jpg" + + /** + * DataSpace of SRGB_ERB. + */ + private val SRGB_XRB = DataSpace.pack( + DataSpace.STANDARD_BT709, + DataSpace.TRANSFER_SRGB, + DataSpace.RANGE_EXTENDED, + ) + + /** + * DataSpace of P3_XRB. + */ + private val P3_XRB = DataSpace.pack( + DataSpace.STANDARD_DCI_P3, + DataSpace.TRANSFER_SRGB, + DataSpace.RANGE_EXTENDED, + ) + } +} \ No newline at end of file diff --git a/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/opengl/UltraHDRWithOpenGLRenderer.kt b/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/opengl/UltraHDRWithOpenGLRenderer.kt new file mode 100644 index 00000000..22fbfcc6 --- /dev/null +++ b/samples/graphics/ultrahdr/src/main/java/com/example/platform/graphics/ultrahdr/opengl/UltraHDRWithOpenGLRenderer.kt @@ -0,0 +1,494 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.platform.graphics.ultrahdr.opengl + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.ColorSpace +import android.opengl.GLES20.* +import android.opengl.GLUtils +import android.opengl.Matrix +import android.os.Build +import androidx.annotation.RequiresApi +import java.lang.IllegalArgumentException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.nio.FloatBuffer +import kotlin.math.abs +import kotlin.math.ln +import kotlin.math.max +import kotlin.math.min +import kotlin.math.pow + +@RequiresApi(Build.VERSION_CODES.UPSIDE_DOWN_CAKE) +class UltraHDRWithOpenGLRenderer( + context: Context, + private val bitmap: Bitmap, +) { + + private val triangleVertices: FloatBuffer = + ByteBuffer.allocateDirect(triangleVerticesData.size * FLOAT_SIZE_BYTES) + .order(ByteOrder.nativeOrder()) + .asFloatBuffer() + + private val mvpMatrix = FloatArray(16) + private val orthoMatrix = FloatArray(16) + private val projMatrix = FloatArray(16) + private val mMatrix = FloatArray(16) + private val vMatrix = FloatArray(16) + private val destTF = FloatArray(7) + + private var program = 0 + private var textureID = 0 + private var gainmapTextureID = 0 + private var muMVPMatrixHandle = 0 + private var maPositionHandle = 0 + private var maTextureHandle = 0 + private var destTFHandle = 0 + private var wHandle = 0 + private var displayRatioSdr = 0f + private var displayRatioHdr = 0f + + val desiredHdrSdrRatio get() = bitmap.gainmap?.displayRatioForFullHdr ?: 1f + + /** + * Location of UltraHDR vertex shader program. + */ + private var ultraHDRVertexShader: String = context.assets + ?.open(FILE_ULTRAHDR_VERTEX_SHADER) + ?.bufferedReader().use { it?.readText() ?: "" } + + /** + * Location of UltraHDR fragment shader program. + */ + private var ultraHDRFragmentShader: String = context.assets + ?.open(FILE_ULTRAHDR_FRAGMENT_SHADER) + ?.bufferedReader().use { it?.readText() ?: "" } + + init { + triangleVertices.put(triangleVerticesData).position(0) + } + + private fun loadShader(shaderType: Int, source: String): Int { + var shader = glCreateShader(shaderType) + if (shader != 0) { + glShaderSource(shader, source) + glCompileShader(shader) + val compiled = IntArray(1) + glGetShaderiv(shader, GL_COMPILE_STATUS, compiled, 0) + if (compiled[0] == 0) { + glDeleteShader(shader) + shader = 0 + } + } + return shader + } + + private fun createProgram(vertexSource: String, fragmentSource: String): Int { + val vertexShader = loadShader(GL_VERTEX_SHADER, vertexSource) + if (vertexShader == 0) return 0 + + val pixelShader = loadShader(GL_FRAGMENT_SHADER, fragmentSource) + if (pixelShader == 0) return 0 + + var program = glCreateProgram() + if (program == 0) return 0 + + glAttachShader(program, vertexShader) + checkGlError("glAttachShader") + glAttachShader(program, pixelShader) + checkGlError("glAttachShader") + + glLinkProgram(program) + val linkStatus = IntArray(1) + glGetProgramiv(program, GL_LINK_STATUS, linkStatus, 0) + + if (linkStatus[0] != GL_TRUE) { + glDeleteProgram(program) + program = 0 + } + + return program + } + + private fun checkGlError(op: String) { + var error: Int + while (glGetError().also { error = it } != GL_NO_ERROR) + throw RuntimeException("$op: glError $error") + } + + private val kSRGB = floatArrayOf( + 2.4f, + (1 / 1.055).toFloat(), + (0.055 / 1.055).toFloat(), + (1 / 12.92).toFloat(), + 0.04045f, + 0.0f, + 0.0f, + ) + + private fun trfnApplyGain(trfn: FloatArray, gain: Float, dest: FloatArray) { + val powGainGinv = gain.toDouble().pow(1.0 / trfn[0]).toFloat() + dest[0] = trfn[0] + dest[1] = trfn[1] * powGainGinv + dest[2] = trfn[2] * powGainGinv + dest[3] = trfn[3] * gain + dest[4] = trfn[4] + dest[5] = trfn[5] * gain + dest[6] = trfn[6] * gain + } + + fun mul3x3(lhs: FloatArray, rhs: FloatArray): FloatArray { + val r = FloatArray(9) + r[0] = lhs[0] * rhs[0] + lhs[3] * rhs[1] + lhs[6] * rhs[2] + r[1] = lhs[1] * rhs[0] + lhs[4] * rhs[1] + lhs[7] * rhs[2] + r[2] = lhs[2] * rhs[0] + lhs[5] * rhs[1] + lhs[8] * rhs[2] + r[3] = lhs[0] * rhs[3] + lhs[3] * rhs[4] + lhs[6] * rhs[5] + r[4] = lhs[1] * rhs[3] + lhs[4] * rhs[4] + lhs[7] * rhs[5] + r[5] = lhs[2] * rhs[3] + lhs[5] * rhs[4] + lhs[8] * rhs[5] + r[6] = lhs[0] * rhs[6] + lhs[3] * rhs[7] + lhs[6] * rhs[8] + r[7] = lhs[1] * rhs[6] + lhs[4] * rhs[7] + lhs[7] * rhs[8] + r[8] = lhs[2] * rhs[6] + lhs[5] * rhs[7] + lhs[8] * rhs[8] + return r + } + + private fun skcmsTransferfunctionEval(tf: FloatArray, x: Float): Float { + val sign = if (x < 0) -1.0f else 1.0f + val nx = x * sign + return sign * if (nx < tf[4]) { + tf[3] * nx * tf[6] + } else { + (tf[1] * nx + tf[2]).toDouble().pow(tf[0].toDouble()).toFloat() + tf[5] + } + } + + /** + * This functions calculates the inverse function by finding the inverse of each segment and + * then stitching the segments together. The code also checks to make sure that the inverse + * function is well-defined and that it satisfies the same constraints as the original function. + * + * Finally, the code tweaks the inverse function to make sure that it satisfies the invariant + * inv(src(1.0f)) == 1.0f. + */ + private fun invertTrfn(src: FloatArray, dest: FloatArray) { + // We're inverting this function, solving for x in terms of y. + // y = (cx + f) x < d + // (ax + b)^g + e x ≥ d + // The inverse of this function can be expressed in the same piecewise form. + val inv = floatArrayOf(0f, 0f, 0f, 0f, 0f, 0f, 0f) + + // We'll start by finding the new threshold inv[4]. + // In principle we should be able to find that by solving for y at x=d from either side. + // (If those two d values aren't the same, it's a discontinuous transfer function.) + val dl = src[3] * src[4] + src[6] + val dr = (src[1] * src[4] + src[2]).toDouble().pow(src[0].toDouble()).toFloat() + src[5] + if (abs(dl - dr) > 1 / 512.0f) throw IllegalArgumentException() + inv[4] = dl + + // When d=0, the linear section collapses to a point. We leave c,d,f all zero in that case. + if (inv[4] > 0) { + // Inverting the linear section is pretty straightforward: + // y = cx + f + // y - f = cx + // (1/c)y - f/c = x + inv[3] = 1.0f / src[3] + inv[6] = -src[6] / src[3] + } + + // The interesting part is inverting the nonlinear section: + // y = (ax + b)^g + e. + // y - e = (ax + b)^g + // (y - e)^1/g = ax + b + // (y - e)^1/g - b = ax + // (1/a)(y - e)^1/g - b/a = x + // + // To make that fit our form, we need to move the (1/a) term inside the exponentiation: + // let k = (1/a)^g + // (1/a)( y - e)^1/g - b/a = x + // (ky - ke)^1/g - b/a = x + val k = src[1].toDouble().pow((-src[0]).toDouble()).toFloat() // (1/a)^g == a^-g + inv[0] = 1.0f / src[0] + inv[1] = k + inv[2] = -k * src[5] + inv[5] = -src[2] / src[1] + + // We need to enforce the same constraints here that we do when fitting a curve, + // a >= 0 and ad+b >= 0. These constraints are checked by classify(), so they're true + // of the source function if we're here. + + // Just like when fitting the curve, there's really no way to rescue a < 0. + if (inv[1] < 0) { + throw IllegalArgumentException() + } + + // On the other hand we can rescue an ad+b that's gone slightly negative here. + if (inv[1] * inv[4] + inv[2] < 0) { + inv[2] = -inv[1] * inv[4] + } + + assert(inv[1] >= 0) + assert(inv[1] * inv[4] + inv[2] >= 0) + + // Now in principle we're done. + // But to preserve the valuable invariant inv(src(1.0f)) == 1.0f, we'll tweak + // e or f of the inverse, depending on which segment contains src(1.0f). + var s = skcmsTransferfunctionEval(src, 1.0f) + + val sign = if (s < 0) -1.0f else 1.0f + s *= sign + + if (s < inv[4]) { + inv[6] = 1.0f - sign * inv[3] * s + } else { + inv[5] = + (1.0f - sign * (inv[1] * s + inv[2]).toDouble().pow(inv[0].toDouble())).toFloat() + } + + System.arraycopy(inv, 0, dest, 0, inv.size) + } + + fun onContextCreated(destColorSpace: ColorSpace) { + program = createProgram(ultraHDRVertexShader, ultraHDRFragmentShader) + if (program == 0) return + + maPositionHandle = glGetAttribLocation(program, "aPosition") + checkGlError("glGetAttribLocation aPosition") + + if (maPositionHandle == -1) + throw RuntimeException("Could not get attrib location for aPosition") + + maTextureHandle = glGetAttribLocation(program, "aTextureCoord") + checkGlError("glGetAttribLocation aTextureCoord") + + if (maTextureHandle == -1) + throw java.lang.RuntimeException("Could not get attrib location for aTextureCoord") + + muMVPMatrixHandle = glGetUniformLocation(program, "uMVPMatrix") + checkGlError("glGetUniformLocation uMVPMatrix") + + if (muMVPMatrixHandle == -1) + throw java.lang.RuntimeException("Could not get attrib location for uMVPMatrix") + + destTFHandle = glGetUniformLocation(program, "destTF") + wHandle = uniform("W") + + val textures = IntArray(2) + glGenTextures(2, textures, 0) + + if (bitmap.config == Bitmap.Config.HARDWARE || + bitmap.gainmap!!.gainmapContents.config == Bitmap.Config.HARDWARE + ) + throw IllegalArgumentException("Cannot handle HARDWARE bitmaps") + + textureID = textures[0] + gainmapTextureID = textures[1] + glBindTexture(GL_TEXTURE_2D, textureID) + setupTexture(bitmap) + glBindTexture(GL_TEXTURE_2D, gainmapTextureID) + setupTexture(bitmap.gainmap!!.gainmapContents) + + // Bind the base & gainmap textures + glUseProgram(program) + val textureLoc = glGetUniformLocation(program, "base") + glUniform1i(textureLoc, 0) + val gainmapLoc = glGetUniformLocation(program, "gainmap") + glUniform1i(gainmapLoc, 1) + + // Bind the base transfer function + val srcTF = FloatArray(7) + System.arraycopy(kSRGB, 0, srcTF, 0, srcTF.size) + + val srcColorspace: ColorSpace.Rgb = bitmap.colorSpace!! as ColorSpace.Rgb + srcColorspace.transferParameters!!.let { params -> + srcTF[0] = params.g.toFloat() + srcTF[1] = params.a.toFloat() + srcTF[2] = params.b.toFloat() + srcTF[3] = params.c.toFloat() + srcTF[4] = params.d.toFloat() + srcTF[5] = params.e.toFloat() + srcTF[6] = params.f.toFloat() + } + + val srcTfHandle = glGetUniformLocation(program, "srcTF") + glUniform1fv(srcTfHandle, 7, srcTF, 0) + + val srcD50 = ColorSpace.adapt(srcColorspace, ColorSpace.ILLUMINANT_D50) as ColorSpace.Rgb + val destD50 = ColorSpace.adapt(destColorSpace, ColorSpace.ILLUMINANT_D50) as ColorSpace.Rgb + + val gamutTransform = mul3x3(destD50.inverseTransform, srcD50.transform) + val gamutHandle = glGetUniformLocation(program, "gamutTransform") + glUniformMatrix3fv(gamutHandle, 1, false, gamutTransform, 0) + + val gainmap = bitmap.gainmap!! + val isAlpha = gainmap.gainmapContents.config == Bitmap.Config.ALPHA_8 + val gainmapGamma = gainmap.gamma + val noGamma = gainmapGamma[0] == 1f && gainmapGamma[1] == 1f && gainmapGamma[2] == 1f + + glUniform1i(uniform("gainmapIsAlpha"), if (isAlpha) 1 else 0) + glUniform1i(uniform("noGamma"), if (noGamma) 1 else 0) + setVec3Uniform("gainmapGamma", gainmapGamma) + setLogVec3Uniform("logRatioMin", gainmap.ratioMin) + setLogVec3Uniform("logRatioMax", gainmap.ratioMax) + setVec3Uniform("epsilonSdr", gainmap.epsilonSdr) + setVec3Uniform("epsilonHdr", gainmap.epsilonHdr) + + displayRatioSdr = gainmap.minDisplayRatioForHdrTransition + displayRatioHdr = gainmap.displayRatioForFullHdr + + Matrix.setLookAtM( + vMatrix, + 0, + 0f, 0f, -5f, + 0f, 0f, 0f, + 0f, 1.0f, 0.0f, + ) + } + + fun onContextDestroyed() { + } + + private fun uniform(name: String) = glGetUniformLocation(program, name) + + private fun setVec3Uniform(name: String, vec3: FloatArray) = + glUniform3f(uniform(name), vec3[0], vec3[1], vec3[2]) + + private fun setLogVec3Uniform(name: String, vec3: FloatArray) { + val log = floatArrayOf( + ln(vec3[0].toDouble()).toFloat(), + ln(vec3[1].toDouble()).toFloat(), + ln(vec3[2].toDouble()).toFloat(), + ) + setVec3Uniform(name, log) + } + + private fun setupTexture(bitmap: Bitmap?) { + glTexParameterf( + GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, + GL_LINEAR.toFloat(), + ) + + glTexParameterf( + GL_TEXTURE_2D, + GL_TEXTURE_MAG_FILTER, + GL_LINEAR.toFloat(), + ) + + glTexParameteri( + GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, + GL_REPEAT, + ) + + glTexParameteri( + GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, + GL_REPEAT, + ) + + glPixelStorei(GL_UNPACK_ALIGNMENT, 1) + GLUtils.texImage2D(GL_TEXTURE_2D, 0, bitmap, 0) + } + + fun onDrawFrame( + width: Int, + height: Int, + hdrSdrRatio: Float, + bufferWidth: Int, + bufferHeight: Int, + transform: FloatArray, + ) { + glViewport(0, 0, bufferWidth, bufferHeight) + + Matrix.orthoM( + orthoMatrix, 0, 0f, bufferWidth.toFloat(), 0f, + bufferHeight.toFloat(), -1f, 1f, + ) + + Matrix.multiplyMM(projMatrix, 0, orthoMatrix, 0, transform, 0) + + glClearColor(0.0f, 0.0f, 0.0f, 1.0f) + glClear(GL_DEPTH_BUFFER_BIT or GL_COLOR_BUFFER_BIT) + glUseProgram(program) + checkGlError("glUseProgram") + + // This isn't a good assumption to make, and applying a color gamut matrix from the source + // to the destination should be added. + trfnApplyGain(kSRGB, hdrSdrRatio, destTF) + invertTrfn(destTF, destTF) + glUniform1fv(destTFHandle, 7, destTF, 0) + + glActiveTexture(GL_TEXTURE0) + glBindTexture(GL_TEXTURE_2D, textureID) + glActiveTexture(GL_TEXTURE1) + glBindTexture(GL_TEXTURE_2D, gainmapTextureID) + + val targetRatio = ln(hdrSdrRatio.toDouble()) - ln(displayRatioSdr.toDouble()) + val maxRatio = ln(displayRatioHdr.toDouble()) - ln(displayRatioSdr.toDouble()) + val wUnclamped = targetRatio / maxRatio + val w = max(min(wUnclamped, 1.0), 0.0).toFloat() + glUniform1f(wHandle, w) + + triangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET) + glVertexAttribPointer( + maPositionHandle, 3, GL_FLOAT, false, + TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices, + ) + checkGlError("glVertexAttribPointer maPosition") + triangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET) + glEnableVertexAttribArray(maPositionHandle) + checkGlError("glEnableVertexAttribArray maPositionHandle") + glVertexAttribPointer( + maTextureHandle, 2, GL_FLOAT, false, + TRIANGLE_VERTICES_DATA_STRIDE_BYTES, triangleVertices, + ) + checkGlError("glVertexAttribPointer maTextureHandle") + glEnableVertexAttribArray(maTextureHandle) + checkGlError("glEnableVertexAttribArray maTextureHandle") + + Matrix.setRotateM(mMatrix, 0, 0f, 0f, 0f, 1.0f) + Matrix.scaleM(mvpMatrix, 0, mMatrix, 0, width.toFloat(), height.toFloat(), 1f) + Matrix.multiplyMM(mvpMatrix, 0, projMatrix, 0, mvpMatrix, 0) + + glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mvpMatrix, 0) + glDrawArrays(GL_TRIANGLES, 0, 6) + checkGlError("glDrawArrays") + } + + companion object { + /** + * Location of UltraHDR vertex shader program. + */ + private const val FILE_ULTRAHDR_VERTEX_SHADER = "shaders/vs_uhdr_texture_sampling.vert" + + /** + * Location of UltraHDR fragment shader program. + */ + private const val FILE_ULTRAHDR_FRAGMENT_SHADER = "shaders/fs_uhdr_tonemapper.frag" + + private const val FLOAT_SIZE_BYTES = 4 + private const val TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES + private const val TRIANGLE_VERTICES_DATA_POS_OFFSET = 0 + private const val TRIANGLE_VERTICES_DATA_UV_OFFSET = 3 + + private val triangleVerticesData = floatArrayOf( + // X, Y, Z, U, V + 0f, 0f, .5f, 0f, 0f, + 0f, 1f, .5f, 0f, 1f, + 1f, 0f, .5f, 1f, 0f, + 1f, 1f, .5f, 1f, 1f, + 0f, 1f, .5f, 0f, 1f, + 1f, 0f, .5f, 1f, 0f, + ) + } +} \ No newline at end of file diff --git a/samples/graphics/ultrahdr/src/main/res/layout/ultrahdr_with_graphics.xml b/samples/graphics/ultrahdr/src/main/res/layout/ultrahdr_with_graphics.xml new file mode 100644 index 00000000..e7c62744 --- /dev/null +++ b/samples/graphics/ultrahdr/src/main/res/layout/ultrahdr_with_graphics.xml @@ -0,0 +1,91 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/graphics/ultrahdr/src/main/res/values/strings.xml b/samples/graphics/ultrahdr/src/main/res/values/strings.xml index 0c906ef0..9de2da53 100644 --- a/samples/graphics/ultrahdr/src/main/res/values/strings.xml +++ b/samples/graphics/ultrahdr/src/main/res/values/strings.xml @@ -42,10 +42,20 @@ Compressed UltraHDR image Displayed image size + + Reference Image + Custom UltraHDR render though GLES to SurfaceView: + GLES SurfaceView desired brightness range: %f + 0% + 30% + 70% + 100% + SDR HDR - HDR | SDR/HDR Ratio = %f + Display.ColorMode: SDR | Ratio = %f + Display.ColorMode: HDR | Ratio = %f Color Mode = %s Unknown