Per-node UBO population and per-frame cam state access

src/main/kotlin/graphics/scenery/Camera.kt

@@ -2,14 +2,16 @@
 
 import graphics.scenery.primitives.TextBoard
 import graphics.scenery.attribute.material.HasMaterial
+import graphics.scenery.attribute.populatesubo.DefaultPopulatesUBO
+import graphics.scenery.attribute.populatesubo.HasCustomPopulatesUBO
+import graphics.scenery.attribute.populatesubo.HasPopulatesUBO
+import graphics.scenery.attribute.populatesubo.PopulatesUBO
 import graphics.scenery.attribute.renderable.HasRenderable
 import graphics.scenery.attribute.spatial.DefaultSpatial
 import graphics.scenery.attribute.spatial.HasCustomSpatial
+import graphics.scenery.backends.UBO
 import graphics.scenery.net.Networkable
-import graphics.scenery.utils.extensions.minus
-import graphics.scenery.utils.extensions.plus
-import graphics.scenery.utils.extensions.times
-import graphics.scenery.utils.extensions.xyz
+import graphics.scenery.utils.extensions.*
 import org.joml.*
 import java.lang.Math
 import java.util.concurrent.atomic.AtomicInteger
@@ -26,7 +28,11 @@
  * @constructor Creates a new camera with default position and right-handed
  *  coordinate system.
  */
-open class Camera : DefaultNode("Camera"), HasRenderable, HasMaterial, HasCustomSpatial<Camera.CameraSpatial> {
+open class Camera : DefaultNode("Camera"),
+                    HasRenderable,
+                    HasMaterial,
+                    HasCustomSpatial<Camera.CameraSpatial>,
+                    HasCustomPopulatesUBO<Camera.CameraUBOPopulator> {
 
     /** Enum class for camera projection types */
     enum class ProjectionType {
@@ -88,22 +94,30 @@
     /** Disables culling for this camera. */
     var disableCulling: Boolean = false
 
+    /** The eye count (aka number of generated viewports) of the current camera. */
+    var eyeCount = 1
+        protected set
+
     var wantsSync = true
     override fun wantsSync(): Boolean = wantsSync
 
     init {
-        this.nodeType = "Camera"
         this.viewSpaceTripod = cameraTripod()
         this.name = "Camera-${counter.incrementAndGet()}"
         addSpatial()
         addRenderable()
         addMaterial()
+        addPopulatesUBO()
     }
 
     override fun createSpatial(): CameraSpatial {
         return CameraSpatial(this)
     }
 
+    override fun createPopulatesUBO(): PopulatesUBO {
+        return CameraUBOPopulator(this)
+    }
+
     override fun update(fresh: Networkable, getNetworkable: (Int) -> Networkable, additionalData: Any?) {
         if (fresh !is Camera) throw IllegalArgumentException("Update called with object of foreign class")
         super.update(fresh, getNetworkable, additionalData)
@@ -367,6 +381,19 @@
         protected val counter = AtomicInteger(0)
     }
 
+    open class CameraUBOPopulator(open val cam: Camera): DefaultPopulatesUBO() {
+        override fun populate(ubo: UBO) {
+            val camSpatial = cam.spatial()
+
+            ubo.add("projection0", { camSpatial.projection.applyVulkanCoordinateSystem() })
+            ubo.add("projection1", { camSpatial.projection.applyVulkanCoordinateSystem() })
+            ubo.add("inverseProjection0", { camSpatial.projection.applyVulkanCoordinateSystem().invert() })
+            ubo.add("inverseProjection1", { camSpatial.projection.applyVulkanCoordinateSystem().invert() })
+            ubo.add("headShift", { Matrix4f().identity() })
+            ubo.add("IPD", { 0.05f })
+            ubo.add("stereoEnabled", { 0 })
+        }
+    }
 
     open class CameraSpatial(val camera: Camera): DefaultSpatial(camera) {
 

src/main/kotlin/graphics/scenery/DefaultNode.kt

@@ -48,7 +48,6 @@ open class DefaultNode(name: String = "Node") : Node, Networkable {
         return true
     }
 
-    override var nodeType = "Node"
     override var boundingBox: OrientedBoundingBox? = null
     override val logger by lazyLogger()
 

src/main/kotlin/graphics/scenery/DetachedHeadCamera.kt

@@ -1,7 +1,11 @@
 package graphics.scenery
 
+import graphics.scenery.attribute.populatesubo.HasCustomPopulatesUBO
+import graphics.scenery.attribute.populatesubo.PopulatesUBO
 import graphics.scenery.backends.Display
+import graphics.scenery.backends.UBO
 import graphics.scenery.controls.TrackerInput
+import graphics.scenery.utils.extensions.applyVulkanCoordinateSystem
 import graphics.scenery.utils.extensions.plus
 import graphics.scenery.utils.extensions.times
 import org.joml.Matrix4f
@@ -20,7 +24,11 @@
  * @author Ulrik Günther <[email protected]>
  */
 
-class DetachedHeadCamera(@Transient var tracker: TrackerInput? = null) : Camera() {
+open class DetachedHeadCamera(@Transient var tracker: TrackerInput? = null) : Camera(), HasCustomPopulatesUBO<Camera.CameraUBOPopulator> {
+
+    init {
+        eyeCount = 2
+    }
 
     override var width: Int = 0
         get() = if (tracker != null && tracker is Display && tracker?.initializedAndWorking() == true) {
@@ -104,12 +112,36 @@
     val headOrientation: Quaternionf by HeadOrientationDelegate()
 
     init {
-        this.nodeType = "Camera"
-        this.name = "DetachedHeadCamera-${tracker ?: "${counter.getAndIncrement()}"}"
+        name = "DetachedHeadCamera-${tracker ?: "${counter.getAndIncrement()}"}"
     }
 
     override fun createSpatial(): CameraSpatial = DetachedHeadCameraSpatial(this)
-
+
+    override fun createPopulatesUBO(): PopulatesUBO = DetachedHeadCameraUBOPopulator(this)
+    open class DetachedHeadCameraUBOPopulator(override val cam: DetachedHeadCamera): Camera.CameraUBOPopulator(cam) {
+        override fun populate(ubo: UBO) {
+            val camSpatial = cam.spatial()
+            val hmd = (cam.tracker as? Display)
+
+            (0 until cam.eyeCount).forEach { eye ->
+                ubo.add("projection$eye", {
+                    (hmd?.getEyeProjection(eye, cam.nearPlaneDistance, cam.farPlaneDistance)
+                        ?: camSpatial.projection).applyVulkanCoordinateSystem()
+                })
+                ubo.add("inverseProjection$eye", {
+                    (hmd?.getEyeProjection(eye, cam.nearPlaneDistance, cam.farPlaneDistance)
+                        ?: camSpatial.projection).applyVulkanCoordinateSystem().invert()
+                })
+            }
+            ubo.add("headShift", { hmd?.getHeadToEyeTransform(0) ?: Matrix4f().identity() })
+            ubo.add("IPD", { hmd?.getIPD() ?: 0.05f })
+            ubo.add("stereoEnabled", { cam.stereoEnabled })
+        }
+    }
+
+    var stereoEnabled = false
+        internal set
+
     class DetachedHeadCameraSpatial(private val cam: DetachedHeadCamera) : Camera.CameraSpatial(cam) {
 
         override var projection: Matrix4f = Matrix4f().identity()

src/main/kotlin/graphics/scenery/Node.kt

@@ -24,7 +24,6 @@ import kotlin.collections.ArrayList
 
 interface Node : Networkable {
     var name: String
-    var nodeType: String
     /** Children of the Node. */
     var children: CopyOnWriteArrayList<Node>
     /** Other nodes that have linked transforms. */

src/main/kotlin/graphics/scenery/attribute/populatesubo/DefaultPopulatesUBO.kt

@@ -0,0 +1,8 @@
+package graphics.scenery.attribute.populatesubo
+
+import graphics.scenery.backends.UBO
+
+open class DefaultPopulatesUBO: PopulatesUBO {
+    override fun populate(ubo: UBO) {
+    }
+}

src/main/kotlin/graphics/scenery/attribute/populatesubo/HasCustomPopulatesUBO.kt

@@ -0,0 +1,15 @@
+package graphics.scenery.attribute.populatesubo
+
+import graphics.scenery.Node
+
+interface HasCustomPopulatesUBO<T: PopulatesUBO>: Node {
+    fun createPopulatesUBO(): PopulatesUBO
+
+    fun addPopulatesUBO() {
+        addAttribute(PopulatesUBO::class.java, createPopulatesUBO())
+    }
+
+    fun populatesUBO(): T {
+        return getAttribute(PopulatesUBO::class.java)
+    }
+}

src/main/kotlin/graphics/scenery/attribute/populatesubo/HasPopulatesUBO.kt

@@ -0,0 +1,7 @@
+package graphics.scenery.attribute.populatesubo
+
+interface HasPopulatesUBO: HasCustomPopulatesUBO<PopulatesUBO> {
+    override fun createPopulatesUBO(): PopulatesUBO {
+        return DefaultPopulatesUBO()
+    }
+}

src/main/kotlin/graphics/scenery/attribute/populatesubo/PopulatesUBO.kt

@@ -0,0 +1,7 @@
+package graphics.scenery.attribute.populatesubo
+
+import graphics.scenery.backends.UBO
+
+interface PopulatesUBO {
+    fun populate(ubo: UBO)
+}

src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderer.kt

@@ -8,6 +8,7 @@ import graphics.scenery.attribute.renderable.Renderable
 import graphics.scenery.backends.*
 import graphics.scenery.textures.Texture
 import graphics.scenery.utils.*
+import graphics.scenery.utils.extensions.applyVulkanCoordinateSystem
 import kotlinx.coroutines.*
 import org.joml.*
 import org.lwjgl.PointerBuffer
@@ -28,7 +29,6 @@ import org.lwjgl.vulkan.KHRWin32Surface.VK_KHR_WIN32_SURFACE_EXTENSION_NAME
 import org.lwjgl.vulkan.KHRXlibSurface.VK_KHR_XLIB_SURFACE_EXTENSION_NAME
 import org.lwjgl.vulkan.MVKMacosSurface.VK_MVK_MACOS_SURFACE_EXTENSION_NAME
 import org.lwjgl.vulkan.VK10.*
-import java.awt.BorderLayout
 import java.awt.image.BufferedImage
 import java.awt.image.DataBufferByte
 import java.io.File
@@ -39,7 +39,6 @@ import java.util.*
 import java.util.concurrent.*
 import java.util.concurrent.locks.ReentrantLock
 import javax.imageio.ImageIO
-import javax.swing.JFrame
 import kotlin.concurrent.thread
 import kotlin.concurrent.withLock
 import kotlin.reflect.full.*
@@ -387,12 +386,6 @@ open class VulkanRenderer(hub: Hub,
     private var renderConfig: RenderConfigReader.RenderConfig
     private var flow: List<String> = listOf()
 
-    private val vulkanProjectionFix =
-        Matrix4f(
-            1.0f,  0.0f, 0.0f, 0.0f,
-            0.0f, -1.0f, 0.0f, 0.0f,
-            0.0f,  0.0f, 0.5f, 0.0f,
-            0.0f,  0.0f, 0.5f, 1.0f)
 
     final override var renderConfigFile: String = ""
         set(config) {
@@ -1727,12 +1720,21 @@ open class VulkanRenderer(hub: Hub,
 
         profiler?.end()
 
+        val cameraConfig = (0 until activeCamera.eyeCount).map { eye ->
+            VulkanRenderpass.CameraConfig(
+                view = Matrix4f(activeCamera.spatial().getTransformationForEye(eye)),
+                projection = Matrix4f(activeCamera.spatial().projection),
+                eye = eye
+            )
+        }.toMutableList()
+
         flow.take(flow.size - 1).forEachIndexed { i, t ->
             val si = submitInfo[i]
             profiler?.begin("Renderer.$t")
             logger.trace("Running pass {}", t)
             val target = renderpasses[t]!!
             val commandBuffer = target.commandBuffer
+            target.cameraConfiguration = cameraConfig
 
             if (commandBuffer.submitted) {
                 commandBuffer.waitForFence()
@@ -1793,6 +1795,7 @@ open class VulkanRenderer(hub: Hub,
         profiler?.begin("Renderer.${renderpasses.keys.last()}")
         val viewportPass = renderpasses.values.last()
         val viewportCommandBuffer = viewportPass.commandBuffer
+        viewportPass.cameraConfiguration = cameraConfig
 
         logger.trace("Running viewport pass {}", renderpasses.keys.last())
 
@@ -2064,14 +2067,6 @@ open class VulkanRenderer(hub: Hub,
         instanceMasters.isNotEmpty()
     }
 
-    fun Matrix4f.applyVulkanCoordinateSystem(): Matrix4f {
-        val m = Matrix4f(vulkanProjectionFix)
-        m.mul(this)
-
-        return m
-    }
-
-
     private fun getDescriptorCache(): TimestampedConcurrentHashMap<String, SimpleTimestamped<Long>> {
         @Suppress("UNCHECKED_CAST")
         return scene.metadata.getOrPut("DescriptorCache") {
@@ -2109,26 +2104,8 @@ open class VulkanRenderer(hub: Hub,
 
         buffers.VRParameters.reset()
         val vrUbo = defaultUBOs["VRParameters"]!!
-        vrUbo.add("projection0", {
-            (hmd?.getEyeProjection(0, cam.nearPlaneDistance, cam.farPlaneDistance)
-                ?: camSpatial.projection).applyVulkanCoordinateSystem()
-        })
-        vrUbo.add("projection1", {
-            (hmd?.getEyeProjection(1, cam.nearPlaneDistance, cam.farPlaneDistance)
-                ?: camSpatial.projection).applyVulkanCoordinateSystem()
-        })
-        vrUbo.add("inverseProjection0", {
-            (hmd?.getEyeProjection(0, cam.nearPlaneDistance, cam.farPlaneDistance)
-                ?: camSpatial.projection).applyVulkanCoordinateSystem().invert()
-        })
-        vrUbo.add("inverseProjection1", {
-            (hmd?.getEyeProjection(1, cam.nearPlaneDistance, cam.farPlaneDistance)
-                ?: camSpatial.projection).applyVulkanCoordinateSystem().invert()
-        })
-        vrUbo.add("headShift", { hmd?.getHeadToEyeTransform(0) ?: Matrix4f().identity() })
-        vrUbo.add("IPD", { hmd?.getIPD() ?: 0.05f })
-        vrUbo.add("stereoEnabled", { renderConfig.stereoEnabled.toInt() })
-
+        (cam as? DetachedHeadCamera)?.stereoEnabled = renderConfig.stereoEnabled
+        cam.populatesUBO().populate(vrUbo)
         updated = vrUbo.populate()
 
         buffers.UBOs.reset()

src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderpass.kt

@@ -10,6 +10,7 @@ import graphics.scenery.attribute.material.Material
 import graphics.scenery.backends.*
 import graphics.scenery.utils.lazyLogger
 import graphics.scenery.utils.RingBuffer
+import org.joml.Matrix4f
 import org.joml.Vector2f
 import org.joml.Vector4f
 import org.lwjgl.system.MemoryUtil.*
@@ -107,6 +108,24 @@ open class VulkanRenderpass(val name: String, var config: RenderConfigReader.Ren
     /** Whether this renderpass will render to the viewport or to a [VulkanFramebuffer] */
     var isViewportRenderpass = false
 
+    /** Data class for storing this passes' camera matrices. These are stored per-eye of the camera. */
+    data class CameraConfig(val view: Matrix4f, val projection: Matrix4f, val eye: Int)
+    /** A list of this passes' camera configuration, backed by a RingBuffer.
+      * For each element in the ring buffer, there will be multiple [CameraConfig]s stored,
+      * each corresponding to an eye of the camera at hand.
+      */
+    var cameraConfiguration: MutableList<CameraConfig>
+        get() {
+            return cameraConfigurationBacking.get()
+        }
+
+        set(cc) {
+            cameraConfigurationBacking.put(cc)
+        }
+
+    private var cameraConfigurationBacking = RingBuffer(size = ringBufferSize,
+                                                        default = { mutableListOf<CameraConfig>() })
+
     /** The number of command buffers to keep in the [RingBuffer] [commandBufferBacking]. */
     var commandBufferCount = 3
         set(count) {