From 40c978c712bf5980dc83f46f5c45e075af6b4032 Mon Sep 17 00:00:00 2001 From: AlvaroHG Date: Thu, 21 Nov 2024 23:47:44 -0800 Subject: [PATCH] Working distortion map --- test_distortion.py | 25 ++- unity/Assets/Scripts/AgentManager.cs | 184 ++++++++++++++---- unity/Assets/Scripts/CapturePass.cs | 177 ++++++++++++++++- .../Shaders/BarrelDistortionImageBlend.shader | 2 +- .../Shaders/BarrelDistortionMap.shader | 126 ++++++++++++ .../Shaders/BarrelDistortionMap.shader.meta | 10 + unity/Assets/Scripts/RenderingManager.cs | 91 +++++++-- .../Prefabs/FPSController.prefab | 1 + 8 files changed, 552 insertions(+), 64 deletions(-) create mode 100644 unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionMap.shader create mode 100644 unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionMap.shader.meta diff --git a/test_distortion.py b/test_distortion.py index d0cbdbec9c..12901cc6cb 100644 --- a/test_distortion.py +++ b/test_distortion.py @@ -4,7 +4,7 @@ import sys import os from ai2thor.interact import InteractiveControllerPrompt - +import numpy as np def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None, local_build=False, commit_id=None, fov=120, distortion=False, image_dir=None, width=300, height=300): if image_dir is not None: @@ -40,6 +40,7 @@ def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None, renderDistortionImage=distortion, renderSemanticSegmentation=True, renderInstanceSegmentation=True, + enableDistortionMap=distortion, fieldOfView=120, **args, ) @@ -103,6 +104,24 @@ def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None, intensityY=0.93 ) + evt = controller.step( + action="GetDistortionMaps", + mainCamera=True, + thidPartyCameraIndices=[0] + ) + result = evt.metadata['actionReturn'] + # keys = [key for (key, val) in result.items()] + maps = [] + + print(f"---Action {controller.last_action['action']} success: {evt.metadata['lastActionSuccess']} result {result.keys()}") + print(f"[x,y] at (0,0) (bottom left corner) len {result['mainCamera'][0][0]}") + tex_height = len(result['mainCamera']) + tex_width = len(result['mainCamera'][0]) + print(f"[x,y] at (height, width) (top right corner) len {result['thirdPartyCameras'][0][tex_height-1][tex_width-1]}") + + print(f'Error: {evt.metadata["errorMessage"]}') + + # xpos = dict(x=0.0, y=0.900992214679718, z=0.0786) # # sr = controller.step( # # action="Teleport", position=xpos, rotation=dict(x=0, y=0, z=0), forceAction=True @@ -217,6 +236,6 @@ def load_scene(scene_name, house_path=None, run_in_editor=False, platform=None, fov=float(args.fov), distortion=args.distortion, image_dir=args.output, - width=float(args.width), - height=float(args.height) + width=int(args.width), + height=int(args.height) ) # platform="CloudRendering") diff --git a/unity/Assets/Scripts/AgentManager.cs b/unity/Assets/Scripts/AgentManager.cs index 600e0cfb9e..ead0e94233 100644 --- a/unity/Assets/Scripts/AgentManager.cs +++ b/unity/Assets/Scripts/AgentManager.cs @@ -18,6 +18,7 @@ using Newtonsoft.Json.Serialization; using Thor.Procedural.Data; using Thor.Rendering; +using UnityEditor.AssetImporters; using UnityEngine; using UnityEngine.Experimental.Rendering; using UnityEngine.Networking; @@ -53,6 +54,7 @@ public class AgentManager : MonoBehaviour, ActionInvokable { private bool renderNormalsImage; private bool renderFlowImage; private bool renderDistortionImage; + private bool enableDistortionMap; private IEnumerable activeCapturePassList; private Socket sock = null; @@ -93,7 +95,8 @@ private enum serverTypes { "ChangeResolution", "CoordinateFromRaycastThirdPartyCamera", "ChangeQuality", - "SetDistortionShaderParams" + "SetDistortionShaderParams", + "GetDistortionMaps" }; public HashSet errorAllowedActions = new HashSet { "Reset" }; @@ -341,19 +344,9 @@ public void Initialize(ServerAction action) { action.renderInstanceSegmentation; this.renderFlowImage = action.renderFlowImage; this.renderDistortionImage = action.renderDistortionImage; + this.enableDistortionMap = action.enableDistortionMap; this.fastActionEmit = action.fastActionEmit; - // TODO Refactor so that we know where these strings come from - activeCapturePassList = new Dictionary() { - {"_img", true}, - {"_depth", this.renderDepthImage}, - {"_id", this.renderInstanceSegmentation}, - {"_class", this.renderSemanticSegmentation}, - {"_normals", this.renderNormalsImage}, - {"_flow", this.renderFlowImage}, - {"_distortion", this.renderDistortionImage} - }.Where(x => x.Value).Select(x => x.Key); - PhysicsSceneManager.SetDefaultSimulationParams(action.defaultPhysicsSimulationParams); Time.fixedDeltaTime = ( action.defaultPhysicsSimulationParams?.fixedDeltaTime @@ -386,7 +379,8 @@ public void Initialize(ServerAction action) { ResetSceneBounds(); } // this.updateImageSynthesis(true, activeCapturePassList); - this.updateRenderingManagers(activeCapturePassList, true); + this.UpdateActivePasses(); + this.UpdateRenderingManagers(activeCapturePassList, true); this.agentManagerState = AgentState.ActionComplete; } @@ -578,7 +572,7 @@ private float ClampFieldOfView( return (fov <= min || fov > max) ? defaultVal : fov; } - public void updateRenderingManagers(IEnumerable activePassList, bool cameraChange = false) { + public void UpdateRenderingManagers(IEnumerable activePassList, bool cameraChange = false) { var renderingManagers = agents.Select(agent => agent.m_Camera.GetComponent()) .Concat(thirdPartyCameras.Select(cam => cam.GetComponent())); foreach (var renderingManager in renderingManagers) { @@ -587,6 +581,21 @@ public void updateRenderingManagers(IEnumerable activePassList, bool cam } } + private void UpdateActivePasses() { + // TODO Refactor so that we know where these strings come from + Debug.Log($"--------- enableDistortionMap {this.enableDistortionMap}"); + activeCapturePassList = new Dictionary() { + {"_img", true}, + {"_depth", this.renderDepthImage}, + {"_id", this.renderInstanceSegmentation}, + {"_class", this.renderSemanticSegmentation}, + {"_normals", this.renderNormalsImage}, + {"_flow", this.renderFlowImage}, + {"_distortion", this.renderDistortionImage}, + {"_distortion_map", this.enableDistortionMap} + }.Where(x => x.Value).Select(x => x.Key); + } + public void updateImageSynthesis(bool status, IEnumerable activePassList, bool cameraChange = false) { foreach (var agent in this.agents) { agent.updateImageSynthesis(status, activePassList); @@ -744,15 +753,7 @@ private void updateCameraProperties( var renderingManager = camera.GetComponent(); // renderingManager.OnCameraChange(); - activeCapturePassList = new Dictionary() { - {"_img", true}, - {"_depth", this.renderDepthImage}, - {"_id", this.renderInstanceSegmentation}, - {"_class", this.renderSemanticSegmentation}, - {"_normals", this.renderNormalsImage}, - {"_flow", this.renderFlowImage}, - {"_distortion", this.renderDistortionImage} - }.Where(x => x.Value).Select(x => x.Key); + this.UpdateActivePasses(); renderingManager.EnablePasses(activeCapturePassList, cameraChange: true); // this.updateRenderingManagers(activeCapturePassList, true); @@ -2096,6 +2097,7 @@ public void SetCriticalErrorState() { this.agentManagerState = AgentState.Error; } + public ActionFinished SetDistortionShaderParams(bool mainCamera = true, IEnumerable thidPartyCameraIndices = null, float zoomPercent = 1.0f, float k1 = 0.0f, float k2 = 0.0f, float k3 = 0.0f, float k4 = 0.0f, float strength = 1.0f, float intensityX = 1.0f, float intensityY = 1.0f) { IEnumerable renderingManagers = mainCamera ? new List() {this.primaryAgent.m_Camera.GetComponent()} : new List(); @@ -2105,23 +2107,138 @@ public ActionFinished SetDistortionShaderParams(bool mainCamera = true, IEnumera // return new ActionFinished(success: false, errorMessage: "No RenderingManager, make sure you pass 'renderDistortionImage = true' to the agent constructor."); // } foreach (var renderingManager in renderingManagers) { - var distortion = renderingManager.GetCapturePass("_distortion"); + var distortion = renderingManager.GetCapturePass("_distortion"); if (distortion == null) { return new ActionFinished(success: false, errorMessage: "No Distortion pass, make sure you pass 'renderDistortionImage = true' to the agent constructor."); } - var material = distortion.material; - material.SetFloat("_ZoomPercent", zoomPercent); - material.SetFloat("_k1", k1); - material.SetFloat("_k2", k2); - material.SetFloat("_k3", k3); - material.SetFloat("_k4", k4); - material.SetFloat("_DistortionIntensityX", intensityX); - material.SetFloat("_DistortionIntensityY", intensityY); - material.SetFloat("_LensDistortionStrength", strength); + // var material = distortion.material; + var mats = new List() {distortion.material, renderingManager.distortionMap.material}; + foreach (var material in mats) { + material.SetFloat("_ZoomPercent", zoomPercent); + material.SetFloat("_k1", k1); + material.SetFloat("_k2", k2); + material.SetFloat("_k3", k3); + material.SetFloat("_k4", k4); + material.SetFloat("_DistortionIntensityX", intensityX); + material.SetFloat("_DistortionIntensityY", intensityY); + material.SetFloat("_LensDistortionStrength", strength); + } } return ActionFinished.Success; } + + private (float[] x, float[] y) decode(byte[] bytes, int width, int height) { + + float[] x = new float[width * height]; + float[] y = new float[width * height]; + Debug.Log($"------------- width {width} height {height} bytes {bytes.Length} "); + int floatingPointPrecision = 8; + for (int i = 0; i < bytes.Length / floatingPointPrecision; i++) + { + int byteIndex = i * floatingPointPrecision; + byte[] localBytesX = new byte[] { bytes[i], bytes[i + 1], bytes[i + 2], bytes[i + 3] }; // converts 4 bytes to a float + x[i] = System.BitConverter.ToSingle(localBytesX, 0); + // byte[] localBytesY = new byte[] { bytes[i + 4], bytes[i + 5], bytes[i + 6], bytes[i + 7] }; + // y[i] = System.BitConverter.ToSingle(localBytesY, 0); + if (i == 0 || i == 1) { + // Debug.Log($"--------- i {i} LocalBytesX {string.Join(", ", localBytesX)} decodex {x[i]} localBytesY {string.Join(", ",localBytesY)} decodey {y[i]}"); + Debug.Log($"--------- i {i} LocalBytesX {string.Join(", ", localBytesX)} decodex {x[i]}"); + } + } + // for (int i = 0; i < bytes.Length / 4; i++) + // { + // int byteIndex = i * 4; + // byte[] localBytesX = new byte[] { bytes[i], bytes[i + 1] }; // converts 4 bytes to a float + // x[i] = System.BitConverter.ToSingle(localBytesX, 0); + // byte[] localBytesY = new byte[] { bytes[i + 2], bytes[i + 3] }; + // y[i] = System.BitConverter.ToSingle(localBytesY, 0); + // if (i == 0 || i == 1) { + // Debug.Log($"--------- i {i} LocalBytesX {string.Join(", ", localBytesX)} decodex {x[i]} localBytesY {string.Join(", ",localBytesY)} decodey {y[i]}"); + // } + // } + return (x, y); + } + + public class DistortionMapReturn { + public float[][,] mainCamera; + public List thirdPartyCameras; + + } + + public ActionFinished GetDistortionMaps(bool mainCamera = true, IEnumerable thidPartyCameraIndices = null) { + // TODO multiagent support + var thirdPartyCamResults = new List>(); + // var result = new Dictionary() { {"thidPartyCameras", thirdPartyCamResults}}; + IEnumerable<(int index, RenderingManager rm)> renderingManagers = mainCamera ? new List<(int, RenderingManager)>() {(-1, this.primaryAgent.m_Camera.GetComponent())} : new List<(int, RenderingManager)>(); + renderingManagers = thidPartyCameraIndices != null ? renderingManagers.Concat(this.thirdPartyCameras.Where((cam, i) => thidPartyCameraIndices.Contains(i)).Select((cam, i) => (i, cam.GetComponent()))) : renderingManagers; + renderingManagers = renderingManagers.ToList(); + + var result = new DistortionMapReturn() { + thirdPartyCameras = new List() + }; + foreach (var (index, renderingManager) in renderingManagers) { + var rt = renderingManager.distortionMap.GetRenderTexture(); + var floats = decode(renderingManager.getDistortionMapBytes(), rt.width, rt.height); + // var map = new Dictionary() { + // {"x", floats.x}, + // // {"y", floats.y}, + // {"width", rt.width}, + // {"height", rt.height} + // }; + var bytes = renderingManager.getDistortionMapBytes(); + int i = 0; + float x_0 = System.BitConverter.ToSingle(new byte[]{ bytes[i], bytes[ i+ 1], bytes[i + 2], bytes[i + 3] }, 0); + i = 8; + float x_1 = System.BitConverter.ToSingle(new byte[]{ bytes[i], bytes[ i + 1], bytes[i + 2], bytes[i + 3] }, 0); + + var maxIndex = bytes.Length / 8; + var x_f = Enumerable.Range(0, maxIndex).Select(i => ( + x: System.BitConverter.ToSingle(new byte[]{ bytes[i*8], bytes[ i*8 + 1], bytes[i*8 + 2], bytes[i*8 + 3] }, 0), + y: System.BitConverter.ToSingle(new byte[]{ bytes[i*8 + 4], bytes[ i*8 + 5], bytes[i*8 + 6], bytes[i*8 + 7] }, 0) + ) + ).ToArray(); + + var y_f = Enumerable.Range(0, maxIndex).Select(i => System.BitConverter.ToSingle(new byte[]{ bytes[i*8 + 4], bytes[ i*8 + 5], bytes[i*8 + 6], bytes[i*8 + 7] }, 0) ); + + var map = Enumerable.Range(0, rt.height).Select(i => { + var row = new float[rt.width, 2]; + for (int j = 0; j < rt.width; j++) { + var val = x_f[i*rt.width + j]; + row[j, 0] = val.x; + row[j, 1] = val.y; + } + return row; + }).ToArray(); + // return new ActionFinished(success: true, actionReturn: new DistortionMapReturn() { + // x =bytes.Select(x => (int)x).ToArray(), x_0 = x_0, x_1 = x_1, test = m + // }); + + // var map = new Dictionary() { + // {"x_0", floats.x[0]}, + // {"x_1", floats.x[1]}, + // {"y_0", floats.y[0]}, + // {"y_1", floats.y[1]}, + // {"width", rt.width}, + // {"height", rt.height} + // }; + // var colors = renderingManager.getDistortionMapColors(); + // var map = new Dictionary() { + // {"map", colors.Select(c => new Dictionary() { {"x", c.r / 255.0f}, {"y", c.g / 255.0f}})}, + // {"width", rt.width}, + // {"height", rt.height} + // }; + if (index == -1) { + + result.mainCamera = map; + } + else { + result.thirdPartyCameras.Add(map); + } + } + return new ActionFinished(success: true, actionReturn: result); + // return new ActionFinished(success: true, actionReturn: result); + } } [Serializable] @@ -2875,6 +2992,7 @@ public class ServerAction { public bool renderNormalsImage; public bool renderFlowImage; public bool renderDistortionImage; + public bool enableDistortionMap; public float cameraY = 0.675f; public bool placeStationary = true; // when placing/spawning an object, do we spawn it stationary (kinematic true) or spawn and let physics resolve final position diff --git a/unity/Assets/Scripts/CapturePass.cs b/unity/Assets/Scripts/CapturePass.cs index 7553602b71..6f05201ffe 100644 --- a/unity/Assets/Scripts/CapturePass.cs +++ b/unity/Assets/Scripts/CapturePass.cs @@ -10,6 +10,7 @@ using UnityEngine.Rendering; using UnityEngine.Experimental.Rendering; using System.Security.Cryptography; +using System.Data.SqlTypes; // // @TODO: // // . support custom color wheels in optical flow via lookup textures @@ -58,7 +59,9 @@ public class CaptureConfig { public bool cloudRendering; - + public RenderTextureFormat renderTextureFormat = RenderTextureFormat.ARGB32; + + public int depthBits = 0; } public interface ICapturePass { @@ -103,8 +106,8 @@ public static void SetupCameraWithPostShader( public string shaderName; - private Texture2D tex; - private RenderTexture renderTexture; + protected Texture2D tex; + protected RenderTexture renderTexture; public Material material; protected Shader shader; @@ -118,7 +121,9 @@ public static void SetupCameraWithPostShader( private TextureFormat readTextureFormat; - + private RenderTextureFormat renderTextureFormat; + + private int depthBits; // private Texture2D readTexture; @@ -133,6 +138,8 @@ public RenderToTexture(CaptureConfig config, Camera camera) { this.name = config.name; this.cloudRendering = config.cloudRendering; this.toDisplayId = config.toDisplay; + this.renderTextureFormat = config.renderTextureFormat; + this.depthBits = config.depthBits; // TODO. if config.toDisplay is present then render to display buffer and copy to render texture // for debugging purposes @@ -191,7 +198,7 @@ public virtual void AddToCommandBuffer(CommandBuffer commandBuffer) { } - private RenderTexture CreateRenderTexture(int width, int height) { + protected RenderTexture CreateRenderTexture(int width, int height) { // for cloud rendering GraphicsFormat.R8G8B8A8_UNorm @@ -212,8 +219,8 @@ private RenderTexture CreateRenderTexture(int width, int height) { RenderTexture rt = null; - if (cloudRendering) { + // Why 0 for depth here ? rt = new RenderTexture(Screen.width, Screen.height, 0, GraphicsFormat.R8G8B8A8_UNorm); // TODO: if 0 then RGB24? if not RGB32? @@ -227,8 +234,22 @@ private RenderTexture CreateRenderTexture(int width, int height) { // ); } else { - rt = new RenderTexture(Screen.width, Screen.height, 24, RenderTextureFormat.ARGB32, RenderTextureReadWrite.Default); - readTextureFormat = TextureFormat.RGBA32; + // Todo string matching from enums, only two supported + if (this.renderTextureFormat == RenderTextureFormat.RGFloat) { + readTextureFormat = TextureFormat.RGFloat; + } + else if (this.renderTextureFormat == RenderTextureFormat.RFloat) { + readTextureFormat = TextureFormat.RFloat; + } + else { + readTextureFormat = TextureFormat.RGBA32; + this.renderTextureFormat = RenderTextureFormat.ARGB32; + } + + rt = new RenderTexture(Screen.width, Screen.height, this.depthBits, this.renderTextureFormat, RenderTextureReadWrite.Default); + // readTextureFormat = TextureFormat.RGBA32; + + // RenderTextureFormat.RGFloat } if (this.tex == null) { @@ -325,7 +346,7 @@ public virtual void OnCameraChange(Camera mainCamera) { this.camera.targetDisplay = this.toDisplayId.GetValueOrDefault(); // If set to render to display don't set render texture because display buffer is only written to if targetTexture is null - if (!this.toDisplayId.HasValue) { + if (!this.toDisplayId.HasValue || cloudRendering) { this.camera.targetTexture = this.renderTexture; } @@ -407,6 +428,7 @@ public virtual byte[] GetBytes( } else { bytes = tex.GetRawTextureData(); } + RenderTexture.active = prevActiveRT; return bytes; } @@ -450,6 +472,7 @@ public static Camera CreateHiddenCamera(Transform parent, string name) { var newCamera = go.GetComponent(); newCamera.cullingMask = 1; // render everything, including PlaceableSurfaces + return newCamera; } @@ -463,6 +486,7 @@ public override void AddToCommandBuffer(CommandBuffer commandBuffer) { this.camera.SetReplacementShader(shader, ""); this.camera.backgroundColor = Color.blue; this.camera.clearFlags = CameraClearFlags.SolidColor; + // TODO make optional for editor debugging this.camera.targetTexture = this.GetRenderTexture(); } @@ -499,7 +523,7 @@ public override void AddToCommandBuffer(CommandBuffer commandBuffer) { // Debug.Log($"----------- Blit for multipass"); // If rendering to display - if (this.toDisplayId.HasValue) { + if (this.toDisplayId.HasValue && !cloudRendering) { // if it's not cloudrendering camera.targetTexture is null which means it's rendering to the display buffer // so then we need to copy the display buffer into render texture @@ -551,6 +575,138 @@ public void AddUpdateCapturePass(RenderToTexture pass) { } + + // Does not attatch to camera with command buffers to render every frame, only on demand + public class OnDemandCapture : RenderToTexture { + public OnDemandCapture(CaptureConfig config) : base(config, null) { + } + + public override void OnInitialize(Camera mainCamera) { + base.OnInitialize(mainCamera); + + // this.camera.enabled = false; + + } + + public override void OnCameraChange(Camera mainCamera) { + if (this.camera == null) { + this.camera = ReplacementShaderCapture.CreateHiddenCamera(mainCamera.transform, this.name); + + } + base.OnCameraChange(mainCamera); + this.camera.renderingPath = RenderingPath.Forward; + // camera renders nothing, unity is silly and dosn't allow to not render anything any other way?? + camera.cullingMask = 0; + camera.clearFlags = CameraClearFlags.SolidColor; + camera.targetTexture = this.GetRenderTexture(); + // do not render at update + // this.camera.enabled = false; + // this.camera.SetReplacementShader(this.shader, ""); + } + + // public override void OnCameraChange(Camera mainCamera) { + // // this.camera.RemoveAllCommandBuffers(); + // // if (tex != null) { + // // UnityEngine.Object.Destroy(tex); + // // tex = null; + // // } + // // this.renderTexture = CreateRenderTexture(Screen.width, Screen.height); + // // this.camera.targetDisplay = this.toDisplayId.GetValueOrDefault(); + + // // // If set to render to display don't set render texture because display buffer is only written to if targetTexture is null + // // if (!this.toDisplayId.HasValue) { + // // this.camera.targetTexture = this.renderTexture; + // // } + + // // if (cb != null) { + // // cb.Clear(); + // // } + // // else { + // // cb = new CommandBuffer(); + // // } + + // // this.AddToCommandBuffer(cb); + + // // // this.camera.enabled = false; + + // base.OnCameraChange(mainCamera); + + // } + + public override void AddToCommandBuffer(CommandBuffer commandBuffer) { + + + // commandBuffer.Blit(BuiltinRenderTextureType.CameraTarget, this.GetRenderTexture(), this.material); + + commandBuffer.Blit(null, BuiltinRenderTextureType.CameraTarget, this.material); + + // If data is on a display buffer copy to render texture, just for debugging should not matter + if (!this.toDisplayId.HasValue || cloudRendering) { + commandBuffer.Blit(BuiltinRenderTextureType.CameraTarget, this.GetRenderTexture(), this.material); + } + + this.camera.AddCommandBuffer(CameraEvent.BeforeImageEffects, commandBuffer); + this.camera.depthTextureMode = DepthTextureMode.Depth; + + } + + public override byte[] GetBytes(bool jpeg = false) { + var renderTexture = this.GetRenderTexture(); + var prevActiveRT = RenderTexture.active; + RenderTexture.active = renderTexture; + + // this.camera.RenderWithShader(this.shader, ""); + this.camera.Render(); + // var bytes = base.GetBytes(jpeg); + + // float startTime = Time.realtimeSinceStartup; + + tex.ReadPixels(new Rect(0, 0, tex.width, tex.height), 0, 0); + + + // startTime = Time.realtimeSinceStartup; + + // encode texture into PNG/JPG + byte[] bytes; + if (jpeg) { + bytes = tex.EncodeToJPG(); + } else { + bytes = tex.GetRawTextureData(); + } + + + RenderTexture.active = prevActiveRT; + return bytes; + + } + + public Color32[] GetColors() { + var renderTexture = this.GetRenderTexture(); + var prevActiveRT = RenderTexture.active; + RenderTexture.active = renderTexture; + + // this.camera.RenderWithShader(this.shader, ""); + this.camera.Render(); + // var bytes = base.GetBytes(jpeg); + + // float startTime = Time.realtimeSinceStartup; + + tex.ReadPixels(new Rect(0, 0, tex.width, tex.height), 0, 0); + + + // startTime = Time.realtimeSinceStartup; + + // encode texture into PNG/JPG + Color32[] bytes; + + bytes = tex.GetPixels32(); + + + + RenderTexture.active = prevActiveRT; + return bytes; + + } // public class DistortionCapture : RenderToTexture { // public DistortionCapture(CaptureConfig config, Camera camera) : base(config, camera) { @@ -559,6 +715,7 @@ public void AddUpdateCapturePass(RenderToTexture pass) { // } + } } diff --git a/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionImageBlend.shader b/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionImageBlend.shader index 6ab756a7b1..9e7db4b44b 100644 --- a/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionImageBlend.shader +++ b/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionImageBlend.shader @@ -1,6 +1,6 @@ // latest -Shader "Custom/BarrelDistortion" { +Shader "Custom/BarrelDistortionBlend" { Properties { [MainTexture] _MainTex ("Base (RGB)", 2D) = "white" {} diff --git a/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionMap.shader b/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionMap.shader new file mode 100644 index 0000000000..0d81c8c3be --- /dev/null +++ b/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionMap.shader @@ -0,0 +1,126 @@ +// latest + +Shader "Custom/BarrelDistortionMap" { + Properties + { + [MainTexture] _MainTex ("Base (RGB)", 2D) = "white" {} + + _LensDistortionStrength ("Lens Distortion Strength", Range (-20.0, 20.0)) = 1.0 + // _LensDistortionTightness ("Lens Distortion Power", Range (-20.0, 20.0)) = 7.0 + _ZoomPercent ("Zoom Percent", Range (0.0, 5.0)) = 1.0 + + + _k1 ("K1 polynomial dist coeff", Range (-8.0, 8.0)) = -0.126 + _k2 ("K2 polynomial dist coeff", Range (-8.0, 8.0)) = 0.004 + _k3 ("K3 polynomial dist coeff", Range (-18.0, 18.0)) = 0.0 + _k4 ("K4 polynomial dist coeff", Range (-18.0, 18.0)) = 0.0 + + _DistortionIntensityX ("Distort Strength X", Range (0.0, 6.0)) = 1.0 + _DistortionIntensityY ("Distort Strength Y", Range (0.0, 6.0)) = 1.0 + + + _OutOfBoundColour ("Outline Color", Color) = (0.0, 0.0, 0.0, 1.0) + } + SubShader + { + Pass + { + CGPROGRAM + + #pragma vertex vert + #pragma fragment frag + #include "UnityCG.cginc" + + uniform sampler2D _MainTex; + uniform sampler2D _CameraDepthTexture; + uniform half4 _MainTex_TexelSize; + + uniform float _LensDistortionStrength; + uniform float4 _OutOfBoundColour; + // uniform float _LensDistortionTightness; + + uniform float _ZoomPercent; + + uniform float _DistortionIntensityX; + uniform float _DistortionIntensityY; + + uniform float _k1; + uniform float _k2; + uniform float _k3; + uniform float _k4; + + + // uniform float4 _ScreenParams; + // uniform float4 _ProjectionParams; + + struct input + { + float4 pos : POSITION; + half2 uv : TEXCOORD0; + }; + + struct output + { + float4 pos : SV_POSITION; + half2 uv : TEXCOORD0; + }; + + + output vert(input i) + { + output o; + o.pos = UnityObjectToClipPos(i.pos); + // o.uv = i.uv; + o.uv = MultiplyUV(UNITY_MATRIX_TEXTURE0, i.uv); + // o.uv.y = 1 - o.uv.y; + // // why do we need this? cause sometimes the image I get is flipped. see: http://docs.unity3d.com/Manual/SL-PlatformDifferences.html + // #if UNITY_UV_STARTS_AT_TOP + // if (_MainTex_TexelSize.y < 0) + // o.uv.y = 1 - o.uv.y; + // #endif + // if (_ProjectionParams.x >= 0) + // o.uv.y = 1 - o.uv.y; + + return o; + } + + fixed4 frag(output o) : SV_Target + { + float effect = _LensDistortionStrength; + float2 distortionStrengthXY = float2(_DistortionIntensityX, _DistortionIntensityY); + float zoom_offset = (1.0 - _ZoomPercent) / 2.0; + + float2 centered_uv = o.uv - float2(0.5, 0.5); + centered_uv = o.uv*2.0 - float2(1.0, 1.0); + // centered_uv = o.uv - float2(0.5, 0.5); + + zoom_offset = (1.0 - _ZoomPercent); + centered_uv = (o.uv*2.0 * _ZoomPercent - float2(1.0, 1.0)) + float2(zoom_offset, zoom_offset); + + + + float uv_dot = dot(centered_uv, centered_uv); + float r = sqrt(uv_dot); + + float smoothDistortionMagnitude = 1.0 + _k1 * pow(r, 2.0) + _k2 * pow(r, 4.0) + _k3 * pow(r,6.0) + _k4 * pow(r,8.0); + + float zoom_percent = 0.2; + float translate = (1.0 - zoom_percent) / 2.0; + float2 centered_uv_norm = normalize(centered_uv); + + + + // Works with zoom for poly distortion + float2 uvDistorted = centered_uv * smoothDistortionMagnitude * distortionStrengthXY / 2.0 + float2(0.5, 0.5) ; + + return fixed4(uvDistorted.x, uvDistorted.y, 0, 0); + // return fixed4(o.uv.x, o.uv., 0, 0); + + // return fixed4(1.0, 0.0, 0, 0); + } + + ENDCG + } + } + } + \ No newline at end of file diff --git a/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionMap.shader.meta b/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionMap.shader.meta new file mode 100644 index 0000000000..d2bc8b2e11 --- /dev/null +++ b/unity/Assets/Scripts/ImageSynthesis/Shaders/BarrelDistortionMap.shader.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: a8fe2fd2b926f4af8a14bb845b34872f +ShaderImporter: + externalObjects: {} + defaultTextures: [] + nonModifiableTextures: [] + preprocessorOverride: 0 + userData: + assetBundleName: + assetBundleVariant: diff --git a/unity/Assets/Scripts/RenderingManager.cs b/unity/Assets/Scripts/RenderingManager.cs index 934719437a..1a51e2fe8f 100644 --- a/unity/Assets/Scripts/RenderingManager.cs +++ b/unity/Assets/Scripts/RenderingManager.cs @@ -16,6 +16,9 @@ public class RenderingManager : MonoBehaviour { private MultiCapture mainPass; + // to set _img pass to display 0 in editor and standalone plaforms + public bool IsMainCamera; + private Dictionary activePasses; // Start is called before the first frame update @@ -25,11 +28,13 @@ public class RenderingManager : MonoBehaviour { "_distortion" }; - public Material distortionMat; - private Texture2D readTex; - private static bool isMainCameraPassCreated = false; + + public RenderToTexture distortionMap { + get; + private set; + } void Initialize(Camera camera) { @@ -122,7 +127,13 @@ public void EnablePasses(IEnumerable activePassesNames, bool cameraChang foreach (var pass in onCameraChange) { // && !initialized.Contains(x.GetName()))) { pass.OnCameraChange(mainCamera); } - } + + // Other special passes + // var others = new List() { distortionMap }; + // foreach (var pass in others) { + // pass.OnCameraChange(mainCamera); + // } + } // Debug.Log($"--------- Enabling passes 4 activePasses {string.Join(", ", this.activePasses)}"); } @@ -158,62 +169,77 @@ void Awake() { var depthPass = new RenderToTexture( - new CaptureConfig() { name = "_depth", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/DepthBW" }, + new CaptureConfig() { name = "_depth", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/DepthBW", cloudRendering = cloudRenderingCapture }, camera: camera ); var distPass = new RenderToTexture( - new CaptureConfig() { name = "_distortion", antiAliasLevel = antiAliasLevel, shaderName = "Custom/BarrelDistortion" }, + new CaptureConfig() { name = "_distortion", antiAliasLevel = antiAliasLevel, shaderName = "Custom/BarrelDistortion", cloudRendering = cloudRenderingCapture }, camera: camera ); var idPass = new ReplacementShaderCapture( - new CaptureConfig() { name = "_id", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/UberReplacement", replacementMode = ReplacelementMode.ObjectId }, + new CaptureConfig() { name = "_id", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/UberReplacement", replacementMode = ReplacelementMode.ObjectId, toDisplay = IsMainCamera? 2 : null as int?, cloudRendering = cloudRenderingCapture }, cameraParent: camera.transform ); var classPass = new ReplacementShaderCapture( - new CaptureConfig() { name = "_class", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/UberReplacement", replacementMode = ReplacelementMode.CatergoryId }, + new CaptureConfig() { name = "_class", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/UberReplacement", replacementMode = ReplacelementMode.CatergoryId, toDisplay = IsMainCamera? 3 : null as int?, cloudRendering = cloudRenderingCapture }, cameraParent: camera.transform ); var normalsPass = new ReplacementShaderCapture( - new CaptureConfig() { name = "_normals", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/UberReplacement", replacementMode = ReplacelementMode.Normals }, + new CaptureConfig() { name = "_normals", antiAliasLevel = antiAliasLevel, shaderName = "Hidden/UberReplacement", replacementMode = ReplacelementMode.Normals, cloudRendering = cloudRenderingCapture }, cameraParent: camera.transform ); // make first _img capture created render to Display - int? toDisplay = null; this.mainPass = new MultiCapture( - config: new CaptureConfig() { name = "_img", antiAliasLevel = antiAliasLevel, cloudRendering = cloudRenderingCapture, toDisplay = isMainCameraPassCreated ? toDisplay : 0}, + config: new CaptureConfig() { name = "_img", antiAliasLevel = antiAliasLevel, depthBits = 24, cloudRendering = cloudRenderingCapture, toDisplay = IsMainCamera? 0 : null as int?}, camera: camera, passes: new List() { } ); + this.distortionMap = new OnDemandCapture( + new CaptureConfig() { name = "_distortion_map", antiAliasLevel = antiAliasLevel, shaderName = "Custom/BarrelDistortionMap" , cloudRendering = cloudRenderingCapture, toDisplay = 7, renderTextureFormat = RenderTextureFormat.RGFloat } + ); + + availablePasses = new List() { this.mainPass, depthPass, distPass, + distortionMap, idPass, classPass }.ToDictionary(x => x.GetName(), x => x); + // Todo move to enable passes call this.activePasses = new List() { this.mainPass }.ToDictionary(x => x.GetName(), x => x); - mainPass.OnInitialize(camera); + if (!mainPass.IsInitialized()) { + mainPass.OnInitialize(camera); + } mainPass.OnCameraChange(camera); + + + // Special OnDemand pass not added to available passes + + // if (!distortionMap.IsInitialized()) { + // distortionMap.OnInitialize(camera); + // } + // distortionMap.OnCameraChange(camera); // this.enabled = true; } - public RenderToTexture GetCapturePass(string passName) { + public T GetCapturePass(string passName) where T : ICapturePass { ICapturePass pass; if (!this.activePasses.TryGetValue(passName, out pass)) { Debug.LogError($"No active pass at GetPassRenderTexture {passName}"); - return null; } - return pass as RenderToTexture; + return (T)pass; } public RenderTexture GetPassRenderTexture(string passName) { @@ -253,8 +279,8 @@ string key Debug.LogError($"No active pass at GetPassRenderTexture {passName}"); } RenderTexture tt = pass.GetRenderTexture(); - var prevActiveTex = RenderTexture.active; - RenderTexture.active = tt; + // var prevActiveTex = RenderTexture.active; + // RenderTexture.active = tt; // camera.Render(); AsyncGPUReadback.Request( tt, @@ -270,6 +296,37 @@ string key ); } + // TODDO: unify readbacks to this + public void GetActiveCapturesAsync( + List> payload, + List<(string passName, string payloadKey)> passToPayloadKey + ) { + + var requestReadbacks = passToPayloadKey + .Where(x => activePasses.ContainsKey(x.passName)) + .Select(mapping => (key: mapping.payloadKey, pass: activePasses[mapping.passName])); + foreach (var pair in requestReadbacks) { + AsyncGPUReadback.Request(pair.pass.GetRenderTexture(), 0, (request) => + { + if (!request.hasError) { + var data = request.GetData().ToArray(); + payload.Add(new KeyValuePair(pair.key, data)); + } else { + Debug.LogError("Request error: " + request.hasError); + } + } + ); + } + } + + public byte[] getDistortionMapBytes() { + return this.GetCapturePass("_distortion_map").GetBytes(); + } + + public Color32[] getDistortionMapColors() { + return this.GetCapturePass("_distortion_map").GetColors(); + } + // Update is called once per frame void Update() { } } diff --git a/unity/Assets/Standard Assets/Characters/FirstPersonCharacter/Prefabs/FPSController.prefab b/unity/Assets/Standard Assets/Characters/FirstPersonCharacter/Prefabs/FPSController.prefab index 907cb6de55..16e556618a 100644 --- a/unity/Assets/Standard Assets/Characters/FirstPersonCharacter/Prefabs/FPSController.prefab +++ b/unity/Assets/Standard Assets/Characters/FirstPersonCharacter/Prefabs/FPSController.prefab @@ -477,6 +477,7 @@ MonoBehaviour: m_Script: {fileID: 11500000, guid: ef2c6204ab3bb40a6afb21bd364dbadc, type: 3} m_Name: m_EditorClassIdentifier: + IsMainCamera: 1 distortionMat: {fileID: 0} --- !u!1 &1340499340313870 GameObject: