I'm doing the following depth filter that removes flickering from a depth stereo image.
The filter works perfectly with low Data delta, like 10 - 20 in terms of flickering. but there is ghosting effect that is very noticeable.
The input is two depth images from a stereo camera.
here is the script that does the flickering removal
using UnityEngine;
using Unity.Jobs;
using Unity.Collections;
using Unity.Burst;
using sl;
using UnityEngine.UI;
using System;
public class DepthFilter : MonoBehaviour
{
const int HISTORY_CAPACITY = 3;
readonly static int _dataDeltaID = Shader.PropertyToID("_dataDelta");
readonly static int[] _historyIDs = new int[] {
Shader.PropertyToID("_HistoryA"),
Shader.PropertyToID("_HistoryB"),
Shader.PropertyToID("_HistoryC"),
};
public Material filterMaterialAsset;
Material _filterMaterialInstance;
RenderTexture[] _historyPages;
public ZEDToOpenCVRetriever imageRetriever;
private const int width = 1280;
private const int height = 720;
int _oldestPageLeft = 0;
int _oldestPageRight = 0;
public GameObject frame_left;
public GameObject frame_right;
private RenderTexture src_left;
private RenderTexture src_right;
private RenderTexture _outputTexture;
private RenderTexture dest_left;
private RenderTexture dest_right;
private Texture2D final_depth;
public uint dataDelta = 50;
private void Start()
{
if (!imageRetriever) imageRetriever = ZEDToOpenCVRetriever.GetInstance();
imageRetriever.OnImageUpdated_GPU += ImageUpdated;
_outputTexture = new RenderTexture(width, height, 0);
dest_left = new RenderTexture(width, height, 0);
dest_right = new RenderTexture(width, height, 0);
frame_left.GetComponent<MeshRenderer>().material.SetTexture("_DepthXYZTex", dest_left);
frame_right.GetComponent<MeshRenderer>().material.SetTexture("_DepthXYZTex", dest_right);
src_left = new RenderTexture(width, height, 0);
src_right = new RenderTexture(width, height, 0);
}
private void ImageUpdated(ref Texture2D zedGPULeftTexture, ref Texture2D zedGPURightTexture)
{
RenderTexture.active = src_left;
// Copy your texture ref to the render texture
Graphics.Blit(zedGPULeftTexture, src_left);
Filter(ref src_left, dest_left, dataDelta);
RenderTexture.active = src_right;
// Copy your texture ref to the render texture
Graphics.Blit(zedGPURightTexture, src_right);
Filter(ref src_right, dest_right, dataDelta);
}
public void Filter(ref RenderTexture source, RenderTexture destination, uint dataDelta)
{
// If we haven't initialized our history buffer, do so.
// Match the size/format of whatever render target
// we were passed as a source.
if (_historyPages == null)
{
_filterMaterialInstance = Instantiate(filterMaterialAsset);
_historyPages = new RenderTexture[HISTORY_CAPACITY];
for (int i = 0; i < HISTORY_CAPACITY; i++)
{
_historyPages[i] = new RenderTexture(source);
_filterMaterialInstance.SetTexture(_historyIDs[i], _historyPages[i]);
}
}
_filterMaterialInstance.SetFloat(_dataDeltaID, dataDelta);
// Apply the shader filter to the source and current history,
// and write the result into destination.
Graphics.Blit(source, destination, _filterMaterialInstance);
// Our source becomes the newest sample in our history.
// (Or, if it's supposed to be your destination, use that instead).
// The oldest history page takes its place as a new writable source texture.
var newPage = source;
source = _historyPages[_oldestPageLeft];
_historyPages[_oldestPageLeft] = newPage;
_filterMaterialInstance.SetTexture(_historyIDs[_oldestPageLeft], newPage);
_oldestPageLeft = (_oldestPageLeft + 1) % HISTORY_CAPACITY;
}
}Shader:
Shader "Unlit/DepthFilter"
{
Properties
{
_MainTex("Texture", 2D) = "white" {}
_HistoryA("History A", 2D) = "white" {}
_HistoryB("History B", 2D) = "white" {}
_HistoryC("History C", 2D) = "white" {}
_dataDelta("Data Delta", range(0, 256)) = 256
}
SubShader
{
Tags { "RenderType" = "Opaque" }
LOD 100
Pass
{
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
struct appdata
{
float4 vertex : POSITION;
float2 uv : TEXCOORD0;
};
struct v2f
{
float2 uv : TEXCOORD0;
float4 vertex : SV_POSITION;
};
sampler2D _MainTex;
sampler2D _HistoryA;
sampler2D _HistoryB;
sampler2D _HistoryC;
float _dataDelta;
v2f vert(appdata v)
{
v2f o;
o.vertex = UnityObjectToClipPos(v.vertex);
o.uv.x = v.uv.x;
o.uv.y = v.uv.y;
return o;
}
fixed4 frag(v2f i) : SV_Target
{
fixed source = tex2D(_MainTex, i.uv).r;
// Accumulate the history into a total.
float average = tex2D(_HistoryA, i.uv).r;
average += tex2D(_HistoryB, i.uv).r;
average += tex2D(_HistoryC, i.uv).r;
average += tex2D(_HistoryD, i.uv).r;
// Divide by our sample count to get an average.
average /= 3.0f;
// Write into the output texture if and only if
// abs(source - average) < _dataDelta.
clip(_dataDelta / 256.0f - abs(source - average));
return source;
}
ENDCG
}
}
}I think the problem is with clip the pixel. What I have tried is to add more history pages, but that didn't help.
here is a video https://youtu.be/fXGE_YvCD4M
I asked an expert he suggested the following but I have no idea how to implement it
You could instead estimate your view ray in each previous history frame, and march that ray through the depth buffer to see where it intersects. Then use that ray depth to accumulate into your average. That would be a form of reprojection.
When your camera is stationary, all the historical view rays match your current view ray, so it acts like your current filter. When your camera is in motion, it compensates for the motion.
The same way we update the history buffer each frame, you'd also update a collection of shader variables representing an inverse-view-projection matrix. In your vertex shader, you'd transform the screen position by those matrices to get the directions of your rays, to interpolate across each pixel.
Then in the fragment shader, you'd write a raymarcher that marches the three history rays into their respective depth buffers.
