Ok guys, tricky one!
I'm trying to modify this Unity script + shader to include normal mapping. The tl;dr is, it uses a single texture rather than a cubemap as it is only for reflecting in a planar surface. So it doesn't use the standard ray bounce onto an axis aligned cube. Instead it :
- creates a reflection matrix from the plane (pos and normal)
- reflects the camera position and view matrix through the plane
- renders to texture from the reflected camera (with oblique near clipping plane set to cull everything behind the mirror)
- creates a projection matrix and sends it as uniform to the shader (basically just a standard MVP matrix but with scale cancelled, and biased from [-1, 1] to [0, 1]
- multiplies the object space coordinate by the projection matrix to get a texcoord and then looks up the mirror texture with this
So there are no normals actually used in the shader at all at present. I know how to construct a TBN matrix and use this to warp the per-pixel normals, but I have no idea how this translates into a texture offset in this case. I have a feeling I might need screen-space normals like in the question here but pretty stumped beyond that. Any ideas?
EDIT -- Here's the script
using UnityEngine;
using System.Collections;
[ExecuteInEditMode()]
public class PlanarRealtimeReflection : MonoBehaviour
{
public bool m_DisablePixelLights = true;
public int m_TextureResolution = 1024;
public float m_clipPlaneOffset = 0.07f;
private float m_finalClipPlaneOffset = 0.0f;
public bool m_NormalsFromMesh = false;
public bool m_BaseClipOffsetFromMesh = false;
public bool m_BaseClipOffsetFromMeshInverted = false;
private Vector3 m_calculatedNormal = Vector3.zero;
private Vector3 m_calculatedTangent = Vector4.zero;
public LayerMask m_ReflectLayers = -1;
private Hashtable m_ReflectionCameras = new Hashtable(); //Camera -> Camera table
private RenderTexture m_ReflectionTexture = null;
private int m_OldReflectionTextureSize = 0;
private static bool s_InsideRendering = false;
//This is called when it's known that the object will be rendered by some
//camera. We render reflections and do other updates here.
//Because the script executes in edit mode, reflections for the scene view
//camera will just work!
public void OnWillRenderObject()
{
if(!enabled || !renderer || !renderer.sharedMaterial || !renderer.enabled)
return;
Camera cam = Camera.current;
if(!cam)
return;
if(m_NormalsFromMesh && GetComponent<MeshFilter>() != null)
m_calculatedNormal = transform.TransformDirection(GetComponent<MeshFilter>().sharedMesh.normals[0]);
m_calculatedTangent = transform.TransformDirection(GetComponent<MeshFilter>().sharedMesh.tangents[0]);
if(m_BaseClipOffsetFromMesh && GetComponent<MeshFilter>() != null)
m_finalClipPlaneOffset = (transform.position - transform.TransformPoint(GetComponent<MeshFilter>().sharedMesh.vertices[0])).magnitude + m_clipPlaneOffset;
else if(m_BaseClipOffsetFromMeshInverted && GetComponent<MeshFilter>() != null)
m_finalClipPlaneOffset = -(transform.position - transform.TransformPoint(GetComponent<MeshFilter>().sharedMesh.vertices[0])).magnitude + m_clipPlaneOffset;
else
m_finalClipPlaneOffset = m_clipPlaneOffset;
//Safeguard from recursive reflections.
if(s_InsideRendering)
return;
s_InsideRendering = true;
Camera reflectionCamera;
CreateSurfaceObjects(cam, out reflectionCamera);
//Find out the reflection plane: position and normal in world space
Vector3 pos = transform.position;
Vector3 normal = m_NormalsFromMesh && GetComponent<MeshFilter>() != null ? m_calculatedNormal : transform.up;
Vector3 tangent = m_NormalsFromMesh && GetComponent<MeshFilter>() != null ? m_calculatedTangent : transform.right;
Vector3 bitangent = Vector3.Cross(normal, tangent);
//Optionally disable pixel lights for reflection
int oldPixelLightCount = QualitySettings.pixelLightCount;
if(m_DisablePixelLights)
QualitySettings.pixelLightCount = 0;
UpdateCameraModes(cam, reflectionCamera);
//Render reflection
//Reflect camera around reflection plane
float d = -Vector3.Dot (normal, pos) - m_finalClipPlaneOffset;
Vector4 reflectionPlane = new Vector4 (normal.x, normal.y, normal.z, d);
Matrix4x4 reflection = Matrix4x4.zero;
CalculateReflectionMatrix (ref reflection, reflectionPlane);
Vector3 oldpos = cam.transform.position;
Vector3 newpos = reflection.MultiplyPoint(oldpos);
reflectionCamera.worldToCameraMatrix = cam.worldToCameraMatrix * reflection;
//Setup oblique projection matrix so that near plane is our reflection plane.
//This way we clip everything below/above it for free.
Vector4 clipPlane = CameraSpacePlane(reflectionCamera, pos, normal, 1.0f);
Matrix4x4 projection = cam.projectionMatrix;
CalculateObliqueMatrix (ref projection, clipPlane);
reflectionCamera.projectionMatrix = projection;
reflectionCamera.cullingMask = ~(1<<4) & m_ReflectLayers.value; //never render water layer
reflectionCamera.targetTexture = m_ReflectionTexture;
GL.SetRevertBackfacing (true);
reflectionCamera.transform.position = newpos;
//Vector3 euler = cam.transform.eulerAngles;
//reflectionCamera.transform.eulerAngles = euler; // new Vector3(0, euler.y, euler.z);
reflectionCamera.Render();
reflectionCamera.transform.position = oldpos;
GL.SetRevertBackfacing (false);
Material[] materials = renderer.sharedMaterials;
foreach(Material mat in materials)
{
if (mat.HasProperty("_ReflectionTex"))
mat.SetTexture("_ReflectionTex", m_ReflectionTexture);
mat.SetVector("_Normal", new Vector4(normal.x,normal.y,normal.z,0));
mat.SetVector("_Tangent", new Vector4(tangent.x, tangent.y, tangent.z));
mat.SetVector("_Bitangent", new Vector4(bitangent.x, bitangent.y, bitangent.z));
}
//Set matrix on the shader that transforms UVs from object space into screen
//space. We want to just project reflection texture on screen.
Matrix4x4 scaleOffset = Matrix4x4.TRS(
new Vector3(0.5f,0.5f,0.5f), Quaternion.identity, new Vector3(0.5f,0.5f,0.5f));
Vector3 scale = transform.localScale;//.lossyScale;
Matrix4x4 mtx = transform.localToWorldMatrix * Matrix4x4.Scale(new Vector3(1.0f/scale.x, 1.0f/scale.y, 1.0f/scale.z));
mtx = scaleOffset * cam.projectionMatrix * cam.worldToCameraMatrix * mtx;
foreach(Material mat in materials)
mat.SetMatrix("_ProjMatrix", mtx);
//Restore pixel light count
if(m_DisablePixelLights)
QualitySettings.pixelLightCount = oldPixelLightCount;
s_InsideRendering = false;
}
//Cleanup all the objects we possibly have created
void OnDisable()
{
if(m_ReflectionTexture)
{
DestroyImmediate(m_ReflectionTexture);
m_ReflectionTexture = null;
}
foreach(DictionaryEntry kvp in m_ReflectionCameras)
DestroyImmediate(((Camera)kvp.Value).gameObject);
m_ReflectionCameras.Clear();
}
private void UpdateCameraModes(Camera src, Camera dest)
{
if(dest == null)
return;
//set camera to clear the same way as current camera
dest.clearFlags = src.clearFlags;
dest.backgroundColor = src.backgroundColor;
if(src.clearFlags == CameraClearFlags.Skybox)
{
Skybox sky = src.GetComponent(typeof(Skybox)) as Skybox;
Skybox mysky = dest.GetComponent(typeof(Skybox)) as Skybox;
if(!sky || !sky.material)
{
mysky.enabled = false;
}
else
{
mysky.enabled = true;
mysky.material = sky.material;
}
}
//update other values to match current camera.
//even if we are supplying custom camera&projection matrices,
//some of values are used elsewhere (e.g. skybox uses far plane)
dest.farClipPlane = src.farClipPlane;
dest.nearClipPlane = src.nearClipPlane;
dest.orthographic = src.orthographic;
dest.fieldOfView = src.fieldOfView;
dest.aspect = src.aspect;
dest.orthographicSize = src.orthographicSize;
}
//On-demand create any objects we need
private void CreateSurfaceObjects(Camera currentCamera, out Camera reflectionCamera)
{
reflectionCamera = null;
//Reflection render texture
if(!m_ReflectionTexture || m_OldReflectionTextureSize != m_TextureResolution)
{
if(m_ReflectionTexture)
DestroyImmediate(m_ReflectionTexture);
m_ReflectionTexture = new RenderTexture(m_TextureResolution, m_TextureResolution, 16);
m_ReflectionTexture.name = "__SurfaceReflection" + GetInstanceID();
m_ReflectionTexture.isPowerOfTwo = true;
m_ReflectionTexture.hideFlags = HideFlags.DontSave;
m_OldReflectionTextureSize = m_TextureResolution;
}
//Camera for reflection
reflectionCamera = m_ReflectionCameras[currentCamera] as Camera;
if(!reflectionCamera) //catch both not-in-dictionary and in-dictionary-but-deleted-GO
{
GameObject go = new GameObject("Surface Refl Camera id" + GetInstanceID() + " for " + currentCamera.GetInstanceID(), typeof(Camera), typeof(Skybox));
reflectionCamera = go.camera;
reflectionCamera.enabled = false;
reflectionCamera.transform.position = transform.position;
reflectionCamera.transform.rotation = transform.rotation;
reflectionCamera.gameObject.AddComponent("FlareLayer");
go.hideFlags = HideFlags.HideAndDontSave;
m_ReflectionCameras[currentCamera] = reflectionCamera;
}
}
//Extended sign: returns -1, 0 or 1 based on sign of a
private static float sgn(float a)
{
if (a > 0.0f) return 1.0f;
if (a < 0.0f) return -1.0f;
return 0.0f;
}
//Given position/normal of the plane, calculates plane in camera space.
private Vector4 CameraSpacePlane (Camera cam, Vector3 pos, Vector3 normal, float sideSign)
{
Vector3 offsetPos = pos + normal * m_finalClipPlaneOffset;
Matrix4x4 m = cam.worldToCameraMatrix;
Vector3 cpos = m.MultiplyPoint(offsetPos);
Vector3 cnormal = m.MultiplyVector(normal).normalized * sideSign;
return new Vector4(cnormal.x, cnormal.y, cnormal.z, -Vector3.Dot(cpos,cnormal));
}
//Adjusts the given projection matrix so that near plane is the given clipPlane
//clipPlane is given in camera space. See article in Game Programming Gems 5 and
//http://aras-p.info/texts/obliqueortho.html
private static void CalculateObliqueMatrix (ref Matrix4x4 projection, Vector4 clipPlane)
{
Vector4 q = projection.inverse * new Vector4(
sgn(clipPlane.x),
sgn(clipPlane.y),
1.0f,
1.0f
);
Vector4 c = clipPlane * (2.0F / (Vector4.Dot (clipPlane, q)));
//third row = clip plane - fourth row
projection[2] = c.x - projection[3];
projection[6] = c.y - projection[7];
projection[10] = c.z - projection[11];
projection[14] = c.w - projection[15];
}
//Calculates reflection matrix around the given plane
private static void CalculateReflectionMatrix (ref Matrix4x4 reflectionMat, Vector4 plane)
{
reflectionMat.m00 = (1F - 2F*plane[0]*plane[0]);
reflectionMat.m01 = ( - 2F*plane[0]*plane[1]);
reflectionMat.m02 = ( - 2F*plane[0]*plane[2]);
reflectionMat.m03 = ( - 2F*plane[3]*plane[0]);
reflectionMat.m10 = ( - 2F*plane[1]*plane[0]);
reflectionMat.m11 = (1F - 2F*plane[1]*plane[1]);
reflectionMat.m12 = ( - 2F*plane[1]*plane[2]);
reflectionMat.m13 = ( - 2F*plane[3]*plane[1]);
reflectionMat.m20 = ( - 2F*plane[2]*plane[0]);
reflectionMat.m21 = ( - 2F*plane[2]*plane[1]);
reflectionMat.m22 = (1F - 2F*plane[2]*plane[2]);
reflectionMat.m23 = ( - 2F*plane[3]*plane[2]);
reflectionMat.m30 = 0F;
reflectionMat.m31 = 0F;
reflectionMat.m32 = 0F;
reflectionMat.m33 = 1F;
}
}
And the shader so far
Shader "Realtime Reflections/New Shader"
{
Properties {
_MainAlpha("MainAlpha", Range(0, 1)) = 1
_MapAmount("Distortion Amount", Range(0, 0.05)) = 0
_MainTex ("Base (RGB) Gloss (A)", 2D) = "white" {}
_ReflectionTex ("Reflection", 2D) = "white" {}
_NormalMap ("Normal Map", 2D) = "white" {}
_RefColor("Color",Color) = (1,1,1,1)
}
SubShader {
Tags {
"RenderType"="Opaque"}
LOD 100
Pass {
CGPROGRAM
#pragma vertex vert
#pragma fragment frag
#include "UnityCG.cginc"
uniform float4x4 _ProjMatrix;
uniform float _RefType;
uniform float4 _Normal;
uniform float4 _Tangent;
uniform float4 _Bitangent;
sampler2D _ReflectionTex;
sampler2D _MainTex;
sampler2D _NormalMap;
float4 _MainTex_ST;
float4 _NormalMap_ST;
float4 _RefColor;
float _MainAlpha;
float _MapAmount;
struct outvertex {
float4 pos : SV_POSITION;
float2 colorUV : TEXCOORD0;
float2 bumpUV : TEXCOORD1;
float4 varyingPos : TEXCOORD2;
};
outvertex vert(appdata_full v) {
outvertex o;
o.pos = mul (UNITY_MATRIX_MVP,v.vertex);
o.colorUV = TRANSFORM_TEX(v.texcoord.xy,_MainTex);
o.bumpUV = TRANSFORM_TEX(v.texcoord1.xy,_NormalMap);
o.varyingPos = v.vertex;
o. color = fixed4(mul(float3x3(UNITY_MATRIX_IT_MV),v.normal),1);
return o;
}
float4 frag(outvertex i) : COLOR {
float4 posProj = mul(_ProjMatrix, i.varyingPos);
float2 normPosProj = posProj.xy / posProj.w;
float3 bump = (float3(tex2D(_NormalMap, i.bumpUV));
//normPosProj += ?????????
half4 flecol = tex2D(_ReflectionTex, normPosProj);
half4 maincol = tex2D(_MainTex, i.colorUV);
half4 outcolor = half4(1,1,1,1);
outcolor = maincol*_MainAlpha+flecol*(1-_MainAlpha);
return outcolor*_RefColor;
}
ENDCG
}
}
}
I had the idea to get the normal in screen space (so z is always toward camera) then simply find the delta between the regular normal and the warped one in x,y and use this as the texture offset, but I have no idea if this is feasible or how to do it.
