GLSL - Merge two textures

Question

I would like to display arbitrary 3d meshes with black edges (black outline, black ridges, etc.). Thereby I have created two different textures: One color texture and one anti-aliased edge texture.

Here is an example of the color texture:

And the the corresponding edge texture:

Now I'd like to merge both textures. The edge texture has a white background with black/grey edges. I used fxaa, therefore some pixels are grey.

My question: How can I combine (blend) both textures correctly? And do I have to apply fxaa for the color texture, too?

EDIT: For completeness, here's the fxaa-shader code:

    #version 330 core

in vec2 texPos;
out vec4 color;
uniform sampler2D colorTexture;
uniform float rt_w;
uniform float rt_h;

vec4 PostFX(sampler2D tex);
vec3 FxaaPixelShader(sampler2D tex, vec2 rcpFrame);  

void main()
{
    color = PostFX(colorTexture);
    //color = texelFetch(colorTexture, ivec2(gl_FragCoord), 0 );
}

vec4 PostFX(sampler2D tex)
{
  vec4 c = vec4(0.0);
  vec2 rcpFrame = vec2(1.0/rt_w, 1.0/rt_h);
  c.rgb = FxaaPixelShader(tex, rcpFrame);
  c.a = 1.0;
  return c;
}

vec3 FxaaPixelShader( 
  sampler2D tex, // Input texture.
  vec2 rcpFrame) // Constant {1.0/frameWidth, 1.0/frameHeight}.
{   
/*---------------------------------------------------------*/
    #define FXAA_REDUCE_MIN   (1.0/128.0)
    #define FXAA_REDUCE_MUL   (1.0/8.0)
    #define FXAA_SPAN_MAX     8.0
/*---------------------------------------------------------*/
    vec3 rgbNW = texture(tex, vec2(texPos) + (vec2(-1,-1) * rcpFrame.xy)).xyz;
    vec3 rgbNE = texture(tex, vec2(texPos) + (vec2(1,-1) * rcpFrame.xy)).xyz;
    vec3 rgbSW = texture(tex, vec2(texPos)+ (vec2(-1,1) * rcpFrame.xy)).xyz;
    vec3 rgbSE = texture(tex, vec2(texPos) + (vec2(1,1) * rcpFrame.xy)).xyz;
    vec3 rgbM  = texture(tex, vec2(texPos)).xyz;
/*---------------------------------------------------------*/
    vec3 luma = vec3(0.299, 0.587, 0.114);
    float lumaNW = dot(rgbNW, luma);
    float lumaNE = dot(rgbNE, luma);
    float lumaSW = dot(rgbSW, luma);
    float lumaSE = dot(rgbSE, luma);
    float lumaM  = dot(rgbM,  luma);
/*---------------------------------------------------------*/
    float lumaMin = min(lumaM, min(min(lumaNW, lumaNE), min(lumaSW, lumaSE)));
    float lumaMax = max(lumaM, max(max(lumaNW, lumaNE), max(lumaSW, lumaSE)));
/*---------------------------------------------------------*/
    vec2 dir; 
    dir.x = -((lumaNW + lumaNE) - (lumaSW + lumaSE));
    dir.y =  ((lumaNW + lumaSW) - (lumaNE + lumaSE));
/*---------------------------------------------------------*/
    float dirReduce = max(
        (lumaNW + lumaNE + lumaSW + lumaSE) * (0.25 * FXAA_REDUCE_MUL),
        FXAA_REDUCE_MIN);
    float rcpDirMin = 1.0/(min(abs(dir.x), abs(dir.y)) + dirReduce);
    dir = min(vec2( FXAA_SPAN_MAX,  FXAA_SPAN_MAX), 
          max(vec2(-FXAA_SPAN_MAX, -FXAA_SPAN_MAX), 
          dir * rcpDirMin)) * rcpFrame.xy;
/*--------------------------------------------------------*/
    vec3 rgbA = (1.0/2.0) * (
        texture(tex, texPos.xy + dir * (1.0/3.0 - 0.5)).xyz +
        texture(tex, texPos.xy + dir * (2.0/3.0 - 0.5)).xyz);
    vec3 rgbB = rgbA * (1.0/2.0) + (1.0/4.0) * (
        texture(tex, texPos.xy + dir * (0.0/3.0 - 0.5)).xyz +
        texture(tex, texPos.xy + dir * (3.0/3.0 - 0.5)).xyz);
    float lumaB = dot(rgbB, luma);
    if((lumaB < lumaMin) || (lumaB > lumaMax)) return rgbA;
    return rgbB; 
}

Would it help to replace the edge texture with one that uses pure black and several levels of alpha, rather than using several levels of grey? — trichoplax is on Codidact now, Aug 21 '16 at 23:11
@trichoplax: I'd say yes, since I could use the glsl mix funtion. But I'm not sure how I'd have to change my fxaa-shader to use alpha instead of grey values. I added the fxaa shader code abvoe. — enne87, Aug 21 '16 at 23:23
I haven't tested this so I'll leave it to someone else to post an answer if they are confident it works, but I'd imagine it working something like this: For each pixel in the texture, change the alpha value to the opposite of the grey value, and set the colour to black (0, 0, 0). Black pixels become fully opaque, white pixels become fully transparent, so the final colour can simply be set to black for all pixels, since the ones you want to appear white will be fully transparent preventing the black from showing up. — trichoplax is on Codidact now, Aug 21 '16 at 23:28
Something like: gl_FragColor = mix(colourRGB, vec3(0), 1 - edgeRGB.r); Would blend in your edge outlines as black — PaulHK, Aug 22 '16 at 02:18
I would suggest something very similar to @PaulHK's snippet too: combinedColor = mix(sceneColor, edgeColor, edgeOpacity * (1. - edgeRGB.r));. — Julien Guertault, Aug 22 '16 at 03:17
Thank you both, but just to make clear: How would I retrieve the edgeOpacity? — enne87, Aug 22 '16 at 08:32
@enne87 edge opacity in this case is edge color (so 1 - color) because edge is black — CoffeDeveloper, Aug 22 '16 at 11:44
@enne87: edgeOpacity is meant to be a parameter between 0 (no edges at all) and 1 (opaque edges) that you choose to your liking. — Julien Guertault, Aug 22 '16 at 14:08
I realize this question has been solved in the comments but has no "formal answer", so I am copying my comments as an answer. — Julien Guertault, Dec 24 '18 at 10:50

score 2 · Answer 1 · answered Dec 24 '18 at 10:51

The simplest solution would be just a multiplication, which would give black edges:

combinedColor = sceneColor * edgeRGB.r;

For more control I would suggest something very similar to what @PaulHK proposed in the comments:

float opacity = maxEdgeOpacity * (1. - edgeRGB.r);
combinedColor = mix(sceneColor, edgeColor, opacity);

edgeColor is the desired edge color and maxEdgeOpacity is meant to be a parameter between 0 (no edges at all) and 1 (opaque edges), that you both choose to your liking.

GLSL - Merge two textures

1 Answers1