How stencil buffer and masking work?

Question

I want to draw object in just specific area. Please take a look this image for reference

The 2 triangles (picture A) being draw just in the area inside the quad (picture B), so the result will look clipped (picture C).

First i draw the quad just in stencil buffer.

gl.stencilOp(gl.KEEP, gl.KEEP, gl.REPLACE);

gl.stencilFunc(gl.ALWAYS, 1, 0xff);
gl.stencilMask(0xff);
gl.depthMask(false);
gl.colorMask(false, false, false, false);

drawQuads();

in my understanding, now the stencil buffer has value 1s in the quad area. Then, draw the triangles.

gl.stencilFunc(gl.EQUAL, 1, 0xff);
gl.stencilMask(0x00);
gl.depthMask(true);
gl.colorMask(true, true, true, true);

drawTriagles();

I was expect the result will be like on the picture (C), but it's not. What I am doing wrong?

Please find the complete code here https://jsfiddle.net/z11zhf01/1

Answer 1

Your program works absolute correctly, but you have to tell the getContext function to create a stencil buffer, when the context is created:

gl = glcanvas.getContext("webgl", {stencil:true});

See Khronos WebGL Specification - WebGLContextAttributes:

stencil
If the value is true, the drawing buffer has a stencil buffer of at least 8 bits. If the value is false, no stencil buffer is available.

See the Example:

(function() {
var gl;

var gProgram;

var gVertexAttribLocation;
var gColorAttribLocation;

var gTriangleVertexBuffer;
var gTriangleColorBuffer;
var gQuadVertexBuffer;
var gQuadColorBuffer;


function initGL() {
 var glcanvas = document.getElementById("glcanvas");
 gl = glcanvas.getContext("webgl", {stencil:true});
}

function createAndCompileShader(type, source) {
 var shader = gl.createShader(type);

 gl.shaderSource(shader, source);
 gl.compileShader(shader);

 if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
  throw new Error(gl.getShaderInfoLog(shader));
 }

 return shader;
}

function createAndLinkProgram(glVertexShader, glFragmentShader) {
 var glProgram = gl.createProgram();

 gl.attachShader(glProgram, glVertexShader);
 gl.attachShader(glProgram, glFragmentShader);
 gl.linkProgram(glProgram);

 if (!gl.getProgramParameter(glProgram, gl.LINK_STATUS)) {
     throw new Error("Could not initialise shaders");
 }

 return glProgram;
}

function initShaderPrograms() {
 var gVertexShader = createAndCompileShader(gl.VERTEX_SHADER, [
  "attribute vec3 a_vertex;",
  "attribute vec4 a_color;",

  "varying vec4 v_color;",

  "void main(void) {",
   "v_color = a_color;",
   "gl_Position = vec4(a_vertex, 1.0);",
  "}"
 ].join("\n"));

 var gFragmentShader = createAndCompileShader(gl.FRAGMENT_SHADER, [
  "precision mediump float;",

  "varying vec4 v_color;",
  "void main(void) {",
   "gl_FragColor = v_color;",
  "}"
 ].join("\n"));

 gProgram = createAndLinkProgram(gVertexShader, gFragmentShader);
}

function initGLAttribLocations() {
 gVertexAttribLocation = gl.getAttribLocation(gProgram, "a_vertex");
 gColorAttribLocation = gl.getAttribLocation(gProgram, "a_color");
}

function initBuffers() {
 gTriangleVertexBuffer = gl.createBuffer();
 gTriangleColorBuffer = gl.createBuffer();
 gQuadVertexBuffer = gl.createBuffer();
 gQuadColorBuffer = gl.createBuffer();


 gl.bindBuffer(gl.ARRAY_BUFFER, gTriangleVertexBuffer);
 var vertices = new Float32Array([
      0.0,  1.0,  0.0,
     -1.0, -1.0,  0.0,
      1.0, -1.0,  0.0,

      0.0, -1.0,  0.0,
     -1.0, 1.0,  0.0,
      1.0, 1.0,  0.0
 ]);
 gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW);

 gl.bindBuffer(gl.ARRAY_BUFFER, gTriangleColorBuffer);
 var colors = new Float32Array([
      0.0, 1.0,  0.0, 1.0,
      0.0, 1.0,  0.0, 1.0,
      0.0, 1.0,  0.0, 1.0,

      0.0, 0.0,  1.0, 1.0,
      0.0, 0.0,  1.0, 1.0,
      0.0, 0.0,  1.0, 1.0
 ]);
 gl.bufferData(gl.ARRAY_BUFFER, colors, gl.STATIC_DRAW);


 gl.bindBuffer(gl.ARRAY_BUFFER, gQuadVertexBuffer);
 var vertices = new Float32Array([
      -1.0,  1.0,  0.0,
     -1.0, -1.0,  0.0,
      1.0, 1.0,  0.0,
      1.0, -1.0,  0.0
 ]);
 for(let i = 0, ii = vertices.length; i < ii; ++i) {
  vertices[i] *= 0.75;
 }
 gl.bufferData(gl.ARRAY_BUFFER, vertices, gl.STATIC_DRAW);

 gl.bindBuffer(gl.ARRAY_BUFFER, gQuadColorBuffer);
 var colors = new Float32Array([
      1.0, 0.0, 0.0, 1.0,
      1.0, 0.0, 0.0, 1.0,
      1.0, 0.0, 0.0, 1.0,
      1.0, 0.0, 0.0, 1.0,
 ]);
 gl.bufferData(gl.ARRAY_BUFFER, colors, gl.STATIC_DRAW);

}

function drawQuads() {
 gl.bindBuffer(gl.ARRAY_BUFFER, gQuadVertexBuffer);
 gl.vertexAttribPointer(gVertexAttribLocation, 3, gl.FLOAT, false, 0, 0);

 gl.bindBuffer(gl.ARRAY_BUFFER, gQuadColorBuffer);
 gl.vertexAttribPointer(gColorAttribLocation, 4, gl.FLOAT, false, 0, 0);

 gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
}

function drawTriagles() {
 gl.bindBuffer(gl.ARRAY_BUFFER, gTriangleVertexBuffer);
 gl.vertexAttribPointer(gVertexAttribLocation, 3, gl.FLOAT, false, 0, 0);

 gl.bindBuffer(gl.ARRAY_BUFFER, gTriangleColorBuffer);
 gl.vertexAttribPointer(gColorAttribLocation, 4, gl.FLOAT, false, 0, 0);

 gl.drawArrays(gl.TRIANGLES, 0, 6);
}


function renderScene() {
 gl.enable(gl.STENCIL_TEST);
 gl.enable(gl.DEPTH_TEST);
 // gl.enable(gl.CULL_FACE);
 gl.useProgram(gProgram);

 gl.clearColor(0.5, 0.5, 0.5, 1.0);

 gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight);

 gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT | gl.STENCIL_BUFFER_BIT);

 gl.enableVertexAttribArray(gVertexAttribLocation);
 gl.enableVertexAttribArray(gColorAttribLocation);

 gl.stencilOp(gl.KEEP, gl.KEEP, gl.REPLACE);

 gl.stencilFunc(gl.ALWAYS, 1, 0xff);
 gl.stencilMask(0xff);
 gl.depthMask(false);
 gl.colorMask(false, false, false, false);

 drawQuads();

 gl.stencilFunc(gl.EQUAL, 1, 0xff);
 gl.stencilMask(0x00);
 gl.depthMask(true);
 gl.colorMask(true, true, true, true);

 drawTriagles();

 gl.disableVertexAttribArray(gVertexAttribLocation);
 gl.disableVertexAttribArray(gColorAttribLocation);

 gl.flush();
}


initGL();
initShaderPrograms();
initGLAttribLocations();
initBuffers();
renderScene();


}());

<main>
 <canvas id="glcanvas" width="480" height="360">
  WebGL not supported!
 </canvas>
</main>