Flat Shading does not work correctly (openGL)

Question

So currently I'm trying to implement flat shading into my engine using glsl's flat qualifier, and I have encounterd a really weird bug.

Somehow the lightning on terrain isn't spread correctly, I want to make the lightning to similar to this:

Here is the terrain generation code:

private RawModel generateTerrain(Loader loader, String 
heightMap) {

    BufferedImage image = null;

    try {
        image = ImageIO.read(new File("res/textureMap/" + heightMap + ".png"));
    } catch (IOException e) {
        e.printStackTrace();
    }

    int VERTEX_COUNT = image.getHeight(); // this is the number of vertices along the size of the terrain
    heights = new float[VERTEX_COUNT][VERTEX_COUNT];
    int count = (VERTEX_COUNT * VERTEX_COUNT);
    float[] vertices = new float[count * 3];
    float[] normals = new float[count * 3];
    float[] textureCoords = new float[count * 2];
    int[] indices = new int[6*(VERTEX_COUNT)*(VERTEX_COUNT)];


    int vertexPointer = 0;
    for (int i = 0; i < VERTEX_COUNT; i++) {
        for (int j = 0; j < VERTEX_COUNT; j++) {
            vertices[vertexPointer * 3] = (float) j / ((float) VERTEX_COUNT - 1) * SIZE;
            float height = getHeight(j, i, image);
            heights[j][i] = height;
            vertices[vertexPointer * 3 + 1] = height;
            vertices[vertexPointer * 3 + 2] = (float) i / ((float) VERTEX_COUNT - 1) * SIZE;
            Vector3f normal = calculateNormal(j, i, image);
            normals[vertexPointer * 3] = normal.x;
            normals[vertexPointer * 3 + 1] = normal.y;
            normals[vertexPointer * 3 + 2] = normal.z;
            textureCoords[vertexPointer * 2] = (float) j / ((float) VERTEX_COUNT - 1);
            textureCoords[vertexPointer * 2 + 1] = (float) i / ((float) VERTEX_COUNT - 1);
            vertexPointer++;
        }
    }
    int pointer = 0;
    for (int gz = 0; gz < VERTEX_COUNT - 1; gz++) {
        for (int gx = 0; gx < VERTEX_COUNT - 1; gx++) {
            int topLeft = (gz * VERTEX_COUNT) + gx;
            int topRight = topLeft + 1;
            int bottomLeft = ((gz + 1) * VERTEX_COUNT) + gx;
            int bottomRight = bottomLeft + 1;
            indices[pointer++] = topLeft;
            indices[pointer++] = bottomLeft;
            indices[pointer++] = topRight;
            indices[pointer++] = topRight;
            indices[pointer++] = bottomLeft;
            indices[pointer++] = bottomRight;
        }
    }
    return loader.loadToVAO(vertices, textureCoords, normals, indices);
}

Here is how I generate normals:

private Vector3f calculateNormal(int x, int z, BufferedImage 
image) {
    float heightL = getHeight(x - 1, z, image);
    float heightR = getHeight(x + 1, z, image);
    float heightD = getHeight(x, z - 1, image);
    float heightU = getHeight(x, z + 1, image);

    Vector3f normal = new Vector3f(heightL - heightR, 1f, heightD - heightU);
    normal.normalise();
    return normal;

}
private float getHeight(int x, int z, BufferedImage image) {

    if (x < 0 || x >= image.getHeight() || z < 0 || z >= image.getHeight()) {
        return 0;
    }
    float height = image.getRGB(x, z);

    height += MAX_PIXEL_COLOUR / 2f;
    height /= MAX_PIXEL_COLOUR / 2f;
    height *= MAX_HEIGHT;
    return height;
}

Here is the terrain vertex shader code:

#version 400

in vec3 position;
in vec2 textureCoordinates;
in vec3 normal;

out vec2 pass_textureCoordinates;
flat out vec3 surfaceNormal;
out vec3 toLightVector[4];
out vec3 toCameraVector;
out vec4 shadowCoords;
out float visibility;
flat out vec4 pass_colour;

uniform mat4 transformationMatrix;  // the entity's position         
relative to the world [-1 1 1 -1]
uniform mat4 projectionMatrix;      // transform's 
everything into a viewing frustum (farther away in -z the 
smaller everything gets foreshortening)
uniform mat4 viewMatrix;            // camera's perspective
uniform vec3 lightPosition[4];          // location of the 
light source (only one source of light so far)
uniform mat4 toShadowMapSpace;

const float shadowDistance = 100; // it has to be the same 
value as the shadowbox shadow distance
const float transitionDistance = 10.0;

const float density = 0.0023;
const float gradient = 50.0;;

uniform vec3 lightColour[4];        //light source's color 
(which can change)
uniform vec3 attenuation[4];        //one for each light 
source
uniform float shineDamper;
uniform float reflectivity;

void main(void){

vec4 worldPosition = transformationMatrix * 
vec4(position,1.0); // position is the position of the 
current vertex
vec4 positionRelativeToCam = viewMatrix * worldPosition;
shadowCoords = toShadowMapSpace * worldPosition;

gl_Position = projectionMatrix * positionRelativeToCam;      
// NOTE: viewMatrix must go in between the projectionMatrix 
and transformationMatrix
pass_textureCoordinates = textureCoordinates;
surfaceNormal = (transformationMatrix * vec4(normal, 
0.0)).xyz; //(swizzle it) convert from vec4 back to vec3

for(int i = 0; i < 4; i++) {
    toLightVector[i] = lightPosition[i] - worldPosition.xyz;
}
toCameraVector = (inverse(viewMatrix) * 
vec4(0.0,0.0,0.0,1.0)).xyz - worldPosition.xyz;

float distance = length(positionRelativeToCam.xyz);
visibility = exp(-pow((distance*density), gradient));
visibility = clamp(visibility, 0.0, 1.0);

distance = distance - (shadowDistance - transitionDistance);
distance = distance / transitionDistance;
shadowCoords.w = clamp(1.0-distance, 0.0, 1.0);

vec3 unitNormal  = normalize(surfaceNormal);
vec3 unitVectorToCamera = normalize(toCameraVector);

vec3 totalDiffuse = vec3(0.0);
vec3 totalSpecular = vec3(0.0);
float lightFactor = 0.5;

for(int i = 0; i < 4; i++) {
    float distance = length(toLightVector[i]);
    float attenuationFactor = attenuation[i].x + 
(attenuation[i].y * distance) + (attenuation[i].z * distance 
* distance); 

    vec3 unitLightVector = normalize(toLightVector[i]);  // pointing from the surface to the light source

    float nDot1 = dot(unitNormal, unitLightVector);
    float brightness = max(nDot1, 0.0);

    vec3 lightDirection = -unitLightVector;           // pointing from the light source to the surface
    vec3 reflectedLightDirection = reflect(lightDirection, unitNormal);

    float specularFactor = dot(reflectedLightDirection, unitVectorToCamera);
    specularFactor = max(specularFactor, 0.0);
    float dampedFactor = pow(specularFactor, shineDamper);

    totalDiffuse = totalDiffuse + (brightness*lightColour[i])/attenuationFactor;
    totalSpecular = totalSpecular + (dampedFactor * reflectivity * lightColour[i])/attenuationFactor;
}

totalDiffuse = max(totalDiffuse * lightFactor, 0.3);
pass_colour = vec4(totalDiffuse, 1.3) + vec4(totalSpecular, 1.0);

}

Here is the terrain fragment shader code:

#version 400

in vec2 pass_textureCoordinates;
flat in vec3 surfaceNormal;
in vec3 toLightVector[4];
in vec3 toCameraVector;
in float visibility;
in vec4 shadowCoords;

out vec4 out_Color;
flat in vec4 pass_colour;

uniform sampler2D backgroundTexture;
uniform sampler2D rTexture;
uniform sampler2D gTexture;
uniform sampler2D bTexture;
uniform sampler2D blendMap;
uniform sampler2D shadowMap;

uniform vec3 lightColour[4];        //light source's color 
(which can change)
uniform vec3 attenuation[4];        //one for each light 
source
uniform float shineDamper;
uniform float reflectivity;
uniform vec3 skyColour;

const int pcfCount = 0;
const float totalTexels = (pcfCount * 2.0 + 1.0) * (pcfCount 
* 
2.0 + 1.0);
uniform float usesNormalMap;

vec3 totalDiffuse;

void main(void){

float mapSize = 1024; // must be the same as the SHADOW_MAP_SIZE in the shadow masterRenderer
float texelSize = 1.0 / mapSize;
float total = 0.0;

for(int x=-pcfCount; x<=pcfCount; x++){
        for(int y=-pcfCount; y<=pcfCount; y++){
            float objectNearestLight = texture(shadowMap, shadowCoords.xy + vec2(x,y) * texelSize).r;
                if(shadowCoords.z > objectNearestLight + 0.002){
                    total += 1.0;
                }
    }       
}

total /= totalTexels;

float lightFactor = 1.0 - (total * shadowCoords.w);

vec4 blendMapColour = texture(blendMap, pass_textureCoordinates);

float backTextureAmount = 1 - (blendMapColour.r + blendMapColour.g + blendMapColour.b);
vec2 tiledCoords = pass_textureCoordinates * 40;

vec4 backgroundTextureColour = texture(backgroundTexture, tiledCoords) * backTextureAmount;
vec4 rTextureColour = texture(rTexture, tiledCoords) * blendMapColour.r;
vec4 gTextureColour = texture(gTexture, tiledCoords) * blendMapColour.g;
vec4 bTextureColour = texture(bTexture, tiledCoords) * blendMapColour.b;

vec4 totalColour = backgroundTextureColour + rTextureColour + gTextureColour + bTextureColour;

out_Color = mix(vec4(skyColour, 1.0), pass_colour*totalColour, visibility);

}

What's "incorrect" about this? Is it the large areas of shadow? You might just need some secondary lights or a cubemap ambient source to help fill-in areas that face away from your primary light. — DMGregory
– DMGregory ♦, Commented Jul 30, 2019 at 11:21
Does your transformation matrix include any kind og scaling, rotation or translation? Those can affect the normal in a bad way. — Bálint
– Bálint, Commented Jul 30, 2019 at 12:01

Theraot · Accepted Answer · 2019-07-30 13:04:15Z

I'm not sure about your normals. Taking Calculate Normals from Heightmap as reference, I'd say change it like this:

private Vector3f calculateNormal(int x, int z, BufferedImage image) {
    float heightL = getHeight(x - 1, z, image);
    float heightR = getHeight(x + 1, z, image);
    float heightD = getHeight(x, z - 1, image);
    float heightU = getHeight(x, z + 1, image);

    Vector3f normal = new Vector3f((heightL - heightR), -2f, (heightD - heightU));
    normal.normalise();
    return normal;
}

Changed 1f to -2f.

How do we get that?

Well, first of all, I will no use gradients, because I am not that confident in my vector calculus.

Instead, I will find the normal as the cross product of the tangents. Since the tangents follow the surface, and the cross product is perpendicular to its input vectors... the cross product of the normals is perpendicular to the surface.

We are taking the xtangent and ztangent (I think they are tangent and bitangent, I do not know which is which one, don't care) like this:

Vector3f l = new Vector3f(x - 1f, heightL, z);
Vector3f r = new Vector3f(x + 1f, heightR, z);
Vector3f d = new Vector3f(x, heightD, z - 1f);
Vector3f u = new Vector3f(x, heightU, z + 1f);

//Vector3f xtangent = d - u;
//Vector3f xtangent = new Vector3f(x - x, heightD - heightU, (z - 1f) - (z + 1f));
Vector3f xtangent = new Vector3f(0f, heightD - heightU, -2f);

//Vector3f ytangent = l - r;
//Vector3f xtangent = new Vector3f((x - 1f) - (x + 1f), heightL - heightR, z - z);
Vector3f xtangent = new Vector3f(-2f, heightL - heightR, 0f);

And we do cross product to get the normal:

//Vector3f normal = normalize(cross(ytangent, xtangent));
//Vector3f normal = normalize(new Vector3f
//          (
//              ytangent.y * xtangent.z - ytangent.z * xtangent.y,
//              ytangent.x * xtangent.z - ytangent.z * xtangent.x,
//              ytangent.x * xtangent.y - ytangent.y * xtangent.x
//          ));
//Vector3f normal = normalize(new Vector3f
//          (
//              (heightL - heightR) * (-2f) - (0f) * (heightD - heightU),
//              (-2f) * (-2f) - (0f) * (0f),
//              (-2f) * (heightD - heightU) - (heightL - heightR) * (0f)
//          ));
//Vector3f normal = normalize(new Vector3f
//          (
//              (heightL - heightR) * (-2f),
//              (-2f) * (-2f),
//              (-2f) * (heightD - heightU)
//          ));
Vector3f normal = normalize(new Vector3f
          (
              (-2f) * (heightL - heightR),
              (-2f) * (-2f),
              (-2f) * (heightD - heightU)
          ));

And since we are normalizing anyway, we can get rid of that common scaling factor of -2f:

Vector3f normal = normalize(new Vector3f
          (
              (heightL - heightR),
              (-2f),
              (heightD - heightU)
          ));

That is like what you have, right?

Vector3f normal = new Vector3f((heightL - heightR), -2f, (heightD - heightU));
normal.normalize();

Except, I have a -2f.

Note: fliping the vectors in the cross product, should flip the signs. However, that ends up meaning we remove a common scaling factor of 2f instead. After that, the result is the same as above.

Let us compare with the result from link I had at the top:

So, because we are normalizing, we do not need that 1/4, and we can simplify by 2, and we end up with a -2 (from the -4).

They have it on the z while we have it on the y because that is how the height map was oriented in space, I guess.

And I also notice they have R - L instead of L - R. Perhaps I should have flipped it? Or perhaps that is because of the coordinate system you use? Well, that is a simple test you can do. By the way, their T (top) is our U (up) and their B (bottom) is our D (down). So B - T is D - U, and that is the same we have.

Finally, I suppose that flipping it all makes sense if we are looking at the height map from the other side:

Vector3f normal = new Vector3f((heightR - heightL), 2f, (heightU - heightD));

Stack Exchange Network

Flat Shading does not work correctly (openGL)

1 Answer 1

You must log in to answer this question.

Hot Network Questions

Flat Shading does not work correctly (openGL)

1 Answer 1

You must log in to answer this question.

Related

Hot Network Questions