shader/rectangle_frag.glsl

layout(location = 0) out vec4 diffuse;
in vec2 uvcoord;

uniform vec2 size;
#ifdef FILL
uniform vec4 color;
#endif
#ifdef BORDER
uniform float thickness;
uniform vec4 border_color;
#endif
#ifdef ROUNDED_CORNERS
uniform float radius;
#endif

void main(void) {
#ifdef ROUNDED_CORNERS
    // Calculate distances from each corner
    vec2 corner_distances[4];
    corner_distances[0] = uvcoord - vec2(radius, radius);                         // top-left
    corner_distances[1] = uvcoord - vec2(size.x - radius, radius);                // top-right
    corner_distances[2] = uvcoord - vec2(radius, size.y - radius);                // bottom-left
    corner_distances[3] = uvcoord - vec2(size.x - radius, size.y - radius);       // bottom-right

    // Check if we're in a corner region
    bool in_corner_region = false;
    int corner_idx = -1;

    for (int i = 0; i < 4; i++) {
        vec2 test_point = corner_distances[i];
        vec2 corner_test = vec2(0.0);

        if (i == 0 && test_point.x < 0.0 && test_point.y < 0.0) { // top-left
            corner_test = test_point;
            corner_idx = i;
            in_corner_region = true;
            break;
        } else if (i == 1 && test_point.x > 0.0 && test_point.y < 0.0) { // top-right
            corner_test = test_point;
            corner_idx = i;
            in_corner_region = true;
            break;
        } else if (i == 2 && test_point.x < 0.0 && test_point.y > 0.0) { // bottom-left
            corner_test = test_point;
            corner_idx = i;
            in_corner_region = true;
            break;
        } else if (i == 3 && test_point.x > 0.0 && test_point.y > 0.0) { // bottom-right
            corner_test = test_point;
            corner_idx = i;
            in_corner_region = true;
            break;
        }
    }

    #ifndef BORDER
    // For filled rectangle with rounded corners
    if (in_corner_region) {
        float dist = length(corner_distances[corner_idx]);
        if (dist > radius) {
            discard; // Outside the rounded corner
        }
    } else if (uvcoord.x < 0.0 || uvcoord.y < 0.0 || uvcoord.x > size.x || uvcoord.y > size.y) {
        // TODO: at the moment we don't have fragments here, but we will with glow
        discard; // Outside the rectangle
    }
    diffuse = color;
    #else  // BORDER
    // For outlined rectangle with rounded corners
    if (in_corner_region) {
        float dist = length(corner_distances[corner_idx]);
        if (dist > radius) {
            discard; // Outside the rounded corner
        } else if (dist < radius - thickness) {
            // Inside the outline
            #ifdef FILL
            diffuse = color;
            #else
            discard;
            #endif
        } else {
            // the corner outline
            diffuse = border_color;
        }
    } else if (any(lessThan(uvcoord, vec2(thickness))) || any(greaterThan(uvcoord, size - thickness))) {
        // On a straight edge
        if (uvcoord.x >= 0.0 && uvcoord.y >= 0.0 && uvcoord.x <= size.x && uvcoord.y <= size.y) {
            diffuse = border_color;
        } else {
            discard;
        }
    } else {
        // Inside the outline
        #ifdef FILL
        diffuse = color;
        #else
        discard;
        #endif
    }
    #endif // BORDER
#else // no ROUNDED_CORNERS
    #ifdef BORDER
    if (any(notEqual(1.0-step(thickness, uvcoord)+step(size-thickness, uvcoord), vec2(0.0)))) {
        diffuse = border_color;
    } else {
        // Inside the outline
        #ifdef FILL
        diffuse = color;
        #else
        discard;
        #endif
    }
    #else // no BORDER
    diffuse = color;
    #endif // BORDER
#endif // ROUNDED_CORNERS
}