Avoid texture bleed rendering X11 window

Summary:
We currently see a gap on transformed windows between the window and the
top decoration.

This is partly the atlas bleed on the decoration, and partly a bleed on
the window content itself.

On X11, the window we composite is the frame window - which is a larger
texture containing a transparent border where the frame normally would
be. When we sample with a linear filter we include these texels. Hence
GL_CLAMP_TO_EDGE doesn't work.

Vlad's patch to composite the correct window, not the frame was my
preferred approach, but we had to revert it as it caused an issue with
xwayland :(

Half pixel correction nearly worked, but caused blurry fonts.

This patch resolves it in the fragment shader used by effects doing
transforms. We pass the real texture geometry of the window to the
client with a half pixel correction. Any samples outside the outer half
pixel are then clamped within bounds.

Arguably a hack, but solves the problem in a comparatively
non-invasive way.

BUG: 360549
BUG: 257566

Test Plan:
X11:
Using Vlad's atlas padding for decoration
Slowed animations, wobbled a dark window over a light background
No artifacts

Wayland:
This isn't needed. Now tested that everything still renders the same.

Reviewers: #kwin, zzag

Reviewed By: #kwin, zzag

Subscribers: zzag, jgrulich, kwin

Tags: #kwin

Differential Revision: https://phabricator.kde.org/D25737

libkwineffects/kwinglutils.cpp

@@ -347,6 +347,7 @@ void GLShader::resolveLocations()
     mFloatLocation[Saturation]    = uniformLocation("saturation");
 
     mColorLocation[Color] = uniformLocation("geometryColor");
+    mVec4Location[TextureClamp] = uniformLocation("textureClamp");
 
     mLocationsResolved = true;
 }
@@ -877,14 +878,24 @@ QByteArray ShaderManager::generateFragmentSource(ShaderTraits traits) const
     } else if (traits & ShaderTrait::UniformColor)
         stream << "uniform vec4 geometryColor;\n";
 
+    if (traits & ShaderTrait::ClampTexture) {
+        stream << "uniform vec4 textureClamp;\n";
+    }
+
     if (output != QByteArrayLiteral("gl_FragColor"))
         stream << "\nout vec4 " << output << ";\n";
 
     stream << "\nvoid main(void)\n{\n";
     if (traits & ShaderTrait::MapTexture) {
-        if (traits & (ShaderTrait::Modulate | ShaderTrait::AdjustSaturation)) {
+        stream << "vec2 texcoordC = texcoord0;\n";
-            stream << "    vec4 texel = " << textureLookup << "(sampler, texcoord0);\n";
 
+        if (traits & ShaderTrait::ClampTexture) {
+            stream << "texcoordC.x = clamp(texcoordC.x, textureClamp.x, textureClamp.z);\n";
+            stream << "texcoordC.y = clamp(texcoordC.y, textureClamp.y, textureClamp.w);\n";
+        }
+
+        if (traits & (ShaderTrait::Modulate | ShaderTrait::AdjustSaturation)) {
+            stream << "    vec4 texel = " << textureLookup << "(sampler, texcoordC);\n";
             if (traits & ShaderTrait::Modulate)
                 stream << "    texel *= modulation;\n";
             if (traits & ShaderTrait::AdjustSaturation)
@@ -892,7 +903,7 @@ QByteArray ShaderManager::generateFragmentSource(ShaderTraits traits) const
 
             stream << "    " << output << " = texel;\n";
         } else {
-            stream << "    " << output << " = " << textureLookup << "(sampler, texcoord0);\n";
+            stream << "    " << output << " = " << textureLookup << "(sampler, texcoordC);\n";
         }
     } else if (traits & ShaderTrait::UniformColor)
         stream << "    " << output << " = geometryColor;\n";

libkwineffects/kwinglutils.h

@@ -129,6 +129,7 @@ public:
 
     enum Vec4Uniform {
         ModulationConstant,
+        TextureClamp,
         Vec4UniformCount
     };
 
@@ -186,6 +187,7 @@ enum class ShaderTrait {
     UniformColor     = (1 << 1),
     Modulate         = (1 << 2),
     AdjustSaturation = (1 << 3),
+    ClampTexture     = (1 << 4),
 };
 
 Q_DECLARE_FLAGS(ShaderTraits, ShaderTrait)

plugins/scenes/opengl/scene_opengl.cpp

@@ -1381,9 +1381,30 @@ void SceneOpenGL2Window::performPaint(int mask, QRegion region, WindowPaintData
     const QMatrix4x4 modelViewProjection = modelViewProjectionMatrix(mask, data);
     const QMatrix4x4 mvpMatrix = modelViewProjection * windowMatrix;
 
+
+    bool useX11TextureClamp = false;
+
     GLShader *shader = data.shader;
+    GLenum filter;
+
+    if (waylandServer()) {
+        filter = GL_LINEAR;
+    } else {
+        const bool isTransformed = mask & (Effect::PAINT_WINDOW_TRANSFORMED |
+                                           Effect::PAINT_SCREEN_TRANSFORMED);
+        useX11TextureClamp = isTransformed;
+        if (isTransformed && options->glSmoothScale() != 0) {
+            filter = GL_LINEAR;
+        } else {
+            filter = GL_NEAREST;
+        }
+    }
+
     if (!shader) {
         ShaderTraits traits = ShaderTrait::MapTexture;
+        if (useX11TextureClamp) {
+            traits |= ShaderTrait::ClampTexture;
+        }
 
         if (data.opacity() != 1.0 || data.brightness() != 1.0 || data.crossFadeProgress() != 1.0)
             traits |= ShaderTrait::Modulate;
@@ -1397,19 +1418,6 @@ void SceneOpenGL2Window::performPaint(int mask, QRegion region, WindowPaintData
 
     shader->setUniform(GLShader::Saturation, data.saturation());
 
-    GLenum filter;
-    if (waylandServer()) {
-        filter = GL_LINEAR;
-    } else {
-        const bool isTransformed = mask & (Effect::PAINT_WINDOW_TRANSFORMED |
-                                           Effect::PAINT_SCREEN_TRANSFORMED);
-        if (isTransformed && options->glSmoothScale() != 0) {
-            filter = GL_LINEAR;
-        } else {
-            filter = GL_NEAREST;
-        }
-    }
-
     WindowQuadList quads[LeafCount];
 
     // Split the quads into separate lists for each type
@@ -1506,6 +1514,25 @@ void SceneOpenGL2Window::performPaint(int mask, QRegion region, WindowPaintData
         nodes[i].texture->setWrapMode(GL_CLAMP_TO_EDGE);
         nodes[i].texture->bind();
 
+        if (i == ContentLeaf && useX11TextureClamp) {
+            // X11 windows are reparented to have their buffer in the middle of a larger texture
+            // holding the frame window.
+            // This code passes the texture geometry to the fragment shader
+            // any samples near the edge of the texture will be constrained to be
+            // at least half a pixel in bounds, meaning we don't bleed the transparent border
+            QRectF bufferContentRect = clientShape().boundingRect();
+            bufferContentRect.adjust(0.5, 0.5, -0.5, -0.5);
+            const QRect bufferGeometry = toplevel->bufferGeometry();
+
+            float leftClamp = bufferContentRect.left() / bufferGeometry.width();
+            float topClamp = bufferContentRect.top() / bufferGeometry.height();
+            float rightClamp = bufferContentRect.right() / bufferGeometry.width();
+            float bottomClamp = bufferContentRect.bottom() / bufferGeometry.height();
+            shader->setUniform(GLShader::TextureClamp, QVector4D({leftClamp, topClamp, rightClamp, bottomClamp}));
+        } else {
+            shader->setUniform(GLShader::TextureClamp, QVector4D({0, 0, 1, 1}));
+        }
+
         vbo->draw(region, primitiveType, nodes[i].firstVertex, nodes[i].vertexCount, m_hardwareClipping);
     }