screencast: Fix format announced with SHM enum format

Kwin announces a format with alpha when Dma-Bufs are available, even
when the texture doesn't contain any. This results in clients segfault
when trying to access the buffer assuming a maxsize wrt. the announced
format by ways of dimensions and bpp.

This patch moves the format masking to affect the Dma-Buf transport
only.
This commit is contained in:
Columbariu s 2021-12-05 18:47:45 +01:00
parent 2b628ea412
commit 621c06deda

View file

@ -280,10 +280,10 @@ bool ScreenCastStream::createStream()
auto canCreateDmaBuf = [this] () -> bool {
return QSharedPointer<DmaBufTexture>(kwinApp()->platform()->createDmaBufTexture(m_resolution));
};
const auto format = m_source->hasAlphaChannel() || canCreateDmaBuf() ? SPA_VIDEO_FORMAT_BGRA : SPA_VIDEO_FORMAT_BGR;
const auto format = m_source->hasAlphaChannel() ? SPA_VIDEO_FORMAT_BGRA : SPA_VIDEO_FORMAT_BGR;
if (canCreateDmaBuf()) {
params[0] = buildFormat(&podBuilder, format, &resolution, &defaultFramerate, &minFramerate, &maxFramerate, &modifier, 1);
params[0] = buildFormat(&podBuilder, SPA_VIDEO_FORMAT_BGRA, &resolution, &defaultFramerate, &minFramerate, &maxFramerate, &modifier, 1);
params[1] = buildFormat(&podBuilder, format, &resolution, &defaultFramerate, &minFramerate, &maxFramerate, nullptr, 0);
n_params = 2;
} else {