kwin/screencast/pipewirestream.cpp
Vlad Zahorodnii 9c20df5030 screencast: Use fences to avoid stalling the graphics pipeline
Currently, we use glFinish() to ensure that stream consumers don't see
corrupted or rather incomplete buffers. This is a serious issue because
glFinish() not only prevents the gpu from processing new GL commands,
but it also blocks the compositor.

This change addresses the blocking issue by using native fences. With
the proposed change, after finishing recording a frame, a fence is
inserted in the command stream. When the native fence is signaled, the
pending pipewire buffer will be enqueued.

If the EGL_ANDROID_native_fence_sync extension is not supported, we'll
fall back to using glFinish().
2020-10-19 14:23:06 +00:00

515 lines
19 KiB
C++

/*
SPDX-FileCopyrightText: 2018-2020 Red Hat Inc
SPDX-FileCopyrightText: 2020 Aleix Pol Gonzalez <aleixpol@kde.org>
SPDX-FileContributor: Jan Grulich <jgrulich@redhat.com>
SPDX-License-Identifier: LGPL-2.0-or-later
*/
#include "pipewirestream.h"
#include "cursor.h"
#include "dmabuftexture.h"
#include "eglnativefence.h"
#include "kwingltexture.h"
#include "kwinglutils.h"
#include "kwinscreencast_logging.h"
#include "main.h"
#include "pipewirecore.h"
#include "platform.h"
#include "utils.h"
#include <KLocalizedString>
#include <QLoggingCategory>
#include <QPainter>
#include <spa/buffer/meta.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <unistd.h>
namespace KWin
{
void PipeWireStream::onStreamStateChanged(void *data, pw_stream_state old, pw_stream_state state, const char *error_message)
{
PipeWireStream *pw = static_cast<PipeWireStream*>(data);
qCDebug(KWIN_SCREENCAST) << "state changed"<< pw_stream_state_as_string(old) << " -> " << pw_stream_state_as_string(state) << error_message;
switch (state) {
case PW_STREAM_STATE_ERROR:
qCWarning(KWIN_SCREENCAST) << "Stream error: " << error_message;
break;
case PW_STREAM_STATE_PAUSED:
if (pw->nodeId() == 0 && pw->pwStream) {
pw->pwNodeId = pw_stream_get_node_id(pw->pwStream);
Q_EMIT pw->streamReady(pw->nodeId());
}
break;
case PW_STREAM_STATE_STREAMING:
Q_EMIT pw->startStreaming();
break;
case PW_STREAM_STATE_CONNECTING:
break;
case PW_STREAM_STATE_UNCONNECTED:
if (!pw->m_stopped) {
Q_EMIT pw->stopStreaming();
}
break;
}
}
#define CURSOR_BPP 4
#define CURSOR_META_SIZE(w,h) (sizeof(struct spa_meta_cursor) + \
sizeof(struct spa_meta_bitmap) + w * h * CURSOR_BPP)
void PipeWireStream::newStreamParams()
{
const int bpp = videoFormat.format == SPA_VIDEO_FORMAT_RGB || videoFormat.format == SPA_VIDEO_FORMAT_BGR ? 3 : 4;
auto stride = SPA_ROUND_UP_N (m_resolution.width() * bpp, 4);
uint8_t paramsBuffer[1024];
spa_pod_builder pod_builder = SPA_POD_BUILDER_INIT (paramsBuffer, sizeof (paramsBuffer));
spa_rectangle resolution = SPA_RECTANGLE(uint32_t(m_resolution.width()), uint32_t(m_resolution.height()));
const auto cursorSize = Cursors::self()->currentCursor()->themeSize();
const spa_pod *params[] = {
(spa_pod*) spa_pod_builder_add_object(&pod_builder,
SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers,
SPA_FORMAT_VIDEO_size, SPA_POD_Rectangle(&resolution),
SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(16, 2, 16),
SPA_PARAM_BUFFERS_blocks, SPA_POD_Int (1),
SPA_PARAM_BUFFERS_stride, SPA_POD_Int(stride),
SPA_PARAM_BUFFERS_size, SPA_POD_Int(stride * m_resolution.height()),
SPA_PARAM_BUFFERS_align, SPA_POD_Int(16)),
(spa_pod*) spa_pod_builder_add_object (&pod_builder,
SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta,
SPA_PARAM_META_type, SPA_POD_Id (SPA_META_Cursor),
SPA_PARAM_META_size, SPA_POD_Int (CURSOR_META_SIZE (cursorSize, cursorSize)))
};
pw_stream_update_params(pwStream, params, 2);
}
void PipeWireStream::onStreamParamChanged(void *data, uint32_t id, const struct spa_pod *format)
{
if (!format || id != SPA_PARAM_Format) {
return;
}
PipeWireStream *pw = static_cast<PipeWireStream *>(data);
spa_format_video_raw_parse (format, &pw->videoFormat);
qCDebug(KWIN_SCREENCAST) << "Stream format changed" << pw << pw->videoFormat.format;
pw->newStreamParams();
}
void PipeWireStream::onStreamAddBuffer(void *data, pw_buffer *buffer)
{
PipeWireStream *stream = static_cast<PipeWireStream *>(data);
struct spa_data *spa_data = buffer->buffer->datas;
spa_data->mapoffset = 0;
spa_data->flags = SPA_DATA_FLAG_READWRITE;
QSharedPointer<DmaBufTexture> dmabuf(kwinApp()->platform()->createDmaBufTexture(stream->m_resolution));
if (dmabuf) {
spa_data->type = SPA_DATA_DmaBuf;
spa_data->fd = dmabuf->fd();
spa_data->data = nullptr;
spa_data->maxsize = dmabuf->stride() * stream->m_resolution.height();
stream->m_dmabufDataForPwBuffer.insert(buffer, dmabuf);
#ifdef F_SEAL_SEAL //Disable memfd on systems that don't have it, like BSD < 12
} else {
const int bytesPerPixel = stream->m_hasAlpha ? 4 : 3;
const int stride = SPA_ROUND_UP_N (stream->m_resolution.width() * bytesPerPixel, 4);
spa_data->maxsize = stride * stream->m_resolution.height();
spa_data->type = SPA_DATA_MemFd;
spa_data->fd = memfd_create("kwin-screencast-memfd", MFD_CLOEXEC | MFD_ALLOW_SEALING);
if (spa_data->fd == -1) {
qCCritical(KWIN_SCREENCAST) << "memfd: Can't create memfd";
return;
}
spa_data->mapoffset = 0;
if (ftruncate (spa_data->fd, spa_data->maxsize) < 0) {
qCCritical(KWIN_SCREENCAST) << "memfd: Can't truncate to" << spa_data->maxsize;
return;
}
unsigned int seals = F_SEAL_GROW | F_SEAL_SHRINK | F_SEAL_SEAL;
if (fcntl(spa_data->fd, F_ADD_SEALS, seals) == -1)
qCWarning(KWIN_SCREENCAST) << "memfd: Failed to add seals";
spa_data->data = mmap(nullptr,
spa_data->maxsize,
PROT_READ | PROT_WRITE,
MAP_SHARED,
spa_data->fd,
spa_data->mapoffset);
if (spa_data->data == MAP_FAILED)
qCCritical(KWIN_SCREENCAST) << "memfd: Failed to mmap memory";
else
qCDebug(KWIN_SCREENCAST) << "memfd: created successfully" << spa_data->data << spa_data->maxsize;
#endif
}
}
void PipeWireStream::onStreamRemoveBuffer(void *data, pw_buffer *buffer)
{
PipeWireStream *stream = static_cast<PipeWireStream *>(data);
struct spa_buffer *spa_buffer = buffer->buffer;
struct spa_data *spa_data = spa_buffer->datas;
if (spa_data->type == SPA_DATA_DmaBuf) {
stream->m_dmabufDataForPwBuffer.remove(buffer);
} else if (spa_data->type == SPA_DATA_MemFd) {
munmap (spa_data->data, spa_data->maxsize);
close (spa_data->fd);
}
}
PipeWireStream::PipeWireStream(bool hasAlpha, const QSize &resolution, QObject *parent)
: QObject(parent)
, m_resolution(resolution)
, m_hasAlpha(hasAlpha)
{
pwStreamEvents.version = PW_VERSION_STREAM_EVENTS;
pwStreamEvents.add_buffer = &PipeWireStream::onStreamAddBuffer;
pwStreamEvents.remove_buffer = &PipeWireStream::onStreamRemoveBuffer;
pwStreamEvents.state_changed = &PipeWireStream::onStreamStateChanged;
pwStreamEvents.param_changed = &PipeWireStream::onStreamParamChanged;
}
PipeWireStream::~PipeWireStream()
{
m_stopped = true;
if (pwStream) {
pw_stream_destroy(pwStream);
}
}
bool PipeWireStream::init()
{
pwCore = PipeWireCore::self();
if (!pwCore->m_error.isEmpty()) {
m_error = pwCore->m_error;
return false;
}
connect(pwCore.data(), &PipeWireCore::pipewireFailed, this, &PipeWireStream::coreFailed);
if (!createStream()) {
qCWarning(KWIN_SCREENCAST) << "Failed to create PipeWire stream";
m_error = i18n("Failed to create PipeWire stream");
return false;
}
return true;
}
uint PipeWireStream::framerate()
{
if (pwStream) {
return videoFormat.max_framerate.num / videoFormat.max_framerate.denom;
}
return 0;
}
uint PipeWireStream::nodeId()
{
return pwNodeId;
}
bool PipeWireStream::createStream()
{
const QByteArray objname = "kwin-screencast-" + objectName().toUtf8();
pwStream = pw_stream_new(pwCore->pwCore, objname, nullptr);
uint8_t buffer[1024];
spa_pod_builder podBuilder = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer));
spa_fraction minFramerate = SPA_FRACTION(1, 1);
spa_fraction maxFramerate = SPA_FRACTION(25, 1);
spa_fraction defaultFramerate = SPA_FRACTION(0, 1);
spa_rectangle resolution = SPA_RECTANGLE(uint32_t(m_resolution.width()), uint32_t(m_resolution.height()));
const auto format = m_hasAlpha ? SPA_VIDEO_FORMAT_BGRA : SPA_VIDEO_FORMAT_BGR;
const spa_pod *param = (spa_pod*)spa_pod_builder_add_object(&podBuilder,
SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video),
SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw),
SPA_FORMAT_VIDEO_format, SPA_POD_Id(format),
SPA_FORMAT_VIDEO_size, SPA_POD_Rectangle(&resolution),
SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction(&defaultFramerate),
SPA_FORMAT_VIDEO_maxFramerate, SPA_POD_CHOICE_RANGE_Fraction(&maxFramerate, &minFramerate, &maxFramerate));
pw_stream_add_listener(pwStream, &streamListener, &pwStreamEvents, this);
auto flags = pw_stream_flags(PW_STREAM_FLAG_DRIVER | PW_STREAM_FLAG_ALLOC_BUFFERS);
if (pw_stream_connect(pwStream, PW_DIRECTION_OUTPUT, SPA_ID_INVALID, flags, &param, 1) != 0) {
qCWarning(KWIN_SCREENCAST) << "Could not connect to stream";
pw_stream_destroy(pwStream);
return false;
}
if (m_cursor.mode == KWaylandServer::ScreencastV1Interface::Embedded) {
connect(Cursors::self(), &Cursors::positionChanged, this, [this] {
if (m_cursor.lastFrameTexture) {
m_repainting = true;
recordFrame(m_cursor.lastFrameTexture.data(), QRegion{m_cursor.lastRect} | cursorGeometry(Cursors::self()->currentCursor()));
m_repainting = false;
}
});
}
return true;
}
void PipeWireStream::coreFailed(const QString &errorMessage)
{
m_error = errorMessage;
Q_EMIT stopStreaming();
}
void PipeWireStream::stop()
{
m_stopped = true;
delete this;
}
static GLTexture *copyTexture(GLTexture *texture)
{
GLTexture *copy = new GLTexture(texture->internalFormat(), texture->size());
copy->setFilter(GL_LINEAR);
copy->setWrapMode(GL_CLAMP_TO_EDGE);
const QRect r({}, texture->size());
copy->bind();
glCopyTextureSubImage2D(copy->texture(), 0, 0, 0, 0, 0, r.width(), r.height());
copy->unbind();
return copy;
}
void PipeWireStream::recordFrame(GLTexture *frameTexture, const QRegion &damagedRegion)
{
Q_ASSERT(!m_stopped);
Q_ASSERT(frameTexture);
if (m_pendingBuffer) {
qCWarning(KWIN_SCREENCAST) << "Dropping a screencast frame because the compositor is slow";
return;
}
if (frameTexture->size() != m_resolution) {
m_resolution = frameTexture->size();
newStreamParams();
return;
}
const char *error = "";
auto state = pw_stream_get_state(pwStream, &error);
if (state != PW_STREAM_STATE_STREAMING) {
if (error) {
qCWarning(KWIN_SCREENCAST) << "Failed to record frame: stream is not active" << error;
}
return;
}
struct pw_buffer *buffer = pw_stream_dequeue_buffer(pwStream);
if (!buffer) {
return;
}
struct spa_buffer *spa_buffer = buffer->buffer;
struct spa_data *spa_data = spa_buffer->datas;
uint8_t *data = (uint8_t *) spa_data->data;
if (!data && spa_buffer->datas->type != SPA_DATA_DmaBuf) {
qCWarning(KWIN_SCREENCAST) << "Failed to record frame: invalid buffer data";
pw_stream_queue_buffer(pwStream, buffer);
return;
}
const auto size = frameTexture->size();
spa_data->chunk->offset = 0;
if (data) {
const int bpp = data && !m_hasAlpha ? 3 : 4;
const uint stride = SPA_ROUND_UP_N (size.width() * bpp, 4);
const uint bufferSize = stride * size.height();
if (bufferSize > spa_data->maxsize) {
qCDebug(KWIN_SCREENCAST) << "Failed to record frame: frame is too big";
pw_stream_queue_buffer(pwStream, buffer);
return;
}
spa_data->chunk->size = bufferSize;
spa_data->chunk->stride = stride;
frameTexture->bind();
glGetTextureImage(frameTexture->texture(), 0, m_hasAlpha ? GL_BGRA : GL_BGR, GL_UNSIGNED_BYTE, bufferSize, data);
auto cursor = Cursors::self()->currentCursor();
if (m_cursor.mode == KWaylandServer::ScreencastV1Interface::Embedded && m_cursor.viewport.contains(cursor->pos())) {
QImage dest(data, size.width(), size.height(), QImage::Format_RGBA8888_Premultiplied);
QPainter painter(&dest);
const auto position = (cursor->pos() - m_cursor.viewport.topLeft() - cursor->hotspot()) * m_cursor.scale;
painter.drawImage(QRect{position, cursor->image().size()}, cursor->image());
}
} else {
auto &buf = m_dmabufDataForPwBuffer[buffer];
spa_data->chunk->stride = buf->stride();
spa_data->chunk->size = spa_data->maxsize;
GLRenderTarget::pushRenderTarget(buf->framebuffer());
frameTexture->bind();
QRect r(QPoint(), size);
auto shader = ShaderManager::instance()->pushShader(ShaderTrait::MapTexture);
QMatrix4x4 mvp;
mvp.ortho(r);
shader->setUniform(GLShader::ModelViewProjectionMatrix, mvp);
QRegion dr = damagedRegion;
if (m_cursor.texture) {
dr |= m_cursor.lastRect;
}
frameTexture->render(damagedRegion, r, true);
auto cursor = Cursors::self()->currentCursor();
if (m_cursor.mode == KWaylandServer::ScreencastV1Interface::Embedded && m_cursor.viewport.contains(cursor->pos())) {
if (!m_repainting) //We need to copy the last version of the stream to render the moved cursor on top
m_cursor.lastFrameTexture.reset(copyTexture(frameTexture));
if (!m_cursor.texture || m_cursor.lastKey != cursor->image().cacheKey())
m_cursor.texture.reset(new GLTexture(cursor->image()));
m_cursor.texture->setYInverted(false);
m_cursor.texture->bind();
const auto cursorRect = cursorGeometry(cursor);
mvp.translate(cursorRect.left(), r.height() - cursorRect.top() - cursor->image().height() * m_cursor.scale);
shader->setUniform(GLShader::ModelViewProjectionMatrix, mvp);
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
m_cursor.texture->render(cursorRect, cursorRect, true);
glDisable(GL_BLEND);
m_cursor.texture->unbind();
m_cursor.lastRect = cursorRect;
}
ShaderManager::instance()->popShader();
GLRenderTarget::popRenderTarget();
}
frameTexture->unbind();
if (m_cursor.mode == KWaylandServer::ScreencastV1Interface::Metadata) {
sendCursorData(Cursors::self()->currentCursor(),
(spa_meta_cursor *) spa_buffer_find_meta_data (spa_buffer, SPA_META_Cursor, sizeof (spa_meta_cursor)));
}
tryEnqueue(buffer);
}
void PipeWireStream::tryEnqueue(pw_buffer *buffer)
{
m_pendingBuffer = buffer;
// The GPU doesn't necessarily process draw commands as soon as they are issued. Thus,
// we need to insert a fence into the command stream and enqueue the pipewire buffer
// only after the fence is signaled; otherwise stream consumers will most likely see
// a corrupted buffer.
if (kwinApp()->platform()->supportsNativeFence()) {
Q_ASSERT_X(eglGetCurrentContext(), "tryEnqueue", "no current context");
m_pendingFence = new EGLNativeFence(kwinApp()->platform()->sceneEglDisplay());
if (!m_pendingFence->isValid()) {
qCWarning(KWIN_SCREENCAST) << "Failed to create a native EGL fence";
glFinish();
enqueue();
} else {
m_pendingNotifier = new QSocketNotifier(m_pendingFence->fileDescriptor(),
QSocketNotifier::Read, this);
connect(m_pendingNotifier, &QSocketNotifier::activated, this, &PipeWireStream::enqueue);
}
} else {
// The compositing backend doesn't support native fences. We don't have any other choice
// but stall the graphics pipeline. Otherwise stream consumers may see an incomplete buffer.
glFinish();
enqueue();
}
}
void PipeWireStream::enqueue()
{
Q_ASSERT_X(m_pendingBuffer, "enqueue", "pending buffer must be valid");
delete m_pendingFence;
delete m_pendingNotifier;
pw_stream_queue_buffer(pwStream, m_pendingBuffer);
m_pendingBuffer = nullptr;
m_pendingFence = nullptr;
m_pendingNotifier = nullptr;
}
QRect PipeWireStream::cursorGeometry(Cursor *cursor) const
{
const auto position = (cursor->pos() - m_cursor.viewport.topLeft() - cursor->hotspot()) * m_cursor.scale;
return QRect{position, m_cursor.texture->size()};
}
void PipeWireStream::sendCursorData(Cursor *cursor, spa_meta_cursor *spa_meta_cursor)
{
if (!cursor || !spa_meta_cursor) {
return;
}
const auto position = (cursor->pos() - m_cursor.viewport.topLeft()) * m_cursor.scale;
spa_meta_cursor->id = 1;
spa_meta_cursor->position.x = position.x();
spa_meta_cursor->position.y = position.y();
spa_meta_cursor->hotspot.x = cursor->hotspot().x() * m_cursor.scale;
spa_meta_cursor->hotspot.y = cursor->hotspot().y() * m_cursor.scale;
spa_meta_cursor->bitmap_offset = 0;
const QImage image = cursor->image();
if (image.cacheKey() == m_cursor.lastKey) {
return;
}
m_cursor.lastKey = image.cacheKey();
spa_meta_cursor->bitmap_offset = sizeof (struct spa_meta_cursor);
struct spa_meta_bitmap *spa_meta_bitmap = SPA_MEMBER (spa_meta_cursor,
spa_meta_cursor->bitmap_offset,
struct spa_meta_bitmap);
spa_meta_bitmap->format = SPA_VIDEO_FORMAT_RGBA;
spa_meta_bitmap->offset = sizeof (struct spa_meta_bitmap);
uint8_t *bitmap_data = SPA_MEMBER (spa_meta_bitmap, spa_meta_bitmap->offset, uint8_t);
QImage dest(bitmap_data, image.width(), image.height(), QImage::Format_RGBA8888_Premultiplied);
spa_meta_bitmap->size.width = image.width();
spa_meta_bitmap->size.height = image.height();
spa_meta_bitmap->stride = dest.bytesPerLine();
QPainter painter(&dest);
painter.drawImage(QPoint(), image);
}
void PipeWireStream::setCursorMode(KWaylandServer::ScreencastV1Interface::CursorMode mode, qreal scale, const QRect &viewport)
{
m_cursor.mode = mode;
m_cursor.scale = scale;
m_cursor.viewport = viewport;
}
} // namespace KWin