From ce9531d5499d676a3618bf70b9acef61aaa2efa4 Mon Sep 17 00:00:00 2001 From: Vlad Zahorodnii Date: Tue, 23 Apr 2024 12:06:01 +0300 Subject: [PATCH] plugins/screencast: Throttle cursor updates --- src/plugins/screencast/screencaststream.cpp | 233 ++++++++++---------- src/plugins/screencast/screencaststream.h | 17 +- 2 files changed, 133 insertions(+), 117 deletions(-) diff --git a/src/plugins/screencast/screencaststream.cpp b/src/plugins/screencast/screencaststream.cpp index 08cc415d19..bffab3866f 100644 --- a/src/plugins/screencast/screencaststream.cpp +++ b/src/plugins/screencast/screencaststream.cpp @@ -251,7 +251,9 @@ ScreenCastStream::ScreenCastStream(ScreenCastSource *source, std::shared_ptrtextureSize()) { - connect(source, &ScreenCastSource::frame, this, &ScreenCastStream::recordFrame); + connect(source, &ScreenCastSource::frame, this, [this](const QRegion &damage) { + recordFrame(damage, Content::Video); + }); connect(source, &ScreenCastSource::closed, this, &ScreenCastStream::close); m_pwStreamEvents.version = PW_VERSION_STREAM_EVENTS; @@ -274,7 +276,7 @@ ScreenCastStream::ScreenCastStream(ScreenCastSource *source, std::shared_ptrcurrentCursor())) { + return; + } + } + if (m_videoFormat.max_framerate.num != 0 && m_lastSent.has_value()) { const auto now = std::chrono::steady_clock::now(); const auto frameInterval = std::chrono::milliseconds(1000 * m_videoFormat.max_framerate.denom / m_videoFormat.max_framerate.num); const auto lastSentAgo = std::chrono::duration_cast(now - m_lastSent.value()); if (lastSentAgo < frameInterval) { m_pendingDamages += damagedRegion; + m_pendingContents |= contents; if (!m_pendingFrame.isActive()) { m_pendingFrame.start(frameInterval - lastSentAgo); } @@ -425,6 +436,7 @@ void ScreenCastStream::recordFrame(const QRegion &_damagedRegion) } m_pendingDamages = {}; + m_pendingContents = {}; struct pw_buffer *pwBuffer = pw_stream_dequeue_buffer(m_pwStream); if (!pwBuffer) { @@ -442,77 +454,58 @@ void ScreenCastStream::recordFrame(const QRegion &_damagedRegion) return; } - EglContext *context = static_cast(Compositor::self()->backend())->openglContext(); - context->makeCurrent(); - - spa_data->chunk->flags = SPA_CHUNK_FLAG_NONE; - if (auto memfd = dynamic_cast(buffer)) { - m_source->render(memfd->view.image()); - - auto cursor = Cursors::self()->currentCursor(); - if (m_cursor.mode == ScreencastV1Interface::Embedded && includesCursor(cursor)) { - QPainter painter(memfd->view.image()); - const auto position = (cursor->pos() - m_cursor.viewport.topLeft() - cursor->hotspot()) * m_cursor.scale; - const PlatformCursorImage cursorImage = kwinApp()->cursorImage(); - painter.drawImage(QRect{position.toPoint(), cursorImage.image().size()}, cursorImage.image()); - } - } else if (auto dmabuf = dynamic_cast(buffer)) { - m_source->render(dmabuf->framebuffer.get()); - - auto cursor = Cursors::self()->currentCursor(); - if (m_cursor.mode == ScreencastV1Interface::Embedded && includesCursor(cursor)) { - if (m_cursor.invalid) { - m_cursor.invalid = false; - const PlatformCursorImage cursorImage = kwinApp()->cursorImage(); - if (cursorImage.isNull()) { - m_cursor.texture = nullptr; - } else { - m_cursor.texture = GLTexture::upload(cursorImage.image()); - } - } - if (m_cursor.texture) { - GLFramebuffer::pushFramebuffer(dmabuf->framebuffer.get()); - - auto shader = ShaderManager::instance()->pushShader(ShaderTrait::MapTexture); - - const QRectF cursorRect = scaledRect(cursor->geometry().translated(-m_cursor.viewport.topLeft()), m_cursor.scale); - QMatrix4x4 mvp; - mvp.scale(1, -1); - mvp.ortho(QRectF(QPointF(0, 0), dmabuf->texture->size())); - mvp.translate(cursorRect.x(), cursorRect.y()); - shader->setUniform(GLShader::Mat4Uniform::ModelViewProjectionMatrix, mvp); - - glEnable(GL_BLEND); - glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); - m_cursor.texture->render(cursorRect.size()); - glDisable(GL_BLEND); - - ShaderManager::instance()->popShader(); - GLFramebuffer::popFramebuffer(); - - damagedRegion += QRegion{m_cursor.lastRect.toAlignedRect()} | cursorRect.toAlignedRect(); - m_cursor.lastRect = cursorRect; - } else { - damagedRegion += m_cursor.lastRect.toAlignedRect(); - m_cursor.lastRect = {}; - } - } - - // Implicit sync is broken on Nvidia. - if (context->glPlatform()->isNvidia()) { - glFinish(); - } else { - glFlush(); + Contents effectiveContents = contents; + if (m_cursor.mode != ScreencastV1Interface::Hidden) { + effectiveContents.setFlag(Content::Cursor); + if (m_cursor.mode == ScreencastV1Interface::Embedded) { + effectiveContents.setFlag(Content::Video); } } - if (m_cursor.mode == ScreencastV1Interface::Metadata) { - addCursorMetadata(spa_buffer, Cursors::self()->currentCursor()); + EglContext *context = static_cast(Compositor::self()->backend())->openglContext(); + context->makeCurrent(); + + if (effectiveContents & Content::Video) { + if (auto memfd = dynamic_cast(buffer)) { + m_source->render(memfd->view.image()); + } else if (auto dmabuf = dynamic_cast(buffer)) { + m_source->render(dmabuf->framebuffer.get()); + } + } + + if (effectiveContents & Content::Cursor) { + Cursor *cursor = Cursors::self()->currentCursor(); + switch (m_cursor.mode) { + case ScreencastV1Interface::Hidden: + break; + case ScreencastV1Interface::Embedded: + damagedRegion += addCursorEmbedded(buffer, cursor); + break; + case ScreencastV1Interface::Metadata: + addCursorMetadata(spa_buffer, cursor); + break; + } + } + + // Implicit sync is broken on Nvidia. + if (context->glPlatform()->isNvidia()) { + glFinish(); + } else { + glFlush(); } addDamage(spa_buffer, damagedRegion); addHeader(spa_buffer); - enqueue(pwBuffer); + + if (effectiveContents & Content::Video) { + spa_data->chunk->flags = SPA_CHUNK_FLAG_NONE; + } else { + // in pipewire terms, corrupted means "do not look at the frame contents" and here they're empty. + spa_data->chunk->flags = SPA_CHUNK_FLAG_CORRUPTED; + } + + pw_stream_queue_buffer(m_pwStream, pwBuffer); + m_lastSent = std::chrono::steady_clock::now(); resize(m_source->textureSize()); } @@ -580,48 +573,6 @@ void ScreenCastStream::invalidateCursor() m_cursor.invalid = true; } -void ScreenCastStream::recordCursor() -{ - Q_ASSERT(!m_closed); - - const char *error = ""; - auto state = pw_stream_get_state(m_pwStream, &error); - if (state != PW_STREAM_STATE_STREAMING) { - if (error) { - qCWarning(KWIN_SCREENCAST) << objectName() << "Failed to record cursor position: stream is not active" << error; - } - return; - } - - if (!includesCursor(Cursors::self()->currentCursor()) && !m_cursor.visible) { - return; - } - - pw_buffer *pwBuffer = pw_stream_dequeue_buffer(m_pwStream); - if (!pwBuffer) { - return; - } - - struct spa_buffer *spa_buffer = pwBuffer->buffer; - - // in pipewire terms, corrupted means "do not look at the frame contents" and here they're empty. - spa_buffer->datas[0].chunk->flags = SPA_CHUNK_FLAG_CORRUPTED; - - addCursorMetadata(spa_buffer, Cursors::self()->currentCursor()); - addHeader(spa_buffer); - addDamage(spa_buffer, {}); - enqueue(pwBuffer); -} - -void ScreenCastStream::enqueue(pw_buffer *pwBuffer) -{ - pw_stream_queue_buffer(m_pwStream, pwBuffer); - - if (pwBuffer->buffer->datas[0].chunk->flags != SPA_CHUNK_FLAG_CORRUPTED) { - m_lastSent = std::chrono::steady_clock::now(); - } -} - QList ScreenCastStream::buildFormats(bool fixate, char buffer[2048]) { const auto format = drmFourCCToSpaVideoFormat(m_drmFormat); @@ -759,6 +710,60 @@ void ScreenCastStream::addCursorMetadata(spa_buffer *spaBuffer, Cursor *cursor) } } +QRegion ScreenCastStream::addCursorEmbedded(ScreenCastBuffer *buffer, Cursor *cursor) +{ + if (!includesCursor(cursor)) { + const QRegion damage = m_cursor.lastRect.toAlignedRect(); + m_cursor.visible = false; + m_cursor.lastRect = QRectF(); + return damage; + } + + const QRectF cursorRect = scaledRect(cursor->geometry().translated(-m_cursor.viewport.topLeft()), m_cursor.scale); + if (auto memfd = dynamic_cast(buffer)) { + QPainter painter(memfd->view.image()); + const auto position = (cursor->pos() - m_cursor.viewport.topLeft() - cursor->hotspot()) * m_cursor.scale; + const PlatformCursorImage cursorImage = kwinApp()->cursorImage(); + painter.drawImage(QRect{position.toPoint(), cursorImage.image().size()}, cursorImage.image()); + } else if (auto dmabuf = dynamic_cast(buffer)) { + if (m_cursor.invalid) { + m_cursor.invalid = false; + const PlatformCursorImage cursorImage = kwinApp()->cursorImage(); + if (cursorImage.isNull()) { + m_cursor.texture = nullptr; + } else { + m_cursor.texture = GLTexture::upload(cursorImage.image()); + } + } + + if (m_cursor.texture) { + GLFramebuffer::pushFramebuffer(dmabuf->framebuffer.get()); + + auto shader = ShaderManager::instance()->pushShader(ShaderTrait::MapTexture); + + QMatrix4x4 mvp; + mvp.scale(1, -1); + mvp.ortho(QRectF(QPointF(0, 0), dmabuf->texture->size())); + mvp.translate(cursorRect.x(), cursorRect.y()); + shader->setUniform(GLShader::Mat4Uniform::ModelViewProjectionMatrix, mvp); + + glEnable(GL_BLEND); + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + m_cursor.texture->render(cursorRect.size()); + glDisable(GL_BLEND); + + ShaderManager::instance()->popShader(); + GLFramebuffer::popFramebuffer(); + } + } + + const QRegion damage = QRegion{m_cursor.lastRect.toAlignedRect()} | cursorRect.toAlignedRect(); + m_cursor.visible = true; + m_cursor.lastRect = cursorRect; + + return damage; +} + void ScreenCastStream::setCursorMode(ScreencastV1Interface::CursorMode mode, qreal scale, const QRectF &viewport) { m_cursor.mode = mode; diff --git a/src/plugins/screencast/screencaststream.h b/src/plugins/screencast/screencaststream.h index 573aac0222..261700e18c 100644 --- a/src/plugins/screencast/screencaststream.h +++ b/src/plugins/screencast/screencaststream.h @@ -29,6 +29,7 @@ namespace KWin class Cursor; class GLTexture; class PipeWireCore; +class ScreenCastBuffer; class ScreenCastSource; struct ScreenCastDmaBufTextureParams @@ -47,6 +48,14 @@ public: explicit ScreenCastStream(ScreenCastSource *source, std::shared_ptr pwCore, QObject *parent); ~ScreenCastStream(); + enum class Content { + None, + Video = 0x1, + Cursor = 0x2, + }; + Q_FLAG(Content) + Q_DECLARE_FLAGS(Contents, Content) + bool init(); uint framerate(); uint nodeId(); @@ -61,13 +70,12 @@ public: * Renders @p frame into the current framebuffer into the stream * @p timestamp */ - void recordFrame(const QRegion &damagedRegion); + void recordFrame(const QRegion &damagedRegion, Contents contents = Content::Video); void setCursorMode(ScreencastV1Interface::CursorMode mode, qreal scale, const QRectF &viewport); public Q_SLOTS: void invalidateCursor(); - void recordCursor(); bool includesCursor(Cursor *cursor) const; Q_SIGNALS: @@ -86,11 +94,11 @@ private: void resize(const QSize &resolution); void coreFailed(const QString &errorMessage); void addCursorMetadata(spa_buffer *spaBuffer, Cursor *cursor); + QRegion addCursorEmbedded(ScreenCastBuffer *buffer, Cursor *cursor); void addHeader(spa_buffer *spaBuffer); void corruptHeader(spa_buffer *spaBuffer); void addDamage(spa_buffer *spaBuffer, const QRegion &damagedRegion); void newStreamParams(); - void enqueue(pw_buffer *buffer); spa_pod *buildFormat(struct spa_pod_builder *b, enum spa_video_format format, struct spa_rectangle *resolution, struct spa_fraction *defaultFramerate, struct spa_fraction *minFramerate, struct spa_fraction *maxFramerate, const QList &modifiers, quint32 modifiersFlags); @@ -134,6 +142,9 @@ private: std::optional m_lastSent; QRegion m_pendingDamages; QTimer m_pendingFrame; + Contents m_pendingContents = Content::None; }; } // namespace KWin + +Q_DECLARE_OPERATORS_FOR_FLAGS(KWin::ScreenCastStream::Contents)