diff --git a/autotests/test_colorspaces.cpp b/autotests/test_colorspaces.cpp
index f783bafba7..f96e8017d6 100644
--- a/autotests/test_colorspaces.cpp
+++ b/autotests/test_colorspaces.cpp
@@ -63,8 +63,8 @@ void TestColorspaces::roundtripConversion()
     QFETCH(TransferFunction::Type, dstTransferFunction);
     QFETCH(double, requiredAccuracy);
 
-    const auto src = ColorDescription(srcColorimetry, srcTransferFunction, 100, 0, 100, 100);
-    const auto dst = ColorDescription(dstColorimetry, dstTransferFunction, 100, 0, 100, 100);
+    const auto src = ColorDescription(srcColorimetry, TransferFunction(srcTransferFunction), 100, 0, 100, 100);
+    const auto dst = ColorDescription(dstColorimetry, TransferFunction(dstTransferFunction), 100, 0, 100, 100);
 
     const QVector3D red(1, 0, 0);
     const QVector3D green(0, 1, 0);
@@ -108,7 +108,7 @@ void TestColorspaces::testIdentityTransformation()
 {
     QFETCH(NamedColorimetry, colorimetry);
     QFETCH(TransferFunction::Type, transferFunction);
-    const ColorDescription color(colorimetry, transferFunction, 100, 0, 100, 100);
+    const ColorDescription color(colorimetry, TransferFunction(transferFunction), 100, 0, 100, 100);
 
     const auto pipeline = ColorPipeline::create(color, color);
     if (!pipeline.isIdentity()) {
@@ -125,13 +125,13 @@ void TestColorspaces::testColorPipeline_data()
     QTest::addColumn<QVector3D>("dstGray");
     QTest::addColumn<QVector3D>("dstWhite");
 
-    QTest::addRow("sRGB -> rec.2020") << ColorDescription(NamedColorimetry::BT709, TransferFunction::Type::gamma22, TransferFunction::defaultReferenceLuminanceFor(TransferFunction::Type::gamma22), 0, std::nullopt, std::nullopt)
-                                      << ColorDescription(NamedColorimetry::BT2020, TransferFunction::Type::PerceptualQuantizer, 500, 0, std::nullopt, std::nullopt)
+    QTest::addRow("sRGB -> rec.2020") << ColorDescription(NamedColorimetry::BT709, TransferFunction(TransferFunction::gamma22), TransferFunction::defaultReferenceLuminanceFor(TransferFunction::gamma22), 0, std::nullopt, std::nullopt)
+                                      << ColorDescription(NamedColorimetry::BT2020, TransferFunction(TransferFunction::PerceptualQuantizer), 500, 0, std::nullopt, std::nullopt)
                                       << QVector3D(0.044, 0.044, 0.044)
                                       << QVector3D(0.517, 0.517, 0.517)
                                       << QVector3D(0.677, 0.677, 0.677);
-    QTest::addRow("sRGB -> scRGB") << ColorDescription(NamedColorimetry::BT709, TransferFunction::Type::gamma22, TransferFunction::defaultReferenceLuminanceFor(TransferFunction::Type::gamma22), 0, std::nullopt, std::nullopt)
-                                   << ColorDescription(NamedColorimetry::BT709, TransferFunction(TransferFunction::Type::linear, 0, 80), 80, 0, std::nullopt, std::nullopt)
+    QTest::addRow("sRGB -> scRGB") << ColorDescription(NamedColorimetry::BT709, TransferFunction(TransferFunction::gamma22), TransferFunction::defaultReferenceLuminanceFor(TransferFunction::gamma22), 0, std::nullopt, std::nullopt)
+                                   << ColorDescription(NamedColorimetry::BT709, TransferFunction(TransferFunction::linear, 0, 80), 80, 0, std::nullopt, std::nullopt)
                                    << QVector3D(0.0001, 0.0001, 0.0001)
                                    << QVector3D(0.2177376408240310, 0.2177376408240310, 0.2177376408240310)
                                    << QVector3D(1, 1, 1);
diff --git a/src/backends/drm/drm_egl_layer.cpp b/src/backends/drm/drm_egl_layer.cpp
index 1f3e3c1eb4..7dd65ff891 100644
--- a/src/backends/drm/drm_egl_layer.cpp
+++ b/src/backends/drm/drm_egl_layer.cpp
@@ -56,7 +56,8 @@ std::optional<OutputLayerBeginFrameInfo> EglGbmLayer::doBeginFrame()
 
     m_scanoutBuffer.reset();
     m_colorPipeline = ColorPipeline{};
-    return m_surface.startRendering(targetRect().size(), m_pipeline->output()->transform().combine(OutputTransform::FlipY), m_pipeline->formats(m_type), m_pipeline->colorDescription(), m_pipeline->output()->effectiveChannelFactors(), m_pipeline->iccProfile(), m_pipeline->output()->needsColormanagement());
+    return m_surface.startRendering(targetRect().size(), m_pipeline->output()->transform().combine(OutputTransform::FlipY), m_pipeline->formats(m_type), m_pipeline->output()->scanoutColorDescription(),
+                                    m_pipeline->output()->needsChannelFactorFallback() ? m_pipeline->output()->effectiveChannelFactors() : QVector3D(1, 1, 1), m_pipeline->iccProfile());
 }
 
 bool EglGbmLayer::doEndFrame(const QRegion &renderedRegion, const QRegion &damagedRegion, OutputFrame *frame)
@@ -96,15 +97,11 @@ bool EglGbmLayer::doAttemptScanout(GraphicsBuffer *buffer, const ColorDescriptio
         // TODO make the icc profile output a color pipeline too?
         return false;
     }
-    ColorPipeline pipeline = ColorPipeline::create(color, m_pipeline->colorDescription());
-    if (m_pipeline->output()->needsColormanagement()) {
-        // with color management enabled, the factors have to be applied in linear space
-        // the pipeline will optimize out the unnecessary transformations
-        pipeline.addTransferFunction(m_pipeline->colorDescription().transferFunction());
-    }
-    pipeline.addMultiplier(m_pipeline->output()->effectiveChannelFactors());
-    if (m_pipeline->output()->needsColormanagement()) {
-        pipeline.addInverseTransferFunction(m_pipeline->colorDescription().transferFunction());
+    ColorPipeline pipeline = ColorPipeline::create(color, m_pipeline->output()->scanoutColorDescription());
+    if (m_pipeline->output()->needsChannelFactorFallback()) {
+        pipeline.addTransferFunction(m_pipeline->output()->scanoutColorDescription().transferFunction());
+        pipeline.addMultiplier(m_pipeline->output()->effectiveChannelFactors());
+        pipeline.addInverseTransferFunction(m_pipeline->output()->scanoutColorDescription().transferFunction());
     }
     m_colorPipeline = pipeline;
     // kernel documentation says that
diff --git a/src/backends/drm/drm_egl_layer_surface.cpp b/src/backends/drm/drm_egl_layer_surface.cpp
index 2970036c9e..96ec1cf9a6 100644
--- a/src/backends/drm/drm_egl_layer_surface.cpp
+++ b/src/backends/drm/drm_egl_layer_surface.cpp
@@ -74,7 +74,7 @@ void EglGbmLayerSurface::destroyResources()
     m_oldSurface = {};
 }
 
-std::optional<OutputLayerBeginFrameInfo> EglGbmLayerSurface::startRendering(const QSize &bufferSize, OutputTransform transformation, const QHash<uint32_t, QList<uint64_t>> &formats, const ColorDescription &colorDescription, const QVector3D &channelFactors, const std::shared_ptr<IccProfile> &iccProfile, bool enableColormanagement)
+std::optional<OutputLayerBeginFrameInfo> EglGbmLayerSurface::startRendering(const QSize &bufferSize, OutputTransform transformation, const QHash<uint32_t, QList<uint64_t>> &formats, const ColorDescription &colorDescription, const QVector3D &channelFactors, const std::shared_ptr<IccProfile> &iccProfile)
 {
     if (!checkSurface(bufferSize, formats)) {
         return std::nullopt;
@@ -95,10 +95,9 @@ std::optional<OutputLayerBeginFrameInfo> EglGbmLayerSurface::startRendering(cons
     slot->framebuffer()->colorAttachment()->setContentTransform(transformation);
     m_surface->currentSlot = slot;
 
-    if (m_surface->targetColorDescription != colorDescription || m_surface->channelFactors != channelFactors
-        || m_surface->colormanagementEnabled != enableColormanagement || m_surface->iccProfile != iccProfile) {
+    if (m_surface->targetColorDescription != colorDescription || m_surface->channelFactors != channelFactors || m_surface->iccProfile != iccProfile) {
         m_surface->damageJournal.clear();
-        m_surface->colormanagementEnabled = enableColormanagement;
+        m_surface->needsShadowBuffer = channelFactors != QVector3D(1, 1, 1) || m_surface->iccProfile || colorDescription.transferFunction().type != TransferFunction::gamma22;
         m_surface->targetColorDescription = colorDescription;
         m_surface->channelFactors = channelFactors;
         m_surface->iccProfile = iccProfile;
@@ -109,8 +108,9 @@ std::optional<OutputLayerBeginFrameInfo> EglGbmLayerSurface::startRendering(cons
         } else {
             m_surface->iccShader.reset();
         }
-        if (enableColormanagement) {
-            m_surface->intermediaryColorDescription = ColorDescription(colorDescription.containerColorimetry(), TransferFunction::gamma22,
+        if (m_surface->needsShadowBuffer) {
+            const double maxLuminance = colorDescription.maxHdrLuminance().value_or(colorDescription.referenceLuminance());
+            m_surface->intermediaryColorDescription = ColorDescription(colorDescription.containerColorimetry(), TransferFunction(TransferFunction::gamma22, 0, maxLuminance),
                                                                        colorDescription.referenceLuminance(), colorDescription.minLuminance(),
                                                                        colorDescription.maxAverageLuminance(), colorDescription.maxHdrLuminance(),
                                                                        colorDescription.containerColorimetry(), colorDescription.sdrColorimetry());
@@ -122,29 +122,37 @@ std::optional<OutputLayerBeginFrameInfo> EglGbmLayerSurface::startRendering(cons
     const QRegion repaint = bufferAgeEnabled ? m_surface->damageJournal.accumulate(slot->age(), infiniteRegion()) : infiniteRegion();
     m_surface->compositingTimeQuery = std::make_unique<GLRenderTimeQuery>(m_surface->context);
     m_surface->compositingTimeQuery->begin();
-    if (enableColormanagement) {
+    if (m_surface->needsShadowBuffer) {
         if (!m_surface->shadowSwapchain || m_surface->shadowSwapchain->size() != m_surface->gbmSwapchain->size()) {
             const auto formats = m_eglBackend->eglDisplayObject()->nonExternalOnlySupportedDrmFormats();
             const auto createSwapchain = [&formats, this](bool requireAlpha) {
-                for (auto it = formats.begin(); it != formats.end(); it++) {
-                    const auto info = FormatInfo::get(it.key());
-                    if (!info || info->bitsPerColor != 16 || !info->floatingPoint) {
-                        continue;
-                    }
-                    if (requireAlpha && info->alphaBits == 0) {
-                        continue;
-                    }
-                    auto mods = it.value();
-                    if (m_eglBackend->gpu()->isAmdgpu() && qEnvironmentVariableIntValue("KWIN_DRM_NO_DCC_WORKAROUND") == 0) {
-                        // using modifiers with DCC here causes glitches on amdgpu: https://gitlab.freedesktop.org/mesa/mesa/-/issues/10875
-                        if (!mods.contains(DRM_FORMAT_MOD_LINEAR)) {
+                std::array options = {10, 16, 8};
+                if (m_surface->iccProfile) {
+                    // assumption: if you've got an ICC profile set, you care more about color than about power usage
+                    // TODO add a setting for this sort of preference instead
+                    std::swap(options[0], options[1]);
+                }
+                for (const uint32_t bitsPerColor : options) {
+                    for (auto it = formats.begin(); it != formats.end(); it++) {
+                        const auto info = FormatInfo::get(it.key());
+                        if (!info || info->bitsPerColor != bitsPerColor) {
                             continue;
                         }
-                        mods = {DRM_FORMAT_MOD_LINEAR};
-                    }
-                    m_surface->shadowSwapchain = EglSwapchain::create(m_eglBackend->drmDevice()->allocator(), m_eglBackend->openglContext(), m_surface->gbmSwapchain->size(), it.key(), mods);
-                    if (m_surface->shadowSwapchain) {
-                        break;
+                        if (requireAlpha && info->alphaBits == 0) {
+                            continue;
+                        }
+                        auto mods = it.value();
+                        if (info->floatingPoint && m_eglBackend->gpu()->isAmdgpu() && qEnvironmentVariableIntValue("KWIN_DRM_NO_DCC_WORKAROUND") == 0) {
+                            // using modifiers with DCC here causes glitches on amdgpu: https://gitlab.freedesktop.org/mesa/mesa/-/issues/10875
+                            if (!mods.contains(DRM_FORMAT_MOD_LINEAR)) {
+                                continue;
+                            }
+                            mods = {DRM_FORMAT_MOD_LINEAR};
+                        }
+                        m_surface->shadowSwapchain = EglSwapchain::create(m_eglBackend->drmDevice()->allocator(), m_eglBackend->openglContext(), m_surface->gbmSwapchain->size(), it.key(), mods);
+                        if (m_surface->shadowSwapchain) {
+                            return;
+                        }
                     }
                 }
             };
@@ -170,7 +178,7 @@ std::optional<OutputLayerBeginFrameInfo> EglGbmLayerSurface::startRendering(cons
         m_surface->shadowSwapchain.reset();
         m_surface->currentShadowSlot.reset();
         return OutputLayerBeginFrameInfo{
-            .renderTarget = RenderTarget(m_surface->currentSlot->framebuffer()),
+            .renderTarget = RenderTarget(m_surface->currentSlot->framebuffer(), m_surface->intermediaryColorDescription),
             .repaint = repaint,
         };
     }
@@ -178,7 +186,7 @@ std::optional<OutputLayerBeginFrameInfo> EglGbmLayerSurface::startRendering(cons
 
 bool EglGbmLayerSurface::endRendering(const QRegion &damagedRegion, OutputFrame *frame)
 {
-    if (m_surface->colormanagementEnabled) {
+    if (m_surface->needsShadowBuffer) {
         GLFramebuffer *fbo = m_surface->currentSlot->framebuffer();
         GLFramebuffer::pushFramebuffer(fbo);
         ShaderBinder binder = m_surface->iccShader ? ShaderBinder(m_surface->iccShader->shader()) : ShaderBinder(ShaderTrait::MapTexture | ShaderTrait::TransformColorspace);
diff --git a/src/backends/drm/drm_egl_layer_surface.h b/src/backends/drm/drm_egl_layer_surface.h
index faf60671d3..f501c8c162 100644
--- a/src/backends/drm/drm_egl_layer_surface.h
+++ b/src/backends/drm/drm_egl_layer_surface.h
@@ -56,7 +56,7 @@ public:
     EglGbmLayerSurface(DrmGpu *gpu, EglGbmBackend *eglBackend, BufferTarget target = BufferTarget::Normal, FormatOption formatOption = FormatOption::PreferAlpha);
     ~EglGbmLayerSurface();
 
-    std::optional<OutputLayerBeginFrameInfo> startRendering(const QSize &bufferSize, OutputTransform transformation, const QHash<uint32_t, QList<uint64_t>> &formats, const ColorDescription &colorDescription, const QVector3D &channelFactors, const std::shared_ptr<IccProfile> &iccProfile, bool enableColormanagement);
+    std::optional<OutputLayerBeginFrameInfo> startRendering(const QSize &bufferSize, OutputTransform transformation, const QHash<uint32_t, QList<uint64_t>> &formats, const ColorDescription &colorDescription, const QVector3D &channelFactors, const std::shared_ptr<IccProfile> &iccProfile);
     bool endRendering(const QRegion &damagedRegion, OutputFrame *frame);
 
     bool doesSurfaceFit(const QSize &size, const QHash<uint32_t, QList<uint64_t>> &formats) const;
@@ -95,7 +95,7 @@ private:
         BufferTarget bufferTarget;
 
         // for color management
-        bool colormanagementEnabled = false;
+        bool needsShadowBuffer = false;
         std::shared_ptr<EglSwapchain> shadowSwapchain;
         std::shared_ptr<EglSwapchainSlot> currentShadowSlot;
         ColorDescription targetColorDescription = ColorDescription::sRGB;
diff --git a/src/backends/drm/drm_output.cpp b/src/backends/drm/drm_output.cpp
index 1eb991e7b6..ea35c40d7d 100644
--- a/src/backends/drm/drm_output.cpp
+++ b/src/backends/drm/drm_output.cpp
@@ -168,7 +168,8 @@ bool DrmOutput::setDrmDpmsMode(DpmsMode mode)
         if (active) {
             m_renderLoop->uninhibit();
             m_renderLoop->scheduleRepaint();
-            doSetChannelFactors(m_channelFactors);
+            // re-set KMS color pipeline stuff
+            tryKmsColorOffloading();
         } else {
             m_renderLoop->inhibit();
         }
@@ -338,14 +339,15 @@ bool DrmOutput::queueChanges(const std::shared_ptr<OutputChangeSet> &props)
     m_pipeline->setRgbRange(props->rgbRange.value_or(m_pipeline->rgbRange()));
     m_pipeline->setEnable(props->enabled.value_or(m_pipeline->enabled()));
     m_pipeline->setColorDescription(createColorDescription(props));
-    if (bt2020 || hdr) {
+    if (bt2020 || hdr || props->colorProfileSource.value_or(m_state.colorProfileSource) != ColorProfileSource::ICC) {
         // ICC profiles don't support HDR (yet)
         m_pipeline->setIccProfile(nullptr);
     } else {
         m_pipeline->setIccProfile(props->iccProfile.value_or(m_state.iccProfile));
     }
-    if (bt2020 || hdr || props->colorProfileSource.value_or(m_state.colorProfileSource) != ColorProfileSource::sRGB) {
-        // remove unused gamma ramp and ctm, if present
+    // remove the color pipeline for the atomic test
+    // otherwise it could potentially fail
+    if (m_gpu->atomicModeSetting()) {
         m_pipeline->setCrtcColorPipeline(ColorPipeline{});
     }
     return true;
@@ -359,7 +361,7 @@ ColorDescription DrmOutput::createColorDescription(const std::shared_ptr<OutputC
     const auto iccProfile = props->iccProfile.value_or(m_state.iccProfile);
     if (colorSource == ColorProfileSource::ICC && !hdr && !wcg && iccProfile) {
         const double brightness = iccProfile->brightness().value_or(200);
-        return ColorDescription(iccProfile->colorimetry(), TransferFunction::gamma22, brightness, 0, brightness, brightness);
+        return ColorDescription(iccProfile->colorimetry(), TransferFunction(TransferFunction::gamma22, 0, brightness), brightness, 0, brightness, brightness);
     }
     const bool screenSupportsHdr = m_connector->edid()->isValid() && m_connector->edid()->supportsBT2020() && m_connector->edid()->supportsPQ();
     const bool driverSupportsHdr = m_connector->colorspace.isValid() && m_connector->hdrMetadata.isValid() && (m_connector->colorspace.hasEnum(DrmConnector::Colorspace::BT2020_RGB) || m_connector->colorspace.hasEnum(DrmConnector::Colorspace::BT2020_YCC));
@@ -371,12 +373,12 @@ ColorDescription DrmOutput::createColorDescription(const std::shared_ptr<OutputC
     const Colorimetry masteringColorimetry = (effectiveWcg || colorSource == ColorProfileSource::EDID) ? nativeColorimetry : Colorimetry::fromName(NamedColorimetry::BT709);
     const Colorimetry sdrColorimetry = effectiveWcg ? Colorimetry::fromName(NamedColorimetry::BT709).interpolateGamutTo(nativeColorimetry, props->sdrGamutWideness.value_or(m_state.sdrGamutWideness)) : Colorimetry::fromName(NamedColorimetry::BT709);
     // TODO the EDID can contain a gamma value, use that when available and colorSource == ColorProfileSource::EDID
-    const TransferFunction transferFunction = effectiveHdr ? TransferFunction::PerceptualQuantizer : TransferFunction::gamma22;
+    const TransferFunction transferFunction{effectiveHdr ? TransferFunction::PerceptualQuantizer : TransferFunction::gamma22};
     const double minBrightness = effectiveHdr ? props->minBrightnessOverride.value_or(m_state.minBrightnessOverride).value_or(m_connector->edid()->desiredMinLuminance()) : 0;
     const double maxAverageBrightness = effectiveHdr ? props->maxAverageBrightnessOverride.value_or(m_state.maxAverageBrightnessOverride).value_or(m_connector->edid()->desiredMaxFrameAverageLuminance().value_or(m_state.referenceLuminance)) : 200;
     const double maxPeakBrightness = effectiveHdr ? props->maxPeakBrightnessOverride.value_or(m_state.maxPeakBrightnessOverride).value_or(m_connector->edid()->desiredMaxLuminance().value_or(800)) : 200;
     const double referenceLuminance = effectiveHdr ? props->referenceLuminance.value_or(m_state.referenceLuminance) : maxPeakBrightness;
-    return ColorDescription(containerColorimetry, transferFunction, referenceLuminance, minBrightness, maxAverageBrightness, maxPeakBrightness, masteringColorimetry, sdrColorimetry);
+    return ColorDescription(containerColorimetry, transferFunction.relativeScaledTo(referenceLuminance), referenceLuminance, minBrightness, maxAverageBrightness, maxPeakBrightness, masteringColorimetry, sdrColorimetry);
 }
 
 void DrmOutput::applyQueuedChanges(const std::shared_ptr<OutputChangeSet> &props)
@@ -429,8 +431,7 @@ void DrmOutput::applyQueuedChanges(const std::shared_ptr<OutputChangeSet> &props
     m_renderLoop->setRefreshRate(refreshRate());
     m_renderLoop->scheduleRepaint();
 
-    // re-set the CTM and/or gamma lut, if necessary
-    doSetChannelFactors(m_channelFactors);
+    tryKmsColorOffloading();
 
     Q_EMIT changed();
 }
@@ -444,6 +445,8 @@ void DrmOutput::setBrightnessDevice(BrightnessDevice *device)
         } else {
             device->setBrightness(m_state.brightness);
         }
+        // reset the brightness factors
+        tryKmsColorOffloading();
     }
 }
 
@@ -464,37 +467,52 @@ DrmOutputLayer *DrmOutput::cursorLayer() const
 
 bool DrmOutput::setChannelFactors(const QVector3D &rgb)
 {
-    return m_channelFactors == rgb || doSetChannelFactors(rgb);
+    if (rgb != m_channelFactors) {
+        m_channelFactors = rgb;
+        tryKmsColorOffloading();
+    }
+    return true;
 }
 
-bool DrmOutput::doSetChannelFactors(const QVector3D &rgb)
+void DrmOutput::tryKmsColorOffloading()
 {
-    m_renderLoop->scheduleRepaint();
-    m_channelFactors = rgb;
-    if (m_state.wideColorGamut || m_state.highDynamicRange || m_state.colorProfileSource != ColorProfileSource::sRGB) {
-        // the shader "fallback" is always active
-        return true;
+    if (m_state.colorProfileSource == ColorProfileSource::ICC && m_state.iccProfile) {
+        // offloading color operations doesn't make sense when we have to apply the icc shader anyways
+        m_scanoutColorDescription = colorDescription();
+        m_pipeline->setCrtcColorPipeline(ColorPipeline{});
+        m_pipeline->applyPendingChanges();
+        return;
     }
     if (!m_pipeline->activePending()) {
-        return false;
+        return;
     }
     // TODO this doesn't allow using only a CTM for night light offloading
     // maybe relax correctness in that case and apply night light in non-linear space?
-    ColorPipeline pipeline{ValueRange{}};
-    pipeline.addTransferFunction(m_state.colorDescription.transferFunction());
-    pipeline.addMultiplier(rgb);
-    pipeline.addInverseTransferFunction(m_state.colorDescription.transferFunction());
-    m_pipeline->setCrtcColorPipeline(pipeline);
+    const QVector3D channelFactors = effectiveChannelFactors();
+    const double maxLuminance = colorDescription().maxHdrLuminance().value_or(colorDescription().referenceLuminance());
+    const ColorDescription optimal = colorDescription().transferFunction().type == TransferFunction::gamma22 ? colorDescription() : colorDescription().withTransferFunction(TransferFunction(TransferFunction::gamma22, 0, maxLuminance));
+    ColorPipeline colorPipeline = ColorPipeline::create(optimal, colorDescription());
+    colorPipeline.addTransferFunction(colorDescription().transferFunction());
+    colorPipeline.addMultiplier(channelFactors);
+    colorPipeline.addInverseTransferFunction(colorDescription().transferFunction());
+    m_pipeline->setCrtcColorPipeline(colorPipeline);
     if (DrmPipeline::commitPipelines({m_pipeline}, DrmPipeline::CommitMode::Test) == DrmPipeline::Error::None) {
         m_pipeline->applyPendingChanges();
+        m_scanoutColorDescription = optimal;
         m_channelFactorsNeedShaderFallback = false;
-        return true;
     } else {
+        // fall back to using a shadow buffer for doing blending in gamma 2.2 and the channel factors
+        m_pipeline->revertPendingChanges();
         m_pipeline->setCrtcColorPipeline(ColorPipeline{});
         m_pipeline->applyPendingChanges();
+        m_scanoutColorDescription = colorDescription();
+        m_channelFactorsNeedShaderFallback = (channelFactors - QVector3D(1, 1, 1)).lengthSquared() > 0.0001;
     }
-    m_channelFactorsNeedShaderFallback = m_channelFactors != QVector3D{1, 1, 1};
-    return true;
+}
+
+bool DrmOutput::needsChannelFactorFallback() const
+{
+    return m_channelFactorsNeedShaderFallback;
 }
 
 QVector3D DrmOutput::effectiveChannelFactors() const
@@ -512,10 +530,9 @@ QVector3D DrmOutput::effectiveChannelFactors() const
     }
 }
 
-bool DrmOutput::needsColormanagement() const
+const ColorDescription &DrmOutput::scanoutColorDescription() const
 {
-    static bool forceColorManagement = qEnvironmentVariableIntValue("KWIN_DRM_FORCE_COLOR_MANAGEMENT") != 0;
-    return forceColorManagement || m_state.wideColorGamut || m_state.highDynamicRange || m_state.colorProfileSource != ColorProfileSource::sRGB || m_channelFactorsNeedShaderFallback;
+    return m_scanoutColorDescription;
 }
 }
 
diff --git a/src/backends/drm/drm_output.h b/src/backends/drm/drm_output.h
index 78f7b5afc0..ff4837d896 100644
--- a/src/backends/drm/drm_output.h
+++ b/src/backends/drm/drm_output.h
@@ -62,14 +62,21 @@ public:
      * channel factors adapted to the target color space + brightness setting multiplied in
      */
     QVector3D effectiveChannelFactors() const;
-    bool needsColormanagement() const;
-
     void updateConnectorProperties();
 
+    /**
+     * @returns the color description / encoding that the buffers passed to the CRTC need to have, without a color pipeline to change it
+     */
+    const ColorDescription &scanoutColorDescription() const;
+    /**
+     * @returns whether or not the renderer should apply channel factors
+     */
+    bool needsChannelFactorFallback() const;
+
 private:
     bool setDrmDpmsMode(DpmsMode mode);
     void setDpmsMode(DpmsMode mode) override;
-    bool doSetChannelFactors(const QVector3D &rgb);
+    void tryKmsColorOffloading();
     ColorDescription createColorDescription(const std::shared_ptr<OutputChangeSet> &props) const;
     Capabilities computeCapabilities() const;
     void updateInformation();
@@ -86,6 +93,7 @@ private:
 
     QVector3D m_channelFactors = {1, 1, 1};
     bool m_channelFactorsNeedShaderFallback = false;
+    ColorDescription m_scanoutColorDescription = ColorDescription::sRGB;
 };
 
 }
diff --git a/src/backends/drm/drm_pipeline.cpp b/src/backends/drm/drm_pipeline.cpp
index 2eb74a89de..65f5d05c75 100644
--- a/src/backends/drm/drm_pipeline.cpp
+++ b/src/backends/drm/drm_pipeline.cpp
@@ -216,11 +216,7 @@ DrmPipeline::Error DrmPipeline::prepareAtomicPresentation(DrmAtomicCommit *commi
     if (m_cursorLayer->isEnabled() && m_primaryLayer->colorPipeline() != m_cursorLayer->colorPipeline()) {
         return DrmPipeline::Error::InvalidArguments;
     }
-    if (!m_pending.crtcColorPipeline.isIdentity() && !m_primaryLayer->colorPipeline().isIdentity()) {
-        // TODO merge the pipelines instead?
-        return DrmPipeline::Error::InvalidArguments;
-    }
-    const auto &colorPipeline = m_pending.crtcColorPipeline.isIdentity() ? m_primaryLayer->colorPipeline() : m_pending.crtcColorPipeline;
+    const ColorPipeline colorPipeline = m_primaryLayer->colorPipeline().merged(m_pending.crtcColorPipeline);
     if (!m_pending.crtc->postBlendingPipeline) {
         if (!colorPipeline.isIdentity()) {
             return Error::InvalidArguments;
@@ -324,7 +320,7 @@ bool DrmPipeline::prepareAtomicModeset(DrmAtomicCommit *commit)
     }
     if (m_connector->hdrMetadata.isValid()) {
         commit->addBlob(m_connector->hdrMetadata, createHdrMetadata(m_pending.colorDescription.transferFunction()));
-    } else if (m_pending.colorDescription.transferFunction() != TransferFunction::gamma22) {
+    } else if (m_pending.colorDescription.transferFunction().type != TransferFunction::gamma22) {
         return false;
     }
     if (m_pending.colorDescription.containerColorimetry() == NamedColorimetry::BT2020) {
@@ -680,14 +676,12 @@ void DrmPipeline::setContentType(DrmConnector::DrmContentType type)
 
 void DrmPipeline::setIccProfile(const std::shared_ptr<IccProfile> &profile)
 {
-    if (m_pending.iccProfile != profile) {
-        m_pending.iccProfile = profile;
-    }
+    m_pending.iccProfile = profile;
 }
 
 std::shared_ptr<DrmBlob> DrmPipeline::createHdrMetadata(TransferFunction transferFunction) const
 {
-    if (transferFunction != TransferFunction::PerceptualQuantizer) {
+    if (transferFunction.type != TransferFunction::PerceptualQuantizer) {
         // for sRGB / gamma 2.2, don't send any metadata, to ensure the non-HDR experience stays the same
         return nullptr;
     }
diff --git a/src/backends/x11/standalone/x11_standalone_output.cpp b/src/backends/x11/standalone/x11_standalone_output.cpp
index b0ee95fbb9..ea15b3cf46 100644
--- a/src/backends/x11/standalone/x11_standalone_output.cpp
+++ b/src/backends/x11/standalone/x11_standalone_output.cpp
@@ -47,9 +47,9 @@ bool X11Output::setChannelFactors(const QVector3D &rgb)
         return true;
     }
     ColorPipeline pipeline;
-    pipeline.addTransferFunction(TransferFunction::gamma22);
+    pipeline.addTransferFunction(TransferFunction(TransferFunction::gamma22));
     pipeline.addMultiplier(rgb);
-    pipeline.addInverseTransferFunction(TransferFunction::gamma22);
+    pipeline.addInverseTransferFunction(TransferFunction(TransferFunction::gamma22));
     std::vector<uint16_t> red(m_gammaRampSize);
     std::vector<uint16_t> green(m_gammaRampSize);
     std::vector<uint16_t> blue(m_gammaRampSize);
diff --git a/src/compositor_wayland.cpp b/src/compositor_wayland.cpp
index c285cfe046..4c08a21ccb 100644
--- a/src/compositor_wayland.cpp
+++ b/src/compositor_wayland.cpp
@@ -287,7 +287,7 @@ static bool checkForBlackBackground(SurfaceItem *background)
     }
     const QRgb rgb = view.image()->pixel(0, 0);
     const QVector3D encoded(qRed(rgb) / 255.0, qGreen(rgb) / 255.0, qBlue(rgb) / 255.0);
-    const QVector3D nits = background->colorDescription().mapTo(encoded, ColorDescription(NamedColorimetry::BT709, TransferFunction::linear, 100, 0, std::nullopt, std::nullopt));
+    const QVector3D nits = background->colorDescription().mapTo(encoded, ColorDescription(NamedColorimetry::BT709, TransferFunction(TransferFunction::linear), 100, 0, std::nullopt, std::nullopt));
     // below 0.1 nits, it shouldn't be noticeable that we replace it with black
     return nits.lengthSquared() <= (0.1 * 0.1);
 }
diff --git a/src/core/colorpipeline.cpp b/src/core/colorpipeline.cpp
index f1e6f72e66..65f58e4320 100644
--- a/src/core/colorpipeline.cpp
+++ b/src/core/colorpipeline.cpp
@@ -106,7 +106,7 @@ void ColorPipeline::addTransferFunction(TransferFunction tf)
             }
         }
     }
-    if (tf == TransferFunction::linear) {
+    if (tf.type == TransferFunction::linear) {
         QMatrix4x4 mat;
         mat.translate(tf.minLuminance, tf.minLuminance, tf.minLuminance);
         mat.scale(tf.maxLuminance - tf.minLuminance);
@@ -136,7 +136,7 @@ void ColorPipeline::addInverseTransferFunction(TransferFunction tf)
             }
         }
     }
-    if (tf == TransferFunction::linear) {
+    if (tf.type == TransferFunction::linear) {
         QMatrix4x4 mat;
         mat.scale(1.0 / (tf.maxLuminance - tf.minLuminance));
         mat.translate(-tf.minLuminance, -tf.minLuminance, -tf.minLuminance);
@@ -242,7 +242,7 @@ void ColorPipeline::add(const ColorOp &op)
     }
 }
 
-ColorPipeline ColorPipeline::merge(const ColorPipeline &onTop)
+ColorPipeline ColorPipeline::merged(const ColorPipeline &onTop) const
 {
     ColorPipeline ret{inputRange};
     ret.ops = ops;
diff --git a/src/core/colorpipeline.h b/src/core/colorpipeline.h
index 34fc9740a8..568cf3268d 100644
--- a/src/core/colorpipeline.h
+++ b/src/core/colorpipeline.h
@@ -81,7 +81,7 @@ public:
 
     static ColorPipeline create(const ColorDescription &from, const ColorDescription &to);
 
-    ColorPipeline merge(const ColorPipeline &onTop);
+    ColorPipeline merged(const ColorPipeline &onTop) const;
 
     bool isIdentity() const;
     bool operator==(const ColorPipeline &other) const = default;
diff --git a/src/core/colorspace.cpp b/src/core/colorspace.cpp
index 778d19a98a..0c8a027aa9 100644
--- a/src/core/colorspace.cpp
+++ b/src/core/colorspace.cpp
@@ -198,7 +198,7 @@ const Colorimetry &Colorimetry::fromName(NamedColorimetry name)
     Q_UNREACHABLE();
 }
 
-const ColorDescription ColorDescription::sRGB = ColorDescription(NamedColorimetry::BT709, TransferFunction::gamma22, TransferFunction::defaultReferenceLuminanceFor(TransferFunction::gamma22), TransferFunction::defaultMinLuminanceFor(TransferFunction::gamma22), TransferFunction::defaultMaxLuminanceFor(TransferFunction::gamma22), TransferFunction::defaultMaxLuminanceFor(TransferFunction::gamma22));
+const ColorDescription ColorDescription::sRGB = ColorDescription(NamedColorimetry::BT709, TransferFunction(TransferFunction::gamma22), TransferFunction::defaultReferenceLuminanceFor(TransferFunction::gamma22), TransferFunction::defaultMinLuminanceFor(TransferFunction::gamma22), TransferFunction::defaultMaxLuminanceFor(TransferFunction::gamma22), TransferFunction::defaultMaxLuminanceFor(TransferFunction::gamma22));
 
 ColorDescription::ColorDescription(const Colorimetry &containerColorimetry, TransferFunction tf, double referenceLuminance, double minLuminance, std::optional<double> maxAverageLuminance, std::optional<double> maxHdrLuminance)
     : ColorDescription(containerColorimetry, tf, referenceLuminance, minLuminance, maxAverageLuminance, maxHdrLuminance, std::nullopt, Colorimetry::fromName(NamedColorimetry::BT709))
@@ -275,6 +275,11 @@ QVector3D ColorDescription::mapTo(QVector3D rgb, const ColorDescription &dst) co
     return dst.transferFunction().nitsToEncoded(rgb);
 }
 
+ColorDescription ColorDescription::withTransferFunction(const TransferFunction &func) const
+{
+    return ColorDescription(m_containerColorimetry, func, m_referenceLuminance, m_minLuminance, m_maxAverageLuminance, m_maxHdrLuminance, m_masteringColorimetry, m_sdrColorimetry);
+}
+
 double TransferFunction::defaultMinLuminanceFor(Type type)
 {
     switch (type) {
@@ -410,10 +415,19 @@ bool TransferFunction::isRelative() const
     }
     Q_UNREACHABLE();
 }
+
+TransferFunction TransferFunction::relativeScaledTo(double referenceLuminance) const
+{
+    if (isRelative()) {
+        return TransferFunction(type, minLuminance * referenceLuminance / maxLuminance, referenceLuminance);
+    } else {
+        return *this;
+    }
+}
 }
 
 QDebug operator<<(QDebug debug, const KWin::TransferFunction &tf)
 {
-    debug << "TransferFunction(" << tf.type << ")";
+    debug << "TransferFunction(" << tf.type << ", [" << tf.minLuminance << "," << tf.maxLuminance << "] )";
     return debug;
 }
diff --git a/src/core/colorspace.h b/src/core/colorspace.h
index 266cfbd2ce..df216d400f 100644
--- a/src/core/colorspace.h
+++ b/src/core/colorspace.h
@@ -96,12 +96,13 @@ public:
         PerceptualQuantizer = 2,
         gamma22 = 3,
     };
-    TransferFunction(Type tf);
+    explicit TransferFunction(Type tf);
     explicit TransferFunction(Type tf, double minLuminance, double maxLuminance);
 
     auto operator<=>(const TransferFunction &) const = default;
 
     bool isRelative() const;
+    TransferFunction relativeScaledTo(double referenceLuminance) const;
     double encodedToNits(double encoded) const;
     double nitsToEncoded(double nits) const;
     QVector3D encodedToNits(const QVector3D &encoded) const;
@@ -164,6 +165,7 @@ public:
     bool operator==(const ColorDescription &other) const = default;
 
     QVector3D mapTo(QVector3D rgb, const ColorDescription &other) const;
+    ColorDescription withTransferFunction(const TransferFunction &func) const;
 
     /**
      * This color description describes display-referred sRGB, with a gamma22 transfer function
diff --git a/src/wayland/frog_colormanagement_v1.cpp b/src/wayland/frog_colormanagement_v1.cpp
index e36e87b991..69a361ff3e 100644
--- a/src/wayland/frog_colormanagement_v1.cpp
+++ b/src/wayland/frog_colormanagement_v1.cpp
@@ -89,10 +89,10 @@ void FrogColorManagementSurfaceV1::frog_color_managed_surface_set_known_transfer
     case transfer_function_undefined:
     case transfer_function_srgb:
     case transfer_function_gamma_22:
-        m_transferFunction = TransferFunction::gamma22;
+        m_transferFunction = TransferFunction(TransferFunction::gamma22);
         break;
     case transfer_function_st2084_pq:
-        m_transferFunction = TransferFunction::PerceptualQuantizer;
+        m_transferFunction = TransferFunction(TransferFunction::PerceptualQuantizer);
         break;
     case transfer_function_scrgb_linear:
         m_transferFunction = TransferFunction(TransferFunction::linear, 0.0, 80.0);
diff --git a/src/wayland/frog_colormanagement_v1.h b/src/wayland/frog_colormanagement_v1.h
index 0da96b3120..bb39d6f5cf 100644
--- a/src/wayland/frog_colormanagement_v1.h
+++ b/src/wayland/frog_colormanagement_v1.h
@@ -53,7 +53,7 @@ private:
     void updateColorDescription();
 
     const QPointer<SurfaceInterface> m_surface;
-    TransferFunction m_transferFunction = TransferFunction::sRGB;
+    TransferFunction m_transferFunction{TransferFunction::sRGB};
     NamedColorimetry m_containerColorimetry = NamedColorimetry::BT709;
     std::optional<Colorimetry> m_masteringColorimetry;
     std::optional<double> m_maxAverageLuminance;
diff --git a/src/wayland/xx_colormanagement_v4.cpp b/src/wayland/xx_colormanagement_v4.cpp
index 2092a474cd..6faec836fc 100644
--- a/src/wayland/xx_colormanagement_v4.cpp
+++ b/src/wayland/xx_colormanagement_v4.cpp
@@ -178,7 +178,7 @@ void XXColorParametricCreatorV4::xx_image_description_creator_params_v4_create(R
         wl_resource_post_error(resource->handle, error::error_incomplete_set, "colorimetry or transfer function missing");
         return;
     }
-    if (m_transferFunction != TransferFunction::PerceptualQuantizer && (m_maxCll || m_maxFall)) {
+    if (m_transferFunction->type != TransferFunction::PerceptualQuantizer && (m_maxCll || m_maxFall)) {
         wl_resource_post_error(resource->handle, error::error_inconsistent_set, "max_cll and max_fall must only be set with the PQ transfer function");
         return;
     }
@@ -198,10 +198,10 @@ void XXColorParametricCreatorV4::xx_image_description_creator_params_v4_set_tf_n
     case XX_COLOR_MANAGER_V4_TRANSFER_FUNCTION_SRGB:
     case XX_COLOR_MANAGER_V4_TRANSFER_FUNCTION_BT709:
     case XX_COLOR_MANAGER_V4_TRANSFER_FUNCTION_GAMMA22:
-        m_transferFunction = TransferFunction::gamma22;
+        m_transferFunction = TransferFunction(TransferFunction::gamma22);
         return;
     case XX_COLOR_MANAGER_V4_TRANSFER_FUNCTION_ST2084_PQ:
-        m_transferFunction = TransferFunction::PerceptualQuantizer;
+        m_transferFunction = TransferFunction(TransferFunction::PerceptualQuantizer);
         return;
     default:
         // TODO add more transfer functions