pixelpipe_gpu.c

Go to the documentation of this file.

/*
    Private OpenCL pixelpipe backend.
*/
 
#include "develop/pipeline_notify.h"
#include "system/macros.h"
#include "develop/iop_profile.h"
#include "common/logging.h"
#include "caches/pixelpipe_cache_alloc.h"
#include "develop/iop_order.h"
#include "common/opencl.h"
#include "develop/blend.h"
#include "caches/pixelpipe_cache.h"
#include "develop/pixelpipe_cpu.h"
#include "develop/pixelpipe_gpu.h"
 
#include <math.h>
#include <stdio.h>
 
void dt_dev_pixelpipe_gpu_flush_host_pinned_images(dt_dev_pixelpipe_t *pipe, void *host_ptr,
                                                   dt_pixel_cache_entry_t *cache_entry, const char *reason)
{
#ifdef HAVE_OPENCL
  if(pipe && !pipe->realtime && pipe->devid >= 0 && host_ptr && cache_entry)
  {
    /* Non-realtime host writes invalidate reusable pinned images bound to the previous ROI/hash.
     * Realtime keeps its pinned reuse untouched to avoid stalling the live draw path. */
    if(dt_dev_pixelpipe_cache_flush_host_pinned_image(host_ptr, cache_entry,
                                                      pipe->devid))
      dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] flushed pinned OpenCL images after %s\n",
               reason ? reason : "host write");
  }
#else
  (void)pipe;
  (void)host_ptr;
  (void)cache_entry;
  (void)reason;
#endif
}
 
#ifdef HAVE_OPENCL
 
static int _is_opencl_supported(dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece, dt_iop_module_t *module)
{
  return dt_opencl_is_inited() && piece->process_cl_ready && module->process_cl;
}
 
static int _gpu_init_input(dt_dev_pixelpipe_t *pipe,
                           float **input, void **cl_mem_input,
                           const dt_dev_pixelpipe_iop_t *piece, dt_develop_tiling_t *tiling,
                           dt_pixel_cache_entry_t *input_entry, dt_pixel_cache_entry_t *output_entry)
{
  dt_iop_module_t *module = piece->module;
 
  // An input that only lives in RAM (its producer ran on CPU, or it came from the cache) is
  // already the authoritative copy, and there is no device buffer to read back.
  // dt_dev_pixelpipe_cache_sync_cl_buffer() fails on a NULL device buffer, which sent every such
  // input of an OpenCL-tiled module to the CPU.
  if(!IS_NULL_PTR(*input) && IS_NULL_PTR(*cl_mem_input)) return 0;
 
  if(IS_NULL_PTR(*input))
  {
    dt_dev_pixelpipe_cache_wrlock_entry(TRUE, input_entry);
    *input = dt_pixel_cache_alloc(input_entry);
    dt_dev_pixelpipe_cache_wrlock_entry(FALSE, input_entry);
  }
 
  if(IS_NULL_PTR(*input))
  {
    dt_print(DT_DEBUG_OPENCL,
             "[dev_pixelpipe] %s CPU fallback has no input buffer (cache allocation failed?)\n",
             module->name());
    return 1;
  }
 
  dt_dev_pixelpipe_cache_wrlock_entry(TRUE, input_entry);
  const int fail = dt_dev_pixelpipe_cache_sync_cl_buffer(pipe->devid, *input, *cl_mem_input, &piece->roi_in, CL_MAP_READ,
                                          piece->dsc_in.bpp, module,
                                          "cpu fallback input copy to cache");
  if(!fail) dt_dev_pixelpipe_cache_flag_host_written(input_entry);
  dt_dev_pixelpipe_cache_wrlock_entry(FALSE, input_entry);
 
  if(fail)
  {
    dt_print(DT_DEBUG_OPENCL,
             "[dev_pixelpipe] %s couldn't resync GPU input to cache for CPU fallback\n",
             module->name());
    return 1;
  }
  return 0;
}
 
static int _gpu_early_cpu_fallback_if_unsupported(dt_dev_pixelpipe_t *pipe, float **input,
                                                  void **cl_mem_input,
                                                  gboolean *const borrowed_cl_mem_input,
                                                  const dt_dev_pixelpipe_iop_t *piece,
                                                  const dt_dev_pixelpipe_iop_t *previous_piece,
                                                  dt_develop_tiling_t *tiling,
                                                  dt_pixelpipe_flow_t *pixelpipe_flow,
                                                  gboolean *const cache_output,
                                                  dt_pixel_cache_entry_t *input_entry,
                                                  dt_pixel_cache_entry_t *output_entry)
{
  dt_iop_module_t *module = piece->module;
 
  dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] %s will run directly on CPU\n", module->name());
 
  if(module->flags() & IOP_FLAGS_TAKE_NO_INPUT)
    return pixelpipe_process_on_CPU(pipe, piece, previous_piece, tiling, pixelpipe_flow,
                                    cache_output, input_entry, output_entry);
 
  /* CPU fallback only needs a valid host buffer. If `input` already exists here, the upstream
   * hand-off has already materialized authoritative RAM and re-reading the same pixels back out
   * of the cached OpenCL image is redundant. A host buffer left over from the entry's previous
   * hash is not `input`: dt_pixel_cache_entry_get_data() reported it as NULL. */
  if(input && !IS_NULL_PTR(*input))
  {
    dt_print(DT_DEBUG_OPENCL,
             "[dev_pixelpipe] %s CPU fallback will reuse host input\n",
             module->name());
  }
  else if(cl_mem_input && !IS_NULL_PTR(*cl_mem_input))
  {
    if(input && IS_NULL_PTR(*input))
    {
      dt_dev_pixelpipe_cache_wrlock_entry(TRUE, input_entry);
      *input = dt_pixel_cache_alloc(input_entry);
      dt_dev_pixelpipe_cache_wrlock_entry(FALSE, input_entry);
    }
 
    if(IS_NULL_PTR(input) || IS_NULL_PTR(*input))
    {
      dt_print(DT_DEBUG_OPENCL,
               "[dev_pixelpipe] %s CPU fallback has no input buffer (cache allocation failed?)\n",
               module->name());
      if(borrowed_cl_mem_input && *borrowed_cl_mem_input)
      {
        dt_dev_pixelpipe_cache_return_cl_payload(input_entry, *cl_mem_input);
        *cl_mem_input = NULL;
        *borrowed_cl_mem_input = FALSE;
      }
      else
        dt_dev_pixelpipe_cache_release_cl_buffer(cl_mem_input, input_entry, NULL,
                                          dt_dev_pixelpipe_cache_gpu_device_buffer(pipe, input_entry));
      return 1;
    }
 
    *input = dt_dev_pixelpipe_cache_restore_cl_buffer(pipe, *input, *cl_mem_input, &piece->roi_in, module,
                                        piece->dsc_in.bpp, input_entry,
                                        "cpu fallback input copy to cache");
    if(IS_NULL_PTR(*input))
    {
      dt_print(DT_DEBUG_OPENCL,
               "[dev_pixelpipe] %s couldn't resync GPU input to cache for CPU fallback\n",
               module->name());
      if(borrowed_cl_mem_input && *borrowed_cl_mem_input)
      {
        dt_dev_pixelpipe_cache_return_cl_payload(input_entry, *cl_mem_input);
        *cl_mem_input = NULL;
        *borrowed_cl_mem_input = FALSE;
      }
      else
        dt_dev_pixelpipe_cache_release_cl_buffer(cl_mem_input, input_entry, NULL,
                                          dt_dev_pixelpipe_cache_gpu_device_buffer(pipe, input_entry));
      return 1;
    }
  }
  else if(!input || IS_NULL_PTR(*input))
  {
    dt_print(DT_DEBUG_OPENCL,
             "[dev_pixelpipe] %s CPU fallback has no input buffer (cache allocation failed?)\n",
             module->name());
    return 1;
  }
 
  if(borrowed_cl_mem_input && *borrowed_cl_mem_input)
  {
    /* Device-only inputs borrowed from the cache stay owned by the cache entry.
     * CPU fallback only needs to drop the temporary borrow after the device->host
     * sync, otherwise releasing the cl_mem here leaves a stale cache-side pointer
     * that later thumbnail runs may reopen as corrupted input. */
    dt_dev_pixelpipe_cache_return_cl_payload(input_entry, *cl_mem_input);
    *cl_mem_input = NULL;
    *borrowed_cl_mem_input = FALSE;
  }
  else
    dt_dev_pixelpipe_cache_release_cl_buffer(cl_mem_input, input_entry, *input,
                                      dt_dev_pixelpipe_cache_gpu_device_buffer(pipe, input_entry));
 
  return pixelpipe_process_on_CPU(pipe, piece, previous_piece, tiling, pixelpipe_flow,
                                  cache_output, input_entry, output_entry);
}
 
int pixelpipe_process_on_GPU(dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece,
                             const dt_dev_pixelpipe_iop_t *previous_piece,
                             dt_develop_tiling_t *tiling,
                             dt_pixelpipe_flow_t *pixelpipe_flow,
                             gboolean *const cache_output,
                             dt_pixel_cache_entry_t *input_entry, dt_pixel_cache_entry_t *output_entry)
{
  dt_iop_module_t *module = piece->module;
  /* The module's timed region is exactly this function, so a module that costs more than its
   * own process_cl() is spending the difference in here -- input colourspace transform, buffer
   * acquisition, blend, readback. Split the prologue from the kernel so the log says which. */
  const gint64 gpu_stage_t0 = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
  /* The prologue's two candidate costs, split out because guessing between them has already
   * been wrong once: borrowing the upstream vRAM payload, and allocating this node's own host
   * output -- the latter going through the cache allocator, which evicts to make room. */
  double gpu_in_borrow_ms = 0.0;
  double gpu_out_alloc_ms = 0.0;
  double gpu_in_prepare_ms = 0.0;
  double gpu_out_cl_ms = 0.0;
  double gpu_cst_ms = 0.0;
  gboolean gpu_out_cl_reused = FALSE;
  float *input = input_entry ? dt_pixel_cache_entry_get_data(input_entry) : NULL;
  // The buffer, not its pixels: a rekeyed output's host buffer backs the pinned output image even
  // when the output is not copied back to RAM.
  void *output = dt_pixel_cache_entry_get_buffer(output_entry);
  void *cl_mem_input = NULL;
  void *cl_mem_output = NULL;
  void *cl_mem_process_input = NULL;
  void *cl_mem_blend_input = NULL;
  void *cl_mem_blend_output = NULL;
  void *cl_mem_process_input_temp = NULL;
  void *cl_mem_blend_input_temp = NULL;
  void *cl_mem_blend_output_temp = NULL;
  dt_pixel_cache_entry_t *cpu_input_entry = input_entry;
  dt_pixel_cache_entry_t *locked_input_entry = NULL;
  gboolean borrowed_cl_mem_input = FALSE;
  const dt_iop_buffer_dsc_t actual_input_dsc = previous_piece ? previous_piece->dsc_out : pipe->dev->image_storage.dsc;
  dt_iop_buffer_dsc_t process_input_dsc = actual_input_dsc;
  dt_iop_buffer_dsc_t blend_input_dsc = actual_input_dsc;
  dt_iop_buffer_dsc_t blend_output_dsc = piece->dsc_out;
 
  // Try to reuse the cached vRAM buffer for the input entry if available 
  // except for basebuffer module which takes no input
  if(!(piece->module->flags() & IOP_FLAGS_TAKE_NO_INPUT))
  {
    const gint64 borrow_t0 = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
    cl_mem_input = dt_dev_pixelpipe_cache_borrow_cl_payload(input_entry, pipe->devid,
                                            piece->roi_in.width, piece->roi_in.height,
                                            actual_input_dsc.bpp);
    if(dt_get_debug_flags() & DT_DEBUG_PERF)
      gpu_in_borrow_ms = (g_get_monotonic_time() - borrow_t0) / 1000.0;
    borrowed_cl_mem_input = (!IS_NULL_PTR(cl_mem_input));
    if(IS_NULL_PTR(cl_mem_input))
      dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] %s could not get a cached vRAM input buffer.\n", module->name());
      
    // Note: if that fails, we will attempt resync from RAM cache later
 
    if(IS_NULL_PTR(input) && IS_NULL_PTR(cl_mem_input))
    {
      dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] %s has no RAM nor vRAM input... aborting.\n", module->name());
      return 1;
    }
  }
 
  if(!_is_opencl_supported(pipe, piece, module) || !pipe->opencl_enabled || !(pipe->devid >= 0))
  {
    return _gpu_early_cpu_fallback_if_unsupported(pipe, &input, &cl_mem_input,
                                                  &borrowed_cl_mem_input, piece, previous_piece, tiling,
                                                  pixelpipe_flow, cache_output,
                                                  input_entry, output_entry);
  }
 
  const dt_iop_order_iccprofile_info_t *const work_profile
      = (process_input_dsc.cst != IOP_CS_RAW || piece->dsc_in.cst != IOP_CS_RAW)
            ? dt_ioppr_get_pipe_work_profile_info(pipe)
            : NULL;
 
  const float required_factor_cl
      = fmaxf(1.0f, (!IS_NULL_PTR(cl_mem_input)) ? tiling->factor_cl - 1.0f : tiling->factor_cl);
 
  const size_t precheck_width = ROUNDUPDWD(MAX(piece->roi_in.width, piece->roi_out.width), pipe->devid);
  const size_t precheck_height = ROUNDUPDHT(MAX(piece->roi_in.height, piece->roi_out.height), pipe->devid);
  // Remember *why* the pre-check (dis)allowed OpenCL so the CPU-fallback message on the
  // error path can quote the limit that was actually exceeded instead of a fixed one.
  size_t fit_needed = 0, fit_limit = 0;
  dt_opencl_fit_reason_t fit_reason
      = dt_opencl_image_fits_device_reason(pipe->devid, precheck_width, precheck_height,
                                           MAX(piece->dsc_in.bpp, piece->dsc_out.bpp),
                                           required_factor_cl, tiling->overhead, &fit_needed, &fit_limit);
  gboolean fits_on_device = (fit_reason == DT_OPENCL_FIT_OK);
  if(!fits_on_device)
  {
    dt_print(DT_DEBUG_OPENCL,
             "[dev_pixelpipe] %s pre-check didn't fit on device, flushing cached pinned buffers and retrying\n",
             module->name());
    dt_dev_pixelpipe_cache_flush_clmem(pipe->devid);
    fit_reason = dt_opencl_image_fits_device_reason(pipe->devid, precheck_width, precheck_height,
                                                    MAX(piece->dsc_in.bpp, piece->dsc_out.bpp),
                                                    required_factor_cl, tiling->overhead, &fit_needed, &fit_limit);
    fits_on_device = (fit_reason == DT_OPENCL_FIT_OK);
  }
 
  gboolean possible_cl = !(pipe->type == DT_DEV_PIXELPIPE_PREVIEW
                           && (module->flags() & IOP_FLAGS_PREVIEW_NON_OPENCL))
                         && (fits_on_device || piece->process_tiling_ready);
 
  if(!possible_cl || !fits_on_device) *cache_output = TRUE;
  if(*cache_output && IS_NULL_PTR(output))
  {
    const gint64 alloc_t0 = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
    output = dt_pixel_cache_alloc(output_entry);
    if(dt_get_debug_flags() & DT_DEBUG_PERF)
      gpu_out_alloc_ms = (g_get_monotonic_time() - alloc_t0) / 1000.0;
    if(IS_NULL_PTR(output)) goto error;
  }
 
  if(possible_cl && !fits_on_device)
  {
    // Prepare the input buffer for tiling
    const float cl_px = dt_opencl_get_device_available(pipe->devid)
                        / (sizeof(float) * MAX(piece->dsc_in.bpp, piece->dsc_out.bpp)
                           * ceilf(required_factor_cl));
    const float dx = MAX(piece->roi_in.width, piece->roi_out.width);
    const float dy = MAX(piece->roi_in.height, piece->roi_out.height);
    const float border = tiling->overlap + 1;
    const gboolean possible = (cl_px > dx * border) || (cl_px > dy * border) || (cl_px > border * border);
    if(!possible)
    {
      dt_print(DT_DEBUG_OPENCL | DT_DEBUG_TILING,
               "[dt_dev_pixelpipe_process_rec] CL: tiling impossible in module `%s'. avail=%.1fM, requ=%.1fM (%ix%i). overlap=%i\n",
               module->name(), cl_px / 1e6f, dx * dy / 1e6f, (int)dx, (int)dy, (int)tiling->overlap);
      goto error;
    }
 
    // Ensure the input image is present on RAM cache,
    // tiling on OpenCL will only copy tiles from it to GPU.
    if(_gpu_init_input(pipe, &input, &cl_mem_input, piece, tiling,
                      input_entry, output_entry))
      goto error;
  }
 
  if(!possible_cl) goto error;
 
  if(fits_on_device)
  {
    // Alloc input GPU buffer if we didn't already borrow it
    const gint64 inprep_t0 = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
    if(!(piece->module->flags() & IOP_FLAGS_TAKE_NO_INPUT))
      if(dt_dev_pixelpipe_cache_prepare_cl_input(pipe, module, input, &cl_mem_input,
                              &piece->roi_in, piece->dsc_in.bpp, input_entry,
                              &locked_input_entry, NULL))
        goto error;
    if(dt_get_debug_flags() & DT_DEBUG_PERF)
      gpu_in_prepare_ms = (g_get_monotonic_time() - inprep_t0) / 1000.0;
 
    cl_mem_process_input = cl_mem_input;
 
    /* The output DEVICE buffer. Suspected of being where display encoding's prologue goes:
     * its output is 4 bpp, the only such size in a 16 bpp pipe, so it can never be served by
     * a device buffer another module just released -- unlike colorout, whose prologue is a
     * twentieth of it. Measured rather than assumed, because the two obvious candidates
     * before it (the input borrow, the host output allocation) both came back at 0.00. */
    const gint64 outcl_t0 = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
    // Alloc output GPU buffer - non-optional
    cl_mem_output = dt_dev_pixelpipe_cache_get_cl_buffer(pipe->devid, output, &piece->roi_out, piece->dsc_out.bpp, module,
                                                         "output", output_entry,
                                                         &gpu_out_cl_reused, cl_mem_input);
    if(dt_get_debug_flags() & DT_DEBUG_PERF)
      gpu_out_cl_ms = (g_get_monotonic_time() - outcl_t0) / 1000.0;
    if(IS_NULL_PTR(cl_mem_output)) goto error;
    
    const int cst_before_cl = process_input_dsc.cst;
    const gint64 cst_t0 = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
    if(process_input_dsc.cst != piece->dsc_in.cst
       && !(dt_iop_colorspace_is_rgb(process_input_dsc.cst) && dt_iop_colorspace_is_rgb(piece->dsc_in.cst)))
    {
      cl_mem_process_input_temp = dt_dev_pixelpipe_cache_alloc_cl_device_buffer(pipe->devid, &piece->roi_in, piece->dsc_in.bpp,
                                                               module, "module input colorspace temp",
                                                               cl_mem_input);
      if(IS_NULL_PTR(cl_mem_process_input_temp))
        goto error;
 
      if(!dt_colorspaces_apply_profile_cl(module->op, module->multi_name, pipe->devid, cl_mem_input, cl_mem_process_input_temp,
                                                 piece->roi_in.width, piece->roi_in.height,
                                                 process_input_dsc.cst, piece->dsc_in.cst,
                                                 &process_input_dsc.cst, work_profile))
        goto error;
      cl_mem_process_input = cl_mem_process_input_temp;
    }
    else if(process_input_dsc.cst != piece->dsc_in.cst)
    {
      process_input_dsc.cst = piece->dsc_in.cst;
    }
    if(dt_get_debug_flags() & DT_DEBUG_PERF)
      gpu_cst_ms = (g_get_monotonic_time() - cst_t0) / 1000.0;
    const int cst_after_cl = process_input_dsc.cst;
 
    dt_dev_pixelpipe_debug_dump_module_io(pipe, module, "pre", TRUE, &piece->dsc_in, &piece->dsc_out,
                                          &piece->roi_in, &piece->roi_out,
                                          process_input_dsc.bpp, piece->dsc_out.bpp,
                                          cst_before_cl, cst_after_cl);
 
    const gint64 gpu_prologue_end = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
    if(!module->process_cl(module, pipe, piece, cl_mem_process_input, cl_mem_output))
      goto error;
    if(dt_get_debug_flags() & DT_DEBUG_PERF)
      dt_print(DT_DEBUG_PERF,
               "[dev_pixelpipe] %s gpu prologue=%.2f ms"
               " (inbuf=%.2f outalloc=%.2f inprep=%.2f outcl=%.2f cst=%.2f) process_cl=%.2f ms"
               " (in %dx%d bpp=%" G_GSIZE_FORMAT " -> out bpp=%" G_GSIZE_FORMAT
               " cache_out=%d outcl_reused=%d host=%p)\n",
               module->op, (gpu_prologue_end - gpu_stage_t0) / 1000.0,
               gpu_in_borrow_ms, gpu_out_alloc_ms, gpu_in_prepare_ms, gpu_out_cl_ms, gpu_cst_ms,
               (g_get_monotonic_time() - gpu_prologue_end) / 1000.0,
               piece->roi_in.width, piece->roi_in.height, process_input_dsc.bpp, piece->dsc_out.bpp,
               *cache_output ? 1 : 0, gpu_out_cl_reused ? 1 : 0, output);
 
    *pixelpipe_flow |= PIXELPIPE_FLOW_PROCESSED_ON_GPU;
    *pixelpipe_flow &= ~(PIXELPIPE_FLOW_PROCESSED_ON_CPU | PIXELPIPE_FLOW_PROCESSED_WITH_TILING);
 
    /* Measured on a painting stroke: the pipeline reports `Drawing' at 47.6 ms a frame while
     * the module's own process_cl measures 22.0 -- so ~25 ms is spent between process_cl
     * returning and this module being declared done. Both blend early-outs should fire when
     * nothing is blended (`transform_for_blend' returns NONE on DEVELOP_MASK_DISABLED, and
     * `dt_develop_blend_process_cl' returns on !top_enabled), so the candidates are the
     * colourspace temps around the blend, the blend kernel itself, or the CPU simply blocking
     * here on kernels process_cl only ENQUEUED. Those have nothing in common as fixes. */
    const gint64 blend_t0 = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
    if(module->flags() & IOP_FLAGS_SUPPORTS_BLENDING)
    {
      const dt_pixelpipe_blend_transform_t blend_transforms
          = dt_dev_pixelpipe_transform_for_blend(module, piece, &piece->dsc_out);
      cl_mem_blend_input = cl_mem_process_input;
      cl_mem_blend_output = cl_mem_output;
      blend_input_dsc = process_input_dsc;
      blend_output_dsc = piece->dsc_out;
      if(blend_transforms != DT_DEV_PIXELPIPE_BLEND_TRANSFORM_NONE)
      {
        dt_iop_colorspace_type_t blend_cst = dt_develop_blend_colorspace(piece, piece->dsc_out.cst);
        int success = 1;
        const int blend_in_before = blend_input_dsc.cst;
        if(blend_transforms & DT_DEV_PIXELPIPE_BLEND_TRANSFORM_INPUT)
        {
          cl_mem_blend_input_temp = dt_dev_pixelpipe_cache_alloc_cl_device_buffer(pipe->devid, &piece->roi_in, piece->dsc_in.bpp,
                                                                 module, "blend input colorspace temp",
                                                                 cl_mem_process_input);
          if(IS_NULL_PTR(cl_mem_blend_input_temp))
            goto error;
 
          success &= dt_colorspaces_apply_profile_cl(module->op, module->multi_name, pipe->devid,
                                                            cl_mem_process_input, cl_mem_blend_input_temp,
                                                            piece->roi_in.width, piece->roi_in.height,
                                                            blend_input_dsc.cst, blend_cst,
                                                            &blend_input_dsc.cst, work_profile);
          cl_mem_blend_input = cl_mem_blend_input_temp;
        }
        const int blend_in_after = blend_input_dsc.cst;
        dt_dev_pixelpipe_debug_dump_module_io(pipe, module, "blend-in", TRUE,
                                              &process_input_dsc, &blend_input_dsc,
                                              &piece->roi_in, &piece->roi_in,
                                              process_input_dsc.bpp, blend_input_dsc.bpp,
                                              blend_in_before, blend_in_after);
        const int blend_out_before = blend_output_dsc.cst;
        if(blend_transforms & DT_DEV_PIXELPIPE_BLEND_TRANSFORM_OUTPUT)
        {
          cl_mem_blend_output_temp = dt_dev_pixelpipe_cache_alloc_cl_device_buffer(pipe->devid, &piece->roi_out,
                                                                  piece->dsc_out.bpp, module,
                                                                  "blend output colorspace temp", cl_mem_output);
          if(IS_NULL_PTR(cl_mem_blend_output_temp))
            goto error;
 
          success &= dt_colorspaces_apply_profile_cl(module->op, module->multi_name, pipe->devid, cl_mem_output,
                                                            cl_mem_blend_output_temp, piece->roi_out.width,
                                                            piece->roi_out.height, blend_output_dsc.cst, blend_cst,
                                                            &blend_output_dsc.cst, work_profile);
          cl_mem_blend_output = cl_mem_blend_output_temp;
        }
        const int blend_out_after = blend_output_dsc.cst;
        dt_dev_pixelpipe_debug_dump_module_io(pipe, module, "blend-out", TRUE,
                                              &piece->dsc_out, &blend_output_dsc,
                                              &piece->roi_out, &piece->roi_out,
                                              piece->dsc_out.bpp, blend_output_dsc.bpp,
                                              blend_out_before, blend_out_after);
 
        if(!success)
        {
          dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] couldn't transform blending colorspace for module %s\n",
                   module->name());
          goto error;
        }
      }
 
      const gint64 blend_pre = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
      if(dt_develop_blend_process_cl(module, pipe, piece, cl_mem_blend_input, cl_mem_blend_output))
        goto error;
      if(dt_get_debug_flags() & DT_DEBUG_PERF)
      {
        const dt_develop_blend_params_t *const bp = (const dt_develop_blend_params_t *)piece->blendop_data;
        dt_print(DT_DEBUG_PERF,
                 "[blend] %s transforms_in+out=%.2f ms kernel=%.2f ms mask_mode=%u transforms=%d\n",
                 module->op, (blend_pre - blend_t0) / 1000.0,
                 (g_get_monotonic_time() - blend_pre) / 1000.0,
                 bp ? bp->mask_mode : 0u, (int)blend_transforms);
      }
 
      // a mask or channel preview is converted like any output, see pixelpipe_cpu.c
      if((blend_transforms & DT_DEV_PIXELPIPE_BLEND_TRANSFORM_OUTPUT)
         && !dt_colorspaces_apply_profile_cl(module->op, module->multi_name, pipe->devid, cl_mem_blend_output,
                                             cl_mem_output, piece->roi_out.width,
                                             piece->roi_out.height, blend_output_dsc.cst,
                                             piece->dsc_out.cst, &blend_output_dsc.cst,
                                             work_profile))
        goto error;
 
      *pixelpipe_flow |= PIXELPIPE_FLOW_BLENDED_ON_GPU;
      *pixelpipe_flow &= ~(PIXELPIPE_FLOW_BLENDED_ON_CPU);
    }
 
    if(*cache_output)
    {
      const gint64 readback_t0 = (dt_get_debug_flags() & DT_DEBUG_PERF) ? g_get_monotonic_time() : 0;
      if(dt_dev_pixelpipe_cache_sync_cl_buffer(pipe->devid, output, cl_mem_output, &piece->roi_out, CL_MAP_READ,
                                piece->dsc_out.bpp, module,
                                "output to cache"))
        goto error;
      /* This readback is inside the module's timed region, so it is charged to the module in
       * the "processed `X'" line although the module did not ask for it: the seal sets
       * cache_output_on_ram from a DOWNSTREAM consumer's need for host data. On a painting
       * stroke it accounted for ~25 of `Drawing''s 47.6 ms a frame -- a 48.9 MB device->host
       * copy of a 2144x1427 float4 buffer -- and the same flag separately disables the
       * output cacheline's in-place rekey, which is what makes drawlayer's damage-limited
       * composite gate report devout=0 and fall back to a full resample every frame. One flag,
       * both costs, and neither visible without asking. */
      if(dt_get_debug_flags() & DT_DEBUG_PERF)
        dt_print(DT_DEBUG_PERF, "[dev_pixelpipe] %s output readback %dx%d bpp=%" G_GSIZE_FORMAT " took %.2f ms\n",
                 module->op, piece->roi_out.width, piece->roi_out.height, piece->dsc_out.bpp,
                 (g_get_monotonic_time() - readback_t0) / 1000.0);
      dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] output memory was copied to cache for %s\n", module->name());
    }
  }
  else if(piece->process_tiling_ready && !IS_NULL_PTR(input))
  {
    // FIXME: we don't cover the case (piece->module->flags() & IOP_FLAGS_TAKE_NO_INPUT)
    // in tiling path
    const float *module_input = input;
    const float *blend_input = input;
    float *module_input_temp = NULL;
    float *blend_input_temp = NULL;
    gboolean input_locked = FALSE;
 
    if(borrowed_cl_mem_input)
    {
      dt_dev_pixelpipe_cache_return_cl_payload(input_entry, cl_mem_input);
      cl_mem_input = NULL;
      borrowed_cl_mem_input = FALSE;
    }
    else
      dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_input, input_entry, input,
                                        dt_dev_pixelpipe_cache_gpu_device_buffer(pipe, input_entry));
 
    if(process_input_dsc.cst != piece->dsc_in.cst
       && !(dt_iop_colorspace_is_rgb(process_input_dsc.cst) && dt_iop_colorspace_is_rgb(piece->dsc_in.cst)))
    {
      module_input_temp
          = dt_pixelpipe_cache_alloc_align_float((size_t)piece->roi_in.width * piece->roi_in.height * 4, pipe);
      if(IS_NULL_PTR(module_input_temp))
        goto error;
 
      dt_dev_pixelpipe_cache_rdlock_entry(TRUE, input_entry);
      input_locked = TRUE;
      dt_colorspaces_apply_profile(module->op, module->multi_name, input, module_input_temp, piece->roi_in.width,
                                          piece->roi_in.height, process_input_dsc.cst, piece->dsc_in.cst,
                                          &process_input_dsc.cst, work_profile);
      dt_dev_pixelpipe_cache_rdlock_entry(FALSE, input_entry);
      input_locked = FALSE;
      module_input = module_input_temp;
    }
    else if(process_input_dsc.cst != piece->dsc_in.cst)
    {
      process_input_dsc.cst = piece->dsc_in.cst;
      dt_dev_pixelpipe_cache_rdlock_entry(TRUE, input_entry);
      input_locked = TRUE;
    }
    else
    {
      dt_dev_pixelpipe_cache_rdlock_entry(TRUE, input_entry);
      input_locked = TRUE;
    }
 
    int fail = !module->process_tiling_cl(module, pipe, piece, module_input, output, piece->dsc_in.bpp);
    dt_opencl_finish(pipe->devid);
 
    if(fail)
    {
      if(input_locked)
        dt_dev_pixelpipe_cache_rdlock_entry(FALSE, input_entry);
      dt_pixelpipe_cache_free_align(module_input_temp);
      goto error;
    }
 
    *pixelpipe_flow |= (PIXELPIPE_FLOW_PROCESSED_ON_GPU | PIXELPIPE_FLOW_PROCESSED_WITH_TILING);
    *pixelpipe_flow &= ~(PIXELPIPE_FLOW_PROCESSED_ON_CPU);
 
    blend_input = module_input;
    blend_input_dsc = process_input_dsc;
    void *blend_output = output;
    blend_output_dsc = piece->dsc_out;
 
    const dt_pixelpipe_blend_transform_t blend_transforms
        = dt_dev_pixelpipe_transform_for_blend(module, piece, &piece->dsc_out);
    if(blend_transforms != DT_DEV_PIXELPIPE_BLEND_TRANSFORM_NONE)
    {
      dt_iop_colorspace_type_t blend_cst = dt_develop_blend_colorspace(piece, piece->dsc_out.cst);
 
      if(blend_transforms & DT_DEV_PIXELPIPE_BLEND_TRANSFORM_INPUT)
      {
        blend_input_temp
            = dt_pixelpipe_cache_alloc_align_float((size_t)piece->roi_in.width * piece->roi_in.height * 4, pipe);
        if(IS_NULL_PTR(blend_input_temp))
        {
          if(input_locked)
            dt_dev_pixelpipe_cache_rdlock_entry(FALSE, input_entry);
          dt_pixelpipe_cache_free_align(module_input_temp);
          goto error;
        }
 
        dt_colorspaces_apply_profile(module->op, module->multi_name, module_input, blend_input_temp, piece->roi_in.width,
                                            piece->roi_in.height, blend_input_dsc.cst, blend_cst,
                                            &blend_input_dsc.cst, work_profile);
        blend_input = blend_input_temp;
        if(input_locked)
        {
          dt_dev_pixelpipe_cache_rdlock_entry(FALSE, input_entry);
          input_locked = FALSE;
        }
      }
 
      if(blend_transforms & DT_DEV_PIXELPIPE_BLEND_TRANSFORM_OUTPUT)
      {
        float *blend_output_temp
            = dt_pixelpipe_cache_alloc_align_float((size_t)piece->roi_out.width * piece->roi_out.height * 4, pipe);
        if(IS_NULL_PTR(blend_output_temp))
        {
          if(input_locked)
            dt_dev_pixelpipe_cache_rdlock_entry(FALSE, input_entry);
          dt_pixelpipe_cache_free_align(blend_input_temp);
          dt_pixelpipe_cache_free_align(module_input_temp);
          goto error;
        }
 
        dt_colorspaces_apply_profile(module->op, module->multi_name, output, blend_output_temp, piece->roi_out.width,
                                            piece->roi_out.height, blend_output_dsc.cst, blend_cst,
                                            &blend_output_dsc.cst, work_profile);
        blend_output = blend_output_temp;
      }
    }
 
    dt_develop_blend_process(module, pipe, piece, blend_input, blend_output);
    *pixelpipe_flow |= PIXELPIPE_FLOW_BLENDED_ON_CPU;
    *pixelpipe_flow &= ~(PIXELPIPE_FLOW_BLENDED_ON_GPU);
 
    if((blend_transforms & DT_DEV_PIXELPIPE_BLEND_TRANSFORM_OUTPUT))
    {
      // A mask or channel preview is converted like any output: the preview is authored in the
      // blending space precisely so this conversion lands it where the display expects it.
      dt_colorspaces_apply_profile(module->op, module->multi_name, blend_output, output, piece->roi_out.width,
                                   piece->roi_out.height, blend_output_dsc.cst, piece->dsc_out.cst,
                                   &blend_output_dsc.cst, work_profile);
    }
 
    if(input_locked)
      dt_dev_pixelpipe_cache_rdlock_entry(FALSE, input_entry);
    if(blend_output != output)
      dt_pixelpipe_cache_free_align(blend_output);
    dt_pixelpipe_cache_free_align(blend_input_temp);
    dt_pixelpipe_cache_free_align(module_input_temp);
  }
  else
  {
    dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] could not run module '%s' on gpu. falling back to cpu path\n",
             module->name());
    goto error;
  }
 
  dt_opencl_finish(pipe->devid);
 
  if(locked_input_entry)
    dt_dev_pixelpipe_cache_rdlock_entry(FALSE, locked_input_entry);
 
  /* Borrowed vRAM inputs must stay protected until the current queue completed, otherwise
   * another pipe can flush or recycle the shared device buffer while the queued kernels
   * are still reading it. */
  if(borrowed_cl_mem_input)
  {
    dt_dev_pixelpipe_cache_return_cl_payload(input_entry, cl_mem_input);
    cl_mem_input = NULL;
    borrowed_cl_mem_input = FALSE;
  }
  else
    dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_input, input_entry, input,
                                      dt_dev_pixelpipe_cache_gpu_device_buffer(pipe, input_entry));
 
  /* The backend now owns the authoritative module output payload until publish time.
   * When the output stayed GPU-only, the recursion no longer carries `cl_mem_output`
   * back explicitly, so we must cache it here before returning. Otherwise
   * the caller publishes a cacheline with metadata only and no recoverable payload. */
  dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_output, output_entry, output, TRUE);
 
  dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_blend_output_temp, NULL, NULL, FALSE);
  dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_blend_input_temp, NULL, NULL, FALSE);
  dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_process_input_temp, NULL, NULL, FALSE);
 
  return 0;
 
error:
  dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] %s couldn't process on GPU\n", module->name());
 
  dt_opencl_finish(pipe->devid);
 
  dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_blend_output_temp, NULL, NULL, FALSE);
  dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_blend_input_temp, NULL, NULL, FALSE);
  dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_process_input_temp, NULL, NULL, FALSE);
 
  if(locked_input_entry)
    dt_dev_pixelpipe_cache_rdlock_entry(FALSE, locked_input_entry);
 
  dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_output, output_entry, NULL, FALSE);
 
  // The pipe abandoned this run while the module was on the device: a module that checks the
  // killswitch mid-process (doc/module-abandon.md) fails for it. Rerunning it on CPU would redo work
  // nobody wants and tell the user OpenCL failed, so abort as the killswitch does.
  if(dt_dev_pixelpipe_has_shutdown(pipe))
  {
    if(borrowed_cl_mem_input)
    {
      dt_dev_pixelpipe_cache_return_cl_payload(cpu_input_entry, cl_mem_input);
      cl_mem_input = NULL;
    }
    else
      dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_input, cpu_input_entry, input,
                                        dt_dev_pixelpipe_cache_gpu_device_buffer(pipe, cpu_input_entry));
    return 1;
  }
 
  if(module->flags() & IOP_FLAGS_TAKE_NO_INPUT)
  {
    /* Root modules build their own input from external storage. If the OpenCL pre-check
     * rejects the required allocation, CPU fallback must keep the same no-input contract
     * instead of looking for an upstream cacheline that cannot exist. */
 
    /* The `error:` label is the catch-all for *every* GPU failure (kernel errors, driver
     * allocation faults, colorspace/blend failures...), not only the memory pre-check.
     * Only quote a memory limit when the pre-check actually rejected the buffer, and quote
     * the limit that was really exceeded -- otherwise the numbers contradict the failure
     * (e.g. "needs 100 MiB but device limit is 1991 MiB", issue #878). */
    switch(fit_reason)
    {
      case DT_OPENCL_FIT_ALLOC_LIMIT:
        dt_pipeline_message(_("OpenCL failed for module `%s`: image buffer needs %" G_GSIZE_FORMAT
                         " MiB but the largest allocation the device allows is %" G_GSIZE_FORMAT
                         " MiB; falling back to CPU"),
                       module->name(), (size_t)(fit_needed / (1024 * 1024)),
                       (size_t)(fit_limit / (1024 * 1024)));
        break;
      case DT_OPENCL_FIT_AVAILABLE:
        dt_pipeline_message(_("OpenCL failed for module `%s`: image buffer needs %" G_GSIZE_FORMAT
                         " MiB but only %" G_GSIZE_FORMAT " MiB are free on the device; falling back to CPU"),
                       module->name(), (size_t)(fit_needed / (1024 * 1024)),
                       (size_t)(fit_limit / (1024 * 1024)));
        break;
      case DT_OPENCL_FIT_DIMENSION:
        dt_pipeline_message(_("OpenCL failed for module `%s`: image dimensions %" G_GSIZE_FORMAT "x%" G_GSIZE_FORMAT
                         " exceed the device limits; falling back to CPU"),
                       module->name(), precheck_width, precheck_height);
        break;
      default: // DT_OPENCL_FIT_OK / UNINITED: the buffer fit, the GPU failed for another reason
        dt_pipeline_message(_("OpenCL failed for module `%s`; falling back to CPU"), module->name());
        break;
    }
    return pixelpipe_process_on_CPU(pipe, piece, previous_piece, tiling, pixelpipe_flow,
                                    cache_output, cpu_input_entry, output_entry);
  }
 
  if(!IS_NULL_PTR(input))
  {
    dt_print(DT_DEBUG_OPENCL,
             "[dev_pixelpipe] %s GPU error fallback will reuse host input\n",
             module->name());
  }
  else if(!IS_NULL_PTR(cl_mem_input))
  {
    if(_gpu_init_input(pipe, &input, &cl_mem_input, piece, tiling,
                       cpu_input_entry, output_entry))
    {
      if(borrowed_cl_mem_input)
      {
        dt_dev_pixelpipe_cache_return_cl_payload(cpu_input_entry, cl_mem_input);
        cl_mem_input = NULL;
        borrowed_cl_mem_input = FALSE;
      }
      else
        dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_input, cpu_input_entry, NULL,
                                          dt_dev_pixelpipe_cache_gpu_device_buffer(pipe, cpu_input_entry));
      return 1;
    }
  }
  else if(IS_NULL_PTR(input))
  {
    dt_print(DT_DEBUG_OPENCL,
             "[dev_pixelpipe] %s CPU fallback has no input buffer (cache allocation failed?)\n",
             module->name());
    return 1;
  }
 
  if(borrowed_cl_mem_input)
  {
    dt_dev_pixelpipe_cache_return_cl_payload(cpu_input_entry, cl_mem_input);
    cl_mem_input = NULL;
  }
  else
    dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_input, cpu_input_entry, input,
                                      dt_dev_pixelpipe_cache_gpu_device_buffer(pipe, cpu_input_entry));
 
  return pixelpipe_process_on_CPU(pipe, piece, previous_piece, tiling, pixelpipe_flow,
                                  cache_output, cpu_input_entry, output_entry);
}
 
#else
 
int pixelpipe_process_on_GPU(dt_dev_pixelpipe_t *pipe, const dt_dev_pixelpipe_iop_t *piece,
                             const dt_dev_pixelpipe_iop_t *previous_piece,
                             dt_develop_tiling_t *tiling,
                             dt_pixelpipe_flow_t *pixelpipe_flow,
                             gboolean *const cache_output,
                             dt_pixel_cache_entry_t *input_entry, dt_pixel_cache_entry_t *output_entry)
{
  return pixelpipe_process_on_CPU(pipe, piece, previous_piece, tiling, pixelpipe_flow,
                                  cache_output, input_entry, output_entry);
}
 
#endif