pixelpipe_hb.c

Go to the documentation of this file.

/*
    This file is part of darktable,
    Copyright (C) 2009-2016 johannes hanika.
    Copyright (C) 2010-2012 Henrik Andersson.
    Copyright (C) 2011 Bruce Guenter.
    Copyright (C) 2011 Robert Bieber.
    Copyright (C) 2011 Rostyslav Pidgornyi.
    Copyright (C) 2011-2017, 2019 Ulrich Pegelow.
    Copyright (C) 2012, 2021 Aldric Renaudin.
    Copyright (C) 2012 Richard Wonka.
    Copyright (C) 2012-2019 Tobias Ellinghaus.
    Copyright (C) 2013-2016 Roman Lebedev.
    Copyright (C) 2013 Simon Spannagel.
    Copyright (C) 2014, 2016 Pedro Côrte-Real.
    Copyright (C) 2016 Matthieu Moy.
    Copyright (C) 2017, 2019 luzpaz.
    Copyright (C) 2018, 2020-2026 Aurélien PIERRE.
    Copyright (C) 2018-2019 Edgardo Hoszowski.
    Copyright (C) 2018-2022 Pascal Obry.
    Copyright (C) 2019 Andreas Schneider.
    Copyright (C) 2019-2022 Dan Torop.
    Copyright (C) 2019-2022 Hanno Schwalm.
    Copyright (C) 2019 Heiko Bauke.
    Copyright (C) 2020 Chris Elston.
    Copyright (C) 2020 Diederik Ter Rahe.
    Copyright (C) 2020 GrahamByrnes.
    Copyright (C) 2020-2021 Harold le Clément de Saint-Marcq.
    Copyright (C) 2020-2021 Hubert Kowalski.
    Copyright (C) 2020-2021 Ralf Brown.
    Copyright (C) 2021 Sakari Kapanen.
    Copyright (C) 2022 Martin Bařinka.
    Copyright (C) 2022 Philipp Lutz.
    Copyright (C) 2023-2024 Alynx Zhou.
    Copyright (C) 2023 lologor.
    Copyright (C) 2023 Luca Zulberti.
    Copyright (C) 2024 Alban Gruin.
    Copyright (C) 2024 tatu.
    Copyright (C) 2025-2026 Guillaume Stutin.
    Copyright (C) 2025 Miguel Moquillon.
    
    darktable is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    darktable is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with darktable.  If not, see <http://www.gnu.org/licenses/>.
*/
#include "develop/pipeline_notify.h"
#include "colorprofiles/profile_types.h"
#include "system/capabilities.h"
#include "system/sys_resources.h"
#include "common/global_mutexes.h"
#include "common/hash.h"
#include "common/histogram.h"
#include "imageio/imageio_core.h"
#include "system/atomic.h"
#include "common/opencl.h"
#include "develop/iop_order.h"
#include "control/signal.h"
#include "develop/blend.h"
#include "develop/dev_pixelpipe.h"
#include "develop/dev_history.h"
#include "pixel/format.h"
#include "develop/imageop_math.h"
#include "common/sentry.h"
#include "common/telemetry.h"
#include "develop/pixelpipe.h"
#include "caches/pixelpipe_cache.h"
#include "develop/supervisor.h"
#include "develop/pixelpipe_cpu.h"
#include "develop/pixelpipe_gpu.h"
#include "develop/pixelpipe_process.h"
#include "develop/tiling.h"
#include "develop/masks.h"
 
#include <assert.h>
#include <inttypes.h>
#include <math.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
 
#include "develop/pixelpipe_raster_masks.c"
#include "develop/pixelpipe_rawdetail.c"
 
static void _trace_cache_owner(const dt_dev_pixelpipe_t *pipe, const dt_iop_module_t *module,
                               const char *phase, const char *slot, const uint64_t requested_hash,
                               const void *buffer, const dt_pixel_cache_entry_t *entry,
                               const gboolean verbose)
{
  if(!(dt_get_debug_flags() & DT_DEBUG_PIPECACHE)) return;
  if(!(dt_get_debug_flags() & DT_DEBUG_VERBOSE)) return;
 
  dt_print(DT_DEBUG_PIPECACHE,
           "[pixelpipe_owner] pipe=%s module=%s phase=%s slot=%s req=%" PRIu64
           " entry=%" PRIu64 "/%" PRIu64 " refs=%i auto=%i data=%p buf=%p name=%s\n",
           pipe ? dt_pixelpipe_get_pipe_name(pipe->type) : "-",
           module ? module->op : "base",
           phase ? phase : "-",
           slot ? slot : "-",
           requested_hash,
           entry ? entry->hash : DT_PIXELPIPE_CACHE_HASH_INVALID,
           entry ? entry->serial : 0,
           entry ? dt_atomic_get_int((dt_atomic_int *)&entry->refcount) : -1,
           entry ? entry->auto_destroy : -1,
           entry ? entry->data : NULL,
           buffer,
           (entry && entry->name) ? entry->name : "-");
}
 
 
static void _trace_buffer_content(const dt_dev_pixelpipe_t *pipe, const dt_iop_module_t *module,
                                  const char *phase, const void *buffer,
                                  const dt_iop_buffer_dsc_t *format, const dt_iop_roi_t *roi)
{
  if(!(dt_get_debug_flags() & DT_DEBUG_PIPECACHE)) return;
  if(!(dt_get_debug_flags() & DT_DEBUG_VERBOSE)) return;
  if(IS_NULL_PTR(buffer) || IS_NULL_PTR(format) || IS_NULL_PTR(roi)) return;
  if(roi->width <= 0 || roi->height <= 0) return;
 
  const size_t pixels = (size_t)roi->width * (size_t)roi->height;
  const unsigned int channels = format->channels;
 
  if(format->datatype == TYPE_FLOAT && channels >= 1)
  {
    const float *in = (const float *)buffer;
    float minv[4] = { FLT_MAX, FLT_MAX, FLT_MAX, FLT_MAX };
    float maxv[4] = { -FLT_MAX, -FLT_MAX, -FLT_MAX, -FLT_MAX };
    size_t nonfinite = 0;
    size_t near_black = 0;
 
    for(size_t k = 0; k < pixels; k++, in += channels)
    {
      gboolean finite = TRUE;
      for(unsigned int c = 0; c < MIN(channels, 4U); c++)
      {
        if(!isfinite(in[c]))
        {
          finite = FALSE;
          continue;
        }
        minv[c] = fminf(minv[c], in[c]);
        maxv[c] = fmaxf(maxv[c], in[c]);
      }
 
      if(!finite)
      {
        nonfinite++;
        continue;
      }
 
      const float energy = fabsf(in[0]) + ((channels > 1) ? fabsf(in[1]) : 0.0f)
                           + ((channels > 2) ? fabsf(in[2]) : 0.0f);
      if(energy < 1e-6f) near_black++;
    }
 
    const uint64_t content_hash = dt_hash(5381, (const char *)buffer, pixels * channels * sizeof(float));
 
    dt_print(DT_DEBUG_PIPECACHE,
             "[pixelpipe_stats] pipe=%s module=%s phase=%s type=float ch=%u roi=%dx%d ptr=%p "
             "content_hash=%" PRIu64 " "
             "rgb_min=(%g,%g,%g) rgb_max=(%g,%g,%g) a_min=%g a_max=%g near_black=%" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT " nonfinite=%" G_GSIZE_FORMAT "\n",
             dt_pixelpipe_get_pipe_name(pipe->type), module->op, phase ? phase : "-",
             channels, roi->width, roi->height, buffer, content_hash,
             minv[0], (channels > 1) ? minv[1] : 0.0f, (channels > 2) ? minv[2] : 0.0f,
             maxv[0], (channels > 1) ? maxv[1] : 0.0f, (channels > 2) ? maxv[2] : 0.0f,
             (channels > 3) ? minv[3] : 0.0f, (channels > 3) ? maxv[3] : 0.0f,
             near_black, pixels, nonfinite);
  }
  else if(format->datatype == TYPE_UINT8 && channels >= 1)
  {
    const uint8_t *in = (const uint8_t *)buffer;
    int minv[4] = { 255, 255, 255, 255 };
    int maxv[4] = { 0, 0, 0, 0 };
    size_t near_black = 0;
 
    for(size_t k = 0; k < pixels; k++, in += channels)
    {
      for(unsigned int c = 0; c < MIN(channels, 4U); c++)
      {
        minv[c] = MIN(minv[c], in[c]);
        maxv[c] = MAX(maxv[c], in[c]);
      }
 
      const int energy = in[0] + ((channels > 1) ? in[1] : 0) + ((channels > 2) ? in[2] : 0);
      if(energy == 0) near_black++;
    }
 
    const uint64_t content_hash = dt_hash(5381, (const char *)buffer, pixels * channels * sizeof(uint8_t));
 
    dt_print(DT_DEBUG_PIPECACHE,
             "[pixelpipe_stats] pipe=%s module=%s phase=%s type=u8 ch=%u roi=%dx%d ptr=%p "
             "content_hash=%" PRIu64 " "
             "rgb_min=(%d,%d,%d) rgb_max=(%d,%d,%d) a_min=%d a_max=%d near_black=%" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT "\n",
             dt_pixelpipe_get_pipe_name(pipe->type), module->op, phase ? phase : "-",
             channels, roi->width, roi->height, buffer, content_hash,
             minv[0], (channels > 1) ? minv[1] : 0, (channels > 2) ? minv[2] : 0,
             maxv[0], (channels > 1) ? maxv[1] : 0, (channels > 2) ? maxv[2] : 0,
             (channels > 3) ? minv[3] : 0, (channels > 3) ? maxv[3] : 0,
             near_black, pixels);
  }
}
 
static int _abort_module_shutdown_cleanup(dt_dev_pixelpipe_t *pipe, dt_dev_pixelpipe_iop_t *piece,
                                          dt_iop_module_t *module, const uint64_t input_hash,
                                          const void *input, dt_pixel_cache_entry_t *input_entry,
                                          const uint64_t output_hash, void **output,
                                          void **cl_mem_output, dt_pixel_cache_entry_t *output_entry)
{
  _trace_cache_owner(pipe, module, "shutdown-drop", "input", input_hash, input, input_entry, FALSE);
  _trace_cache_owner(pipe, module, "shutdown-drop", "output", output_hash,
                     output ? *output : NULL, output_entry, FALSE);
 
  _reset_piece_cache_entry(piece);
 
  // An input its producer flagged disposable goes with this release.
  if(!IS_NULL_PTR(input_entry)) dt_dev_pixelpipe_cache_ref_count_entry(FALSE, input_entry);
 
  // Flagged while still held, so its last release removes it.
  if(!IS_NULL_PTR(output_entry))
  {
    dt_dev_pixelpipe_cache_flag_auto_destroy(output_entry);
    dt_dev_pixelpipe_cache_ref_count_entry(FALSE, output_entry);
  }
 
  if(output) *output = NULL;
 
  if(!IS_NULL_PTR(*cl_mem_output))
    dt_dev_pixelpipe_cache_release_cl_buffer(cl_mem_output, NULL, NULL, FALSE);
 
  return 1;
}
 
static inline gboolean _is_focused_realtime_gui_module(const dt_dev_pixelpipe_t *pipe,
                                                       const dt_develop_t *dev,
                                                       const dt_iop_module_t *module)
{
  return (pipe->type == DT_DEV_PIXELPIPE_FULL || pipe->type == DT_DEV_PIXELPIPE_PREVIEW)
         && pipe->realtime && dev && module && dev->gui_module == module;
}
 
 
gboolean dt_dev_pixelpipe_cache_gpu_device_buffer(const dt_dev_pixelpipe_t *pipe,
                                                  const dt_pixel_cache_entry_t *cache_entry)
{
  return (pipe->realtime
          || (cache_entry && IS_NULL_PTR(dt_pixel_cache_entry_get_data((dt_pixel_cache_entry_t *)cache_entry)))
          || !_bypass_cache(pipe, NULL));
}
 
char *dt_pixelpipe_get_pipe_name(dt_dev_pixelpipe_type_t pipe_type)
{
  char *r = NULL;
 
  switch(pipe_type)
  {
    case DT_DEV_PIXELPIPE_PREVIEW:
      r = _("preview");
      break;
    case DT_DEV_PIXELPIPE_FULL:
      r = _("full");
      break;
    case DT_DEV_PIXELPIPE_THUMBNAIL:
      r = _("thumbnail");
      break;
    case DT_DEV_PIXELPIPE_EXPORT:
      r = _("export");
      break;
    default:
      r = _("invalid");
  }
  return r;
}
 
 
inline static void _uint8_to_float(const uint8_t *const input, float *const output,
                                   const size_t width, const size_t height, const size_t chan)
{
  __OMP_FOR_SIMD__(aligned(input, output: 64) )
  for(size_t k = 0; k < height * width; k++)
  {
    const size_t index = k * chan;
    // Warning: we take BGRa and put it back into RGBa
    output[index + 0] = (float)input[index + 2] / 255.f;
    output[index + 1] = (float)input[index + 1] / 255.f;
    output[index + 2] = (float)input[index + 0] / 255.f;
    output[index + 3] = 0.f;
  }
}
 
static const char *_debug_cst_to_string(const int cst)
{
  switch(cst)
  {
    case IOP_CS_RAW:
      return "raw";
    case IOP_CS_LAB:
      return "lab";
    case IOP_CS_RGB:
      return "rgb";
    case IOP_CS_RGB_DISPLAY:
      return "display-rgb";
    case IOP_CS_LCH:
      return "lch";
    case IOP_CS_HSL:
      return "hsl";
    case IOP_CS_JZCZHZ:
      return "jzczhz";
    case IOP_CS_NONE:
      return "none";
    default:
      return "unknown";
  }
}
 
static const char *_debug_type_to_string(const dt_iop_buffer_type_t type)
{
  switch(type)
  {
    case TYPE_FLOAT:
      return "float";
    case TYPE_UINT16:
      return "uint16";
    case TYPE_UINT8:
      return "uint8";
    case TYPE_UNKNOWN:
    default:
      return "unknown";
  }
}
 
void dt_dev_pixelpipe_debug_dump_module_io(dt_dev_pixelpipe_t *pipe, dt_iop_module_t *module, const char *stage,
                                           const gboolean is_cl,
                                           const dt_iop_buffer_dsc_t *in_dsc, const dt_iop_buffer_dsc_t *out_dsc,
                                           const dt_iop_roi_t *roi_in, const dt_iop_roi_t *roi_out,
                                           const size_t in_bpp, const size_t out_bpp,
                                           const int cst_before, const int cst_after)
{
  if(!(dt_get_debug_flags() & DT_DEBUG_PIPE)) return;
  if(!(dt_get_debug_flags() & DT_DEBUG_VERBOSE)) return;
  const char *module_name = module ? module->op : "base";
  const char *pipe_name = dt_pixelpipe_get_pipe_name(pipe->type);
  const char *stage_name = stage ? stage : "process";
 
  if(!IS_NULL_PTR(in_dsc) && !IS_NULL_PTR(out_dsc))
  {
    dt_print(DT_DEBUG_PIPE,
             "[pixelpipe] %s %s %s %s: in cst=%s->%s ch=%d type=%s bpp=%" G_GSIZE_FORMAT " roi=%dx%d | "
             "out cst=%s ch=%d type=%s bpp=%" G_GSIZE_FORMAT " roi=%dx%d\n",
             pipe_name, module_name, is_cl ? "cl" : "cpu", stage_name,
             _debug_cst_to_string(cst_before), _debug_cst_to_string(cst_after),
             in_dsc->channels, _debug_type_to_string(in_dsc->datatype), in_bpp,
             roi_in ? roi_in->width : 0, roi_in ? roi_in->height : 0,
             _debug_cst_to_string(out_dsc->cst), out_dsc->channels, _debug_type_to_string(out_dsc->datatype),
             out_bpp, roi_out ? roi_out->width : 0, roi_out ? roi_out->height : 0);
  }
  else if(!IS_NULL_PTR(out_dsc))
  {
    dt_print(DT_DEBUG_PIPE,
             "[pixelpipe] %s %s %s %s: out cst=%s ch=%d type=%s bpp=%" G_GSIZE_FORMAT " roi=%dx%d\n",
             pipe_name, module_name, is_cl ? "cl" : "cpu", stage_name,
             _debug_cst_to_string(out_dsc->cst), out_dsc->channels, _debug_type_to_string(out_dsc->datatype),
             out_bpp, roi_out ? roi_out->width : 0, roi_out ? roi_out->height : 0);
  }
}
 
 
int dt_dev_pixelpipe_init_export(dt_dev_pixelpipe_t *pipe, dt_develop_t *dev, int levels, gboolean store_masks)
{
  const int res = dt_dev_pixelpipe_init_cached(pipe);
  pipe->type = DT_DEV_PIXELPIPE_EXPORT;
  // Full-size outputs nothing reads again: kept, one export filled the cache with gigabytes and
  // pushed the darkroom's lines out.
  pipe->no_cache = TRUE;
  pipe->gui_observable_source = FALSE;
  pipe->levels = levels;
  pipe->store_all_raster_masks = store_masks;
  pipe->dev = dev;
  return res;
}
 
int dt_dev_pixelpipe_init_thumbnail(dt_dev_pixelpipe_t *pipe, dt_develop_t *dev)
{
  const int res = dt_dev_pixelpipe_init_cached(pipe);
  pipe->type = DT_DEV_PIXELPIPE_THUMBNAIL;
  pipe->no_cache = TRUE;
  pipe->dev = dev;
  return res;
}
 
int dt_dev_pixelpipe_init_dummy(dt_dev_pixelpipe_t *pipe, dt_develop_t *dev)
{
  const int res = dt_dev_pixelpipe_init_cached(pipe);
  pipe->type = DT_DEV_PIXELPIPE_THUMBNAIL;
  pipe->no_cache = TRUE;
  pipe->dev = dev;
  return res;
}
 
int dt_dev_pixelpipe_init_preview(dt_dev_pixelpipe_t *pipe, dt_develop_t *dev)
{
  // Init with the size of MIPMAP_F
  const int res = dt_dev_pixelpipe_init_cached(pipe);
  pipe->type = DT_DEV_PIXELPIPE_PREVIEW;
  pipe->gui_observable_source = TRUE;
 
  // Needed for caching
  pipe->store_all_raster_masks = TRUE;
  pipe->dev = dev;
  return res;
}
 
void dt_dev_pixelpipe_set_mask_rasterization_step(dt_dev_pixelpipe_t *pipe, const int step)
{
  if(IS_NULL_PTR(pipe)) return;
 
  pipe->mask_rasterization_step = MAX(1, step);
}
 
int dt_dev_pixelpipe_init(dt_dev_pixelpipe_t *pipe, dt_develop_t *dev)
{
  const int res = dt_dev_pixelpipe_init_cached(pipe);
  pipe->type = DT_DEV_PIXELPIPE_FULL;
 
  // Needed for caching
  pipe->store_all_raster_masks = TRUE;
  pipe->dev = dev;
  return res;
}
 
int dt_dev_pixelpipe_init_cached(dt_dev_pixelpipe_t *pipe)
{
  // Set everything to 0 = NULL = FALSE
  memset(pipe, 0, sizeof(dt_dev_pixelpipe_t));
 
  // Set only the stuff that doesn't take 0 as default
  pipe->roi_request.value = dt_dev_roi_request_neutral();
  pipe->devid = -1;
  pipe->last_devid = -1;
 
  /* Pixel accuracy, the safe default: only the darkroom ever raises this, from the GUI thread.
   * It is set HERE, in the one function every pipe init calls, and not in each of them: the
   * per-type version of this missed dt_dev_pixelpipe_init_dummy(), which gui/dtgtk/focus.h uses
   * for the focus-ring overlay, and left it at the memset's 0. Zero does not divide by anything
   * -- `use_sparse' is `step > 1' and the interpolation loop is `for(k = 1; k < step; k++)', so
   * both are simply skipped -- but it makes _is_within_pxl_threshold() test `< 0', which is
   * never true, so the outline subdivides until `tmax - tmin < 0.0001' instead of stopping at
   * one pixel: thousands of samples per segment, on an overlay that wanted the cheap path. */
  pipe->mask_rasterization_step = 1;
  dt_dev_pixelpipe_set_changed(pipe, DT_DEV_PIPE_UNCHANGED);
  dt_dev_pixelpipe_set_hash(pipe, DT_PIXELPIPE_CACHE_HASH_INVALID);
  dt_dev_pixelpipe_set_history_hash(pipe, DT_PIXELPIPE_CACHE_HASH_INVALID);
  dt_dev_set_backbuf(&pipe->backbuf, 0, 0, 0, -1, -1);
  pipe->last_history_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
  pipe->rawdetail_mask_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
  pipe->raster_mask_hashes = g_array_new(FALSE, FALSE, sizeof(uint64_t));
 
  pipe->output_imgid = UNKNOWN_IMAGE;
  pipe->iscale = 1.0f;
  dt_atomic_set_int(&pipe->shutdown, FALSE);
  dt_atomic_set_int(&pipe->realtime, FALSE);
  dt_dev_pixelpipe_reset_cache_request(pipe);
 
  pipe->levels = IMAGEIO_RGB | IMAGEIO_INT8;
  dt_pthread_mutex_init(&(pipe->busy_mutex), NULL);
 
  pipe->icc_type = DT_COLORSPACE_NONE;
  pipe->icc_intent = DT_INTENT_LAST;
 
  dt_dev_pixelpipe_reset_reentry(pipe);
  return 1;
}
 
void dt_dev_pixelpipe_set_realtime(dt_dev_pixelpipe_t *pipe, gboolean state)
{
  if(IS_NULL_PTR(pipe)) return;
  dt_atomic_set_int(&pipe->realtime, state ? TRUE : FALSE);
}
 
gboolean dt_dev_pixelpipe_get_realtime(const dt_dev_pixelpipe_t *pipe)
{
  return pipe ? dt_atomic_get_int((dt_atomic_int *)&pipe->realtime) : FALSE;
}
 
void dt_dev_pixelpipe_set_input(dt_dev_pixelpipe_t *pipe, int32_t imgid, int width, int height,
                                float iscale, dt_mipmap_size_t size)
{
  pipe->iwidth = width;
  pipe->iheight = height;
  pipe->iscale = iscale;
  pipe->imgid = imgid;
  pipe->dev->image_storage = pipe->dev->image_storage;
  pipe->size = size;
 
  // Nodes may already exist if this pipe was set up before its input size was known: export
  // pipes call dt_dev_pixelpipe_create_nodes() before dt_dev_pixelpipe_set_input() (see
  // common/imageio.c), and create_nodes() bakes pipe->iwidth/iheight into each piece at
  // node-creation time. Without this, every piece stays stuck with whatever (zero) size the
  // pipe had at that point, so any module reading piece->iwidth/iheight directly (toneequal,
  // soften, ashift) silently gets 0 during export while darkroom pipes (whose input is set
  // before their nodes are created) see the real size -- e.g. toneequal's blending radius
  // collapses to 0 on export only, disabling its local smoothing there regardless of the
  // module's actual settings.
  for(GList *nodes = g_list_first(pipe->nodes); nodes; nodes = g_list_next(nodes))
  {
    dt_dev_pixelpipe_iop_t *piece = (dt_dev_pixelpipe_iop_t *)nodes->data;
    if(IS_NULL_PTR(piece)) continue;
    piece->iwidth = width;
    piece->iheight = height;
  }
}
 
void dt_dev_pixelpipe_set_icc(dt_dev_pixelpipe_t *pipe, dt_colorspaces_color_profile_type_t icc_type,
                              const gchar *icc_filename, dt_iop_color_intent_t icc_intent)
{
  pipe->icc_type = icc_type;
  dt_free(pipe->icc_filename);
  pipe->icc_filename = g_strdup(icc_filename ? icc_filename : "");
  pipe->icc_intent = icc_intent;
}
 
void dt_dev_pixelpipe_cleanup(dt_dev_pixelpipe_t *pipe)
{
  /* Device-side cache payloads are only an acceleration layer. Drop the cl_mem
   * objects this pipe produced on the device it last ran on -- but only that
   * device, so we never touch cache entries another, still-running pipe holds
   * on a different (or the same) OpenCL device. */
  dt_dev_pixelpipe_cache_flush_clmem_for_pipe(pipe->last_devid);
 
  // blocks while busy and sets shutdown bit:
  dt_dev_pixelpipe_cleanup_nodes(pipe);
  // The pipe held a reference on the history item it last synced to (see the field's comment
  // in pixelpipe_hb.h); drop it now that no resync can run on this pipe any more. Exchanged,
  // not read-then-cleared, for the same reason every other write to that slot is.
  dt_dev_free_history_item(dt_atomic_exch_ptr(&pipe->last_history_item, NULL));
  // so now it's safe to clean up cache:
  const uint64_t old_backbuf_hash = dt_dev_backbuf_get_hash(&pipe->backbuf);
  if(old_backbuf_hash != DT_PIXELPIPE_CACHE_HASH_INVALID)
  {
    /* Backbuffer ownership belongs to the pipeline, not its GUI consumers. Once the pipe itself is
     * torn down, always release that keepalive ref and invalidate the published backbuffer metadata.
     * A pipe that keeps no cache takes its last frame with it: flagged while the keepalive still
     * holds it, so the release removes it. */
    if(pipe->no_cache) dt_dev_pixelpipe_cache_flag_auto_destroy(pipe->backbuf.keepalive);
    dt_dev_backbuf_release_keepalive(&pipe->backbuf);
  }
  dt_dev_set_backbuf(&pipe->backbuf, 0, 0, 0, DT_PIXELPIPE_CACHE_HASH_INVALID, DT_PIXELPIPE_CACHE_HASH_INVALID);
  dt_dev_pixelpipe_reset_cache_request(pipe);
  dt_pthread_mutex_destroy(&(pipe->busy_mutex));
  pipe->icc_type = DT_COLORSPACE_NONE;
  dt_free(pipe->icc_filename);
 
  /* Image-derived input profile, if this pipe built one. input_profile_info may point at
   * it or at a shared memo entry, so free through the owning pointer only. */
  dt_ioppr_cleanup_profile_info(&pipe->owned_input_profile_info);
  pipe->input_profile_info = NULL;
 
  pipe->output_imgid = UNKNOWN_IMAGE;
 
  // dt_dev_pixelpipe_cleanup_nodes() above released the references the array held.
  g_array_free(pipe->raster_mask_hashes, TRUE);
  pipe->raster_mask_hashes = NULL;
 
  dt_masks_forms_snapshot_release(&pipe->forms);
}
 
 
gboolean dt_dev_pixelpipe_set_reentry(dt_dev_pixelpipe_t *pipe, uint64_t hash)
{
  if(pipe->reentry_hash == DT_PIXELPIPE_CACHE_HASH_INVALID)
  {
    pipe->reentry = TRUE;
    pipe->reentry_hash = hash;
    dt_print(DT_DEBUG_DEV, "[dev_pixelpipe] re-entry flag set for %" PRIu64 "\n", hash);
    return TRUE;
  }
 
  return FALSE;
}
 
 
gboolean dt_dev_pixelpipe_unset_reentry(dt_dev_pixelpipe_t *pipe, uint64_t hash)
{
  if(pipe->reentry_hash == hash)
  {
    pipe->reentry = FALSE;
    pipe->reentry_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
    dt_print(DT_DEBUG_DEV, "[dev_pixelpipe] re-entry flag unset for %" PRIu64 "\n", hash);
    return TRUE;
  }
 
  return FALSE;
}
 
gboolean dt_dev_pixelpipe_has_reentry(dt_dev_pixelpipe_t *pipe)
{
  return pipe->reentry;
}
 
void dt_dev_pixelpipe_reset_reentry(dt_dev_pixelpipe_t *pipe)
{
  pipe->reentry = FALSE;
  pipe->reentry_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
  pipe->flush_cache = FALSE;
}
 
void dt_dev_pixelpipe_cleanup_nodes(dt_dev_pixelpipe_t *pipe)
{
  /* The side-band masks the graph referenced go with it: left to the next render, they outlive
   * the image on a pipe that does not render again, like the darkroom's after leave(). */
  dt_dev_clear_rawdetail_mask(pipe);
  // Every hash in this array corresponds to one cache reference acquired
  // either while reopening current provider masks or while publishing a new
  // one. Release the exact same number of references.
  for(guint k = 0; k < pipe->raster_mask_hashes->len; k++)
  {
    const uint64_t hash = g_array_index(pipe->raster_mask_hashes, uint64_t, k);
    dt_dev_pixelpipe_cache_unref_hash(hash);
  }
  g_array_set_size(pipe->raster_mask_hashes, 0);
 
  // destroy all nodes
  for(GList *nodes = g_list_first(pipe->nodes); nodes; nodes = g_list_next(nodes))
  {
    dt_dev_pixelpipe_iop_t *piece = (dt_dev_pixelpipe_iop_t *)nodes->data;
    if(IS_NULL_PTR(piece)) continue;
    // printf("cleanup module `%s'\n", piece->module->name());
    if(dt_supervisor_active() && piece->module)
      dt_supervisor_node(DT_SV_DELETE,
                         dt_supervisor_node_key(pipe->type, piece->module->op, piece->module->multi_priority),
                         DT_PIXELPIPE_CACHE_HASH_INVALID, DT_PIXELPIPE_CACHE_HASH_INVALID,
                         piece->module->op, piece->module->multi_priority, piece->module->iop_order,
                         pipe->type, pipe->imgid);
    if(piece->module) dt_iop_cleanup_pipe(piece->module, pipe, piece);
 
    /* Clear the node before releasing it. A freed node whose fields still read as a live one is
     * exactly what makes this class of bug land far from its cause: a walker that outlives the
     * teardown finds a plausible module pointer and a plausible private-data pointer, and the
     * crash surfaces inside whichever module's commit_params() dereferences them. Zeroed, the
     * same node is rejected by _iop_piece_is_committable() and reported by name.
     * This does not make use-after-free safe -- the allocator may hand the bytes out again --
     * it removes the window where the node is stale but still self-consistent. */
    piece->module = NULL;
    piece->data = NULL;
    piece->data_size = 0;
    piece->blendop_data = NULL;
    piece->enabled = 0;
 
    dt_free(piece);
  }
  g_list_free(pipe->nodes);
  pipe->nodes = NULL;
  // and iop order
  g_list_free_full(pipe->iop_order_list, dt_free_gpointer);
  pipe->iop_order_list = NULL;
  dt_dev_pixelpipe_set_history_hash(pipe, DT_PIXELPIPE_CACHE_HASH_INVALID);
}
 
void dt_dev_pixelpipe_create_nodes(dt_dev_pixelpipe_t *pipe)
{
  // check that the pipe was actually properly cleaned up after the last run
  g_assert(IS_NULL_PTR(pipe->nodes));
  g_assert(IS_NULL_PTR(pipe->iop_order_list));
  pipe->iop_order_list = dt_ioppr_iop_order_copy_deep(pipe->dev->iop_order_list);
 
  // for all modules in dev (iterated in iop_order, so each is the predecessor of the next):
  dt_iop_module_t *prev_module = NULL;
  for(GList *modules = g_list_first(pipe->dev->iop); modules; modules = g_list_next(modules))
  {
    dt_iop_module_t *module = (dt_iop_module_t *)modules->data;
    dt_dev_pixelpipe_iop_t *piece = (dt_dev_pixelpipe_iop_t *)calloc(1, sizeof(dt_dev_pixelpipe_iop_t));
    if(IS_NULL_PTR(piece)) continue;
    piece->enabled = module->enabled;
    piece->request_histogram = DT_REQUEST_ONLY_IN_GUI;
    piece->histogram_params.bins_count = 256;
    piece->iwidth = pipe->iwidth;
    piece->iheight = pipe->iheight;
    piece->module = module;
    piece->hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
    piece->blendop_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
    piece->global_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
    piece->global_mask_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
    piece->upstream_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
    _reset_piece_cache_entry(piece);
    piece->cache_output_on_ram = TRUE;
 
    // dsc_mask is static, single channel float image
    piece->dsc_mask.channels = 1;
    piece->dsc_mask.datatype = TYPE_FLOAT;
    dt_iop_buffer_dsc_update_bpp(&piece->dsc_mask);
 
    dt_iop_init_pipe(piece->module, pipe, piece);
 
    pipe->nodes = g_list_append(pipe->nodes, piece);
 
    // Topology node: created once here, immutable until the nodes are torn down.
    // param_hash is bound later, at history synchronization (dt_supervisor_node UPDATE).
    if(dt_supervisor_active())
    {
      const uint64_t pred = prev_module
          ? dt_supervisor_node_key(pipe->type, prev_module->op, prev_module->multi_priority)
          : DT_PIXELPIPE_CACHE_HASH_INVALID;
      dt_supervisor_node(DT_SV_CREATE,
                         dt_supervisor_node_key(pipe->type, module->op, module->multi_priority),
                         DT_PIXELPIPE_CACHE_HASH_INVALID, pred, module->op, module->multi_priority,
                         module->iop_order, pipe->type, pipe->imgid);
    }
    prev_module = module;
  }
}
 
dt_pixelpipe_blend_transform_t dt_dev_pixelpipe_transform_for_blend(const dt_iop_module_t *const self,
                                                                    const dt_dev_pixelpipe_iop_t *const piece,
                                                                    const dt_iop_buffer_dsc_t *const output_dsc)
{
  const dt_develop_blend_params_t *const d = (const dt_develop_blend_params_t *)piece->blendop_data;
  if(IS_NULL_PTR(d)) return DT_DEV_PIXELPIPE_BLEND_TRANSFORM_NONE;
  if(!(self->flags() & IOP_FLAGS_SUPPORTS_BLENDING)) return DT_DEV_PIXELPIPE_BLEND_TRANSFORM_NONE;
  if(d->mask_mode == DEVELOP_MASK_DISABLED) return DT_DEV_PIXELPIPE_BLEND_TRANSFORM_NONE;
 
  const dt_iop_colorspace_type_t blend_cst = dt_develop_blend_colorspace(piece, output_dsc->cst);
  dt_pixelpipe_blend_transform_t transforms = DT_DEV_PIXELPIPE_BLEND_TRANSFORM_NONE;
  if(piece->dsc_in.cst != blend_cst
     && !(dt_iop_colorspace_is_rgb(piece->dsc_in.cst) && dt_iop_colorspace_is_rgb(blend_cst)))
    transforms |= DT_DEV_PIXELPIPE_BLEND_TRANSFORM_INPUT;
  if(output_dsc->cst != blend_cst
     && !(dt_iop_colorspace_is_rgb(output_dsc->cst) && dt_iop_colorspace_is_rgb(blend_cst)))
    transforms |= DT_DEV_PIXELPIPE_BLEND_TRANSFORM_OUTPUT;
 
  return transforms;
}
 
#define KILL_SWITCH_ABORT                                                                                         \
  if(dt_dev_pixelpipe_has_shutdown(pipe))                                                                         \
  {                                                                                                               \
    if(!IS_NULL_PTR(cl_mem_output))                                                                                     \
    {                                                                                                             \
      dt_dev_pixelpipe_cache_release_cl_buffer(&cl_mem_output, NULL, NULL, FALSE);                                      \
    }                                                                                                             \
    return 1;                                                                                                     \
  }
 
// Once we have a cache, stopping computation before full completion
// has good chances of leaving it corrupted. So we invalidate it.
#define KILL_SWITCH_AND_FLUSH_CACHE                                                                               \
  if(dt_dev_pixelpipe_has_shutdown(pipe))                                                                         \
  {                                                                                                               \
    return _abort_module_shutdown_cleanup(pipe, piece, module, input_hash, input, input_entry, hash, &output,    \
                                          &cl_mem_output, output_entry);                                          \
  }
 
static void _print_perf_debug(dt_dev_pixelpipe_t *pipe, const dt_pixelpipe_flow_t pixelpipe_flow,
                              dt_dev_pixelpipe_iop_t *piece, dt_iop_module_t *module,
                              const gboolean recycled_output_cacheline, dt_times_t *start)
{
  char histogram_log[32] = "";
  if(!(pixelpipe_flow & PIXELPIPE_FLOW_HISTOGRAM_NONE))
  {
    snprintf(histogram_log, sizeof(histogram_log), ", collected histogram on %s",
             (pixelpipe_flow & PIXELPIPE_FLOW_HISTOGRAM_ON_GPU
                  ? "GPU"
                  : pixelpipe_flow & PIXELPIPE_FLOW_HISTOGRAM_ON_CPU ? "CPU" : ""));
  }
 
  const char *blend_log = pixelpipe_flow & PIXELPIPE_FLOW_BLENDED_ON_GPU
                               ? "blended on GPU"
                               : pixelpipe_flow & PIXELPIPE_FLOW_BLENDED_ON_CPU ? "blended on CPU" : "no blending";
 
  gchar *module_label = dt_history_item_get_name(module);
  dt_show_times_f(
      start, "[dev_pixelpipe]", "processed `%s' on %s%s%s%s, %s [%s]", module_label,
      pixelpipe_flow & PIXELPIPE_FLOW_PROCESSED_ON_GPU
          ? "GPU"
          : pixelpipe_flow & PIXELPIPE_FLOW_PROCESSED_ON_CPU ? "CPU" : "",
      pixelpipe_flow & PIXELPIPE_FLOW_PROCESSED_WITH_TILING ? " with tiling" : "",
      recycled_output_cacheline ? ", recycled cacheline" : "",
      (!(pixelpipe_flow & PIXELPIPE_FLOW_HISTOGRAM_NONE) && (piece->request_histogram & DT_REQUEST_ON))
          ? histogram_log
          : "",
      blend_log,
      dt_pixelpipe_get_pipe_name(pipe->type));
  dt_free(module_label);
}
 
 
static void _print_nan_debug(dt_dev_pixelpipe_t *pipe, void *cl_mem_output, void *output,
                             const dt_iop_roi_t *roi_out, dt_iop_buffer_dsc_t *out_format,
                             dt_iop_module_t *module)
{
  if(!(dt_get_debug_flags() & DT_DEBUG_NAN)) return;
  if(!(dt_get_debug_flags() & DT_DEBUG_VERBOSE)) return;
  if((dt_get_debug_flags() & DT_DEBUG_NAN) && strcmp(module->op, "gamma") != 0)
  {
    gchar *module_label = dt_history_item_get_name(module);
 
    if(out_format->datatype == TYPE_FLOAT && out_format->channels == 4)
    {
      int hasinf = 0, hasnan = 0;
      dt_aligned_pixel_t min = { FLT_MAX };
      dt_aligned_pixel_t max = { FLT_MIN };
 
      for(int k = 0; k < 4 * roi_out->width * roi_out->height; k++)
      {
        if((k & 3) < 3)
        {
          float f = ((float *)(output))[k];
          if(isnan(f))
            hasnan = 1;
          else if(!isfinite(f))
            hasinf = 1;
          else
          {
            min[k & 3] = fmin(f, min[k & 3]);
            max[k & 3] = fmax(f, max[k & 3]);
          }
        }
      }
      if(hasnan)
        fprintf(stderr, "[dev_pixelpipe] module `%s' outputs NaNs! [%s]\n", module_label,
                dt_pixelpipe_get_pipe_name(pipe->type));
      if(hasinf)
        fprintf(stderr, "[dev_pixelpipe] module `%s' outputs non-finite floats! [%s]\n", module_label,
                dt_pixelpipe_get_pipe_name(pipe->type));
      fprintf(stderr, "[dev_pixelpipe] module `%s' min: (%f; %f; %f) max: (%f; %f; %f) [%s]\n", module_label,
              min[0], min[1], min[2], max[0], max[1], max[2], dt_pixelpipe_get_pipe_name(pipe->type));
    }
    else if(out_format->datatype == TYPE_FLOAT && out_format->channels == 1)
    {
      int hasinf = 0, hasnan = 0;
      float min = FLT_MAX;
      float max = FLT_MIN;
 
      for(int k = 0; k < roi_out->width * roi_out->height; k++)
      {
        float f = ((float *)(output))[k];
        if(isnan(f))
          hasnan = 1;
        else if(!isfinite(f))
          hasinf = 1;
        else
        {
          min = fmin(f, min);
          max = fmax(f, max);
        }
      }
      if(hasnan)
        fprintf(stderr, "[dev_pixelpipe] module `%s' outputs NaNs! [%s]\n", module_label,
                dt_pixelpipe_get_pipe_name(pipe->type));
      if(hasinf)
        fprintf(stderr, "[dev_pixelpipe] module `%s' outputs non-finite floats! [%s]\n", module_label,
                dt_pixelpipe_get_pipe_name(pipe->type));
      fprintf(stderr, "[dev_pixelpipe] module `%s' min: (%f) max: (%f) [%s]\n", module_label, min, max,
                dt_pixelpipe_get_pipe_name(pipe->type));
    }
 
    dt_free(module_label);
  }
}
 
static int dt_dev_pixelpipe_process_rec(dt_dev_pixelpipe_t *pipe,
                                        uint64_t *out_hash, const dt_dev_pixelpipe_iop_t **out_piece,
                                        GList *pieces, int pos)
{
  // The pipeline is executed recursively, from the end. For each module n, starting from the end,
  // if output is cached, take it, else if input is cached, take it, process it and output,
  // else recurse to the previous module n-1 to get a an input.
  void *input = NULL;
  void *output = NULL;
  void *cl_mem_output = NULL;
 
  KILL_SWITCH_ABORT;
 
  if(IS_NULL_PTR(pieces))
  {
    *out_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
    *out_piece = NULL;
    return 0;
  }
 
  dt_iop_module_t *module = NULL;
  dt_dev_pixelpipe_iop_t *piece = (dt_dev_pixelpipe_iop_t *)pieces->data;
  
  KILL_SWITCH_ABORT;
 
  // skip this module?
  if(!piece->enabled)
    return dt_dev_pixelpipe_process_rec(pipe, out_hash, out_piece, g_list_previous(pieces), pos - 1);
 
  module = piece->module;
 
  if(pipe->dev->gui_attached) pipe->dev->progress.total++;
  
  KILL_SWITCH_ABORT;
 
  const uint64_t hash = dt_dev_pixelpipe_node_hash(pipe, piece, piece->roi_out, pos);
  dt_dev_pixelpipe_rawdetail_mask_check_hit(pipe, piece, hash);
 
  // 1) Fast-track:
  // If we have a cache entry for this hash, return it straight away,
  // don't recurse through pipeline and don't process, unless this module still
  // needs GUI-side sampling from its host input or the gamma display histogram
  // needs the upstream cache entry.
  dt_pixel_cache_entry_t *existing_cache = NULL;
  void *existing_output = NULL;
  /* Atomically look up the entry and increment its refcount so the caller receives a
   * fully-owned reference.  peek() + separate ref_count_entry() has a TOCTOU window:
   * peek releases its tryrdlock immediately, so the entry can reach refcount 0 and be
   * evicted between peek() and ref_count_entry(). */
  const gboolean exact_output_cache_hit
      = !_bypass_cache(pipe, piece)
        && dt_dev_pixelpipe_cache_ref_entry_by_hash(hash,
                                                    &existing_output, &existing_cache)
        && !IS_NULL_PTR(existing_output);
 
  if(exact_output_cache_hit)
  {
    _trace_cache_owner(pipe, module, "exact-hit-direct", "output", hash, NULL, existing_cache, FALSE);
    *out_hash = hash;
    *out_piece = piece;
    return 0;
  }
  else if(existing_cache)
  {
    /* ref_entry_by_hash succeeded but reported no host pixels (device-only entry, or a host buffer
     * still holding the hash it was rekeyed from); undo the ref. */
    dt_dev_pixelpipe_cache_ref_count_entry(FALSE, existing_cache);
  }
 
  // 3) now recurse through the pipeline.
  uint64_t input_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
  const dt_dev_pixelpipe_iop_t *previous_piece = NULL;
  if(dt_dev_pixelpipe_process_rec(pipe, &input_hash, &previous_piece, g_list_previous(pieces), pos - 1))
  {
    /* Child recursion failed before this module acquired any output cache entry.
     * Dropping `hash` here underflows cached exact-hit outputs during shutdown. */
    dt_print(DT_DEBUG_DEV,
             "[pipeline] module=%s child recursion failed input_hash=%" PRIu64 " output_hash=%" PRIu64 "\n",
             module->op, input_hash, hash);
    return 1;
  }
 
  // Child recursion just published or exact-hit returned this hash with one ref already reserved for
  // this immediate consumer. Reopen the live cache entry directly instead of going through exact-hit
  // lookup, because exact-hit intentionally rejects auto-destroy entries while the parent recursion
  // still needs to consume transient outputs in the same run.
  dt_pixel_cache_entry_t *input_entry
      = dt_dev_pixelpipe_cache_get_entry(input_hash);
  if(!IS_NULL_PTR(previous_piece))
    input_entry = dt_dev_pixelpipe_cache_get_entry(input_hash);
  if(IS_NULL_PTR(input_entry) && !(module->flags() & IOP_FLAGS_TAKE_NO_INPUT))
  {
    dt_print(DT_DEBUG_DEV,
             "[pipeline] module=%s input cache entry missing input_hash=%" PRIu64 " output_hash=%" PRIu64
             " prev_module=%s prev_hash=%" PRIu64 "\n",
             module->op, input_hash, hash, 
             !IS_NULL_PTR(previous_piece) ? previous_piece->module->op : "", 
             !IS_NULL_PTR(previous_piece) ? previous_piece->global_hash : -1);
    return 1;
  }
 
  // Aborting here hands back the reference the upstream module reserved for this one.
  if(dt_dev_pixelpipe_has_shutdown(pipe))
  {
    if(!IS_NULL_PTR(input_entry)) dt_dev_pixelpipe_cache_ref_count_entry(FALSE, input_entry);
    return 1;
  }
  input = input_entry ? dt_pixel_cache_entry_get_data(input_entry) : NULL;
  _trace_cache_owner(pipe, module, "acquire", "input", input_hash, input, input_entry, FALSE);
  if(input_entry)
    _trace_buffer_content(pipe, module, "input-acquire", input, &piece->dsc_in, &piece->roi_in);
  const size_t bufsize = (size_t)piece->dsc_out.bpp * piece->roi_out.width * piece->roi_out.height;
  // Note: IS_NULL_PTR(input) is valid if we are on a GPU-only path, aka previous module ran on GPU
  // without leaving its output on a RAM cache copy, and current module will also run on GPU.
  // In this case, we rely on cl_mem_input for best performance (avoid memcpy between RAM and GPU).
  // Should the GPU path fail at process time, we will init input and flush cl_mem_input into it.
  // In any case, this avoids carrying a possibly-uninited input buffer, without knowing if it has
  // data on it (or having to blindly copy back from vRAM to RAM).
 
  // 3c) actually process this module BUT treat all bypasses first.
  // special case: user requests to see channel data in the parametric mask of a module, or the blending
  // mask. In that case we skip all modules manipulating pixel content and only process image distorting
  // modules. Finally "gamma" is responsible for displaying channel/mask data accordingly.
  if(pipe->dev->gui_attached
     && (pipe->mask_display != DT_DEV_PIXELPIPE_DISPLAY_NONE)
     && !(module->operation_tags() & IOP_TAG_DISTORT)
     && (piece->dsc_in.bpp == piece->dsc_out.bpp)
     && !memcmp(&piece->roi_in, &piece->roi_out, sizeof(struct dt_iop_roi_t)))
  {
    /* Mask/channel passthrough keeps the exact child buffer alive for the next
     * downstream module. This stage does not publish a new cacheline, so it
     * must forward the child hash and actual previous piece contract exactly as
     * they came from recursion, including the single reserved ref for the next
     * real consumer. */
    *out_hash = input_hash;
    *out_piece = previous_piece;
    return 0;
  }
 
  if(pipe->dev->gui_attached)
  {
    gchar *module_label = dt_history_item_get_name(module);
    dt_pipeline_busy_printf(_("Processing module `%s` for pipeline %s (%ix%i px @ %0.f%%)..."),
                            module_label, dt_pixelpipe_get_pipe_name(pipe->type),
                            piece->roi_out.width, piece->roi_out.height,
                            piece->roi_out.scale * 100.f);
    dt_free(module_label);
  }
 
  dt_pixel_cache_entry_t *output_entry = NULL;
  gchar *type = dt_pixelpipe_get_pipe_name(pipe->type);
 
  dt_pixelpipe_flow_t pixelpipe_flow = (PIXELPIPE_FLOW_NONE | PIXELPIPE_FLOW_HISTOGRAM_NONE);
 
  gchar *name = g_strdup_printf("module %s (%s) for pipe %s", module->op, module->multi_name, type);
  // Cache bypass makes intermediate module outputs disposable, but the final output is the
  // backbuffer consumed by Cairo. It must keep a host payload even when an edit mode (crop,
  // clipping, ashift...) bypasses the downstream cache. Otherwise the GUI requests that host
  // cacheline after every GPU render, while each cache-request pass publishes another device-only
  // backbuffer and feeds an infinite preview recompute loop.
  const gboolean keep_final_output = (hash == dt_dev_pixelpipe_get_hash(pipe));
  gboolean cache_ram_output
      = piece->cache_output_on_ram && (!_bypass_cache(pipe, piece) || keep_final_output);
 
  /* `piece->cache_entry` is the writable-reuse hint: the cacheline this piece wrote last time,
   * offered back so the next run rekeys it in place instead of taking a fresh arena slot.
   *
   * This used to exclude any output published to RAM (`&& !cache_ram_output'), because such an
   * output is what a GUI consumer displays and nothing stopped a later run from overwriting it
   * mid-frame. Taking a new slot every time kept the pipe out of the GUI's way by never reusing
   * anything -- measured on the backbuffer, 65 frames gave 65 distinct host pointers, and 0%
   * reuse of its output device buffer against 92-96% for every module that does rekey. That is
   * what made the pipe re-register 12.2 MB of pinned host pages every frame: 5.6 ms of a 26 ms
   * frame, for a buffer it threw away immediately.
   *
   * Both halves of the protection that replaces it are now explicit, and neither existed when
   * this exclusion was written:
   *  - LIFETIME: a displayed cacheline is refcounted by the consumer displaying it
   *    (views/dev_backbuf.c), and refcount > 0 is what the LRU, the vRAM flush and the removal
   *    path all refuse to touch. It used to borrow the pipeline's keepalive, which covered it
   *    only until the pipeline published the next frame.
   *  - EXCLUSION: dt_dev_render_locked_surface() read-locks the entry around its cairo blit and
   *    the rekey takes the entry's WRITE lock -- held from here until this module publishes, at
   *    the release further down -- so a run reusing the displayed line waits for the blit, and a
   *    blit starting mid-render waits for the publish.
   *
   * The third thing this needed was not a lock at all: every long-lived reference is taken by
   * pointer, and releasing it by hash silently lost it once rekey could move that hash. See
   * dt_dev_pixelpipe_cache_unref_entry() -- without that, this line would leak one reference per
   * frame on the very entry it is meant to reuse.
   *
   * The wait is mutual and bounded by one module's render: measured at 3.10 ms for the
   * backbuffer's producer, against the 7.68 ms the GUI already spends painting a frame. If that
   * ever reads as a stall rather than a wait, the answer is a try-read-lock on the GUI side, not
   * going back to allocating a buffer per frame to avoid the question. */
  /* Not after a switch between history states (pipe->keep_outputs): this cacheline then holds
   * the output of the state the user is likely to switch back to. */
  const gboolean allow_rekey_reuse = !(dt_get_debug_flags() & DT_DEBUG_NOCACHE_REUSE) && !pipe->keep_outputs;
  // From here to the end of its processing, what the cache allocates is this module's -- starting
  // with its output, allocated right below when it stays in RAM: the cache names the module and its
  // image when it refuses. It knows nothing of images: it gets the image as text, as it gets the
  // module as its name. The file name is quoted: it can hold spaces, and nothing else would show
  // where it ends.
  const char *prev_module = dt_pixelpipe_cache_set_current_module(module->op);
  gchar *image = g_strdup_printf(_("`%s` (image id %d)"), pipe->dev->image_storage.filename,
                                 pipe->dev->image_storage.id);
  const char *prev_image = dt_pixelpipe_cache_set_current_image(image);
  const dt_dev_pixelpipe_cache_writable_status_t acquire_status
      = dt_dev_pixelpipe_cache_get_writable(hash, bufsize, name, pipe->type,
                                            cache_ram_output, allow_rekey_reuse,
                                            &piece->cache_entry, &piece->cache_entry_prev,
                                            &output, &output_entry);
  dt_free(name);
  if(acquire_status == DT_DEV_PIXELPIPE_CACHE_WRITABLE_EXACT_HIT)
  {
    /* Another pipe already owns a cacheline for this exact hash. If that cacheline is
     * still write-locked, this is not a processing error: it only means the concurrent
     * publisher has not finished exposing the exact-hit payload yet. Wait for that
     * publication to complete instead of aborting the whole recursion.
     * The cache hands the entry back referenced: that reference is the one this output reserves
     * for its consumer. */
    dt_pixel_cache_entry_t *exact_entry = output_entry;
    dt_dev_pixelpipe_cache_rdlock_entry(TRUE, exact_entry);
    dt_dev_pixelpipe_cache_rdlock_entry(FALSE, exact_entry);
 
    dt_print(DT_DEBUG_DEV,
             "[pipeline] module=%s writable-exact-hit output_hash=%" PRIu64 " has_host_data=%d"
             " cache_ram_output=%d devid=%d\n",
             module->op, hash, !IS_NULL_PTR(dt_pixel_cache_entry_get_data(exact_entry)), cache_ram_output,
             pipe->devid);
 
    /* This entry may predate the current run's host-caching requirement: it was created device-only
     * (OpenCL, no RAM copy) back when nothing needed a host copy of it, or rekeyed from another hash
     * and left device-only, its host buffer still holding the old pixels. An exact-hash hit here
     * short-circuits without ever re-running the module, so `cache_ram_output` newly turning TRUE
     * (e.g. a color picker or histogram just started sampling this piece) would otherwise never take
     * effect on an already-cached entry. Materialize the host copy now from the device payload we
     * already own the OpenCL context for, instead of silently leaving GUI host-only consumers
     * (dt_dev_pixelpipe_cache_peek_gui, preferred_devid = -1) missing this hash forever. */
    if(cache_ram_output && IS_NULL_PTR(dt_pixel_cache_entry_get_data(exact_entry)))
    {
      void *restored = NULL;
      const gboolean restored_ok
          = dt_dev_pixelpipe_cache_restore_host_payload(exact_entry, pipe->devid,
                                                        &restored);
      dt_print(DT_DEBUG_DEV,
               "[pipeline] module=%s exact-hit was device-only, host materialize %s output_hash=%" PRIu64 "\n",
               module->op, restored_ok ? "succeeded" : "failed", hash);
    }
 
    _trace_cache_owner(pipe, module, "exact-hit-wait", "output", hash,
                       dt_pixel_cache_entry_get_data(exact_entry), exact_entry, FALSE);
 
    if(input_entry)
      dt_dev_pixelpipe_cache_ref_count_entry(FALSE, input_entry);
    *out_hash = hash;
    *out_piece = piece;
    dt_pixelpipe_cache_set_current_image(prev_image);
    dt_free(image);
    dt_pixelpipe_cache_set_current_module(prev_module);
    return 0;
  }
  if(IS_NULL_PTR(output_entry))
  {
    dt_print(DT_DEBUG_DEV,
             "[pipeline] module=%s writable output acquisition failed output_hash=%" PRIu64
             " acquire_status=%d\n",
             module->op, hash, acquire_status);
    if(input_entry)
      dt_dev_pixelpipe_cache_ref_count_entry(FALSE, input_entry);
    dt_pixelpipe_cache_set_current_image(prev_image);
    dt_free(image);
    dt_pixelpipe_cache_set_current_module(prev_module);
    return 1;
  }
  const gboolean new_entry = (acquire_status == DT_DEV_PIXELPIPE_CACHE_WRITABLE_CREATED);
  _trace_cache_owner(pipe, module, (acquire_status == DT_DEV_PIXELPIPE_CACHE_WRITABLE_REKEYED) ? "acquire-rekeyed"
                                                         : (new_entry ? "acquire-new" : "acquire-existing"),
                     "output", hash, output, output_entry, FALSE);
 
  /* get tiling requirement of module */
  dt_develop_tiling_t tiling = { 0 };
  tiling.factor_cl = tiling.maxbuf_cl = -1; // set sentinel value to detect whether callback set sizes
  module->tiling_callback(module, pipe, piece, &tiling);
  if (tiling.factor_cl < 0) tiling.factor_cl = tiling.factor; // default to CPU size if callback didn't set GPU
  if (tiling.maxbuf_cl < 0) tiling.maxbuf_cl = tiling.maxbuf;
 
  /* does this module involve blending? */
  if(piece->blendop_data && ((dt_develop_blend_params_t *)piece->blendop_data)->mask_mode != DEVELOP_MASK_DISABLED)
  {
    /* get specific memory requirement for blending */
    dt_develop_tiling_t tiling_blendop = { 0 };
    tiling_callback_blendop(module, pipe, piece, &tiling_blendop);
 
    /* aggregate in structure tiling */
    tiling.factor = fmax(tiling.factor, tiling_blendop.factor);
    tiling.factor_cl = fmax(tiling.factor_cl, tiling_blendop.factor);
    tiling.maxbuf = fmax(tiling.maxbuf, tiling_blendop.maxbuf);
    tiling.maxbuf_cl = fmax(tiling.maxbuf_cl, tiling_blendop.maxbuf);
    tiling.overhead = fmax(tiling.overhead, tiling_blendop.overhead);
  }
 
  /* remark: we do not do tiling for blendop step, neither in opencl nor on cpu. if overall tiling
     requirements (maximum of module and blendop) require tiling for opencl path, then following blend
     step is anyhow done on cpu. we assume that blending itself will never require tiling in cpu path,
     because memory requirements will still be low enough. */
 
  assert(tiling.factor > 0.0f);
  assert(tiling.factor_cl > 0.0f);
 
  // Actual pixel processing for this module
  int error = 0;
  dt_times_t start;
  dt_get_times(&start);
 
#ifdef HAVE_OPENCL
  error = pixelpipe_process_on_GPU(pipe, piece, previous_piece, &tiling, &pixelpipe_flow,
                                   &cache_ram_output,
                                   input_entry, output_entry);
#else
  error = pixelpipe_process_on_CPU(pipe, piece, previous_piece, &tiling, &pixelpipe_flow,
                                   &cache_ram_output,
                                   input_entry, output_entry);
#endif
 
  dt_pixelpipe_cache_set_current_image(prev_image);
  dt_free(image);
  dt_pixelpipe_cache_set_current_module(prev_module);
 
  // Every CPU and tiled path writes the host buffer, and the OpenCL path copies its output back
  // when `cache_ram_output` asked for it (or when it had to fall back and raised it).
  if(!error
     && ((pixelpipe_flow & (PIXELPIPE_FLOW_PROCESSED_ON_CPU | PIXELPIPE_FLOW_PROCESSED_WITH_TILING))
         || cache_ram_output))
    dt_dev_pixelpipe_cache_flag_host_written(output_entry);
  output = dt_pixel_cache_entry_get_data(output_entry);
 
  _print_perf_debug(pipe, pixelpipe_flow, piece, module,
                    (acquire_status != DT_DEV_PIXELPIPE_CACHE_WRITABLE_CREATED), &start);
 
  if(pipe->dev->gui_attached) pipe->dev->progress.completed++;
 
  if(error)
  {
    dt_print(DT_DEBUG_DEV,
             "[pipeline] module=%s backend processing failed input_hash=%" PRIu64 " output_hash=%" PRIu64
             " input_cst=%d output_cst=%d roi_in=%dx%d roi_out=%dx%d\n",
             module->op, input_hash, hash, piece->dsc_in.cst, piece->dsc_out.cst,
             piece->roi_in.width, piece->roi_in.height, piece->roi_out.width, piece->roi_out.height);
    _trace_cache_owner(pipe, module, "error-cleanup", "input", input_hash, input, input_entry, FALSE);
    _trace_cache_owner(pipe, module, "error-cleanup", "output", hash, output, output_entry, FALSE);
    // Ensure we always release locks and cache references on error, otherwise cache eviction/GC will stall.
    _reset_piece_cache_entry(piece);
    // No point in keeping garbled output: flagged while still held and before the write lock goes,
    // so it is never published as a regular line, and its last release removes it.
    dt_dev_pixelpipe_cache_flag_auto_destroy(output_entry);
    dt_dev_pixelpipe_cache_wrlock_entry(FALSE, output_entry);
    if(!IS_NULL_PTR(input_entry)) dt_dev_pixelpipe_cache_ref_count_entry(FALSE, input_entry);
    dt_dev_pixelpipe_cache_ref_count_entry(FALSE, output_entry);
    return 1;
  }
 
  // Publish the module output descriptor authored for this stage. The cache entry keeps the
  // descriptor of the pixels this module actually published, not the stale descriptor of its input.
  _trace_cache_owner(pipe, module, "publish", "output", hash, output, output_entry, FALSE);
  _trace_buffer_content(pipe, module, "publish", output, &piece->dsc_out, &piece->roi_out);
 
  if(!IS_NULL_PTR(output_entry))
  {
    /* Shift, so the slot the next run falls back on is the cacheline from two runs ago -- by
     * then released by whoever was displaying it, where the one just published is not. */
    piece->cache_entry_prev = piece->cache_entry;
    piece->cache_entry = *output_entry;
  }
  else
    _reset_piece_cache_entry(piece);
 
  if(!IS_NULL_PTR(output_entry) && !IS_NULL_PTR(output) && !(module->flags() & IOP_FLAGS_CPU_WRITES_OPENCL)
     && ((pixelpipe_flow & PIXELPIPE_FLOW_PROCESSED_ON_CPU)
         || (pixelpipe_flow & PIXELPIPE_FLOW_PROCESSED_WITH_TILING)))
  {
    dt_dev_pixelpipe_gpu_flush_host_pinned_images(pipe, output, output_entry, "module output host rewrite");
    /* CPU/tiling processing rewrote the whole host buffer for this output. If the cache entry was rekeyed
     * from an older GPU stage, any cached device-only images still hanging off the same entry now point to
     * obsolete pixels. Drop them here so later mixed GPU/CPU modules cannot resurrect stale device payloads. */
    dt_dev_pixelpipe_cache_flush_entry_clmem(output_entry);
  }
 
  // Flag to throw away intermediate outputs as soon as the next module consumes them.
  // `cache_output` only means the backend had to keep a host-authoritative payload for this
  // stage (for example because the next module may need RAM or because the current stage ran
  // through an OpenCL cache path). In no-cache/bypass pipelines, that does not make the
  // published cacheline long-lived: once the downstream module takes its input ref, this
  // stage is transient and must disappear on release. Only the final published output needs
  // to survive long enough for dt_dev_pixelpipe_process() to promote it to the backbuffer,
  // otherwise thumbnail/export callers only see a missing exact-hit and fall back to invalid
  // placeholder pixels.
  if(_bypass_cache(pipe, piece) && !keep_final_output)
    dt_dev_pixelpipe_cache_flag_auto_destroy(output_entry);
 
  if(pipe->dev->gui_attached)
    dt_pipeline_busy_clear();
 
  // From here on we only publish/inspect the finished output. Keep the writable lock strictly
  // around cacheline allocation and backend processing, then release it at one visible point
  // before the generic tail cleanup shared by darkroom and headless paths.
  if(dt_supervisor_active() && !IS_NULL_PTR(output_entry))
    dt_supervisor_cacheline_create(hash,
                                   dt_supervisor_node_key(pipe->type, module->op, module->multi_priority),
                                   piece->hash, input_hash, module->op, module->multi_priority,
                                   module->iop_order, pipe->type, pipe->imgid, piece->roi_out.width,
                                   piece->roi_out.height, pipe->devid,
                                   dt_pixel_cache_entry_get_size(output_entry), NULL);
  // Stamp the producing node identity so the CACHELINE_READY raised on the write-lock
  // release below carries it: GUI cache-wait consumers can then be served by producer
  // node even when the exact output hash drifted between their request and this publish
  // (doc/pipeline-cache.md §8). Set on every publish so a rekey-reused entry never keeps
  // a stale producer.
  if(!IS_NULL_PTR(output_entry))
    output_entry->producer_node_key
        = dt_supervisor_node_key(pipe->type, module->op, module->multi_priority);
  dt_dev_pixelpipe_cache_wrlock_entry(FALSE, output_entry);
  
  KILL_SWITCH_AND_FLUSH_CACHE;
 
  // Print min/max/Nan in debug mode only
  if((dt_get_debug_flags() & DT_DEBUG_NAN) && strcmp(module->op, "gamma") != 0 && !IS_NULL_PTR(output))
  {
    dt_dev_pixelpipe_cache_rdlock_entry(TRUE, output_entry);
    _print_nan_debug(pipe, cl_mem_output, output, &piece->roi_out, &piece->dsc_out, module);
    dt_dev_pixelpipe_cache_rdlock_entry(FALSE, output_entry);
  }
 
  KILL_SWITCH_AND_FLUSH_CACHE;
 
  // Released past the last abort point, which releases it itself. An input its producer flagged
  // disposable goes with this release.
  _trace_cache_owner(pipe, module, "release", "input", input_hash, input, input_entry, FALSE);
  if(!IS_NULL_PTR(input_entry)) dt_dev_pixelpipe_cache_ref_count_entry(FALSE, input_entry);
 
  *out_hash = hash;
  *out_piece = piece;
  return 0;
}
 
void dt_dev_pixelpipe_disable_after(dt_dev_pixelpipe_t *pipe, const char *op)
{
  GList *nodes = g_list_last(pipe->nodes);
  dt_dev_pixelpipe_iop_t *piece = (dt_dev_pixelpipe_iop_t *)nodes->data;
  while(strcmp(piece->module->op, op))
  {
    piece->enabled = 0;
    piece = NULL;
    nodes = g_list_previous(nodes);
    if(!nodes) break;
    piece = (dt_dev_pixelpipe_iop_t *)nodes->data;
  }
}
 
void dt_dev_pixelpipe_disable_before(dt_dev_pixelpipe_t *pipe, const char *op)
{
  GList *nodes = pipe->nodes;
  dt_dev_pixelpipe_iop_t *piece = (dt_dev_pixelpipe_iop_t *)nodes->data;
  while(strcmp(piece->module->op, op))
  {
    piece->enabled = 0;
    piece = NULL;
    nodes = g_list_next(nodes);
    if(!nodes) break;
    piece = (dt_dev_pixelpipe_iop_t *)nodes->data;
  }
}
 
#define KILL_SWITCH_PIPE                                                                                          \
  if(dt_dev_pixelpipe_has_shutdown(pipe))                                                                         \
  {                                                                                                               \
    if(pipe->devid >= 0)                                                                                          \
    {                                                                                                             \
      dt_opencl_release_device(pipe->devid);                                                                       \
      pipe->devid = -1;                                                                                           \
    }                                                                                                             \
    dt_masks_forms_snapshot_release(&pipe->forms);                                                                \
    dt_pthread_mutex_unlock(dt_pipeline_threadsafe_mutex());                                                      \
    return 1;                                                                                                     \
  }
 
 
static void _print_opencl_errors(int error, dt_dev_pixelpipe_t *pipe)
{
  switch(error)
  {
    case 1:
      dt_print(DT_DEBUG_OPENCL, "[opencl] Opencl errors; disabling opencl for %s pipeline!\n", dt_pixelpipe_get_pipe_name(pipe->type));
      dt_pipeline_message(_("Ansel discovered problems with your OpenCL setup; disabling OpenCL for %s pipeline!"), dt_pixelpipe_get_pipe_name(pipe->type));
      break;
    case 2:
      dt_print(DT_DEBUG_OPENCL,
                 "[opencl] Too many opencl errors; disabling opencl for this session!\n");
      dt_pipeline_message(_("Ansel discovered problems with your OpenCL setup; disabling OpenCL for this session!"));
      break;
    default:
      break;
  }
}
 
/* `entry` must be held by the caller: this reads it and takes the backbuffer's keepalive from it. */
static void _update_backbuf_cache_reference(dt_dev_pixelpipe_t *pipe, dt_iop_roi_t roi, dt_pixel_cache_entry_t *entry)
{
  const uint64_t requested_hash = dt_dev_pixelpipe_get_hash(pipe);
  const uint64_t entry_hash = entry->hash;
 
  _trace_cache_owner(pipe, NULL, "backbuf-update", "backbuf", requested_hash,
                     entry ? entry->data : NULL, entry, FALSE);
 
  if(requested_hash == DT_PIXELPIPE_CACHE_HASH_INVALID 
     || entry_hash == DT_PIXELPIPE_CACHE_HASH_INVALID
     || entry_hash != requested_hash)
  {
    dt_dev_backbuf_release_keepalive(&pipe->backbuf);
    dt_dev_set_backbuf(&pipe->backbuf, 0, 0, 0, DT_PIXELPIPE_CACHE_HASH_INVALID,
                       dt_dev_pixelpipe_get_history_hash(pipe));
    return;
  }
 
  // Keep exactly one cache reference to the last valid output ("backbuf") for display.
  // This prevents the cache entry from being evicted while still in use by the GUI,
  // without leaking references on repeated cache hits.
  /* Keyed on the ENTRY, not on the hash it currently answers to. A rekey-reused entry keeps
   * its identity while its hash moves, so "same cacheline, new hash" must not re-take a
   * reference -- and "new cacheline" must release the old one by pointer, since the hash it was
   * taken at may no longer resolve to it. The helper is idempotent, so this is unconditional. */
  dt_dev_backbuf_take_keepalive(&pipe->backbuf, entry);
 
  /* The backbuf advertises the ROI THIS RUN ASKED FOR, paired with whatever cacheline it
   * resolved. Those two are supposed to describe the same image, and every consumer trusts that
   * they do: the display paths compute a cairo stride from the width and walk the cacheline with
   * it. When they disagree the picture is drawn with the wrong row length -- the diagonal
   * striping of a stride error -- and nothing downstream can detect it, because a cacheline is
   * just bytes and carries no shape of its own.
   *
   * They can disagree. Cache entries are reused in place by rekeying rather than reallocated
   * (caches/pixelpipe_cache.c), so an entry produced at one ROI can be handed back under a key
   * planned at another.
   *
   * The invariant worth checking is that the cacheline is BIG ENOUGH for the advertised
   * dimensions -- not that it is exactly that size. Buffers come from an aligned allocator and
   * from a reuse pool, so a cacheline is routinely larger than width x height x bpp: measured on
   * an ordinary darkroom entry, the display buffers run about 0.2 % over and a thumbnail's can
   * be several times over. That slack is why `bpp' below, a division by the requested pixel
   * count, is not a pixel size and should not be read as one -- nothing does; the field exists
   * for a consumer that no longer has one.
   *
   * Too SMALL is the dangerous direction, and the only one a consumer cannot survive: every
   * display path computes a cairo stride from the width and walks that many rows, so a short
   * cacheline is an out-of-bounds read, and a wrong width is the diagonal striping of a stride
   * error. Report it; do not try to repair it here, where neither the true shape of the entry nor
   * the reason it was selected is known. */
  int bpp = 0;
  if(roi.width > 0 && roi.height > 0)
  {
    const size_t entry_size = dt_pixel_cache_entry_get_size(entry);
    const size_t pixels = (size_t)roi.width * (size_t)roi.height;
    bpp = (int)(entry_size / pixels);
 
    /* 4 bytes per pixel: this is the final, display-encoded backbuffer. */
    if(entry_size < pixels * 4)
      dt_print(DT_DEBUG_ALWAYS,
               // %llu, not %zu: MinGW's printf does not implement the C99 length modifier.
               "[pixelpipe] BACKBUF TOO SMALL on pipe %s: roi %dx%d needs %llu bytes, cacheline holds "
               "%llu; hash %" PRIu64 "\n",
               dt_pixelpipe_get_pipe_name(pipe->type), roi.width, roi.height,
               (unsigned long long)(pixels * 4), (unsigned long long)entry_size, (uint64_t)entry_hash);
 
    /* And the shape itself. `roi' is what this run ASKED the pipe for; the pixels in the
     * cacheline are whatever the last node actually produced, which is its roi_out. Those are
     * supposed to be the same rectangle, and the backbuffer advertises the former while handing
     * consumers the latter -- so if they differ, every display path computes its cairo stride
     * from a width the data does not have, and draws the diagonal striping of a stride error.
     *
     * The cache cannot answer this on its own: an entry records how many BYTES it holds and not
     * what shape they are in, so a cacheline reused at another size is indistinguishable from a
     * correct one by size alone -- large-but-wrong passes every check a consumer can make. The
     * last node's planned output is the only place the true shape survives. */
    const GList *const last = g_list_last(pipe->nodes);
    if(!IS_NULL_PTR(last))
    {
      const dt_dev_pixelpipe_iop_t *const tail = (const dt_dev_pixelpipe_iop_t *)last->data;
      if(tail->roi_out.width != roi.width || tail->roi_out.height != roi.height)
        dt_print(DT_DEBUG_ALWAYS,
                 "[pixelpipe] BACKBUF SHAPE MISMATCH on pipe %s: advertising %dx%d but %s produced "
                 "%dx%d; hash %" PRIu64 "\n",
                 dt_pixelpipe_get_pipe_name(pipe->type), roi.width, roi.height, tail->module->op,
                 tail->roi_out.width, tail->roi_out.height, (uint64_t)entry_hash);
    }
  }
 
  // Always refresh backbuf geometry/state, even when the cache key is unchanged.
  // Realtime drawing can update pixels in-place in the same cacheline, so width/height/history
  // must stay synchronized independently from key changes.
  dt_dev_set_backbuf(&pipe->backbuf, roi.width, roi.height, bpp, entry_hash,
                     dt_dev_pixelpipe_get_history_hash(pipe));
 
  if(dt_supervisor_active())
    dt_supervisor_backbuf(DT_SV_UPDATE, entry_hash, dt_dev_pixelpipe_get_history_hash(pipe),
                          roi.width, roi.height, bpp, pipe->type,
                          pipe->devid >= 0 ? pipe->devid : pipe->last_devid);
}
 
static GList *_get_requested_piece_node(const dt_dev_pixelpipe_t *pipe, const dt_iop_module_t *module, int *pos)
{
  if(pos) *pos = 0;
  if(IS_NULL_PTR(pipe) || IS_NULL_PTR(module)) return NULL;
 
  int current_pos = 1;
  for(GList *node = g_list_first(pipe->nodes); node; node = g_list_next(node), current_pos++)
  {
    dt_dev_pixelpipe_iop_t *const piece = node->data;
    if(piece && piece->enabled && piece->module == module)
    {
      if(pos) *pos = current_pos;
      return node;
    }
  }
 
  return NULL;
}
 
int dt_dev_pixelpipe_process(dt_dev_pixelpipe_t *pipe, dt_iop_roi_t roi)
{
  /* `pipe->devid` is only valid while the current run owns the OpenCL device lock.
   * Reset it before any cache probe so callers never reuse a stale device id from a
   * previous pipeline pass. */
  pipe->devid = -1;
 
  /* Record what image / pipeline is being processed, for crash reports. */
  if(pipe->dev)
  {
    const char *pl = "other";
    switch(pipe->type)
    {
      case DT_DEV_PIXELPIPE_FULL:      pl = "darkroom";         break;
      case DT_DEV_PIXELPIPE_PREVIEW:   pl = "darkroom-preview"; break;
      case DT_DEV_PIXELPIPE_EXPORT:    pl = "export";           break;
      case DT_DEV_PIXELPIPE_THUMBNAIL: pl = "thumbnail";        break;
      default: break;
    }
    dt_sentry_set_processed_image(&pipe->dev->image_storage, pl);
    dt_telemetry_record_file_type(&pipe->dev->image_storage, pl);
  }
 
  if(dt_get_debug_flags() & DT_DEBUG_MEMORY)
  {
    fprintf(stderr, "[memory] before pixelpipe process\n");
    dt_print_mem_usage();
  }
 
  dt_dev_pixelpipe_cache_print();
 
  if(pipe->dev->gui_attached)
  {
    pipe->dev->color_picker.pending_module = NULL;
    pipe->dev->color_picker.pending_pipe = NULL;
    pipe->dev->color_picker.piece_hash = DT_PIXELPIPE_CACHE_HASH_INVALID;
  }
 
  // get a snapshot of the mask list: shared by reference (dt_masks_cow_touch on the GUI side
  // clones instead of mutating in place whenever refcount > 1, so this snapshot stays frozen
  // even if dev->forms is edited mid-run), not deep-copied on every single pipeline recompute.
  // Taken before ROI planning, so modify_roi_in() and process() see the same shapes.
  pipe->forms = dt_masks_snapshot_current_forms(pipe->dev, FALSE);
 
  // Get the roi_out hash of all nodes.
  // Get the previous output size of the module, for cache invalidation.
  dt_dev_pixelpipe_get_roi_in(pipe, roi);
  dt_pixelpipe_get_global_hash(pipe);
  const guint pos = g_list_length(pipe->dev->iop);
 
  const guint previous_raster_refs = pipe->raster_mask_hashes->len;
 
  for(GList *node = g_list_first(pipe->nodes); node; node = g_list_next(node))
  {
    const dt_dev_pixelpipe_iop_t *piece = (dt_dev_pixelpipe_iop_t *)node->data;
    if(!piece->enabled) continue;
 
    GHashTableIter mask_iter;
    gpointer mask_key = NULL;
    gpointer mask_name = NULL;
    g_hash_table_iter_init(&mask_iter, piece->module->raster_mask.source.masks);
    while(g_hash_table_iter_next(&mask_iter, &mask_key, &mask_name))
    {
      const int mask_id = GPOINTER_TO_INT(mask_key);
      if(!pipe->store_all_raster_masks
         && !dt_iop_is_raster_mask_used(piece->module, mask_id))
        continue;
 
      const uint64_t mask_hash = dt_dev_pixelpipe_raster_mask_hash(piece, mask_id);
      if(dt_dev_pixelpipe_cache_ref_entry_by_hash(
             mask_hash, NULL, NULL))
        g_array_append_val(pipe->raster_mask_hashes, mask_hash);
    }
  }
 
  for(guint k = 0; k < previous_raster_refs; k++)
  {
    const uint64_t hash = g_array_index(pipe->raster_mask_hashes, uint64_t, k);
    dt_dev_pixelpipe_cache_unref_hash(hash);
  }
  if(previous_raster_refs > 0)
    g_array_remove_range(pipe->raster_mask_hashes, 0, previous_raster_refs);
 
  dt_dev_pixelpipe_cache_request_t cache_request = dt_dev_pixelpipe_get_cache_request(pipe);
  const dt_iop_module_t *const requested_module = dt_dev_pixelpipe_get_cache_request_module(pipe);
  dt_dev_pixelpipe_reset_cache_request(pipe);
 
  GList *requested_pieces = g_list_last(pipe->nodes);
  int requested_pos = (int)pos;
  dt_dev_pixelpipe_iop_t *requested_piece = NULL;
  gboolean requested_backbuf = TRUE;
 
  if(cache_request == DT_DEV_PIXELPIPE_CACHE_REQUEST_MODULE && !IS_NULL_PTR(requested_module))
  {
    requested_pieces = _get_requested_piece_node(pipe, requested_module, &requested_pos);
    if(requested_pieces)
    {
      requested_piece = requested_pieces->data;
      requested_backbuf = FALSE;
    }
    else
    {
      dt_print(DT_DEBUG_DEV, "[pixelpipe/gui] requested module cache target disappeared pipe=%s module=%s\n",
               dt_pixelpipe_get_pipe_name(pipe->type), requested_module->op);
    }
  }
 
  /* GUI cache requests can target either the final backbuffer or one module output in the middle of the
     current synchronized graph. Exact-hit checks must therefore look at the requested target instead of
     always assuming the run goes to the pipe end. */
  const uint64_t requested_hash = requested_backbuf ? dt_dev_pixelpipe_get_hash(pipe)
                                                    : requested_piece ? requested_piece->global_hash
                                                                      : DT_PIXELPIPE_CACHE_HASH_INVALID;
  /* Retained: the hit may be an output nobody holds, such as the state a module toggle switches back
   * to, and the backbuffer takes its keepalive from it. */
  dt_pixel_cache_entry_t *entry = NULL;
  if(!_bypass_cache(pipe, requested_piece)
     && dt_dev_pixelpipe_cache_ref_host_entry_by_hash(requested_hash, NULL, &entry))
  {
    if(requested_backbuf)
      _update_backbuf_cache_reference(pipe, roi, entry);
    dt_dev_pixelpipe_cache_unref_entry(entry);
 
    /* A GUI consumer explicitly requested this target (color picker, histogram, autoset) and is
     * blocked on it in the cache-wait manager. The output is already host-cached, so the run stops
     * here without ever taking a write lock — hence no DT_SIGNAL_CACHELINE_READY would fire and the
     * waiter would hang forever, even though its buffer is right there (doc/pipeline-cache.md §8).
     * This is the "no node update, nothing" deadlock: it bites when the hash the GUI predicted
     * (H_gui) drifted from the one now cached (H_pipe), so the waiter never sees its exact hash and
     * the pipe never republishes. Wake it here, carrying the producing node key so the manager's
     * producer-node match serves the waiter even under that drift; its restart then re-reads the
     * module's current hash and hits. Gated on an actual pending request so ordinary cache-hit
     * renders add no signal traffic. */
    if(cache_request != DT_DEV_PIXELPIPE_CACHE_REQUEST_NONE)
    {
      const uint64_t ready_node_key
          = (!requested_backbuf && !IS_NULL_PTR(requested_piece) && !IS_NULL_PTR(requested_piece->module))
                ? dt_supervisor_node_key(pipe->type, requested_piece->module->op,
                                         requested_piece->module->multi_priority)
                : DT_PIXELPIPE_CACHE_HASH_INVALID;
      DT_DEBUG_CONTROL_SIGNAL_RAISE(dt_control_signal_get_global(), DT_SIGNAL_CACHELINE_READY, requested_hash,
                                    ready_node_key);
    }
    dt_masks_forms_snapshot_release(&pipe->forms);
    return 0;
  }
 
  dt_print(DT_DEBUG_DEV, "[pixelpipe] Started %s pipeline recompute at %i×%i px\n", 
           dt_pixelpipe_get_pipe_name(pipe->type), roi.width, roi.height);
 
  // go through the list of modules from the end:
  GList *pieces = g_list_last(pipe->nodes);
 
  // Because it's possible here that we export at full resolution,
  // and our memory planning doesn't account for several concurrent pipelines
  // at full size, we allow only one pipeline at a time to run.
  // This is because wavelets decompositions and such use 6 copies,
  // so the RAM usage can go out of control here.
  dt_pthread_mutex_lock(dt_pipeline_threadsafe_mutex());
 
  if(dt_dev_pixelpipe_has_reentry(pipe))
  {
    const guint node_count = g_list_length(pipe->nodes);
    uint64_t *invalidated_hashes = g_new(uint64_t, node_count);
    size_t invalidated_count = 0;
    gboolean reached_provider = FALSE;
 
    for(GList *node = g_list_first(pipe->nodes); node; node = g_list_next(node))
    {
      dt_dev_pixelpipe_iop_t *piece = (dt_dev_pixelpipe_iop_t *)node->data;
      if(!piece->enabled) continue;
 
      reached_provider |= (piece->global_hash == pipe->reentry_hash);
      if(reached_provider)
        invalidated_hashes[invalidated_count++] = piece->global_hash;
    }
 
    const int retained = dt_dev_pixelpipe_cache_invalidate_hashes(
        invalidated_hashes, invalidated_count);
    dt_print(DT_DEBUG_DEV,
             "[raster masks] invalidated %" G_GSIZE_FORMAT " cache states at retry from provider=%" PRIu64
             " retained=%d pipe=%s\n",
             invalidated_count, pipe->reentry_hash, retained,
             dt_pixelpipe_get_pipe_name(pipe->type));
    dt_free(invalidated_hashes);
  }
 
  pipe->opencl_enabled = dt_opencl_update_settings(); // update enabled flag and profile from preferences
  pipe->devid = (pipe->opencl_enabled) ? dt_opencl_reserve_device_for_pipe(pipe->type)
                                       : -1; // try to get/lock opencl resource
 
  if(pipe->devid > -1)
  {
    dt_opencl_events_reset(pipe->devid);
    pipe->last_devid = pipe->devid;
  }
  dt_print(DT_DEBUG_OPENCL, "[pixelpipe_process] [%s] using device %d\n", dt_pixelpipe_get_pipe_name(pipe->type),
           pipe->devid);
 
  KILL_SWITCH_PIPE
 
  gboolean keep_running = TRUE;
  int runs = 0;
  int opencl_error = 0;
  int err = 0;
 
  while(keep_running && runs < 3)
  {
    ++runs;
 
    /* Mask preview is authored while the current run advances through blend.c.
     * Reset it for each retry so a stale state from a previous pass cannot leak
     * into the next recursion before the active module reaches its own blend. */
    pipe->mask_display = DT_DEV_PIXELPIPE_DISPLAY_NONE;
 
#ifdef HAVE_OPENCL
    dt_opencl_check_tuning(pipe->devid);
#endif
 
    KILL_SWITCH_PIPE
 
    dt_times_t start;
    dt_get_times(&start);
    uint64_t final_hash = -1;
    const dt_dev_pixelpipe_iop_t *final_piece = NULL;
    err = dt_dev_pixelpipe_process_rec(pipe, &final_hash, &final_piece,
                                       requested_backbuf ? pieces : requested_pieces,
                                       requested_backbuf ? pos : requested_pos);
    (void)final_piece;
    gchar *msg = g_strdup_printf("[pixelpipe] %s internal pixel pipeline processing", dt_pixelpipe_get_pipe_name(pipe->type));
    dt_show_times(&start, msg);
    dt_free(msg);
 
    // get status summary of opencl queue by checking the eventlist
    const int oclerr = (pipe->devid > -1) ? dt_opencl_events_flush(pipe->devid, TRUE) != 0 : 0;
 
    // Check if we had opencl errors ....
    // remark: opencl errors can come in two ways: pipe->opencl_error is TRUE (and err is TRUE) OR oclerr is
    // TRUE
    keep_running = (oclerr || (err && pipe->opencl_error));
    const gboolean shut_down = dt_dev_pixelpipe_has_shutdown(pipe);
    // A completed run whose output is not published hands back the reference reserved on it.
    if(!err && (keep_running || shut_down)) dt_dev_pixelpipe_cache_unref_hash(final_hash);
    if(keep_running)
    {
      // Report it and be told what it means: 1 = retry this run on CPU, 2 = OpenCL is off
      // for the rest of the session. The count and the threshold are the OpenCL module's.
      opencl_error = dt_opencl_report_pipe_error();
 
      // Disable OpenCL for this pipe
      dt_opencl_release_device(pipe->devid);
      pipe->opencl_enabled = 0;
      pipe->opencl_error = 0;
      pipe->devid = -1;
 
      _print_opencl_errors(opencl_error, pipe);
    }
    else if(!shut_down)
    {
      // The state switched to is now rendered: the next run may overwrite its outputs again.
      if(!err) pipe->keep_outputs = FALSE;
 
      // No opencl errors, no killswitch triggered: we should have a valid output buffer now.
      /* Retained too: the run's own reference covers this entry only while the planned hash is still
       * the one it produced.
       * And not required to be in RAM yet: an export pipe never goes through the darkroom's cache
       * policy seal, so its last node keeps cache_output_on_ram = 0 and an OpenCL run leaves the
       * final output in vRAM only. The device is still reserved here, so the host copy every
       * backbuffer consumer reads is downloaded now, on the referenced entry. */
      dt_pixel_cache_entry_t *final_entry = NULL;
      const gboolean published
          = requested_backbuf
            && dt_dev_pixelpipe_cache_ref_entry_by_hash(dt_dev_pixelpipe_get_hash(pipe), NULL, &final_entry)
            && dt_dev_pixelpipe_cache_restore_host_payload(final_entry, pipe->devid, NULL);
      if(published)
        _update_backbuf_cache_reference(pipe, roi, final_entry);
      else if(requested_backbuf)
        dt_print(DT_DEBUG_DEV,
                 "[picker/rec] final output cache missing pipe=%s hash=%" PRIu64 " history=%" PRIu64
                 " devid=%d err=%d\n",
                 dt_pixelpipe_get_pipe_name(pipe->type), dt_dev_pixelpipe_get_hash(pipe),
                 dt_dev_pixelpipe_get_history_hash(pipe), pipe->devid, err);
 
      // Taken above whenever the entry was found, restored to RAM or not.
      if(!IS_NULL_PTR(final_entry)) dt_dev_pixelpipe_cache_unref_entry(final_entry);
      dt_dev_pixelpipe_cache_unref_hash(final_hash);
 
      // Note : the last output (backbuf) of the pixelpipe cache is internally locked
      // Whatever consuming it will need to unlock it.
    }
  }
 
  dt_pthread_mutex_unlock(dt_pipeline_threadsafe_mutex());
 
  // release resources:
  dt_masks_forms_snapshot_release(&pipe->forms);
  if(pipe->devid >= 0)
  {
    dt_opencl_release_device(pipe->devid);
    pipe->devid = -1;
  }
 
  // terminate
  dt_dev_pixelpipe_cache_print();
 
  // If an intermediate module set that, be sure to reset it at the end
  pipe->flush_cache = FALSE;
  return err;
}
 
// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on