pixelpipe_cache_cl.c

Go to the documentation of this file.

/*
    This file is part of the Ansel project.
    Copyright (C) 2026 Aurélien PIERRE.
    
    Ansel is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    Ansel is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with Ansel.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#include "common/atomic.h"
#include "common/darktable.h"
#include "common/opencl.h"
#include "develop/pixelpipe.h"
#include "develop/pixelpipe_cache.h"
 
#include <stddef.h>
 
#ifdef HAVE_OPENCL
 
static gboolean _cl_is_zero_copy_image(const int devid, cl_mem mem, void *host_ptr, const dt_iop_roi_t *roi,
                                       const size_t bpp)
{
  if(devid < 0 || !mem || !host_ptr || !roi || roi->width <= 0 || roi->height <= 0 || bpp == 0) return FALSE;
 
  void *mapped = dt_opencl_map_image(devid, mem, TRUE, CL_MAP_READ, roi->width, roi->height, (int)bpp);
  if(!mapped) return FALSE;
 
  const gboolean ptr_matches = (mapped == host_ptr);
  const gboolean is_zero_copy = ptr_matches;
  const cl_int unmap_err = dt_opencl_unmap_mem_object(devid, mem, mapped);
  if(unmap_err != CL_SUCCESS) return FALSE;
 
  // Use clFinish rather than event wait: some drivers disable event tracking, but we still need to guarantee
  // the unmap (and implicit sync) is complete before touching host memory or unlocking the cache entry.
  dt_opencl_finish(devid);
 
  return is_zero_copy;
}
 
static void *_gpu_try_reuse_pinned_from_cache(dt_pixel_cache_entry_t *cache_entry, void *host_ptr, int devid,
                                              const dt_iop_roi_t *roi, const size_t bpp, const int flags,
                                              int *out_cst, gboolean *out_reused)
{
  if(out_reused) *out_reused = FALSE;
  if(!cache_entry || !host_ptr || devid < 0) return NULL;
 
  int cached_cst = IOP_CS_NONE;
  void *mem = dt_pixel_cache_clmem_get(cache_entry, host_ptr, devid, roi->width, roi->height, (int)bpp, flags,
                                       &cached_cst);
  if(mem)
  {
    if(out_reused) *out_reused = TRUE;
    if(out_cst && cached_cst != IOP_CS_NONE) *out_cst = cached_cst;
  }
 
  return mem;
}
 
static inline gboolean _is_gamma_rgba8_output(const dt_iop_module_t *module, const size_t bpp,
                                              const char *message)
{
  return module && message && bpp == 4 * sizeof(uint8_t) && strcmp(module->op, "gamma") == 0
         && strcmp(message, "output") == 0;
}
 
static void *_gpu_alloc_device_with_flush(int devid, const dt_iop_roi_t *roi, const size_t bpp,
                                          const dt_iop_module_t *module, const char *message,
                                          void *keep);
static void *_gpu_try_reuse_device_from_cache(dt_pixel_cache_entry_t *cache_entry, int devid,
                                              const dt_iop_roi_t *roi, const size_t bpp,
                                              gboolean *out_reused);
 
static void *_gpu_get_pinned_or_alloc(int devid, void *host_ptr, const dt_iop_roi_t *roi, const size_t bpp,
                                      dt_pixel_cache_entry_t *cache_entry, const gboolean reuse_pinned,
                                      int *out_cst, gboolean *out_reused,
                                      const dt_iop_module_t *module, const char *message)
{
  const int flags = CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR;
  void *mem = NULL;
  const gboolean gamma_rgba8 = _is_gamma_rgba8_output(module, bpp, message);
  const int cl_bpp = gamma_rgba8 ? DT_OPENCL_BPP_ENCODE_RGBA8((int)bpp) : (int)bpp;
 
  if(out_reused) *out_reused = FALSE;
 
  if(reuse_pinned)
    mem = _gpu_try_reuse_pinned_from_cache(cache_entry, host_ptr, devid, roi, bpp, flags, out_cst, out_reused);
 
  if(!mem)
    mem = dt_opencl_alloc_device_use_host_pointer(devid, roi->width, roi->height, cl_bpp, host_ptr, flags);
 
  if(!mem)
  {
    dt_dev_pixelpipe_cache_flush_clmem(darktable.pixelpipe_cache, devid, NULL);
    if(reuse_pinned)
      mem = _gpu_try_reuse_pinned_from_cache(cache_entry, host_ptr, devid, roi, bpp, flags, out_cst, out_reused);
    if(!mem)
      mem = dt_opencl_alloc_device_use_host_pointer(devid, roi->width, roi->height, cl_bpp, host_ptr, flags);
  }
 
  return mem;
}
 
static void *_gpu_alloc_device_with_flush(int devid, const dt_iop_roi_t *roi, const size_t bpp,
                                          const dt_iop_module_t *module, const char *message,
                                          void *keep)
{
  const gboolean gamma_rgba8 = _is_gamma_rgba8_output(module, bpp, message);
  const int cl_bpp = gamma_rgba8 ? DT_OPENCL_BPP_ENCODE_RGBA8((int)bpp) : (int)bpp;
  void *mem = dt_opencl_alloc_device(devid, roi->width, roi->height, cl_bpp);
  if(!mem)
  {
    dt_dev_pixelpipe_cache_flush_clmem(darktable.pixelpipe_cache, devid, keep);
    mem = dt_opencl_alloc_device(devid, roi->width, roi->height, cl_bpp);
  }
  return mem;
}
 
static void *_gpu_try_reuse_device_from_cache(dt_pixel_cache_entry_t *cache_entry, int devid,
                                              const dt_iop_roi_t *roi, const size_t bpp,
                                              gboolean *out_reused)
{
  if(out_reused) *out_reused = FALSE;
  if(!cache_entry || devid < 0 || !roi || roi->width <= 0 || roi->height <= 0 || bpp == 0) return NULL;
 
  // Device-only allocations are tracked with a NULL host_ptr key and a
  // normalized READ_WRITE flag so we can reliably match across drivers.
  void *mem = dt_pixel_cache_clmem_get(cache_entry, NULL, devid, roi->width, roi->height, (int)bpp,
                                       CL_MEM_READ_WRITE, NULL);
  if(mem && out_reused) *out_reused = TRUE;
  return mem;
}
 
static void _gpu_log_pinned_reuse(dt_iop_module_t *module, const gboolean reused_from_cache)
{
  static dt_atomic_int clmem_reuse_hits;
  static dt_atomic_int clmem_reuse_misses;
 
  if(reused_from_cache)
  {
    const int hits = dt_atomic_add_int(&clmem_reuse_hits, 1) + 1;
    const int misses = dt_atomic_get_int(&clmem_reuse_misses);
    dt_print(DT_DEBUG_OPENCL,
             "[opencl_pixelpipe] %s reused pinned input from cache (hits=%d, misses=%d)\n",
             module ? module->name() : "unknown", hits, misses);
  }
  else
  {
    (void)dt_atomic_add_int(&clmem_reuse_misses, 1);
  }
}
 
static void *_gpu_init_buffer(int devid, void *const host_ptr, const dt_iop_roi_t *roi, const size_t bpp,
                              dt_iop_module_t *module, const char *message,
                              dt_pixel_cache_entry_t *cache_entry, const gboolean reuse_pinned,
                              const gboolean reuse_device,
                              int *out_cst, gboolean *out_reused, void *keep)
{
  // Need to use read-write mode because of in-place color space conversions.
  void *cl_mem_input = NULL;
  gboolean reused_from_cache = FALSE;
  const gboolean allow_reuse_pinned = reuse_pinned;
  const gboolean allow_reuse_device = reuse_device;
 
  if(out_reused) *out_reused = FALSE;
 
  if(host_ptr)
  {
    cl_mem_input = _gpu_get_pinned_or_alloc(devid, host_ptr, roi, bpp, cache_entry, allow_reuse_pinned,
                                           out_cst, &reused_from_cache, module, message);
  }
  else
  {
    if(allow_reuse_device)
      cl_mem_input = _gpu_try_reuse_device_from_cache(cache_entry, devid, roi, bpp, &reused_from_cache);
 
    if(!cl_mem_input)
      cl_mem_input = _gpu_alloc_device_with_flush(devid, roi, bpp, module, message, keep);
  }
 
  if(cl_mem_input == NULL)
  {
    dt_print(DT_DEBUG_OPENCL, "[opencl_pixelpipe] couldn't generate %s buffer for module %s\n", message,
             module ? module->op : "unknown");
  }
  else if(allow_reuse_pinned && cache_entry && host_ptr)
  {
    if(out_reused) *out_reused = reused_from_cache;
    _gpu_log_pinned_reuse(module, reused_from_cache);
  }
  else if(allow_reuse_device && cache_entry && !host_ptr && out_reused)
  {
    *out_reused = reused_from_cache;
  }
 
  return cl_mem_input;
}
 
static void _gpu_clear_buffer(void **cl_mem_buffer, dt_pixel_cache_entry_t *cache_entry, void *host_ptr, int cst,
                              const gboolean cache_device)
{
  if(cl_mem_buffer && *cl_mem_buffer != NULL)
  {
    cl_mem mem = *cl_mem_buffer;
    const cl_mem_flags flags = dt_opencl_get_mem_flags(mem);
    const gboolean can_cache_pinned = (cache_entry && host_ptr && (flags & CL_MEM_USE_HOST_PTR));
    const gboolean can_cache_device = (cache_entry && !host_ptr && cache_device && !(flags & CL_MEM_USE_HOST_PTR));
    const gboolean can_cache = (can_cache_pinned || can_cache_device);
    if(can_cache)
    {
      const int devid = dt_opencl_get_mem_context_id(mem);
      const int width = dt_opencl_get_image_width(mem);
      const int height = dt_opencl_get_image_height(mem);
      const int bpp = dt_opencl_get_image_element_size(mem);
      const int tracked_flags = can_cache_device ? CL_MEM_READ_WRITE : (int)flags;
      dt_pixel_cache_clmem_put(cache_entry, host_ptr, devid, width, height, bpp, tracked_flags, cst, mem);
    }
    else
    {
      if(cache_entry) dt_pixel_cache_clmem_remove(cache_entry, mem);
      dt_opencl_release_mem_object(mem);
    }
    *cl_mem_buffer = NULL;
  }
}
 
static int _cl_pinned_memory_copy(const int devid, void *host_ptr, void *cl_mem_buffer, const dt_iop_roi_t *roi,
                                  int cl_mode, size_t bpp, dt_iop_module_t *module, const char *message)
{
  if(!host_ptr || !cl_mem_buffer) return 1;
 
  const cl_mem mem = (cl_mem)cl_mem_buffer;
  const cl_mem_flags flags = dt_opencl_get_mem_flags(mem);
 
  // Fast path for true zero-copy pinned images: map/unmap is enough to synchronize host<->device.
  if(flags & CL_MEM_USE_HOST_PTR)
  {
    void *mapped = dt_opencl_map_image(devid, mem, TRUE, cl_mode, roi->width, roi->height, (int)bpp);
    if(mapped)
    {
      const gboolean ptr_matches = (mapped == host_ptr);
      const cl_int unmap_err = dt_opencl_unmap_mem_object(devid, mem, mapped);
      if(unmap_err != CL_SUCCESS) return 1;
 
      // Ensure unmap (and any implicit sync) completed before we possibly enqueue explicit transfers.
      // When event tracking is disabled, clFinish is the only reliable barrier.
      dt_opencl_finish(devid);
 
      if(ptr_matches)
      {
        dt_print(DT_DEBUG_OPENCL,
                 "[opencl_pixelpipe] successfully synced image %s via map/unmap for module %s (%s)\n",
                 (cl_mode == CL_MAP_WRITE) ? "host to device" : "device to host",
                 (module) ? module->op : "base buffer", message);
        return 0;
      }
    }
  }
 
  // Fallback: explicit blocking transfers (safe on all drivers).
  cl_int err = CL_SUCCESS;
  if(cl_mode == CL_MAP_WRITE)
    err = dt_opencl_write_host_to_device(devid, host_ptr, mem, roi->width, roi->height, (int)bpp);
  else if(cl_mode == CL_MAP_READ)
    err = dt_opencl_read_host_from_device(devid, host_ptr, mem, roi->width, roi->height, (int)bpp);
  else
    return 1;
 
  if(err != CL_SUCCESS)
  {
    dt_print(DT_DEBUG_OPENCL, "[opencl_pixelpipe] couldn't copy image %s for module %s (%s)\n",
             (cl_mode == CL_MAP_WRITE) ? "host to device" : "device to host",
             (module) ? module->op : "base buffer", message);
    return 1;
  }
 
  dt_print(DT_DEBUG_OPENCL, "[opencl_pixelpipe] successfully copied image %s for module %s (%s)\n",
           (cl_mode == CL_MAP_WRITE) ? "host to device" : "device to host",
           (module) ? module->op : "base buffer", message);
  return 0;
}
 
static float *_resync_input_gpu_to_cache(dt_dev_pixelpipe_t *pipe, float *input, void *cl_mem_input,
                                         dt_iop_buffer_dsc_t *input_format, const dt_iop_roi_t *roi_in,
                                         dt_iop_module_t *module, dt_iop_colorspace_type_t input_cst_cl,
                                         const size_t in_bpp, dt_pixel_cache_entry_t *input_entry,
                                         const char *message)
{
  if(!cl_mem_input) return input;
  dt_dev_pixelpipe_cache_wrlock_entry(darktable.pixelpipe_cache, DT_PIXELPIPE_CACHE_HASH_INVALID, TRUE, input_entry);
 
  int fail = _cl_pinned_memory_copy(pipe->devid, input, cl_mem_input, roi_in, CL_MAP_READ, in_bpp, module, message);
 
  // Color conversions happen inplace, so we need to ensure colorspace metadata are up-to-date.
  if(!fail) input_format->cst = input_cst_cl;
 
  // Enforce the OpenCL pipe to run in sync with CPU RAM cache so lock validity is guaranteed.
  dt_opencl_finish(pipe->devid);
  dt_dev_pixelpipe_cache_wrlock_entry(darktable.pixelpipe_cache, DT_PIXELPIPE_CACHE_HASH_INVALID, FALSE, input_entry);
 
  // Update colorspace tag (again, for safety).
  input_format->cst = input_cst_cl;
  return input;
}
 
static int _gpu_prepare_cl_input(dt_dev_pixelpipe_t *pipe, dt_iop_module_t *module,
                                 float *input, void **cl_mem_input, dt_iop_colorspace_type_t *input_cst_cl,
                                 const dt_iop_roi_t *roi_in, const size_t in_bpp,
                                 dt_pixel_cache_entry_t *input_entry, dt_pixel_cache_entry_t **locked_input_entry,
                                 void *keep)
{
  if(!locked_input_entry) return 1;
  *locked_input_entry = NULL;
 
  if(*cl_mem_input != NULL)
  {
    // We passed the OpenCL memory buffer through directly on vRAM from previous module.
    // This is fast and efficient.
    // If it's a true zero-copy pinned image, keep the input cache entry read-locked until kernels complete,
    // otherwise another thread may overwrite host memory while the GPU is still reading it.
    dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] %s will use its input directly from vRAM\n", module->name());
    const cl_mem mem = (cl_mem)*cl_mem_input;
    const cl_mem_flags flags = dt_opencl_get_mem_flags(mem);
    if(flags & CL_MEM_USE_HOST_PTR)
      if(_cl_is_zero_copy_image(pipe->devid, mem, input, roi_in, in_bpp))
      {
        dt_dev_pixelpipe_cache_rdlock_entry(darktable.pixelpipe_cache, DT_PIXELPIPE_CACHE_HASH_INVALID, TRUE, input_entry);
        *locked_input_entry = input_entry;
      }
    return 0;
  }
 
  if(!input)
  {
    dt_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] %s has no input (cache)\n", module->name());
    return 1;
  }
 
  dt_dev_pixelpipe_cache_rdlock_entry(darktable.pixelpipe_cache, DT_PIXELPIPE_CACHE_HASH_INVALID, TRUE, input_entry);
 
  // Try to reuse a cached pinned buffer; otherwise allocate a new pinned image backed by `input`.
  gboolean input_reused_from_cache = FALSE;
  *cl_mem_input = _gpu_init_buffer(pipe->devid, input, roi_in, in_bpp, module, "input", input_entry,
                                  TRUE, FALSE, input_cst_cl, &input_reused_from_cache, keep);
  int fail = (*cl_mem_input == NULL);
 
  // If the input is true zero-copy, the GPU will access host memory asynchronously: keep the cache
  // entry read-locked until all kernels have completed. If not, drivers may use a device-side copy
  // which must be synchronized from the host before running kernels.
  gboolean keep_lock = FALSE;
  cl_mem mem = NULL;
  if(!fail && *cl_mem_input)
  {
    mem = (cl_mem)*cl_mem_input;
    const cl_mem_flags flags = dt_opencl_get_mem_flags(mem);
    if(flags & CL_MEM_USE_HOST_PTR)
      keep_lock = _cl_is_zero_copy_image(pipe->devid, mem, input, roi_in, in_bpp);
  }
 
  if(!fail && mem && !keep_lock)
  {
    const cl_int err = dt_opencl_write_host_to_device(pipe->devid, input, mem, roi_in->width, roi_in->height,
                                                      (int)in_bpp);
    if(err != CL_SUCCESS)
    {
      dt_print(DT_DEBUG_OPENCL, "[opencl_pixelpipe] couldn't copy image host to device for module %s (%s)\n",
               (module) ? module->op : "base buffer", "cache to input");
      fail = TRUE;
    }
    else
    {
      dt_print(DT_DEBUG_OPENCL, "[opencl_pixelpipe] successfully copied image host to device for module %s (%s)\n",
               (module) ? module->op : "base buffer", "cache to input");
    }
  }
 
  // Enforce sync with the CPU/RAM cache so lock validity is guaranteed.
  dt_opencl_events_wait_for(pipe->devid);
 
  if(keep_lock)
    *locked_input_entry = input_entry;
  else
    dt_dev_pixelpipe_cache_rdlock_entry(darktable.pixelpipe_cache, DT_PIXELPIPE_CACHE_HASH_INVALID, FALSE, input_entry);
 
  return fail ? 1 : 0;
}
 
#else // HAVE_OPENCL
 
static inline void _gpu_clear_buffer(void **cl_mem_buffer, dt_pixel_cache_entry_t *cache_entry, void *host_ptr, int cst,
                                     const gboolean cache_device)
{
  (void)cache_entry;
  (void)host_ptr;
  (void)cst;
  (void)cache_device;
  if(cl_mem_buffer) *cl_mem_buffer = NULL;
}
 
#endif // HAVE_OPENCL
 
// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on