pixelpipe_cache.c

Go to the documentation of this file.

/*
    This file is part of darktable,
    Copyright (C) 2009-2012, 2015 johannes hanika.
    Copyright (C) 2010-2011 Henrik Andersson.
    Copyright (C) 2011 Robert Bieber.
    Copyright (C) 2011 Rostyslav Pidgornyi.
    Copyright (C) 2012 Richard Wonka.
    Copyright (C) 2012-2014, 2016 Tobias Ellinghaus.
    Copyright (C) 2013-2014, 2016 Roman Lebedev.
    Copyright (C) 2014 Ulrich Pegelow.
    Copyright (C) 2019, 2023-2026 Aurélien PIERRE.
    Copyright (C) 2019-2021 Pascal Obry.
    Copyright (C) 2020, 2022 Hanno Schwalm.
    Copyright (C) 2020 Ralf Brown.
    Copyright (C) 2021 Aldric Renaudin.
    Copyright (C) 2021 Dan Torop.
    Copyright (C) 2022 Martin Bařinka.
    Copyright (C) 2023 lologor.
    Copyright (C) 2024 Alynx Zhou.
    Copyright (C) 2025-2026 Guillaume Stutin.
    Copyright (C) 2025 Miguel Moquillon.
    
    darktable is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    darktable is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with darktable.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#include <inttypes.h>
#include <stdarg.h>
#include <glib.h>
#include <stdlib.h>
#include <signal.h>
#include <string.h>
 
#include "system/sys_resources.h"
#include "caches/pixelpipe_cache_pressure.h"
#include "develop/pixelpipe_hb.h"
#include "caches/pixelpipe_cache.h"
#include "common/opencl.h"
#include "pixel/format.h"
/* For dt_iop_module_t: the cache reads `module->op` to special-case the gamma module
 * and calls `module->name()` for its diagnostics. That is the last edge keeping this
 * file above develop/; taking a name string instead of a module would cut it. */
#include "develop/imageop.h"
 
/* Set once by dt_dev_pixelpipe_cache_init(); see the header for why they are not read live. */
static gboolean _verbose = FALSE;
static gboolean _verbose_detail = FALSE;
 
/* Statement-safe: a bare `if(_verbose) dt_print(...)` swallows a following `else`. */
#define _cache_print(channel, ...)                    \
  do {                                                \
    if(_verbose) dt_print((channel), __VA_ARGS__);     \
  } while(0)
 
 
 
/* PRIVATE. Nothing outside this file reads a field of it -- the header exposes the
 * type as an opaque handle so that it cannot. */
typedef struct dt_dev_pixelpipe_cache_t
{
  GHashTable *entries;
  // External (temporary) buffers keyed by address hash, separate from pipeline cache entries.
  GHashTable *external_entries;
  uint64_t next_serial;
  uint64_t queries;
  uint64_t hits;
  size_t max_memory;
  size_t current_memory;
  // System memory-pressure probe cache, guarded by `lock` (see the pressure valve
  // in pixelpipe_cache.c): last probed system-wide available RAM, decremented by
  // our own allocations between two rate-limited probes. The estimate legitimately
  // reaches 0 under pressure, so whether the platform answers at all is a separate
  // flag rather than an `est == 0` sentinel.
  gint64 sys_probe_time_us;
  size_t sys_available_est;
  gboolean sys_probe_valid;
  /* Kernel memory pressure: how much the cache may hold while the machine stalls, and what
   * decides it -- caches/pixelpipe_cache_pressure.c, which this file feeds through _pressure_sink()
   * and which never touches an entry itself. Guarded by `lock`, like the fields above. */
  dt_pixelpipe_cache_pressure_monitor_t psi;
  dt_pthread_mutex_t lock; // mutex to protect the cache entries
  dt_cache_arena_t arena;
} dt_dev_pixelpipe_cache_t;
 
 
/* The cache instance, owned HERE. darktable.c used to hold it on the application struct and
 * implement the accessor; it still decides HOW BIG the cache is and still retries smaller on
 * failure -- that is resource policy, not a cache decision -- but the pointer lives here. */
static dt_dev_pixelpipe_cache_t *_pixelpipe_cache = NULL;
 
gboolean dt_dev_pixelpipe_cache_is_ready(void)
{
  return !IS_NULL_PTR(_pixelpipe_cache);
}
 
/* Installed by the orchestrator; see dt_dev_pixelpipe_cache_set_handlers(). NULL means
 * nobody is listening, which is a working configuration, not an error. */
static dt_pixelpipe_cache_warn_handler_t _warn_handler = NULL;
static dt_pixelpipe_cache_ready_handler_t _ready_handler = NULL;
static const dt_pixelpipe_cache_observer_t *_observer = NULL;
 
void dt_dev_pixelpipe_cache_set_handlers(dt_pixelpipe_cache_warn_handler_t warn,
                                         dt_pixelpipe_cache_ready_handler_t ready,
                                         const dt_pixelpipe_cache_observer_t *observer)
{
  _warn_handler = warn;
  _ready_handler = ready;
  _observer = observer;
}
 
/* printf-style, so the call sites keep reading as they did; the formatting happens here and
 * the handler receives a finished string. */
static void _warn_user(const char *format, ...) __attribute__((format(printf, 1, 2)));
static void _warn_user(const char *format, ...)
{
  if(!_warn_handler) return;
  va_list ap;
  va_start(ap, format);
  char *message = g_strdup_vprintf(format, ap);
  va_end(ap);
  if(message) _warn_handler(message);
  g_free(message);
}
 
static inline gboolean _observed(void)
{
  return _observer && _observer->active && _observer->active();
}
 
/* Each member is checked at its own call: `active` returning TRUE says the supervisor is
 * watching, not that it implements every hook. */
static inline void _observe_read(uint64_t hash, size_t size)
{
  if(_observed() && _observer->cacheline_read) _observer->cacheline_read(hash, size);
}
static inline void _observe_delete(uint64_t hash, size_t size, int owner_pipe_id, const char *name)
{
  if(_observed() && _observer->cacheline_delete) _observer->cacheline_delete(hash, size, owner_pipe_id, name);
}
static inline void _observe_rekey(uint64_t old_hash, uint64_t new_hash)
{
  if(_observed() && _observer->rekey) _observer->rekey(old_hash, new_hash);
}
 
 
static __thread const char *dt_pixelpipe_cache_current_module = NULL;
 
static dt_pixel_cache_entry_t *_non_threadsafe_cache_get_entry(dt_dev_pixelpipe_cache_t *cache, GHashTable *table,
                                                               const uint64_t key);
 
static inline const char *_cache_debug_module_name(void)
{
  return dt_pixelpipe_cache_current_module ? dt_pixelpipe_cache_current_module : "-";
}
 
static void _trace_exact_hit(const char *phase, const uint64_t hash, dt_pixel_cache_entry_t *cache_entry,
                             void *data, void *cl_mem_output, const int preferred_devid, const gboolean verbose)
{
  if(!_verbose) return;
  if(verbose && !_verbose_detail) return;
 
  _cache_print(DT_DEBUG_PIPECACHE,
           "[pixelpipe_cache] exact-hit %s req=%" PRIu64 " entry=%" PRIu64 "/%" PRIu64
           " data=%p cl=%p refs=%i auto=%i dev=%i module=%s name=%s\n",
           phase, hash, cache_entry ? cache_entry->hash : DT_PIXELPIPE_CACHE_HASH_INVALID,
           cache_entry ? cache_entry->serial : 0, data, cl_mem_output,
           cache_entry ? dt_atomic_get_int(&cache_entry->refcount) : -1,
           cache_entry ? cache_entry->auto_destroy : -1, preferred_devid, _cache_debug_module_name(),
           (cache_entry && cache_entry->name) ? cache_entry->name : "-");
}
 
const char *dt_pixelpipe_cache_set_current_module(const char *module)
{
  const char *previous = dt_pixelpipe_cache_current_module;
  dt_pixelpipe_cache_current_module = module;
  return previous;
}
 
typedef struct dt_cache_clmem_t
{
  void *host_ptr;
  void *mem;
  int refs;
} dt_cache_clmem_t;
 
typedef enum dt_pixel_cache_materialize_source_rank_t
{
  DT_PIXEL_CACHE_MATERIALIZE_SOURCE_NONE = 0,
  DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_ANY = 1,
  DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_PREFERRED = 2,
  DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_ANY = 3,
  DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_PREFERRED = 4,
} dt_pixel_cache_materialize_source_rank_t;
 
 
void _non_thread_safe_cache_ref_count_entry(dt_dev_pixelpipe_cache_t *cache, gboolean lock,
                                            dt_pixel_cache_entry_t *cache_entry);
static void _free_cache_entry(dt_pixel_cache_entry_t *cache_entry);
static void _pixelpipe_cache_finalize_entry(dt_pixel_cache_entry_t *cache_entry, void **data,
                                            const char *message);
int _non_thread_safe_cache_remove(dt_dev_pixelpipe_cache_t *cache, const gboolean force,
                                  dt_pixel_cache_entry_t *cache_entry, GHashTable *table);
 
static dt_pixel_cache_entry_t *_pixelpipe_cache_create_entry_locked(dt_dev_pixelpipe_cache_t *cache,
                                                                    const uint64_t hash, const size_t size,
                                                                    const char *name, const int id);
static dt_pixel_cache_entry_t *dt_pixel_cache_new_entry(const uint64_t hash, const size_t size,
                                                        const char *name, const int id,
                                                        dt_dev_pixelpipe_cache_t *cache, gboolean alloc,
                                                        GHashTable *table);
static gboolean _cache_entry_clmem_flush_device(dt_pixel_cache_entry_t *entry, const int devid);
static gboolean _cache_entry_materialize_host_data_locked(dt_pixel_cache_entry_t *entry, int preferred_devid,
                                                          gboolean prefer_device_payload);
static int dt_dev_pixelpipe_cache_flush_old(dt_dev_pixelpipe_cache_t *cache);
static int _memory_pressure_shedder(dt_dev_pixelpipe_cache_t *cache);
 
#ifdef HAVE_OPENCL
static gboolean _cache_entry_clmem_flush_host_pinned_locked(dt_pixel_cache_entry_t *entry, void *host_ptr, int devid);
#endif
 
/* Single writer of `cache_entry->age`, and the only way an entry's MRU timestamp moves. EVERY
 * path that hands an entry over to a consumer must call this -- the intra-run producer->consumer
 * handoff included, since that is still a use as far as eviction is concerned. An entry whose age
 * is only ever set at creation is aged by the eviction sweeps as if it had never been read, so the
 * cacheline the pipeline consumes on every frame becomes the first LRU victim.
 *
 * This deliberately does NOT touch `hits`. `age` answers "when was this last needed" and every
 * use, reuse or passthrough, answers it. `hits` answers "has this ever been reused ASYNCHRONOUSLY"
 * and passthrough does not: a module publishing its output with one reference reserved for the
 * next module, which reopens it as its input and releases it, is the normal pipeline flow, not a
 * cache win. Only the lookups that also count a `cache->queries`/`cache->hits` pair may bump it. */
static inline void _pixel_cache_touch(dt_pixel_cache_entry_t *cache_entry)
{
  if(IS_NULL_PTR(cache_entry)) return;
  dt_atomic_set_uint64(&cache_entry->age, (uint64_t)g_get_monotonic_time());
}
 
static inline int64_t _pixel_cache_get_age(dt_pixel_cache_entry_t *cache_entry)
{
  return (int64_t)dt_atomic_get_uint64(&cache_entry->age);
}
 
 
static dt_pixel_cache_entry_t *_cache_entry_for_host_ptr_locked(dt_dev_pixelpipe_cache_t *cache, void *host_ptr)
{
  if(IS_NULL_PTR(cache) || IS_NULL_PTR(host_ptr)) return NULL;
 
  const uint64_t hash = (uint64_t)(uintptr_t)host_ptr;
  dt_pixel_cache_entry_t *entry = _non_threadsafe_cache_get_entry(cache, cache->external_entries, hash);
  if(entry && entry->external_alloc && entry->data == host_ptr) return entry;
  return NULL;
}
 
dt_pixel_cache_entry_t *_non_threadsafe_cache_get_entry(dt_dev_pixelpipe_cache_t *cache, GHashTable *table,
                                                        const uint64_t key)
{
  dt_pixel_cache_entry_t *entry = (dt_pixel_cache_entry_t *)g_hash_table_lookup(table, &key);
  return entry;
}
 
 
dt_pixel_cache_entry_t *dt_dev_pixelpipe_cache_get_entry(const uint64_t hash)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(hash == DT_PIXELPIPE_CACHE_HASH_INVALID) return NULL;
  dt_pthread_mutex_lock(&cache->lock);
  dt_pixel_cache_entry_t *entry = _non_threadsafe_cache_get_entry(cache, cache->entries, hash);
  /* Refresh recency only. `develop/pixelpipe_hb.c` reopens each module's INPUT cacheline through
   * here on every recursion step: that is the producer->consumer handoff of one pipeline run, a
   * use for eviction purposes but not an asynchronous reuse, so neither `hits` counter moves. */
  _pixel_cache_touch(entry);
  dt_pthread_mutex_unlock(&cache->lock);
  return entry;
}
 
 
dt_pixel_cache_entry_t *dt_dev_pixelpipe_cache_get_entry_by_data(void *data)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(!cache || !data) return NULL;
 
  dt_pthread_mutex_lock(&cache->lock);
 
  GHashTableIter iter;
  gpointer key, value;
 
  /* Search regular entries table */
  g_hash_table_iter_init(&iter, cache->entries);
  while(g_hash_table_iter_next(&iter, &key, &value))
  {
    dt_pixel_cache_entry_t *entry = (dt_pixel_cache_entry_t *)value;
    if(entry && entry->data == data)
    {
      _pixel_cache_touch(entry);
      dt_pthread_mutex_unlock(&cache->lock);
      return entry;
    }
  }
 
  /* Search external entries table */
  g_hash_table_iter_init(&iter, cache->external_entries);
  while(g_hash_table_iter_next(&iter, &key, &value))
  {
    dt_pixel_cache_entry_t *entry = (dt_pixel_cache_entry_t *)value;
    if(entry && entry->data == data)
    {
      _pixel_cache_touch(entry);
      dt_pthread_mutex_unlock(&cache->lock);
      return entry;
    }
  }
 
  dt_pthread_mutex_unlock(&cache->lock);
  return NULL;
}
 
 
static size_t _pixel_cache_get_size(dt_pixel_cache_entry_t *cache_entry)
{
  return cache_entry->size / (1024 * 1024);
}
 
 
static void _pixel_cache_message(dt_pixel_cache_entry_t *cache_entry, const char *message, gboolean verbose)
{
  if(!_verbose) return;
  if(verbose && !_verbose_detail) return;
  _cache_print(DT_DEBUG_PIPECACHE,
           "[pixelpipe] cache entry %" PRIu64 "/%" PRIu64 ": %s (data=%p - %" G_GSIZE_FORMAT " MiB - age %" PRId64
           " - hits %i - refs %i - auto %i - ext %i - id %i - module %s) %s\n",
           cache_entry->hash, cache_entry->serial,
           cache_entry->name ? cache_entry->name : "-", cache_entry->data,
           _pixel_cache_get_size(cache_entry), _pixel_cache_get_age(cache_entry), cache_entry->hits,
           dt_atomic_get_int(&cache_entry->refcount), cache_entry->auto_destroy,
           cache_entry->external_alloc, cache_entry->id, _cache_debug_module_name(), message);
}
 
static void _pixelpipe_cache_finalize_entry(dt_pixel_cache_entry_t *cache_entry, void **data,
                                            const char *message)
{
  _pixel_cache_touch(cache_entry);
  if(data)
    *data = cache_entry->data ? __builtin_assume_aligned(cache_entry->data, DT_CACHELINE_BYTES) : NULL;
  _pixel_cache_message(cache_entry, message, FALSE);
}
 
gboolean dt_dev_pixelpipe_cache_ref_entry_by_hash(const uint64_t hash,
                                                  void **data,
                                                  dt_pixel_cache_entry_t **entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(!IS_NULL_PTR(data)) *data = NULL;
  if(!IS_NULL_PTR(entry)) *entry = NULL;
  if(IS_NULL_PTR(cache) || hash == DT_PIXELPIPE_CACHE_HASH_INVALID) return FALSE;
 
  dt_pthread_mutex_lock(&cache->lock);
  cache->queries++;
 
  dt_pixel_cache_entry_t *cache_entry = _non_threadsafe_cache_get_entry(cache, cache->entries, hash);
  if(!IS_NULL_PTR(cache_entry) && !cache_entry->auto_destroy)
  {
    cache->hits++;
    cache_entry->hits++;
    _non_thread_safe_cache_ref_count_entry(cache, TRUE, cache_entry);
    _pixelpipe_cache_finalize_entry(cache_entry, data, "ref-by-hash");
    if(!IS_NULL_PTR(entry)) *entry = cache_entry;
  }
 
  const gboolean found = !IS_NULL_PTR(cache_entry) && !cache_entry->auto_destroy;
  const size_t found_size = found ? cache_entry->size : 0;
  dt_pthread_mutex_unlock(&cache->lock);
 
  if(found ) _observe_read(hash, found_size);
    
  return found;
}
 
 
gboolean dt_dev_pixelpipe_cache_ref_host_entry_by_hash(const uint64_t hash,
                                                       void **data,
                                                       dt_pixel_cache_entry_t **entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(!IS_NULL_PTR(data)) *data = NULL;
  if(!IS_NULL_PTR(entry)) *entry = NULL;
  if(IS_NULL_PTR(cache) || hash == DT_PIXELPIPE_CACHE_HASH_INVALID) return FALSE;
 
  gboolean found = FALSE;
  size_t found_size = 0;
 
  dt_pthread_mutex_lock(&cache->lock);
  cache->queries++;
 
  dt_pixel_cache_entry_t *cache_entry = _non_threadsafe_cache_get_entry(cache, cache->entries, hash);
  if(!IS_NULL_PTR(cache_entry) && !cache_entry->auto_destroy && !IS_NULL_PTR(cache_entry->data))
  {
    /* The same rejection `dt_dev_pixelpipe_cache_peek()` applies, for the same reason: a reusable
     * output cacheline is rekeyed to its new hash BEFORE the recompute that fills it starts, so an
     * entry somebody currently holds the write lock on carries stale or half-written pixels under a
     * key that already claims to describe the new ones. Never WAIT on that lock -- report a miss and
     * let the caller queue for the publication instead. */
    if(dt_pthread_rwlock_tryrdlock(&cache_entry->lock) == 0)
    {
      dt_pthread_rwlock_unlock(&cache_entry->lock);
      cache->hits++;
      cache_entry->hits++;
      _non_thread_safe_cache_ref_count_entry(cache, TRUE, cache_entry);
      _pixelpipe_cache_finalize_entry(cache_entry, data, "ref-host-by-hash");
      if(!IS_NULL_PTR(entry)) *entry = cache_entry;
      found = TRUE;
      found_size = cache_entry->size;
    }
  }
 
  dt_pthread_mutex_unlock(&cache->lock);
 
  if(found) _observe_read(hash, found_size);
 
  return found;
}
 
// remove the cache entry with the given hash and update the cache memory usage
// WARNING: not internally thread-safe, protect its calls with mutex lock
// return 0 on success, 1 on error
int _non_thread_safe_cache_remove(dt_dev_pixelpipe_cache_t *cache, const gboolean force,
                                  dt_pixel_cache_entry_t *cache_entry, GHashTable *table)
{
  if(!IS_NULL_PTR(cache_entry))
  {
    // Returns 1 if the lock is captured by another thread
    // 0 if WE capture the lock, and then need to release it
    gboolean locked = dt_pthread_rwlock_trywrlock(&cache_entry->lock);
    if(!locked) dt_pthread_rwlock_unlock(&cache_entry->lock);
    gboolean used = dt_atomic_get_int(&cache_entry->refcount) > 0;
 
    /* Force-removal may bypass caller lifecycle checks but must never destroy
     * an entry that still has active readers/writers. Active users can still
     * access cl_mem_list after this call (for example borrowed GPU payloads),
     * so removing a referenced entry here would create dangling pointers. */
    if(!used && (!locked || force))
    {
      // Note: the free callback takes care of flushing OpenCL buffers too
      g_hash_table_remove(table, &cache_entry->hash);
      return 0;
    }
    else if(used)
      _pixel_cache_message(cache_entry, "cannot remove: used", TRUE);
    else if(locked)
      _pixel_cache_message(cache_entry, "cannot remove: locked", TRUE);
  }
  else
  {
    _cache_print(DT_DEBUG_PIPECACHE, "[pixelpipe] cache entry not found, will not be removed\n");
  }
  return 1;
}
 
 
int dt_dev_pixelpipe_cache_remove(const gboolean force,
                                  dt_pixel_cache_entry_t *cache_entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  dt_pthread_mutex_lock(&cache->lock);
  int error = _non_thread_safe_cache_remove(cache, force, cache_entry, cache->entries);
  dt_pthread_mutex_unlock(&cache->lock);
  return error;
}
 
#ifdef HAVE_OPENCL
static gboolean _cache_entry_materialize_host_data_locked(dt_pixel_cache_entry_t *entry, int preferred_devid,
                                                          gboolean prefer_device_payload)
{
  dt_cache_clmem_t *source = NULL;
  gboolean ok = FALSE;
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
 
  /* We materialize RAM from the most authoritative cached payload in one pass instead of
   * walking the list multiple times with slightly different predicates:
   * - when RAM existed before, prefer pinned host-backed payloads first because they should
   *   already alias the cacheline or be the cheapest path back to host,
   * - when RAM has just been allocated for a GPU-only cacheline, prefer device payloads first,
   * - if a preferred OpenCL device is known, rank payloads from that device ahead of the rest.
   * This keeps the fallback order explicit without scattering it over six loops. */
  dt_pixel_cache_materialize_source_rank_t best_rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_NONE;
  for(GList *l = g_list_first(entry->cl_mem_list); l; l = g_list_next(l))
  {
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(IS_NULL_PTR(c) || IS_NULL_PTR(c->mem)) continue;
 
    /* We are looking for one authoritative cached payload to materialize back to RAM.
     * Only consider records whose live OpenCL context still matches the recorded device.
     * Other cached payloads may belong to a different pipeline/device and must stay untouched. */
    const int mem_devid = dt_opencl_get_mem_context_id((cl_mem)c->mem);
    if(mem_devid != preferred_devid) continue;
 
    const gboolean host_backed = (c->host_ptr == entry->data);
    const gboolean device_only = (IS_NULL_PTR(c->host_ptr));
    if(!host_backed && !device_only) continue;
 
    dt_pixel_cache_materialize_source_rank_t rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_NONE;
    if(!prefer_device_payload)
    {
      if(host_backed && (preferred_devid < 0 || mem_devid == preferred_devid))
        rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_PREFERRED;
      else if(device_only && preferred_devid >= 0 && mem_devid == preferred_devid)
        rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_PREFERRED;
      else if(device_only)
        rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_ANY;
      else if(host_backed)
        rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_ANY;
    }
    else
    {
      if(device_only && preferred_devid >= 0 && mem_devid == preferred_devid)
        rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_PREFERRED;
      else if(device_only)
        rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_ANY;
      else if(host_backed && (preferred_devid < 0 || mem_devid == preferred_devid))
        rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_PREFERRED;
      else if(host_backed)
        rank = DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_ANY;
    }
 
    if(rank > best_rank)
    {
      best_rank = rank;
      source = c;
      if(rank == DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_PREFERRED) break;
    }
  }
 
  if(source)
  {
 
    const int devid = dt_opencl_get_mem_context_id(source->mem);
    const int width = dt_opencl_get_image_width(source->mem);
    const int height = dt_opencl_get_image_height(source->mem);
    const int bpp = dt_opencl_get_image_element_size(source->mem);
 
    if(dt_opencl_is_pinned_memory((cl_mem)source->mem) && source->host_ptr == entry->data)
    {
      void *mapped = dt_opencl_map_image(devid, (cl_mem)source->mem, TRUE, CL_MAP_READ,
                                         width, height, bpp);
      ok = (dt_opencl_unmap_mem_object(devid, (cl_mem)source->mem, mapped) == CL_SUCCESS);
    }
    if(!ok)
    {
      ok = (dt_opencl_read_host_from_device(devid, entry->data, source->mem,
                                            width, height, bpp) == CL_SUCCESS);
    }
  }
 
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
  return ok;
}
#else
static gboolean _cache_entry_materialize_host_data_locked(dt_pixel_cache_entry_t *entry, int preferred_devid,
                                                          gboolean prefer_device_payload)
{
  (void)preferred_devid;
  (void)prefer_device_payload;
  return entry && !IS_NULL_PTR(entry->data);
}
#endif
 
static gboolean _cache_entry_materialize_host_data(dt_dev_pixelpipe_cache_t *cache, int preferred_devid,
                                                   dt_pixel_cache_entry_t *entry)
{
  if(IS_NULL_PTR(cache) || IS_NULL_PTR(entry)) return FALSE;
  if(preferred_devid < 0 && dt_pixel_cache_entry_get_data(entry) == NULL) return FALSE;
 
  dt_dev_pixelpipe_cache_wrlock_entry(TRUE, entry);
  gboolean use_host_ptr = TRUE;
  if(IS_NULL_PTR(dt_pixel_cache_entry_get_data(entry)))
  {
    dt_pixel_cache_alloc(entry);
    use_host_ptr = FALSE;
  }
  const gboolean ok = _cache_entry_materialize_host_data_locked(entry, preferred_devid, use_host_ptr);
  dt_dev_pixelpipe_cache_wrlock_entry(FALSE, entry);
 
  return ok;
}
 
#ifdef HAVE_OPENCL
static gboolean _cache_entry_clmem_has_host_pinned_locked(dt_pixel_cache_entry_t *entry, void *host_ptr, int devid)
{
  if(IS_NULL_PTR(entry) || IS_NULL_PTR(host_ptr)) return FALSE;
 
  gboolean found = FALSE;
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
  for(GList *l = g_list_first(entry->cl_mem_list); l; l = g_list_next(l))
  {
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(IS_NULL_PTR(c) || IS_NULL_PTR(c->mem)) continue;
 
    if(c->refs == 0 && devid == dt_opencl_get_mem_context_id((cl_mem)c->mem))
    {
      found = TRUE;
      break;
    }
  }
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
 
  return found;
}
 
static gboolean _cache_entry_clmem_flush_host_pinned_locked(dt_pixel_cache_entry_t *entry, void *host_ptr, int devid)
{
  // If host_ptr is NULL, we don't have RAM cache for this buffer, 
  // so we can't flush the vRAM cache or we would loose it forever.
  if(IS_NULL_PTR(entry) || IS_NULL_PTR(host_ptr)) return FALSE;
 
  gboolean flushed = FALSE;
 
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
  for(GList *l = g_list_first(entry->cl_mem_list); l;)
  {
    GList *next = g_list_next(l);
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(IS_NULL_PTR(c->mem))
    {
      // Current cacheline holds an empty buffer, no point keeping it
      entry->cl_mem_list = g_list_delete_link(entry->cl_mem_list, l);
      dt_free(c);
      l = next;
      continue;
    }
    if(dt_opencl_get_mem_context_id(c->mem) != devid)
    {
      // Current cacheline doesn't belong to current OpenCL devide: don't touch it
      l = next;
      continue;
    }
 
    entry->cl_mem_list = g_list_delete_link(entry->cl_mem_list, l);
    dt_opencl_release_mem_object(c->mem);
    dt_free(c);
    flushed = TRUE;
    l = next;
  }
 
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
 
  return flushed;
}
#endif
 
void dt_dev_pixelpipe_cache_flush_clmem(const int devid)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  // devid < 0 means the calling pipe never used OpenCL: it owns no device-side
  // payload in the cache, so there is nothing of its own to release here.
  //
  // This used to also support devid == -1 as "drop cl_mem from every device,
  // regardless of who else is using it", called from per-pipe cleanup. That ran
  // without holding any dev[].lock, so it could race the eventlist/cl_mem
  // bookkeeping of whichever OTHER pixelpipe was concurrently running on that
  // device, corrupting it and crashing inside clGetEventInfo/clWaitForEvents
  // (see #859 and #864). A global, all-devices teardown is never needed during
  // normal operation: dt_cleanup() already finishes every device and
  // dt_dev_pixelpipe_cache_cleanup() unconditionally releases all remaining
  // cl_mem objects at application exit, when nothing else is running.
  if(devid < 0) return;
 
  // NOTE: the caller must hold the device lock (dt_opencl_reserve_device_by_id()) -- either
  // because it IS the pixelpipe currently running on that device (the lock
  // dt_opencl_reserve_device_for_pipe() handed it for the duration of its run), or because
  // it explicitly took that lock to safely flush this device's cache entries
  // after its own run finished (see dt_dev_pixelpipe_cache_flush_clmem_for_pipe()).
  dt_opencl_events_wait_for(devid);
 
  dt_pthread_mutex_lock(&cache->lock);
  GHashTableIter iter;
  gpointer key, value;
  g_hash_table_iter_init(&iter, cache->entries);
  while(g_hash_table_iter_next(&iter, &key, &value))
  {
    dt_pixel_cache_entry_t *entry = (dt_pixel_cache_entry_t *)value;
 
    /* Only idle cachelines may have their vRAM reclaimed. An entry that is referenced or
     * write-locked is somebody's live (or about-to-be-consumed) buffer: the recursion reserves
     * an entry-level ref for the next consumer before that consumer borrows the cl_mem payload,
     * so a payload can be unborrowed (per-payload refs == 0) yet still belong to an in-flight
     * pipe. Honoring the same protection the LRU/removal paths use (refcount + non-blocking
     * write-lock probe) keeps us from yanking the sole vRAM copy of another pipe's input out
     * from under it -- which left a husk and produced skull thumbnails (issue #817). The
     * trywrlock never waits, so this stays lightweight and cannot deadlock against renders that
     * already hold entry locks. */
    const gboolean used = dt_atomic_get_int(&entry->refcount) > 0;
    gboolean locked = dt_pthread_rwlock_trywrlock(&entry->lock);
    if(!locked) dt_pthread_rwlock_unlock(&entry->lock);
    if(used || locked)
    {
      if(_verbose && _verbose_detail)
        _cache_print(DT_DEBUG_OPENCL,
          "[dt_dev_pixelpipe_cache_flush_clmem] entry %" PRIu64 " is in use (refcount=%i locked=%i), "
          "keeping its vRAM\n", entry->hash, dt_atomic_get_int(&entry->refcount), locked);
      continue;
    }
 
    if(_verbose && _verbose_detail)
      _cache_print(DT_DEBUG_OPENCL,
        "[dt_dev_pixelpipe_cache_flush_clmem] trying to flush vRAM for entry %" PRIu64 " on device %d...\n",
        entry->hash, devid);
 
    /* If reclaiming this device's vRAM leaves the entry with no buffer at all, delete it now
     * instead of letting a payload-less husk persist as a cache hit. We hold cache->lock for the
     * whole iteration, and lookups bump the consumer ref under that same lock, so no consumer can
     * be mid-acquisition of this (refcount == 0) entry. iter_remove runs _free_cache_entry, which
     * releases any remaining resources. */
    if(_cache_entry_clmem_flush_device(entry, devid))
      g_hash_table_iter_remove(&iter);
  }
  dt_pthread_mutex_unlock(&cache->lock);
}
 
#ifdef HAVE_OPENCL
void dt_dev_pixelpipe_cache_flush_clmem_for_pipe(const int devid)
{
  // Like dt_dev_pixelpipe_cache_flush_clmem(), but for callers that do NOT
  // currently hold the device lock (dt_opencl_reserve_device_by_id()) -- typically a pipe's own
  // cleanup, running after dt_dev_pixelpipe_process() already released that
  // lock. Taking it here ensures we can't race the eventlist/cl_mem bookkeeping
  // of whichever OTHER pixelpipe is now running on that device.
  if(devid < 0 || !dt_opencl_is_inited()) return;
 
  dt_opencl_reserve_device_by_id(devid);
  dt_dev_pixelpipe_cache_flush_clmem(devid);
  dt_opencl_release_device(devid);
}
#else
void dt_dev_pixelpipe_cache_flush_clmem_for_pipe(const int devid)
{
  (void)devid;
}
#endif
 
typedef struct _cache_lru_t
{
  int64_t max_age;
  uint64_t hash;
  dt_pixel_cache_entry_t *cache_entry;
} _cache_lru_t;
 
 
// find the cache entry hash with the oldest use
static void _cache_get_oldest(gpointer key, gpointer value, gpointer user_data)
{
  dt_pixel_cache_entry_t *cache_entry = (dt_pixel_cache_entry_t *)value;
  _cache_lru_t *lru = (_cache_lru_t *)user_data;
 
  // Don't remove LRU entries that are still in use
  // NOTE: with all the killswitches mechanisms and safety measures,
  // we might have more things decreasing refcount than increasing it.
  // It's no big deal though, as long as the (final output) backbuf
  // is checked for NULL and not reused if pipeline is DIRTY.
  const int64_t age = _pixel_cache_get_age(cache_entry);
  if(age < lru->max_age)
  {
    // Returns 1 if the lock is captured by another thread
    // 0 if WE capture the lock, and then need to release it
    gboolean locked = dt_pthread_rwlock_trywrlock(&cache_entry->lock);
    if(!locked) dt_pthread_rwlock_unlock(&cache_entry->lock);
    gboolean used = dt_atomic_get_int(&cache_entry->refcount) > 0;
 
    if(!locked && !used)
    {
      lru->max_age = age;
      lru->hash = cache_entry->hash;
      lru->cache_entry = cache_entry;
      _pixel_cache_message(cache_entry, "candidate for deletion", TRUE);
    }
    else if(used)
      _pixel_cache_message(cache_entry, "cannot be deleted: used", TRUE);
    else if(locked)
      _pixel_cache_message(cache_entry, "cannot be deleted: locked", TRUE);
  }
}
 
static void _print_cache_lines(gpointer key, gpointer value, gpointer user_data)
{
  dt_pixel_cache_entry_t *cache_entry = (dt_pixel_cache_entry_t *)value;
  _pixel_cache_message(cache_entry, "", FALSE);
}
 
 
// remove the least used cache entry
// return 0 on success, 1 on error
// error is : we couldn't find a candidate for deletion because all entries are either locked or in use
// or we found one but failed to remove it.
static int _non_thread_safe_pixel_pipe_cache_remove_lru(dt_dev_pixelpipe_cache_t *cache)
{
  _cache_lru_t *lru = (_cache_lru_t *)malloc(sizeof(_cache_lru_t));
  lru->max_age = g_get_monotonic_time();
  lru->hash = 0;
  lru->cache_entry = NULL;
  int error = 1;
  g_hash_table_foreach(cache->entries, _cache_get_oldest, lru);
 
  if(lru->hash > 0)
  {
    error = _non_thread_safe_cache_remove(cache, FALSE, lru->cache_entry, cache->entries);
    if(error)
      _cache_print(DT_DEBUG_PIPECACHE, "[pixelpipe] couldn't remove LRU %" PRIu64 "\n", lru->hash);
    else
      _cache_print(DT_DEBUG_PIPECACHE, "[pixelpipe] LRU %" PRIu64 " removed. Total cache size: %" G_GSIZE_FORMAT " MiB\n",
               lru->hash, cache->current_memory / (1024 * 1024));
  }
  else
  {
    _cache_print(DT_DEBUG_PIPECACHE, "[pixelpipe] couldn't remove LRU, %i items and all are used\n", g_hash_table_size(cache->entries));
    g_hash_table_foreach(cache->entries, _print_cache_lines, NULL);
  }
 
  dt_free(lru);
  return error;
}
 
// return 0 on success 1 on error
int dt_dev_pixel_pipe_cache_remove_lru(void)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  dt_pthread_mutex_lock(&cache->lock);
  int error = _non_thread_safe_pixel_pipe_cache_remove_lru(cache);
  dt_pthread_mutex_unlock(&cache->lock);
  return error;
}
 
#ifdef HAVE_OPENCL
static void *_pixel_cache_clmem_get(dt_pixel_cache_entry_t *entry, void *host_ptr, int devid,
                                    int width, int height, int bpp, int flags)
{
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
 
  if(_verbose && _verbose_detail)
  _cache_print(DT_DEBUG_OPENCL, 
    "[_pixel_cache_clmem_get] %u output entries in %" PRIu64 "\n", 
    g_list_length(entry->cl_mem_list), entry->hash);
 
  for(GList *l = g_list_first(entry->cl_mem_list); l;)
  {
    GList *next = g_list_next(l);
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(IS_NULL_PTR(c->mem))
    {
      // No point in keeping buffer-less cachelines
      entry->cl_mem_list = g_list_delete_link(entry->cl_mem_list, l);
      dt_free(c);
      l = next;
      continue;
    }
 
    // Buffer reuse must stay on the same OpenCL device and ensure proper size
    if(dt_opencl_get_mem_context_id(c->mem) == devid 
       && dt_opencl_get_image_width(c->mem) == width
       && dt_opencl_get_image_height(c->mem) == height
       && dt_opencl_get_image_element_size(c->mem) == bpp
       && c->refs == 0)
    {
      // Destroy the current OpenCL cacheline and return the buffer, the cacheline will be recreated
      // when we are done consuming the buffer
      entry->cl_mem_list = g_list_delete_link(entry->cl_mem_list, l);
      void *mem = c->mem;
      dt_free(c);
      // Reusing the entry's vRAM payload is a use of the entry, host side read or not.
      _pixel_cache_touch(entry);
      dt_pthread_mutex_unlock(&entry->cl_mem_lock);
      return mem;
    }
 
    l = next;
  }
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
 
  return NULL;
}
#endif
 
void *dt_dev_pixelpipe_cache_borrow_cl_payload(dt_pixel_cache_entry_t *entry, int devid,
                                               int width, int height, int bpp)
{
#ifdef HAVE_OPENCL
 
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
 
  for(GList *l = g_list_first(entry->cl_mem_list); l; l = g_list_next(l))
  {
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(dt_opencl_get_mem_context_id(c->mem) == devid 
       && dt_opencl_get_image_width(c->mem) == width
       && dt_opencl_get_image_height(c->mem) == height
       && dt_opencl_get_image_element_size(c->mem) == bpp)
    {
      c->refs++;
      void *mem = c->mem;
      _pixel_cache_touch(entry);
      dt_pthread_mutex_unlock(&entry->cl_mem_lock);
      return mem;
    }
  }
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
 
#endif
 
  return NULL;
}
 
void dt_dev_pixelpipe_cache_return_cl_payload(dt_pixel_cache_entry_t *entry, void *mem)
{
#ifdef HAVE_OPENCL
 
  if(IS_NULL_PTR(entry) || IS_NULL_PTR(mem)) return;
 
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
  for(GList *l = entry->cl_mem_list; l; l = g_list_next(l))
  {
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(c && c->mem == mem)
    {
      if(c->refs > 0) c->refs--;
      break;
    }
  }
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
 
#else
  (void)entry;
  (void)mem;
#endif
}
 
#ifdef HAVE_OPENCL
static int _pixel_cache_clmem_put(dt_pixel_cache_entry_t *entry, void *host_ptr, void *mem)
{
  cl_mem clmem = (cl_mem)mem;
  const int devid = dt_opencl_get_mem_context_id(clmem);
 
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
  for(GList *l = g_list_first(entry->cl_mem_list); l; l = g_list_next(l))
  {
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(c->mem == mem)
    {
      dt_pthread_mutex_unlock(&entry->cl_mem_lock);
      return 3;
    }
    if(dt_opencl_get_mem_context_id(c->mem) == devid)
    {
      // We keep one GPU cacheline per GPU device per pipeline cache entry
      // If refs > 0 here, we have a problem earlier.
      if(c->refs > 0) continue;
 
      void *old = c->mem;
      c->mem = mem;
      c->host_ptr = host_ptr;
      dt_pthread_mutex_unlock(&entry->cl_mem_lock);
      dt_opencl_release_mem_object(old);
      return 2;
    }
  }
 
  dt_cache_clmem_t *c = (dt_cache_clmem_t *)g_malloc0(sizeof(*c));
  if(IS_NULL_PTR(c))
  {
    dt_pthread_mutex_unlock(&entry->cl_mem_lock);
    dt_opencl_release_mem_object(mem);
    return 0;
  }
 
  c->host_ptr = host_ptr;
  c->mem = mem;
  entry->cl_mem_list = g_list_prepend(entry->cl_mem_list, c);
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
  return 1;
}
 
static void _pixel_cache_clmem_remove(dt_pixel_cache_entry_t *entry, void *mem)
{
  if(IS_NULL_PTR(entry) || IS_NULL_PTR(mem)) return;
 
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
  for(GList *l = entry->cl_mem_list; l;)
  {
    GList *next = g_list_next(l);
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(c && c->mem == mem)
    {
      entry->cl_mem_list = g_list_delete_link(entry->cl_mem_list, l);
      dt_free(c);
    }
    l = next;
  }
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
}
#endif
 
void dt_dev_pixelpipe_cache_flush_entry_clmem(dt_pixel_cache_entry_t *entry)
{
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
  for(GList *l = entry->cl_mem_list; l;)
  {
    GList *next = g_list_next(l);
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(c->refs > 0)
    {
      l = next;
      continue;
    }
 
    entry->cl_mem_list = g_list_delete_link(entry->cl_mem_list, l);
    dt_opencl_release_mem_object(c->mem);
    dt_free(c);
    l = next;
  }
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
}
 
#ifdef HAVE_OPENCL
void *dt_dev_pixelpipe_cache_get_pinned_image(void *host_ptr,
                                              dt_pixel_cache_entry_t *entry_hint, int devid,
                                              int width, int height, int bpp, int flags,
                                              gboolean *out_reused)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(!IS_NULL_PTR(out_reused)) *out_reused = FALSE;
  if(devid < 0 || width <= 0 || height <= 0 || bpp <= 0) return NULL;
 
  // Pinning is enabled if the calling function requests it and if it is allowed by user for this device
  gboolean use_pinned = dt_opencl_use_pinned_memory(devid) && (flags & CL_MEM_USE_HOST_PTR);
 
  // If no pinning, remove the allocation flag now because pinning happens at vRAM alloc time
  if(!use_pinned) flags &= ~CL_MEM_USE_HOST_PTR;
 
  // Reuse the entry hint if available, else find the cache entry attached to the host_ptr
  dt_pixel_cache_entry_t *entry = entry_hint;
  if(IS_NULL_PTR(entry))
  {
    dt_pthread_mutex_lock(&cache->lock);
    entry = _cache_entry_for_host_ptr_locked(cache, host_ptr);
    dt_pthread_mutex_unlock(&cache->lock);
  }
 
  // Reuse the vRAM buffer attached to the cache entry if any
  void *mem = NULL;
  if(entry)
  {
    mem = _pixel_cache_clmem_get(entry, host_ptr, devid, width, height, bpp, flags);
    if(!IS_NULL_PTR(mem) && !IS_NULL_PTR(out_reused)) *out_reused = TRUE;
  }
 
  // If no vRAM buffer was found, allocate a new one, pinning the host_ptr memory if the option is enabled
  if(IS_NULL_PTR(mem))
  {
    mem = dt_opencl_alloc_device_use_host_pointer(devid, width, height, bpp, use_pinned ? host_ptr : NULL, flags);
    if(IS_NULL_PTR(mem)) return NULL;
  }
 
  gboolean synced = FALSE;
 
  // Synchronize host_ptr with mem
  if(dt_opencl_is_pinned_memory(mem))
  {
    // Zero-copy for pinned buffers : note that some drivers may still use non-zero-copy,
    // in which case that degrades to basic memory copy.
    void *mapped = dt_opencl_map_image(devid, mem, TRUE, CL_MAP_WRITE, width, height, bpp);
    synced = (dt_opencl_unmap_mem_object(devid, mem, mapped) == CL_SUCCESS);
  }
 
  if(!synced)
  {
    // Zero-copy failed or pinned memory is disabled for this device : use plain memory transfer
    if(dt_opencl_write_host_to_device(devid, host_ptr, mem, width, height, bpp) != CL_SUCCESS)
    {
      // Clean everything up on error and abort
      if(entry) _pixel_cache_clmem_remove(entry, mem);
      dt_opencl_release_mem_object(mem);
      _cache_print(DT_DEBUG_OPENCL, "[dt_dev_pixelpipe_cache_get_pinned_image] failed to synchronize\n");
      return NULL;
    }
    else
    {
      _cache_print(DT_DEBUG_OPENCL, "[dt_dev_pixelpipe_cache_get_pinned_image] synchronized with write_host_to_device\n");
    }
  }
  else
  {
    _cache_print(DT_DEBUG_OPENCL, "[dt_dev_pixelpipe_cache_get_pinned_image] synchronized with mapping/unmapping\n");
  }
 
  return mem;
}
 
void dt_dev_pixelpipe_cache_put_pinned_image(void *host_ptr,
                                             dt_pixel_cache_entry_t *entry_hint, void **mem)
{
  if(IS_NULL_PTR(mem) || IS_NULL_PTR(*mem) || IS_NULL_PTR(host_ptr)) return;
  dt_pixel_cache_entry_t *entry = entry_hint;
  if(IS_NULL_PTR(entry)) 
  {
    if(_verbose && _verbose_detail)
      _cache_print(DT_DEBUG_OPENCL, "[dt_dev_pixelpipe_cache_put_pinned_image] no cache entry to put the vRAM buffer\n");
    return;
  }
 
  // FIXME: is it safe to cache non-pinned vRAM buffers (aka no CL_MEM_USE_HOST_PTR in flags) ?
  const int state = _pixel_cache_clmem_put(entry, host_ptr, (cl_mem)*mem);
  *mem = NULL;
  if(_verbose && _verbose_detail)
    _cache_print(DT_DEBUG_OPENCL, "[dt_dev_pixelpipe_cache_put_pinned_image] cache entry put the vRAM buffer (state=%i) in %p\n", state, entry);
}
 
gboolean dt_dev_pixelpipe_cache_flush_host_pinned_image(void *host_ptr,
                                                        dt_pixel_cache_entry_t *entry_hint, int devid)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(IS_NULL_PTR(cache) || IS_NULL_PTR(host_ptr)) return FALSE;
 
  dt_pixel_cache_entry_t *entry = entry_hint;
  if(IS_NULL_PTR(entry))
  {
    dt_pthread_mutex_lock(&cache->lock);
    entry = _cache_entry_for_host_ptr_locked(cache, host_ptr);
    dt_pthread_mutex_unlock(&cache->lock);
  }
 
  if(IS_NULL_PTR(entry)) return FALSE;
  if(!_cache_entry_clmem_has_host_pinned_locked(entry, host_ptr, devid)) return FALSE;
 
  if(devid >= 0) dt_opencl_events_wait_for(devid);
  dt_dev_pixelpipe_cache_ref_count_entry(TRUE, entry);
  const gboolean flushed = _cache_entry_clmem_flush_host_pinned_locked(entry, host_ptr, devid);
  dt_dev_pixelpipe_cache_ref_count_entry(FALSE, entry);
  return flushed;
}
 
#else
 
void dt_dev_pixelpipe_cache_put_pinned_image(void *host_ptr,
                                             dt_pixel_cache_entry_t *entry_hint, void **mem)
{
  (void)host_ptr;
  (void)entry_hint;
  if(mem) *mem = NULL;
}
 
gboolean dt_dev_pixelpipe_cache_flush_host_pinned_image(void *host_ptr,
                                                        dt_pixel_cache_entry_t *entry_hint, int devid)
{
  (void)host_ptr;
  (void)entry_hint;
  (void)devid;
  return FALSE;
}
 
void dt_dev_pixelpipe_cache_resync_host_pinned_image(dt_dev_pixelpipe_cache_t *cache, void *host_ptr,
                                                     dt_pixel_cache_entry_t *entry_hint, int devid)
{
  (void)host_ptr;
  (void)entry_hint;
  (void)devid;
}
#endif
 
#ifdef HAVE_OPENCL
static inline gboolean _is_gamma_rgba8_output(const dt_iop_module_t *module, const size_t bpp,
                                              const char *message)
{
  return module && message && bpp == 4 * sizeof(uint8_t) && strcmp(module->op, "gamma") == 0
         && strcmp(message, "output") == 0;
}
 
void *dt_dev_pixelpipe_cache_alloc_cl_device_buffer(int devid, const dt_iop_roi_t *roi, const size_t bpp,
                                                    const dt_iop_module_t *module, const char *message,
                                                    void *keep)
{
  const gboolean gamma_rgba8 = _is_gamma_rgba8_output(module, bpp, message);
  const int cl_bpp = gamma_rgba8 ? DT_OPENCL_BPP_ENCODE_RGBA8((int)bpp) : (int)bpp;
  return dt_opencl_alloc_device(devid, roi->width, roi->height, cl_bpp);
}
 
void *dt_dev_pixelpipe_cache_get_cl_buffer(int devid, void *const host_ptr, const dt_iop_roi_t *roi,
                                           const size_t bpp, dt_iop_module_t *module,
                                           const char *message, dt_pixel_cache_entry_t *cache_entry,
                                           gboolean *out_reused, void *keep)
{
  // Need to use read-write mode because of in-place color space conversions.
  void *cl_mem_input = NULL;
  gboolean reused_from_cache = FALSE;
  const gboolean gamma_rgba8 = _is_gamma_rgba8_output(module, bpp, message);
  const int cl_bpp = gamma_rgba8 ? DT_OPENCL_BPP_ENCODE_RGBA8((int)bpp) : (int)bpp;
  static dt_atomic_int clmem_reuse_hits;
  static dt_atomic_int clmem_reuse_misses;
 
  if(out_reused) *out_reused = FALSE;
 
  if(host_ptr && dt_opencl_use_pinned_memory(devid))
  {
    const int flags = CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR;
 
    // Try to reuse existing buffer
    if(cache_entry)
    {
      cl_mem_input = _pixel_cache_clmem_get(cache_entry, host_ptr, devid, roi->width, roi->height,
                                            (int)bpp, flags);
      reused_from_cache = (!IS_NULL_PTR(cl_mem_input));
    }
 
    // This will internally try to free up cache space if first alloc fails
    if(IS_NULL_PTR(cl_mem_input))
    {
      cl_mem_input = dt_opencl_alloc_device_use_host_pointer(devid, roi->width, roi->height, cl_bpp,
                                                             host_ptr, flags);
      if(_verbose && _verbose_detail)
        _cache_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] allocated a pinned GPU buffer for %s %s\n", module->name(), message);
    }
  }
  else
  {
    if(cache_entry)
    {
      /* Device-only allocations are tracked with a NULL host_ptr key and a normalized READ_WRITE
       * flag so scratch buffers can be reused deterministically across drivers. */
      cl_mem_input = _pixel_cache_clmem_get(cache_entry, NULL, devid, roi->width, roi->height,
                                            (int)bpp, CL_MEM_READ_WRITE);
      reused_from_cache = (!IS_NULL_PTR(cl_mem_input));
    }
 
    // This will internally try to free up cache space if first alloc fails
    if(IS_NULL_PTR(cl_mem_input))
    {
      cl_mem_input = dt_dev_pixelpipe_cache_alloc_cl_device_buffer(devid, roi, bpp, module, message, keep);
 
      if(_verbose && _verbose_detail)
        _cache_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] allocated a device-only GPU buffer for %s %s\n", module->name(), message);
    }
  }
 
  if(IS_NULL_PTR(cl_mem_input))
  {
    _cache_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] couldn't allocate GPU buffer for module %s %s\n", module->name(), message);
  }
  else if(reused_from_cache)
  {
    const int hits = dt_atomic_add_int(&clmem_reuse_hits, 1) + 1;
    const int misses = dt_atomic_get_int(&clmem_reuse_misses);
    if(_verbose && _verbose_detail)
      _cache_print(DT_DEBUG_OPENCL,
              "[dev_pixelpipe] reused GPU buffer from cache (hits=%d, misses=%d) for module %s %s\n",
              hits, misses, module->name(), message);
  }
  else
  {
    dt_atomic_add_int(&clmem_reuse_misses, 1);
  }
 
  if(out_reused) *out_reused = reused_from_cache;
  return cl_mem_input;
}
 
void dt_dev_pixelpipe_cache_release_cl_buffer(void **cl_mem_buffer, dt_pixel_cache_entry_t *cache_entry,
                                              void *host_ptr, const gboolean cache_device)
{
  if(!IS_NULL_PTR(cl_mem_buffer) && !IS_NULL_PTR(*cl_mem_buffer))
  {
    cl_mem mem = *cl_mem_buffer;
    if(cache_device && !IS_NULL_PTR(cache_entry))
    {
      _pixel_cache_clmem_put(cache_entry, host_ptr, mem);
    }
    else
    {
      if(!IS_NULL_PTR(cache_entry)) _pixel_cache_clmem_remove(cache_entry, mem);
      dt_opencl_release_mem_object(mem);
    }
    *cl_mem_buffer = NULL;
  }
}
 
int dt_dev_pixelpipe_cache_sync_cl_buffer(const int devid, void *host_ptr, void *cl_mem_buffer,
                                          const dt_iop_roi_t *roi, int cl_mode, size_t bpp,
                                          dt_iop_module_t *module, const char *message)
{
  if(IS_NULL_PTR(host_ptr) || IS_NULL_PTR(cl_mem_buffer)) return 1;
 
  const cl_mem mem = (cl_mem)cl_mem_buffer;
 
  // Fast path for true zero-copy pinned images: map/unmap is enough to synchronize host<->device.
  if(dt_opencl_is_pinned_memory(mem))
  {
    void *mapped = dt_opencl_map_image(devid, mem, TRUE, cl_mode, roi->width, roi->height, (int)bpp);
    if(dt_opencl_unmap_mem_object(devid, mem, mapped) == CL_SUCCESS)
    {
      _cache_print(DT_DEBUG_OPENCL,
                "[dev_pixelpipe] successfully synced image %s via map/unmap for module %s (%s)\n",
                (cl_mode == CL_MAP_WRITE) ? "host to device" : "device to host",
                (module) ? module->op : "base buffer", message);
      return 0;
    }
  }
 
  // Fallback: explicit blocking transfers (safe on all drivers).
  cl_int err = CL_SUCCESS;
  if(cl_mode == CL_MAP_WRITE)
    err = dt_opencl_write_host_to_device(devid, host_ptr, mem, roi->width, roi->height, (int)bpp);
  else if(cl_mode == CL_MAP_READ)
    err = dt_opencl_read_host_from_device(devid, host_ptr, mem, roi->width, roi->height, (int)bpp);
  else
    return 1;
 
  if(err != CL_SUCCESS)
  {
    _cache_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] couldn't copy image %s for module %s (%s)\n",
             (cl_mode == CL_MAP_WRITE) ? "host to device" : "device to host",
             (module) ? module->op : "base buffer", message);
    return 1;
  }
 
  _cache_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] successfully copied image %s for module %s (%s)\n",
           (cl_mode == CL_MAP_WRITE) ? "host to device" : "device to host",
           (module) ? module->op : "base buffer", message);
  return 0;
}
 
float *dt_dev_pixelpipe_cache_restore_cl_buffer(dt_dev_pixelpipe_t *pipe, float *input, void *cl_mem_input,
                                                const dt_iop_roi_t *roi_in, dt_iop_module_t *module,
                                                const size_t in_bpp, dt_pixel_cache_entry_t *input_entry,
                                                const char *message)
{
  if(IS_NULL_PTR(cl_mem_input)) return input;
  dt_dev_pixelpipe_cache_wrlock_entry(TRUE, input_entry);
 
  const int fail = dt_dev_pixelpipe_cache_sync_cl_buffer(pipe->devid, input, cl_mem_input, roi_in,
                                                         CL_MAP_READ, in_bpp, module, message);
  dt_dev_pixelpipe_cache_wrlock_entry(FALSE, input_entry);
  return fail ? NULL : input;
}
 
int dt_dev_pixelpipe_cache_prepare_cl_input(dt_dev_pixelpipe_t *pipe, dt_iop_module_t *module,
                                            float *input, void **cl_mem_input,
                                            const dt_iop_roi_t *roi_in, const size_t in_bpp,
                                            dt_pixel_cache_entry_t *input_entry,
                                            dt_pixel_cache_entry_t **locked_input_entry, void *keep)
{
  if(IS_NULL_PTR(locked_input_entry)) return 1;
  *locked_input_entry = NULL;
 
  if(!IS_NULL_PTR(*cl_mem_input))
  {
    // We passed the OpenCL memory buffer through directly on vRAM from previous module.
    // This is fast and efficient.
    // If it's a true zero-copy pinned image, keep the input cache entry read-locked until kernels complete,
    // otherwise another thread may overwrite host memory while the GPU is still reading it.
    _cache_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] %s will use its input directly from vRAM\n", module->name());
    const cl_mem mem = (cl_mem)*cl_mem_input;
    if(dt_opencl_is_pinned_memory(mem))
    {
      dt_dev_pixelpipe_cache_rdlock_entry(TRUE, input_entry);
      *locked_input_entry = input_entry;
    }
    return 0;
  }
 
  if(IS_NULL_PTR(input))
  {
    _cache_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] %s has no input (cache)\n", module->name());
    return 1;
  }
 
  dt_dev_pixelpipe_cache_rdlock_entry(TRUE, input_entry);
 
  // Try to reuse a cached pinned buffer; otherwise allocate a new pinned image backed by `input`.
  gboolean input_reused_from_cache = FALSE;
  *cl_mem_input = dt_dev_pixelpipe_cache_get_cl_buffer(pipe->devid, input, roi_in, in_bpp, module,
                                                       "input", input_entry,
                                                       &input_reused_from_cache, keep);
  int fail = (IS_NULL_PTR(*cl_mem_input));
 
  // If the input is true zero-copy, the GPU will access host memory asynchronously: keep the cache
  // entry read-locked until all kernels have completed. If not, drivers may use a device-side copy
  // which must be synchronized from the host before running kernels.
  gboolean keep_lock = FALSE;
  cl_mem mem = NULL;
  if(!fail && *cl_mem_input)
  {
    mem = (cl_mem)*cl_mem_input;
    keep_lock = dt_opencl_is_pinned_memory(mem);
  }
 
  /* A reused cached pinned image already carries the authoritative device payload from the
   * previous module output. Re-uploading host RAM here would overwrite that valid vRAM state
   * with whatever stale contents the host buffer still has when the previous stage stayed GPU-only.
   * Only freshly allocated pinned inputs need an explicit host->device copy. */
  if(!fail && mem && !keep_lock && !input_reused_from_cache)
  {
    const cl_int err = dt_opencl_write_host_to_device(pipe->devid, input, mem, roi_in->width, roi_in->height,
                                                      (int)in_bpp);
    if(err != CL_SUCCESS)
    {
      _cache_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] couldn't copy image host to device for module %s (%s)\n",
               (module) ? module->op : "base buffer", "cache to input");
      fail = TRUE;
    }
    else
    {
      _cache_print(DT_DEBUG_OPENCL, "[dev_pixelpipe] successfully copied image host to device for module %s (%s)\n",
               (module) ? module->op : "base buffer", "cache to input");
    }
  }
 
  // Enforce sync with the CPU/RAM cache so lock validity is guaranteed.
  dt_opencl_events_wait_for(pipe->devid);
 
  if(keep_lock)
    *locked_input_entry = input_entry;
  else
    dt_dev_pixelpipe_cache_rdlock_entry(FALSE, input_entry);
 
  return fail ? 1 : 0;
}
#else
void *dt_dev_pixelpipe_cache_get_cl_buffer(int devid, void *host_ptr, const dt_iop_roi_t *roi,
                                           size_t bpp, dt_iop_module_t *module, const char *message,
                                           dt_pixel_cache_entry_t *entry,
                                           gboolean *out_reused, void *keep)
{
  (void)devid;
  (void)host_ptr;
  (void)roi;
  (void)bpp;
  (void)module;
  (void)message;
  (void)entry;
  (void)keep;
  if(out_reused) *out_reused = FALSE;
  return NULL;
}
 
void *dt_dev_pixelpipe_cache_alloc_cl_device_buffer(int devid, const dt_iop_roi_t *roi, size_t bpp,
                                                    const dt_iop_module_t *module,
                                                    const char *message, void *keep)
{
  (void)devid;
  (void)roi;
  (void)bpp;
  (void)module;
  (void)message;
  (void)keep;
  return NULL;
}
 
void dt_dev_pixelpipe_cache_release_cl_buffer(void **cl_mem_buffer, dt_pixel_cache_entry_t *entry,
                                              void *host_ptr, gboolean cache_device)
{
  (void)entry;
  (void)host_ptr;
  (void)cache_device;
  if(cl_mem_buffer) *cl_mem_buffer = NULL;
}
 
int dt_dev_pixelpipe_cache_sync_cl_buffer(int devid, void *host_ptr, void *cl_mem_buffer,
                                          const dt_iop_roi_t *roi, int cl_mode, size_t bpp,
                                          dt_iop_module_t *module, const char *message)
{
  (void)devid;
  (void)host_ptr;
  (void)cl_mem_buffer;
  (void)roi;
  (void)cl_mode;
  (void)bpp;
  (void)module;
  (void)message;
  return 1;
}
 
float *dt_dev_pixelpipe_cache_restore_cl_buffer(dt_dev_pixelpipe_t *pipe, float *input,
                                                void *cl_mem_input, const dt_iop_roi_t *roi_in,
                                                dt_iop_module_t *module, size_t in_bpp,
                                                dt_pixel_cache_entry_t *input_entry,
                                                const char *message)
{
  (void)pipe;
  (void)cl_mem_input;
  (void)roi_in;
  (void)module;
  (void)in_bpp;
  (void)input_entry;
  (void)message;
  return input;
}
 
int dt_dev_pixelpipe_cache_prepare_cl_input(dt_dev_pixelpipe_t *pipe, dt_iop_module_t *module,
                                            float *input, void **cl_mem_input,
                                            const dt_iop_roi_t *roi_in, size_t in_bpp,
                                            dt_pixel_cache_entry_t *input_entry,
                                            dt_pixel_cache_entry_t **locked_input_entry,
                                            void *keep)
{
  (void)pipe;
  (void)module;
  (void)input;
  (void)cl_mem_input;
  (void)roi_in;
  (void)in_bpp;
  (void)input_entry;
  (void)locked_input_entry;
  (void)keep;
  return 1;
}
#endif
 
dt_pixel_cache_entry_t *dt_dev_pixelpipe_cache_ref_entry_for_host_ptr(void *host_ptr)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(IS_NULL_PTR(cache) || IS_NULL_PTR(host_ptr)) return NULL;
 
  dt_pthread_mutex_lock(&cache->lock);
  dt_pixel_cache_entry_t *entry = _cache_entry_for_host_ptr_locked(cache, host_ptr);
  if(entry)
    _non_thread_safe_cache_ref_count_entry(cache, TRUE, entry);
  dt_pthread_mutex_unlock(&cache->lock);
 
  return entry;
}
 
/* Allocations at or below this size bypass the system-pressure valve: they cannot
 * change the pressure meaningfully, and refusing them breaks functionality for no
 * benefit. Deliberately small -- this is an escape hatch for scratch and bookkeeping
 * buffers, not a hole large enough for pipeline tiles. */
#define DT_PIXELPIPE_CACHE_PRESSURE_EXEMPT_SIZE ((size_t)1024 * 1024)
 
/* System memory-pressure valve (issue #1083).
 *
 * The internal budget (max_memory) is only a plan made at startup: it says nothing
 * about what the system can actually back RIGHT NOW, with other applications competing
 * for the same physical RAM. Before growing our committed footprint by `request_size`,
 * make sure the system-wide available memory keeps the configured floor
 * (dt_get_memory_pressure_floor()); when it doesn't, evict LRU entries to cover the
 * deficit, hand their pages back to the OS for real, and re-read what that actually
 * bought — so the system OOM-killer never has a reason to look at us.
 *
 * The probe is rate-limited and the cached value is decremented by our own
 * allocations in between, so the hot path pays one probe per PROBE_PERIOD at most.
 * That running estimate legitimately reaches 0 under sustained pressure, which is why
 * "the platform answers at all" is tracked separately (`sys_probe_valid`) instead of
 * being read off a 0 estimate: conflating the two would disable the valve at exactly
 * the moment it matters.
 *
 * Returns TRUE when the allocation may proceed. Returns FALSE when, even after
 * shedding everything evictable, the system could not take `request_size` more bytes
 * without dropping under HALF the floor: the caller must fail the allocation cleanly —
 * a failed pipeline with a message beats a silent SIGKILL from the OOM-killer.
 */
static gboolean _system_memory_pressure_valve(dt_dev_pixelpipe_cache_t *cache, size_t request_size)
{
  const size_t pressure_floor = dt_get_memory_pressure_floor();
  if(pressure_floor == 0) return TRUE;
 
  dt_pthread_mutex_lock(&cache->lock);
 
  const gint64 now = g_get_monotonic_time();
  const gint64 PROBE_PERIOD_US = 100000; // 100 ms
  if(cache->sys_probe_time_us == 0 || now - cache->sys_probe_time_us > PROBE_PERIOD_US)
  {
    const size_t probed = dt_get_system_available_mem();
    cache->sys_probe_valid = (probed > 0);
    cache->sys_available_est = probed;
    cache->sys_probe_time_us = now;
  }
 
  // The platform gives us no way to know: pressure handling disabled, never refuse.
  if(!cache->sys_probe_valid)
  {
    dt_pthread_mutex_unlock(&cache->lock);
    return TRUE;
  }
 
  if(cache->sys_available_est < request_size + pressure_floor)
  {
    const size_t deficit = request_size + pressure_floor - cache->sys_available_est;
    size_t freed = 0;
    while(freed < deficit && g_hash_table_size(cache->entries) > 0)
    {
      const size_t before = cache->current_memory;
      if(_non_thread_safe_pixel_pipe_cache_remove_lru(cache)) break;
      freed += before - cache->current_memory;
    }
 
    if(freed)
    {
      /* Evicting only returns the pages to the ARENA. How much of that reaches the OS
       * is not ours to guess: the per-free lazy release is a no-op on Windows until a
       * decommit runs, and even where MADV_FREE applies, the kernel decides when those
       * pages stop counting against us. So hand them over for real, then take the new
       * number from the OS instead of crediting what we think we released — an
       * over-credit here would let the allocation through on a system that is still
       * just as full, which is precisely the OOM this valve exists to prevent.
       * Ordering note: cache->lock is held and dt_cache_arena_trim() takes arena.lock,
       * matching the cache->lock → arena.lock order the rest of this file already uses
       * (see the defrag path below). */
      dt_cache_arena_trim(&cache->arena);
      dt_invalidate_system_available_mem();
      const size_t probed = dt_get_system_available_mem();
      cache->sys_probe_valid = (probed > 0);
      cache->sys_available_est = probed;
      cache->sys_probe_time_us = g_get_monotonic_time();
 
      dt_print(DT_DEBUG_MEMORY | DT_DEBUG_PIPECACHE,
               "[pixelpipe_cache] system memory pressure: shed %" G_GSIZE_FORMAT " MiB of cache "
               "(%" G_GSIZE_FORMAT " MiB available after trim, floor %" G_GSIZE_FORMAT " MiB)\n",
               freed / (1024 * 1024), cache->sys_available_est / (1024 * 1024),
               pressure_floor / (1024 * 1024));
    }
  }
 
  // The post-trim re-probe can itself come back unanswered; absence of information is
  // never a reason to refuse (see the dt_get_system_available_mem() header contract).
  /* A request too small to move the needle must never be refused. The valve exists to
   * stop the cache COMMITTING LARGE BUFFERS into the memory the OS and other
   * applications need -- not to make Ansel fail. Because the floor term dominates the
   * comparison below, a system already under the floor refused EVERY allocation
   * regardless of size, printing the giveaway line
   *
   *   [pixelpipe_cache] refusing to allocate 0 MiB: the system has only 1638 MiB ...
   *
   * and turning a transient shortage into a hard failure (it segfaulted startup through
   * an unchecked allocation in common/points.h). Anything at or below this size cannot
   * meaningfully change system pressure, and refusing it buys nothing. Larger requests
   * remain fully gated.
   *
   * The comparison below uses the WHOLE floor, not half of it. The floor used to be a
   * large derived number where a half-way hard limit made sense as hysteresis; it is now
   * an absolute reserve (DT_MEMORY_PRESSURE_FLOOR_DEFAULT, 200 MiB), and "always leave
   * 200 MiB to the OS" has to mean 200, not 100. The shed/trim step above already ran at
   * this same threshold, so reaching here means trimming did not recover enough. */
  const gboolean negligible = request_size <= DT_PIXELPIPE_CACHE_PRESSURE_EXEMPT_SIZE;
 
  const gboolean allowed = negligible
                           || !cache->sys_probe_valid
                           || (cache->sys_available_est >= request_size + pressure_floor);
 
  if(allowed)
  {
    // Optimistically debit the estimate now that the caller will commit these pages;
    // self-corrects at the next probe if the arena allocation fails afterwards.
    cache->sys_available_est
        = (cache->sys_available_est > request_size) ? cache->sys_available_est - request_size : 0;
  }
  else
  {
    // Warn the user at most every 10 s: this fires per failed allocation, on a
    // system that is already drowning.
    static gint64 last_warning_us = 0;
    if(now - last_warning_us > 10000000)
    {
      last_warning_us = now;
      _warn_user(_("Your system is running out of memory. "
                       "Close other applications or add more RAM to your system."));
    }
    fprintf(stdout,
            "[pixelpipe_cache] refusing to allocate %" G_GSIZE_FORMAT " MiB: the system has only "
            "%" G_GSIZE_FORMAT " MiB of available RAM left (pressure floor: %" G_GSIZE_FORMAT " MiB)\n",
            request_size / (1024 * 1024), cache->sys_available_est / (1024 * 1024),
            pressure_floor / (1024 * 1024));
  }
 
  dt_pthread_mutex_unlock(&cache->lock);
  return allowed;
}
 
// Attempt to allocate from the arena; if fragmentation prevents it, evict LRU cache lines
// until a sufficiently large contiguous run is available (or nothing remains to evict).
static inline void *_arena_alloc_with_defrag(dt_dev_pixelpipe_cache_t *cache, size_t request_size,
                                             size_t *actual_size)
{
  // Never grow the committed footprint past what the system can actually take,
  // whatever our internal budget still allows.
  if(!_system_memory_pressure_valve(cache, request_size)) return NULL;
 
  void *buf = dt_cache_arena_alloc(&cache->arena, request_size, actual_size);
  if(!IS_NULL_PTR(buf)) return buf;
 
  uint32_t pages_needed = 0;
  if(dt_cache_arena_calc(&cache->arena, request_size, &pages_needed, NULL))
  {
    dt_pthread_mutex_lock(&cache->lock);
    uint32_t total_free_pages = 0, largest_free_run_pages = 0;
    dt_cache_arena_stats(&cache->arena, &total_free_pages, &largest_free_run_pages);
 
    while(largest_free_run_pages < pages_needed && g_hash_table_size(cache->entries) > 0)
    {
      if(_non_thread_safe_pixel_pipe_cache_remove_lru(cache)) break;
      dt_cache_arena_stats(&cache->arena, &total_free_pages, &largest_free_run_pages);
    }
    dt_pthread_mutex_unlock(&cache->lock);
  }
 
  return dt_cache_arena_alloc(&cache->arena, request_size, actual_size);
}
 
static inline void _arena_stats_bytes(dt_dev_pixelpipe_cache_t *cache, uint32_t *total_pages,
                                      uint32_t *largest_pages, size_t *total_bytes, size_t *largest_bytes)
{
  dt_cache_arena_stats(&cache->arena, total_pages, largest_pages);
  const size_t page_size = cache->arena.page_size ? cache->arena.page_size : 1;
  if(total_bytes) *total_bytes = (size_t)(*total_pages) * page_size;
  if(largest_bytes) *largest_bytes = (size_t)(*largest_pages) * page_size;
}
 
/* Largest contiguous free run in the arena, in bytes. The arena serves every
 * allocation from ONE contiguous run, and pinned entries partition its
 * address space — so at tiling-planning time this, not the byte headroom, is
 * the number a module's working set must actually fit (see the available-
 * memory cap in develop/tiling.c). Locked because the arena bitmap mutates
 * under concurrent pipes. */
size_t dt_pixelpipe_cache_get_largest_free_run(void)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  uint32_t total_pages = 0, largest_pages = 0;
  size_t total_bytes = 0, largest_bytes = 0;
  dt_pthread_mutex_lock(&cache->lock);
  _arena_stats_bytes(cache, &total_pages, &largest_pages, &total_bytes, &largest_bytes);
  dt_pthread_mutex_unlock(&cache->lock);
  return largest_bytes;
}
 
static inline void _log_arena_allocation_failure(dt_dev_pixelpipe_cache_t *cache, size_t request_size,
                                                 const char *entry_name, const char *module, uint64_t hash,
                                                 gboolean name_is_file)
{
  uint32_t total_free_pages = 0, largest_free_run_pages = 0;
  size_t total_free_bytes = 0, largest_free_bytes = 0;
  _arena_stats_bytes(cache, &total_free_pages, &largest_free_run_pages, &total_free_bytes, &largest_free_bytes);
 
  if(entry_name)
    fprintf(stdout,
            "[pixelpipe_cache] failed to allocate %" G_GSIZE_FORMAT " bytes for entry %" PRIu64 " (%s, module=%s) "
            "[arena largest=%" G_GSIZE_FORMAT " MiB, total=%" G_GSIZE_FORMAT " MiB, cache=%" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT " MiB]\n",
            request_size, hash, entry_name, module ? module : "unknown",
            largest_free_bytes / (1024 * 1024), total_free_bytes / (1024 * 1024),
            cache->current_memory / (1024 * 1024), cache->max_memory / (1024 * 1024));
  else
    fprintf(stdout,
            "[pixelpipe_cache] failed to allocate %" G_GSIZE_FORMAT " bytes for entry %" PRIu64 " (module=%s) "
            "[arena largest=%" G_GSIZE_FORMAT " MiB, total=%" G_GSIZE_FORMAT " MiB, cache=%" G_GSIZE_FORMAT "/%" G_GSIZE_FORMAT " MiB]\n",
            request_size, hash, module ? module : "unknown",
            largest_free_bytes / (1024 * 1024), total_free_bytes / (1024 * 1024),
            cache->current_memory / (1024 * 1024), cache->max_memory / (1024 * 1024));
 
  if(!IS_NULL_PTR(entry_name) && !IS_NULL_PTR(module))
    _warn_user(_("The pipeline cache is full while allocating `%s` (module `%s`). Either your RAM settings are too frugal or your RAM is too small."),
                   entry_name, module);
  else if(!IS_NULL_PTR(entry_name))
    _warn_user(_("The pipeline cache is full while allocating `%s`. Either your RAM settings are too frugal or your RAM is too small."),
                   entry_name);
  else if(!IS_NULL_PTR(module))
    _warn_user(_("The pipeline cache is full while processing module `%s`. Either your RAM settings are too frugal or your RAM is too small."),
                   module);
  else
    _warn_user(_("The pipeline cache is full. Either your RAM settings are too frugal or your RAM is too small."));
 
  (void)name_is_file; // kept for signature symmetry if future callers need it.
}
 
// keep: OpenCL buffer to NOT release
#ifdef HAVE_OPENCL
// Release this device's vRAM payloads for one entry. The caller has already established that
// the entry is idle (refcount == 0, not write-locked), so the device buffers are nobody's live
// input and reclaiming them honors the flush's purpose: free vRAM for later allocations.
// Returns TRUE if the entry holds no buffer at all afterwards (no host RAM, no vRAM on any
// device) and should therefore be evicted entirely instead of lingering as a husk.
static gboolean _cache_entry_clmem_flush_device(dt_pixel_cache_entry_t *entry, const int devid)
{
  // devid is always >= 0 here: dt_dev_pixelpipe_cache_flush_clmem() early-returns
  // otherwise. Only cachelines living on this specific device are candidates.
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
 
  for(GList *l = g_list_first(entry->cl_mem_list); l;)
  {
    GList *next = g_list_next(l);
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(IS_NULL_PTR(c->mem))
    {
      // Don't keep cacheline with NULL buffer
      entry->cl_mem_list = g_list_delete_link(entry->cl_mem_list, l);
      dt_free(c);
      l = next;
      continue;
    }
 
    gboolean referenced = c->refs > 0;
    gboolean not_ours = dt_opencl_get_mem_context_id(c->mem) != devid;
 
    if(referenced || not_ours)
    {
      // Don't flush cachelines that don't belong to the current OpenCL device,
      // or are still borrowed by an in-flight GPU module (per-payload refs > 0).
      if(_verbose && _verbose_detail)
        _cache_print(DT_DEBUG_OPENCL,
          "[dt_dev_pixelpipe_cache_flush_clmem] for entry %" PRIu64 ": couldn't flush %p "
          "(referenced=%i not ours=%i)\n",
          entry->hash, c->mem, referenced, not_ours);
      l = next;
      continue;
    }
 
    entry->cl_mem_list = g_list_delete_link(entry->cl_mem_list, l);
    dt_opencl_release_mem_object(c->mem);
    dt_free(c);
    l = next;
  }
 
  // A cacheline that now carries neither a host buffer nor any vRAM is a husk: the cache would
  // still hand it out as a hit, making a later consumer abort with "has no RAM nor vRAM input"
  // (issue #817 skull thumbnails). Signal the caller to delete it entirely.
  const gboolean empty = IS_NULL_PTR(entry->data) && IS_NULL_PTR(entry->cl_mem_list);
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
  return empty;
}
#else
static gboolean _cache_entry_clmem_flush_device(dt_pixel_cache_entry_t *entry, const int devid)
{
  return FALSE;
}
#endif
 
void *dt_pixel_cache_alloc(dt_pixel_cache_entry_t *cache_entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  // allocate the data buffer
  if(IS_NULL_PTR(cache_entry->data))
  {
    cache_entry->data = _arena_alloc_with_defrag(cache, cache_entry->size, &cache_entry->size);
 
    if(IS_NULL_PTR(cache_entry->data))
    {
      const char *module = dt_pixelpipe_cache_current_module;
      _log_arena_allocation_failure(cache, cache_entry->size, cache_entry->name, module,
                                    cache_entry->hash, FALSE);
    }
  }
 
  return cache_entry->data;
}
 
void *dt_pixel_cache_entry_get_data(dt_pixel_cache_entry_t *entry)
{
  return entry ? entry->data : NULL;
}
 
size_t dt_pixel_cache_entry_get_size(dt_pixel_cache_entry_t *entry)
{
  return entry ? entry->size : 0;
}
 
/* Kernel memory pressure: the cache's half of it.
 *
 * caches/pixelpipe_cache_pressure.c decides WHEN memory must go back to the system and HOW MUCH,
 * from the kernel counters system/memory_pressure.c reads; this is the part only the cache can
 * answer -- what it holds, and what giving some of it back costs. Both run under `lock`, taken by
 * whoever called into the monitor, the watcher's own thread included. */
static size_t _pressure_held(void *user)
{
  return ((dt_dev_pixelpipe_cache_t *)user)->current_memory;
}
 
static size_t _pressure_shed(void *user, const size_t target, size_t *given_back)
{
  dt_dev_pixelpipe_cache_t *cache = (dt_dev_pixelpipe_cache_t *)user;
 
  while(cache->current_memory > target && g_hash_table_size(cache->entries) > 0)
    if(_non_thread_safe_pixel_pipe_cache_remove_lru(cache)) break;
 
  // Hand the pages over now rather than when the kernel gets to MADV_FREE, and make the
  // available-RAM valve re-read the system instead of trusting its pre-shed estimate.
  *given_back = dt_cache_arena_trim(&cache->arena);
  dt_invalidate_system_available_mem();
  cache->sys_probe_time_us = 0;
  return cache->current_memory;
}
 
static dt_pixelpipe_cache_pressure_sink_t _pressure_sink(dt_dev_pixelpipe_cache_t *cache)
{
  return (dt_pixelpipe_cache_pressure_sink_t){ .held = _pressure_held,
                                               .shed = _pressure_shed,
                                               .user = cache };
}
 
// The kernel woke the watcher, on a thread of its own: take the lock the monitor is guarded by.
static void _pressure_wake(void *user)
{
  dt_dev_pixelpipe_cache_t *cache = (dt_dev_pixelpipe_cache_t *)user;
  const dt_pixelpipe_cache_pressure_sink_t sink = _pressure_sink(cache);
  dt_pthread_mutex_lock(&cache->lock);
  dt_pixelpipe_cache_pressure_triggered(&cache->psi, &sink);
  dt_pthread_mutex_unlock(&cache->lock);
}
 
// What allocations evict down to: the plan, or less under kernel memory pressure.
// WARNING: non thread-safe
static inline size_t _cache_budget_locked(const dt_dev_pixelpipe_cache_t *cache)
{
  return dt_pixelpipe_cache_pressure_budget(&cache->psi.budget);
}
 
// Close the kernel-pressure window and act on it. WARNING: non thread-safe
static void _pressure_react_locked(dt_dev_pixelpipe_cache_t *cache)
{
  const dt_pixelpipe_cache_pressure_sink_t sink = _pressure_sink(cache);
  dt_pixelpipe_cache_pressure_react(&cache->psi, &sink);
}
 
// WARNING: non thread-safe
static int _free_space_to_alloc(dt_dev_pixelpipe_cache_t *cache, const size_t size, const uint64_t hash,
                                const char *name)
{
  /* Under kernel memory pressure, keep to the lowered budget as far as eviction allows. It is a
   * target, never a reason to fail: when everything left is in use the allocation still goes
   * ahead, and only the plan below is a hard limit. */
  _pressure_react_locked(cache);
  const size_t budget = _cache_budget_locked(cache);
  while(cache->current_memory + size > budget && g_hash_table_size(cache->entries) > 0)
    if(_non_thread_safe_pixel_pipe_cache_remove_lru(cache)) break;
 
  // Free up space if needed to match the max memory limit
  // If error, all entries are currently locked or in use, so we cannot free space to allocate a new entry.
  int error = 0;
  while(cache->current_memory + size > cache->max_memory && g_hash_table_size(cache->entries) > 0 && !error)
    error = _non_thread_safe_pixel_pipe_cache_remove_lru(cache);
 
  if(cache->current_memory + size > cache->max_memory)
  {
    const char *module = dt_pixelpipe_cache_current_module;
    const gboolean name_is_file = (!IS_NULL_PTR(name)) && (strchr(name, '/') != NULL) && (strchr(name, ':') != NULL);
    if(IS_NULL_PTR(name)) name = g_strdup("unknown");
    
    if(hash)
      fprintf(stdout, "[pixelpipe] cache is full, cannot allocate new entry %" PRIu64 " (%s)\n", hash, name);
    else
      fprintf(stdout, "[pixelpipe] cache is full, cannot allocate new entry (%s)\n", name);
    if(!IS_NULL_PTR(name) && !IS_NULL_PTR(module) && name_is_file)
      _warn_user(_("The pipeline cache is full while allocating `%s` (module `%s`). Either your RAM settings are too frugal or your RAM is too small."), name, module);
    else if(!IS_NULL_PTR(name))
      _warn_user(_("The pipeline cache is full while allocating `%s`. Either your RAM settings are too frugal or your RAM is too small."), name);
    else if(!IS_NULL_PTR(module))
      _warn_user(_("The pipeline cache is full while processing module `%s`. Either your RAM settings are too frugal or your RAM is too small."), module);
    else
      _warn_user(_("The pipeline cache is full. Either your RAM settings are too frugal or your RAM is too small."));
  }
 
  return error;
}
 
void *dt_pixelpipe_cache_alloc_align_cache_impl(size_t size, int id,
                                                const char *name)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  // Free up space if needed to match the max memory limit
  // If error, all entries are currently locked or in use, so we cannot free space to allocate a new entry.
  dt_pthread_mutex_lock(&cache->lock);
  int error = _free_space_to_alloc(cache, size, 0, name);
  dt_pthread_mutex_unlock(&cache->lock);
 
  if(error) return NULL;
 
  // Page size is the desired size + AVX/SSE rounding
  size_t page_size = 0;
  void *buf = _arena_alloc_with_defrag(cache, size, &page_size);
 
  if(IS_NULL_PTR(buf))
  {
    _log_arena_allocation_failure(cache, size, name, NULL, 0, FALSE);
    return NULL;
  }
 
  void *aligned = __builtin_assume_aligned(buf, DT_CACHELINE_BYTES);
 
  const uint64_t hash = (uint64_t)(uintptr_t)(aligned);
 
  dt_pthread_mutex_lock(&cache->lock);
  dt_pixel_cache_entry_t *cache_entry
      = dt_pixel_cache_new_entry(hash, page_size, name, id, cache, FALSE, cache->external_entries);
 
  if(IS_NULL_PTR(cache_entry))
  {
    dt_pthread_mutex_unlock(&cache->lock);
    dt_cache_arena_free(&cache->arena, buf, page_size);
    return NULL;
  }
 
  // Keep this entry marked as "used" for diagnostics/bookkeeping.
  // Note that external_entries are not subject to LRU eviction, so we must not keep
  // a thread-owned rwlock held across the lifetime of the buffer (it may be freed
  // from a different thread during cleanup paths).
  _non_thread_safe_cache_ref_count_entry(cache, TRUE, cache_entry);
  cache_entry->data = aligned;
  _pixel_cache_touch(cache_entry);
  cache_entry->external_alloc = TRUE;
  dt_pthread_mutex_unlock(&cache->lock);
  return aligned;
}
 
void dt_pixelpipe_cache_free_align_cache(void **mem, const char *message)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(IS_NULL_PTR(mem) || !*mem) return;
 
  dt_pthread_mutex_lock(&cache->lock);
  const uint64_t hash = (uint64_t)(uintptr_t)(*mem);
  dt_pixel_cache_entry_t *cache_entry = _non_threadsafe_cache_get_entry(cache, cache->external_entries, hash);
  if(IS_NULL_PTR(cache_entry) || !cache_entry->external_alloc)
  {
    dt_pthread_mutex_unlock(&cache->lock);
    fprintf(stdout, "error while freeing cache entry: no entry found but we have a buffer, %s.\n", message);
    raise(SIGSEGV); // triggers dt_set_signal_handlers() backtrace on Unix
    return;
  }
 
  _non_thread_safe_cache_ref_count_entry(cache, FALSE, cache_entry);
  g_hash_table_remove(cache->external_entries, &cache_entry->hash);
  *mem = NULL;
 
  dt_pthread_mutex_unlock(&cache->lock);
}
 
 
// WARNING: not thread-safe, protect its calls with mutex lock
static dt_pixel_cache_entry_t *dt_pixel_cache_new_entry(const uint64_t hash, const size_t size,
                                                        const char *name, const int id,
                                                        dt_dev_pixelpipe_cache_t *cache, gboolean alloc,
                                                        GHashTable *table)
{
  uint32_t pages_needed = 0;
  size_t rounded_size = 0;
  if(!dt_cache_arena_calc(&cache->arena, size, &pages_needed, &rounded_size))
  {
    fprintf(stderr, "[pixelpipe] invalid cache entry size %" G_GSIZE_FORMAT " for %s\n", size, name);
    return NULL;
  }
 
  int error = _free_space_to_alloc(cache, rounded_size, hash, name);
  if(error) return NULL;
 
  dt_pixel_cache_entry_t *cache_entry = (dt_pixel_cache_entry_t *)malloc(sizeof(dt_pixel_cache_entry_t));
  if(IS_NULL_PTR(cache_entry)) return NULL;
 
  // Metadata, easy to free in batch if need be
  cache_entry->size = rounded_size;
  /* A fresh entry is the MOST recently used one, not the oldest: leaving `age` at 0 until the
   * caller reaches _pixelpipe_cache_finalize_entry() makes it the unconditional LRU victim and
   * gives _for_each_remove_old() a `delta` of the whole process uptime in between. */
  dt_atomic_set_uint64(&cache_entry->age, (uint64_t)g_get_monotonic_time());
  cache_entry->hits = 0;
  cache_entry->hash = hash;
  cache_entry->serial = cache->next_serial++;
  cache_entry->id = id;
  cache_entry->producer_node_key = DT_PIXELPIPE_CACHE_HASH_INVALID;
  cache_entry->refcount = 0;
  cache_entry->auto_destroy = FALSE;
  cache_entry->external_alloc = FALSE;
  cache_entry->data = NULL;
  cache_entry->cache = cache;
  cache_entry->cl_mem_list = NULL;
  dt_pthread_mutex_init(&cache_entry->cl_mem_lock, NULL);
 
  // Optionally alloc the actual buffer, but still record its size in cache
  if(alloc) dt_pixel_cache_alloc(cache_entry);
 
  if(alloc && IS_NULL_PTR(cache_entry->data))
  {
    dt_free(cache_entry);
    return NULL;
  }
  
  // Metadata that need alloc
  cache_entry->name = g_strdup(name);
  dt_pthread_rwlock_init(&cache_entry->lock, NULL);
 
  uint64_t *key = g_malloc(sizeof(*key));
  if(IS_NULL_PTR(key))
  {
    dt_pthread_rwlock_destroy(&cache_entry->lock);
    dt_free(cache_entry->name);
    dt_pthread_mutex_destroy(&cache_entry->cl_mem_lock);
    dt_free(cache_entry);
    return NULL;
  }
  *key = hash;
  g_hash_table_insert(table, key, cache_entry);
 
  // Note : we grow the cache size even though the data buffer is not yet allocated
  // This is planning
  cache->current_memory += rounded_size;
 
  return cache_entry;
}
 
 
static void _free_cache_entry(dt_pixel_cache_entry_t *cache_entry)
{
  if(IS_NULL_PTR(cache_entry)) return;
 
  _pixel_cache_message(cache_entry, "freed", FALSE);
 
  _observe_delete(cache_entry->hash, cache_entry->size, cache_entry->id,
                                   cache_entry->name);
 
  /* Every live entry belongs to the one and only global pixelpipe cache, so its back-reference
   * must match it. If it doesn't, the entry struct has been corrupted (we have seen a single
   * flipped bit in the pointer from faulty RAM) or is stale: reaching cache->arena or
   * cache->current_memory through it would dereference a wild pointer and turn an innocuous
   * teardown into a SIGSEGV. Skip the arena free and the accounting in that case -- the arena is
   * unmapped wholesale right after, so nothing actually leaks. */
  dt_dev_pixelpipe_cache_t *cache = cache_entry->cache;
  if(cache != _pixelpipe_cache)
  {
    fprintf(stderr, "[pixelpipe] cache entry %p has a corrupted back-reference (%p, expected %p); "
                    "skipping arena free to avoid a crash\n",
            (void *)cache_entry, (void *)cache, (void *)_pixelpipe_cache);
    cache = NULL;
  }
 
  if(cache_entry->data && cache)
  {
#ifdef HAVE_OPENCL
    dt_pthread_mutex_lock(&cache_entry->cl_mem_lock);
    for(GList *l = cache_entry->cl_mem_list; l; l = g_list_next(l))
    {
      dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
      if(IS_NULL_PTR(c) || c->host_ptr != cache_entry->data) continue;
      
      /* Host-backed OpenCL images may still dereference `cache_entry->data` asynchronously until their
        * queued work completes. We therefore wait for the owning device before releasing the host arena slot,
        * otherwise an auto-destroyed intermediate can be recycled into another module output while the GPU
        * is still reading the previous pixels.
        *
        * dt_opencl_finish() flushes that device's event list, which is only thread-safe while we hold
        * the device lock (dt_opencl_reserve_device_by_id()). Touching it without that lock races the eventlist/cl_mem
        * bookkeeping of whichever pixelpipe is currently running on that device and crashes inside
        * clWaitForEvents (issues #859, #864, #131742439 -- the 3-min GUI garbage collection timeout
        * dt_dev_pixelpipe_cache_flush_old() owns no device lock). _free_cache_entry() also runs from LRU
        * eviction under a live pipe that already holds this very lock, so we can neither assume we hold it
        * nor block on it: trylock. If we acquire the device it is idle and we drain it safely; if we don't,
        * the owner is draining it itself at the end of its run and this refcount==0 entry is not one of its
        * live borrows, so skipping the finish is safe. */
      const int mem_devid = dt_opencl_get_mem_context_id((cl_mem)c->mem);
      if(mem_devid >= 0 && dt_opencl_is_inited()
         && !dt_opencl_try_reserve_device_by_id(mem_devid))
      {
        dt_opencl_finish(mem_devid);
        dt_opencl_release_device(mem_devid);
      }
    }
    dt_pthread_mutex_unlock(&cache_entry->cl_mem_lock);
#endif
 
    dt_dev_pixelpipe_cache_flush_entry_clmem(cache_entry);
    dt_cache_arena_free(&cache->arena, cache_entry->data, cache_entry->size);
  }
  else
  {
    dt_dev_pixelpipe_cache_flush_entry_clmem(cache_entry);
  }
 
  cache_entry->data = NULL;
  if(cache) cache->current_memory -= cache_entry->size;
  dt_pthread_rwlock_destroy(&cache_entry->lock);
  dt_pthread_mutex_destroy(&cache_entry->cl_mem_lock);
  dt_free(cache_entry->name);
  dt_free(cache_entry);
}
 
static int garbage_collection = 0;
static int pressure_shedding = 0;
 
gboolean dt_dev_pixelpipe_cache_init(size_t max_memory, const gboolean verbose,
                                     const gboolean verbose_detail)
{
  _verbose = verbose;
  _verbose_detail = verbose_detail;
  dt_dev_pixelpipe_cache_t *cache = (dt_dev_pixelpipe_cache_t *)malloc(sizeof(dt_dev_pixelpipe_cache_t));
  dt_pthread_mutex_init(&cache->lock, NULL);
  cache->entries = g_hash_table_new_full(g_int64_hash, g_int64_equal, dt_free_gpointer, (GDestroyNotify)_free_cache_entry);
  cache->external_entries = g_hash_table_new_full(g_int64_hash, g_int64_equal, dt_free_gpointer, (GDestroyNotify)_free_cache_entry);
  cache->max_memory = max_memory;
  cache->current_memory = 0;
  cache->next_serial = 1;
  cache->queries = cache->hits = 0;
  cache->sys_probe_time_us = 0;
  cache->sys_available_est = 0;
  cache->sys_probe_valid = FALSE;
  dt_pixelpipe_cache_pressure_monitor_init(&cache->psi, max_memory);
 
  if(IS_NULL_PTR(cache->entries) || IS_NULL_PTR(cache->external_entries))
  {
    if(cache->entries) g_hash_table_destroy(cache->entries);
    if(cache->external_entries) g_hash_table_destroy(cache->external_entries);
    dt_pthread_mutex_destroy(&cache->lock);
    dt_free(cache);
    return FALSE;
  }
 
  if(dt_cache_arena_init(&cache->arena, cache->max_memory))
  {
    dt_pthread_mutex_destroy(&cache->lock);
    g_hash_table_destroy(cache->external_entries);
    g_hash_table_destroy(cache->entries);
    dt_free(cache);
    return FALSE;
  }
 
  // Run every 3 minutes
  garbage_collection = g_timeout_add(3 * 60 * 1000, (GSourceFunc)dt_dev_pixelpipe_cache_flush_old, cache);
 
  // React within seconds when ANOTHER application's allocations push the system
  // toward memory starvation while we sit idle (the alloc-time pressure valve only
  // runs when we allocate). No-ops when the system has RAM to spare. Every 2 s, one
  // kernel-pressure window (DT_PIXELPIPE_CACHE_PSI_WINDOW_US).
  pressure_shedding = g_timeout_add_seconds(2, (GSourceFunc)_memory_pressure_shedder, cache);
 
  // And react without waiting for either, since a thrashing machine runs neither.
  dt_pixelpipe_cache_pressure_watch_start(&cache->psi, _pressure_wake, cache);
 
  _pixelpipe_cache = cache;
  return TRUE;
}
 
 
void dt_dev_pixelpipe_cache_cleanup(void)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
 
  // Before anything it holds goes away: the watcher takes `lock` and walks the entries.
  dt_pixelpipe_cache_pressure_watch_stop(&cache->psi);
 
  g_hash_table_destroy(cache->external_entries);
  g_hash_table_destroy(cache->entries);
  cache->external_entries = NULL;
  cache->entries = NULL;
  dt_pthread_mutex_destroy(&cache->lock);
  dt_cache_arena_cleanup(&cache->arena);
 
  if(garbage_collection != 0)
  {
    g_source_remove(garbage_collection);
    garbage_collection = 0;
  }
 
  if(pressure_shedding != 0)
  {
    g_source_remove(pressure_shedding);
    pressure_shedding = 0;
  }
 
  if(_pixelpipe_cache == cache) _pixelpipe_cache = NULL;
}
 
static dt_pixel_cache_entry_t *_pixelpipe_cache_create_entry_locked(dt_dev_pixelpipe_cache_t *cache,
                                                                    const uint64_t hash, const size_t size,
                                                                    const char *name, const int id)
{
  dt_pixel_cache_entry_t *cache_entry = dt_pixel_cache_new_entry(hash, size, name, id, cache, FALSE, cache->entries);
  if(IS_NULL_PTR(cache_entry)) return NULL;
 
  // Increase ref_count, consumer will have to decrease it
  _non_thread_safe_cache_ref_count_entry(cache, TRUE, cache_entry);
 
  // Acquire write lock so caller can populate data safely
  dt_dev_pixelpipe_cache_wrlock_entry(TRUE, cache_entry);
 
  return cache_entry;
}
 
static dt_pixel_cache_entry_t *_cache_try_rekey_reuse_locked(dt_dev_pixelpipe_cache_t *cache,
                                                             const uint64_t new_hash, const size_t size,
                                                             const dt_pixel_cache_entry_t *reuse_hint)
{
  if(IS_NULL_PTR(cache) || IS_NULL_PTR(reuse_hint)) return NULL;
 
  const uint64_t old_hash = reuse_hint->hash;
  if(old_hash == DT_PIXELPIPE_CACHE_HASH_INVALID || old_hash == new_hash) return NULL;
  if(reuse_hint->size < size) return NULL;
 
  dt_pixel_cache_entry_t *cache_entry = _non_threadsafe_cache_get_entry(cache, cache->entries, old_hash);
  if(IS_NULL_PTR(cache_entry)) return NULL;
  if(cache_entry->serial != reuse_hint->serial) return NULL;
  if(cache_entry->auto_destroy) return NULL;
  if(cache_entry->size < size) return NULL;
  if(_non_threadsafe_cache_get_entry(cache, cache->entries, new_hash)) return NULL;
 
  _non_thread_safe_cache_ref_count_entry(cache, TRUE, cache_entry);
  dt_dev_pixelpipe_cache_wrlock_entry(TRUE, cache_entry);
 
  /* Rekey reuse transfers the RAM arena slot to a completely different hash. Any cached OpenCL payload
   * still attached to the previous owner would otherwise remain reachable through the new hash and could
   * later be materialized as if it belonged to the new module output. Bail out if some GPU path is still
   * borrowing one of those payloads, otherwise flush the stale bookkeeping before publishing the new hash. */
  dt_pthread_mutex_lock(&cache_entry->cl_mem_lock);
  for(GList *l = cache_entry->cl_mem_list; l; l = g_list_next(l))
  {
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(c && c->refs > 0)
    {
      dt_pthread_mutex_unlock(&cache_entry->cl_mem_lock);
      dt_dev_pixelpipe_cache_wrlock_entry(FALSE, cache_entry);
      // _non_thread_safe_: our caller holds cache->lock (see the function name), and the
      // locking variant would take it again on a non-recursive mutex.
      _non_thread_safe_cache_ref_count_entry(cache, FALSE, cache_entry);
      return NULL;
    }
  }
  dt_pthread_mutex_unlock(&cache_entry->cl_mem_lock);
 
  gpointer stolen_key = NULL;
  gpointer stolen_value = NULL;
  if(!g_hash_table_steal_extended(cache->entries, &old_hash, &stolen_key, &stolen_value)
     || stolen_value != cache_entry)
  {
    if(stolen_key && stolen_value) g_hash_table_insert(cache->entries, stolen_key, stolen_value);
    dt_dev_pixelpipe_cache_wrlock_entry(FALSE, cache_entry);
    // _non_thread_safe_: same reason as the bail path above -- cache->lock is already ours.
    _non_thread_safe_cache_ref_count_entry(cache, FALSE, cache_entry);
    return NULL;
  }
 
  *(uint64_t *)stolen_key = new_hash;
  cache_entry->hash = new_hash;
  g_hash_table_insert(cache->entries, stolen_key, cache_entry);
 
  _observe_rekey(old_hash, new_hash);
 
  _cache_print(DT_DEBUG_PIPECACHE,
           "[pixelpipe_cache] writable rekey old=%" PRIu64 " new=%" PRIu64 " entry=%" PRIu64 "/%" PRIu64
           " refs=%i auto=%i data=%p module=%s\n",
           old_hash, new_hash, cache_entry->hash, cache_entry->serial,
           dt_atomic_get_int(&cache_entry->refcount), cache_entry->auto_destroy, cache_entry->data,
           _cache_debug_module_name());
  return cache_entry;
}
 
 
int dt_dev_pixelpipe_cache_get(const uint64_t hash,
                               const size_t size, const char *name, const int id,
                               const gboolean alloc, void **data,
                               dt_pixel_cache_entry_t **entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(hash == DT_PIXELPIPE_CACHE_HASH_INVALID)
  {
    _cache_print(DT_DEBUG_PIPECACHE, "[pixelpipe_cache] refusing invalid hash allocation for %s\n",
             name ? name : "unknown");
    if(data) *data = NULL;
    if(entry) *entry = NULL;
    return 1;
  }
 
  // Search or create cache entry (under cache lock)
  dt_pthread_mutex_lock(&cache->lock);
  cache->queries++;
 
  dt_pixel_cache_entry_t *cache_entry = _non_threadsafe_cache_get_entry(cache, cache->entries, hash);
  if(!IS_NULL_PTR(cache_entry) && cache_entry->auto_destroy)
  {
    _pixel_cache_message(cache_entry, "dropping auto-destroy entry before cache_get reuse", FALSE);
    if(_non_thread_safe_cache_remove(cache, FALSE, cache_entry, cache->entries) == 0)
      cache_entry = NULL;
  }
 
  if(!IS_NULL_PTR(cache_entry))
  {
    cache->hits++;
    cache_entry->hits++;
    _non_thread_safe_cache_ref_count_entry(cache, TRUE, cache_entry);
    dt_pthread_mutex_unlock(&cache->lock);
 
    // Allocate on demand if requested (e.g. when falling back from vRAM-only buffers).
    if(alloc && IS_NULL_PTR(cache_entry->data))
    {
      dt_dev_pixelpipe_cache_wrlock_entry(TRUE, cache_entry);
      dt_pixel_cache_alloc(cache_entry);
      dt_dev_pixelpipe_cache_wrlock_entry(FALSE, cache_entry);
    }
 
    _pixelpipe_cache_finalize_entry(cache_entry, data, "found");
    if(entry) *entry = cache_entry;
    // existing output reused: a cache hit (entry pinned by the ref above, safe to read)
    _observe_read(hash, cache_entry->size);
      
    return 0;
  }
 
  cache_entry = _pixelpipe_cache_create_entry_locked(cache, hash, size, name, id);
  if(IS_NULL_PTR(cache_entry))
  {
    _cache_print(DT_DEBUG_PIPECACHE, "couldn't allocate new cache entry %" PRIu64 "\n", hash);
    dt_pthread_mutex_unlock(&cache->lock);
    if(entry) *entry = NULL;
    return 1;
  }
 
  // Release cache lock AFTER acquiring entry locks to prevent other threads to capture it in-between
  dt_pthread_mutex_unlock(&cache->lock);
 
  // Alloc after releasing the lock for better runtimes
  if(alloc) dt_pixel_cache_alloc(cache_entry);
 
  _cache_print(DT_DEBUG_PIPECACHE, "[pixelpipe_cache] Write-lock on entry (new cache entry %" PRIu64 " for %s pipeline)\n",
           hash, name);
  _pixelpipe_cache_finalize_entry(cache_entry, data, "created");
 
  if(entry) *entry = cache_entry;
  return 1;
}
 
dt_dev_pixelpipe_cache_writable_status_t
dt_dev_pixelpipe_cache_get_writable(const uint64_t hash,
                                    const size_t size, const char *name, const int id,
                                    const gboolean alloc, const gboolean allow_rekey_reuse,
                                    const dt_pixel_cache_entry_t *reuse_hint,
                                    void **data,
                                    dt_pixel_cache_entry_t **entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(hash == DT_PIXELPIPE_CACHE_HASH_INVALID)
  {
    if(data) *data = NULL;
    if(entry) *entry = NULL;
    return DT_DEV_PIXELPIPE_CACHE_WRITABLE_ERROR;
  }
 
  dt_pthread_mutex_lock(&cache->lock);
  cache->queries++;
 
  dt_pixel_cache_entry_t *cache_entry = _non_threadsafe_cache_get_entry(cache, cache->entries, hash);
  if(!IS_NULL_PTR(cache_entry) && cache_entry->auto_destroy)
  {
    _pixel_cache_message(cache_entry, "dropping auto-destroy entry before writable reuse", FALSE);
    if(_non_thread_safe_cache_remove(cache, FALSE, cache_entry, cache->entries) == 0)
      cache_entry = NULL;
  }
 
  if(!IS_NULL_PTR(cache_entry))
  {
    /* Another pipe already owns this exact hash: the caller consumes that cacheline instead of
     * writing its own, so this is a genuine hit. `queries` was already counted above; count the
     * hit and refresh the MRU too, or a cacheline shared between the preview and the full pipe
     * ages as if only its original producer ever touched it. */
    cache->hits++;
    cache_entry->hits++;
    _pixel_cache_touch(cache_entry);
    dt_pthread_mutex_unlock(&cache->lock);
    if(data) *data = NULL;
    if(entry) *entry = NULL;
    return DT_DEV_PIXELPIPE_CACHE_WRITABLE_EXACT_HIT;
  }
 
  if(allow_rekey_reuse)
  {
    cache_entry = _cache_try_rekey_reuse_locked(cache, hash, size, reuse_hint);
    if(!IS_NULL_PTR(cache_entry))
    {
      dt_pthread_mutex_unlock(&cache->lock);
      if(alloc && IS_NULL_PTR(cache_entry->data)) dt_pixel_cache_alloc(cache_entry);
      _pixelpipe_cache_finalize_entry(cache_entry, data, "writable-rekeyed");
      if(entry) *entry = cache_entry;
      return DT_DEV_PIXELPIPE_CACHE_WRITABLE_REKEYED;
    }
  }
 
  cache_entry = _pixelpipe_cache_create_entry_locked(cache, hash, size, name, id);
  if(IS_NULL_PTR(cache_entry))
  {
    dt_pthread_mutex_unlock(&cache->lock);
    if(data) *data = NULL;
    if(entry) *entry = NULL;
    return DT_DEV_PIXELPIPE_CACHE_WRITABLE_ERROR;
  }
 
  dt_pthread_mutex_unlock(&cache->lock);
 
  if(alloc) dt_pixel_cache_alloc(cache_entry);
  _pixelpipe_cache_finalize_entry(cache_entry, data, "writable-created");
  if(entry) *entry = cache_entry;
  return DT_DEV_PIXELPIPE_CACHE_WRITABLE_CREATED;
}
 
static dt_pixel_cache_entry_t *_cache_lookup_existing(dt_dev_pixelpipe_cache_t *cache, const uint64_t hash,
                                                      void **data)
{
  dt_pthread_mutex_lock(&cache->lock);
  cache->queries++;
  dt_pixel_cache_entry_t *cache_entry = _non_threadsafe_cache_get_entry(cache, cache->entries, hash);
 
  const gboolean hit = !IS_NULL_PTR(cache_entry);
  const size_t hit_size = hit ? cache_entry->size : 0;
  if(hit)
  {
    cache->hits++;
    cache_entry->hits++;
    _pixelpipe_cache_finalize_entry(cache_entry, data, "found");
  }
 
  dt_pthread_mutex_unlock(&cache->lock);
 
  if(hit ) _observe_read(hash, hit_size);
 
  return cache_entry;
}
 
#ifdef HAVE_OPENCL
static gboolean _cache_try_restore_device_payload(dt_pixel_cache_entry_t *cache_entry,
                                                  const int preferred_devid, void **cl_mem_output)
{
  if(IS_NULL_PTR(cache_entry) || IS_NULL_PTR(cl_mem_output) || !IS_NULL_PTR(*cl_mem_output) || preferred_devid < 0)
    return FALSE;
 
  dt_pthread_mutex_lock(&cache_entry->cl_mem_lock);
  for(GList *l = cache_entry->cl_mem_list; l;)
  {
    GList *next = g_list_next(l);
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(!IS_NULL_PTR(c->mem) && c->refs == 0
       && dt_opencl_get_mem_context_id((cl_mem)c->mem) == preferred_devid)
    {
      cache_entry->cl_mem_list = g_list_delete_link(cache_entry->cl_mem_list, l);
      *cl_mem_output = c->mem;
      dt_free(c);
      _pixel_cache_touch(cache_entry);
      break;
    }
    l = next;
  }
  dt_pthread_mutex_unlock(&cache_entry->cl_mem_lock);
 
  return !IS_NULL_PTR(*cl_mem_output);
}
#else
static gboolean _cache_try_restore_device_payload(dt_pixel_cache_entry_t *cache_entry,
                                                  const int preferred_devid, void **cl_mem_output)
{
  return FALSE;
}
#endif
 
gboolean dt_dev_pixelpipe_cache_restore_host_payload(dt_pixel_cache_entry_t *cache_entry,
                                                     const int preferred_devid, void **data)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(data) *data = NULL;
  if(IS_NULL_PTR(cache) || IS_NULL_PTR(cache_entry)) return FALSE;
 
  if(dt_pixel_cache_entry_get_data(cache_entry) != NULL)
  {
    _pixel_cache_touch(cache_entry);
    if(!IS_NULL_PTR(data)) *data = dt_pixel_cache_entry_get_data(cache_entry);
    return TRUE;
  }
 
  if(!_cache_entry_materialize_host_data(cache, preferred_devid, cache_entry))
    return FALSE;
 
  _pixel_cache_touch(cache_entry);
  if(!IS_NULL_PTR(data)) *data = dt_pixel_cache_entry_get_data(cache_entry);
  return dt_pixel_cache_entry_get_data(cache_entry) != NULL;
}
 
gboolean dt_dev_pixelpipe_cache_peek(const uint64_t hash, void **data,
                                     dt_pixel_cache_entry_t **entry, const int preferred_devid,
                                     void **cl_mem_output)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(data) *data = NULL;
  if(entry) *entry = NULL;
  if(cl_mem_output) *cl_mem_output = NULL;
 
  if(hash == DT_PIXELPIPE_CACHE_HASH_INVALID)
    return FALSE;
 
  dt_pixel_cache_entry_t *cache_entry = _cache_lookup_existing(cache, hash, data);
  if(IS_NULL_PTR(cache_entry)) return FALSE;
 
  if(data) *data = dt_pixel_cache_entry_get_data(cache_entry);
 
  /* Exact-hit callers treat the returned payload as already published. Reject
   * cachelines that are still write-locked: reusable output cachelines are
   * rekeyed to their new hash before recompute starts, so exposing them here
   * would let concurrent pipes consume stale or half-written buffers. */
  if(dt_pthread_rwlock_tryrdlock(&cache_entry->lock) != 0)
  {
    _trace_exact_hit("locked", hash, cache_entry, data ? *data : NULL,
                     cl_mem_output ? *cl_mem_output : NULL, preferred_devid, FALSE);
    if(data) *data = NULL;
    return FALSE;
  }
  dt_pthread_rwlock_unlock(&cache_entry->lock);
 
  if(IS_NULL_PTR(data) && IS_NULL_PTR(cl_mem_output))
  {
    if(entry) *entry = cache_entry;
    return TRUE;
  }
 
  /* Picker-triggered aborts can leave a cacheline temporarily present under its
   * hash while it is already marked auto-destroy. Those entries must never exact-hit:
   * they belong to the aborted lifecycle and must force a rebuild on the next run. */
  if(cache_entry->auto_destroy)
  {
    _trace_exact_hit("auto-destroy", hash, cache_entry, data ? *data : NULL,
                     cl_mem_output ? *cl_mem_output : NULL, preferred_devid, FALSE);
    if(data) *data = NULL;
    return FALSE;
  }
 
  if(dt_pixel_cache_entry_get_data(cache_entry) != NULL)
  {
    if(data) *data = dt_pixel_cache_entry_get_data(cache_entry);
    _cache_try_restore_device_payload(cache_entry, preferred_devid, cl_mem_output);
 
    _trace_exact_hit("host", hash, cache_entry, data ? *data : NULL,
                     cl_mem_output ? *cl_mem_output : NULL, preferred_devid, FALSE);
    if(entry) *entry = cache_entry;
    return TRUE;
  }
 
  /* `preferred_devid < 0` means the caller is on a CPU path and does not own any
   * OpenCL device. In that case, hostless cachelines are not consumable here:
   * reopening device-only payloads would enqueue hidden GPU work without a locked
   * device, while reporting a device-only exact-hit would let CPU callers sample
   * an uninitialized host buffer. */
  if(preferred_devid < 0)
  {
    _trace_exact_hit("cpu-no-device", hash, cache_entry, NULL, NULL, preferred_devid, FALSE);
    return FALSE;
  }
 
  if(_cache_try_restore_device_payload(cache_entry, preferred_devid, cl_mem_output))
  {
    _trace_exact_hit("device", hash, cache_entry, data ? *data : NULL,
                     cl_mem_output ? *cl_mem_output : NULL, preferred_devid, FALSE);
    if(entry) *entry = cache_entry;
    return TRUE;
  }
 
  if(!IS_NULL_PTR(data) && dt_dev_pixelpipe_cache_restore_host_payload(cache_entry, preferred_devid, data))
  {
    _trace_exact_hit("restore-host", hash, cache_entry, data ? *data : NULL,
                     cl_mem_output ? *cl_mem_output : NULL, preferred_devid, FALSE);
    if(entry) *entry = cache_entry;
    return TRUE;
  }
 
  _trace_exact_hit("drop-invalid", hash, cache_entry, data ? *data : NULL,
                   cl_mem_output ? *cl_mem_output : NULL, preferred_devid, FALSE);
  _cache_print(DT_DEBUG_PIPECACHE,
           "[pixelpipe] cache entry %" PRIu64 " has no authoritative RAM nor vRAM payload and will be removed\n",
           hash);
  // If the entry removal fails, flag it for auto-destroy.
  if(dt_dev_pixelpipe_cache_remove(TRUE, cache_entry))
    dt_dev_pixelpipe_cache_flag_auto_destroy(cache_entry);
  if(data) *data = NULL;
  return FALSE;
}
 
 
static gboolean _for_each_remove(gpointer key, gpointer value, gpointer user_data)
{
  dt_pixel_cache_entry_t *cache_entry = (dt_pixel_cache_entry_t *)value;
  const int id = GPOINTER_TO_INT(user_data);
 
  // Returns 1 if the lock is captured by another thread
  // 0 if WE capture the lock, and then need to release it
  gboolean locked = dt_pthread_rwlock_trywrlock(&cache_entry->lock);
  if(!locked) dt_pthread_rwlock_unlock(&cache_entry->lock);
 
  return (cache_entry->id == id || id == -1) && !locked;
}
 
 
void dt_dev_pixelpipe_cache_flush(const int id)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  dt_pthread_mutex_lock(&cache->lock);
  g_hash_table_foreach_remove(cache->entries, _for_each_remove, GINT_TO_POINTER(id));
  dt_pthread_mutex_unlock(&cache->lock);
}
 
int dt_dev_pixelpipe_cache_invalidate_hashes(const uint64_t *hashes,
                                             const size_t count)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  int retained = 0;
  dt_pthread_mutex_lock(&cache->lock);
 
  // We are invalidating the cumulative outputs from one pipeline stage onward.
  // Look them up under the same cache lock used for removal so a shared preview
  // pipe cannot replace an entry between lookup and invalidation.
  for(size_t k = 0; k < count; k++)
  {
    if(hashes[k] == DT_PIXELPIPE_CACHE_HASH_INVALID) continue;
 
    dt_pixel_cache_entry_t *entry
        = _non_threadsafe_cache_get_entry(cache, cache->entries, hashes[k]);
    if(IS_NULL_PTR(entry)) continue;
 
    // A displayed backbuffer or an in-flight consumer may still own this
    // shared state. Leave it valid; cache bypass on the retry still walks
    // through downstream stages after the provider has been regenerated.
    if(_non_thread_safe_cache_remove(cache, FALSE, entry, cache->entries))
      retained++;
  }
 
  dt_pthread_mutex_unlock(&cache->lock);
  return retained;
}
 
 
// Find the age of the most recently used entry in the table.
static void _cache_get_newest(gpointer key, gpointer value, gpointer user_data)
{
  dt_pixel_cache_entry_t *cache_entry = (dt_pixel_cache_entry_t *)value;
  int64_t *newest = (int64_t *)user_data;
  const int64_t age = _pixel_cache_get_age(cache_entry);
  if(age > *newest) *newest = age;
}
 
/* `user_data` is the age cutoff computed by dt_dev_pixelpipe_cache_flush_old(): entries last used
 * before it are candidates for collection. */
static gboolean _for_each_remove_old(gpointer key, gpointer value, gpointer user_data)
{
  dt_pixel_cache_entry_t *cache_entry = (dt_pixel_cache_entry_t *)value;
  const int64_t cutoff = *(const int64_t *)user_data;
 
  // Returns 1 if the lock is captured by another thread
  // 0 if WE capture the lock, and then need to release it
  gboolean locked = dt_pthread_rwlock_trywrlock(&cache_entry->lock);
  if(!locked) dt_pthread_rwlock_unlock(&cache_entry->lock);
  gboolean used = dt_atomic_get_int(&cache_entry->refcount) > 0;
 
  /* Two independent signals, and both must agree before an entry is dropped: it was last used
   * before the cutoff, i.e. the cache has moved on from it, and `hits` says it was never
   * asynchronously reused either, so a cacheline that has repeatedly proven reusable across runs
   * is kept for the next one. */
  const gboolean too_old = (_pixel_cache_get_age(cache_entry) < cutoff) && (cache_entry->hits < 4);
 
  return too_old && !used && !locked;
}
 
static int dt_dev_pixelpipe_cache_flush_old(dt_dev_pixelpipe_cache_t *cache)
{
  // Don't hang the GUI thread if the cache is locked by a pipeline.
  // Better luck next time.
  if(dt_pthread_mutex_trylock(&cache->lock)) return G_SOURCE_CONTINUE;
 
  /* Age entries against the cache's OWN most recent activity, not against wall-clock now.
   *
   * "Older than now - 5 min" measures how long the user has been away from the application, which
   * is not what this sweep is about: leave the darkroom open over a coffee break and it wipes a
   * perfectly warm working set the next interaction would have reused, paying a full recompute for
   * a delay the cache had no say in. Anchoring on the newest cacheline instead makes the window
   * relative to the PIPELINE's activity: "the cache has moved on by more than 10 minutes of work
   * since this entry was last needed". An idle cache's newest entry ages alongside everything
   * else, so the whole set stays inside the window and nothing is dropped -- the sweep only bites
   * while something is actively producing newer cachelines, which is exactly when the memory is
   * worth reclaiming.
   *
   * Both passes run under the same lock hold, so the cutoff describes the table it is applied to. */
  int64_t newest = INT64_MIN;
  g_hash_table_foreach(cache->entries, _cache_get_newest, &newest);
  if(newest != INT64_MIN)
  {
    // 10 min in microseconds
    const int64_t ten_min = 10 * 60 * 1000 * 1000;
    int64_t cutoff = newest - ten_min;
    g_hash_table_foreach_remove(cache->entries, _for_each_remove_old, &cutoff);
  }
 
  dt_pthread_mutex_unlock(&cache->lock);
 
  // Hand free pages back to the OS while we're at it, but only when the system
  // actually runs lowish: the per-free MADV_FREE already makes them reclaimable
  // (they count as available), and a hard trim of a large resident free set is
  // O(resident pages) under the arena lock — a needless stall of this (GUI)
  // thread when RAM is plentiful.
  const size_t available = dt_get_system_available_mem();
  const size_t pressure_floor = dt_get_memory_pressure_floor();
  if(available > 0 && pressure_floor > 0 && available < 2 * pressure_floor)
    dt_cache_arena_trim(&cache->arena);
  return G_SOURCE_CONTINUE;
}
 
/* Periodic system memory-pressure shedder (issue #1083), the idle-time counterpart of
 * the alloc-time _system_memory_pressure_valve(): when OTHER applications push the
 * system toward starvation while we are not allocating anything, nobody runs the valve,
 * so watch the system-wide available RAM here and shed cache below the floor. The
 * hard trim matters: the per-free lazy release (MADV_FREE) technically keeps the pages
 * ours until the kernel scavenges them, while decommitting hands them over NOW. */
static int _memory_pressure_shedder(dt_dev_pixelpipe_cache_t *cache)
{
  // Kernel memory pressure first. A pipeline holding the lock is allocating, and allocations
  // react to it themselves (_free_space_to_alloc()).
  if(!dt_pthread_mutex_trylock(&cache->lock))
  {
    _pressure_react_locked(cache);
    dt_pthread_mutex_unlock(&cache->lock);
  }
 
  const size_t pressure_floor = dt_get_memory_pressure_floor();
  if(pressure_floor == 0) return G_SOURCE_CONTINUE;
 
  const size_t available = dt_get_system_available_mem();
  if(available == 0 || available >= pressure_floor) return G_SOURCE_CONTINUE;
 
  // Don't hang the GUI thread if the cache is locked by a pipeline.
  // The valve covers pressure handling while pipelines are allocating anyway.
  if(dt_pthread_mutex_trylock(&cache->lock)) return G_SOURCE_CONTINUE;
 
  const size_t deficit = pressure_floor - available;
  size_t freed = 0;
  while(freed < deficit && g_hash_table_size(cache->entries) > 0)
  {
    const size_t before = cache->current_memory;
    if(_non_thread_safe_pixel_pipe_cache_remove_lru(cache)) break;
    freed += before - cache->current_memory;
  }
 
  // Same reasoning as in the valve: hand the freed pages over for real, then let the
  // OS — not our own bookkeeping — say what that bought, and seed the valve's probe
  // cache with that ground truth.
  const size_t trimmed = dt_cache_arena_trim(&cache->arena);
  dt_invalidate_system_available_mem();
  const size_t available_after = dt_get_system_available_mem();
  cache->sys_probe_valid = (available_after > 0);
  cache->sys_available_est = available_after;
  cache->sys_probe_time_us = g_get_monotonic_time();
  dt_pthread_mutex_unlock(&cache->lock);
 
  dt_print(DT_DEBUG_MEMORY | DT_DEBUG_PIPECACHE,
           "[pixelpipe_cache] system memory pressure while idle: %" G_GSIZE_FORMAT " MiB available "
           "under the %" G_GSIZE_FORMAT " MiB floor — shed %" G_GSIZE_FORMAT " MiB of cache, "
           "returned %" G_GSIZE_FORMAT " MiB to the OS, now %" G_GSIZE_FORMAT " MiB available\n",
           available / (1024 * 1024), pressure_floor / (1024 * 1024), freed / (1024 * 1024),
           trimmed / (1024 * 1024), available_after / (1024 * 1024));
  return G_SOURCE_CONTINUE;
}
 
typedef struct _cache_invalidate_t
{
  void *data;
  size_t size;
} _cache_invalidate_t;
 
 
void _non_thread_safe_cache_ref_count_entry(dt_dev_pixelpipe_cache_t *cache, gboolean lock,
                                            dt_pixel_cache_entry_t *cache_entry)
{
  if(IS_NULL_PTR(cache_entry)) return;
 
  if(lock)
  {
    dt_atomic_add_int(&cache_entry->refcount, 1);
    // Pinning an entry is a use: refresh the MRU timestamp so it does not age while held.
    _pixel_cache_touch(cache_entry);
    _pixel_cache_message(cache_entry, "ref count ++", TRUE);
  }
  else
  {
    dt_atomic_sub_int(&cache_entry->refcount, 1);
    _pixel_cache_message(cache_entry, "ref count --", TRUE);
  }
}
 
 
void dt_dev_pixelpipe_cache_ref_count_entry(gboolean lock,
                                            dt_pixel_cache_entry_t *cache_entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  dt_pthread_mutex_lock(&cache->lock);
  _non_thread_safe_cache_ref_count_entry(cache, lock, cache_entry);
  dt_pthread_mutex_unlock(&cache->lock);
}
 
 
void dt_dev_pixelpipe_cache_wrlock_entry(gboolean lock,
                                         dt_pixel_cache_entry_t *cache_entry)
{
  if(lock)
  {
    dt_pthread_rwlock_wrlock(&cache_entry->lock);
    _pixel_cache_message(cache_entry, "write lock", TRUE);
  }
  else
  {
    dt_pthread_rwlock_unlock(&cache_entry->lock);
    _pixel_cache_message(cache_entry, "write unlock", TRUE);
    // The producer node key travels alongside the hash so GUI waiters can match by
    // the node that produced this output even when the exact output hash the GUI
    // predicted has drifted from the one the worker actually published (the
    // never-served case, doc/pipeline-cache.md §8). INVALID for non-module outputs
    // (raster masks, republished inputs): waiters simply fall back to hash match.
    if(cache_entry && cache_entry->hash != DT_PIXELPIPE_CACHE_HASH_INVALID && _ready_handler)
      _ready_handler(cache_entry->hash, cache_entry->producer_node_key);
  }
}
 
 
void dt_dev_pixelpipe_cache_rdlock_entry(gboolean lock,
                                         dt_pixel_cache_entry_t *cache_entry)
{
  if(lock)
  {
    dt_pthread_rwlock_rdlock(&cache_entry->lock);
    _pixel_cache_message(cache_entry, "read lock", TRUE);
  }
  else
  {
    dt_pthread_rwlock_unlock(&cache_entry->lock);
    _pixel_cache_message(cache_entry, "read unlock", TRUE);
  }
}
 
 
void dt_dev_pixelpipe_cache_flag_auto_destroy(dt_pixel_cache_entry_t *cache_entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  dt_pthread_mutex_lock(&cache->lock);
  if(IS_NULL_PTR(cache_entry))
  {
    dt_pthread_mutex_unlock(&cache->lock);
    return;
  }
 
  cache_entry->auto_destroy = TRUE;
  _pixel_cache_message(cache_entry, "auto destroy flagged", TRUE);
  dt_pthread_mutex_unlock(&cache->lock);
}
 
 
void dt_dev_pixelpipe_cache_auto_destroy_apply(dt_pixel_cache_entry_t *cache_entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  dt_pthread_mutex_lock(&cache->lock);
  if(IS_NULL_PTR(cache_entry))
  {
    dt_pthread_mutex_unlock(&cache->lock);
    return;
  }
 
  if(cache_entry->auto_destroy)
  {
    /* `auto_destroy` is still a normal cache lifecycle: the creator flags a transient entry, then the final
     * consumer decrements its refcount and asks the cache to reap it. Only remove it once no consumer owns
     * it anymore and nobody still holds the entry lock, otherwise teardown paths can free cachelines that
     * still report `refs>0` and hide ownership bugs instead of exposing them. */
    const gboolean locked = dt_pthread_rwlock_trywrlock(&cache_entry->lock);
    if(!locked) dt_pthread_rwlock_unlock(&cache_entry->lock);
    const gboolean used = dt_atomic_get_int(&cache_entry->refcount) > 0;
 
    if(!used && !locked)
    {
      _pixel_cache_message(cache_entry, "auto destroy removing", FALSE);
      g_hash_table_remove(cache->entries, &cache_entry->hash);
    }
    else if(used)
    {
      _pixel_cache_message(cache_entry, "auto destroy postponed: used", TRUE);
    }
    else
    {
      _pixel_cache_message(cache_entry, "auto destroy postponed: locked", TRUE);
    }
  }
  else
  {
    _pixel_cache_message(cache_entry, "auto destroy skipped", TRUE);
  }
  
  dt_pthread_mutex_unlock(&cache->lock);
}
 
void dt_dev_pixelpipe_cache_unref_hash(const uint64_t hash)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(hash == DT_PIXELPIPE_CACHE_HASH_INVALID) return;
 
  dt_pthread_mutex_lock(&cache->lock);
  cache->queries++;
  dt_pixel_cache_entry_t *cache_entry = _non_threadsafe_cache_get_entry(cache, cache->entries, hash);
  dt_pthread_mutex_unlock(&cache->lock);
 
  if(cache_entry)
    dt_dev_pixelpipe_cache_ref_count_entry(FALSE, cache_entry);
}
 
int dt_dev_pixelpipe_cache_rekey(const uint64_t old_hash,
                                 const uint64_t new_hash, dt_pixel_cache_entry_t *entry)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(IS_NULL_PTR(cache)) return 1;
  if(old_hash == new_hash) return 0;
 
  dt_pthread_mutex_lock(&cache->lock);
 
  if(IS_NULL_PTR(entry)) entry = _non_threadsafe_cache_get_entry(cache, cache->entries, old_hash);
  if(IS_NULL_PTR(entry))
  {
    _cache_print(DT_DEBUG_PIPECACHE,
             "[pixelpipe_cache] rekey miss old=%" PRIu64 " new=%" PRIu64 " module=%s\n",
             old_hash, new_hash, _cache_debug_module_name());
    dt_pthread_mutex_unlock(&cache->lock);
    return 1;
  }
 
  dt_pixel_cache_entry_t *conflict = _non_threadsafe_cache_get_entry(cache, cache->entries, new_hash);
  if(conflict && conflict != entry)
  {
    _cache_print(DT_DEBUG_PIPECACHE,
             "[pixelpipe_cache] rekey conflict old=%" PRIu64 " new=%" PRIu64
             " entry=%" PRIu64 "/%" PRIu64 " conflict=%" PRIu64 "/%" PRIu64 " module=%s\n",
             old_hash, new_hash, entry->hash, entry->serial, conflict->hash, conflict->serial,
             _cache_debug_module_name());
    dt_pthread_mutex_unlock(&cache->lock);
    return 1;
  }
 
  gpointer stolen_key = NULL;
  gpointer stolen_value = NULL;
  if(!g_hash_table_steal_extended(cache->entries, &old_hash, &stolen_key, &stolen_value))
  {
    _cache_print(DT_DEBUG_PIPECACHE,
             "[pixelpipe_cache] rekey steal-miss old=%" PRIu64 " new=%" PRIu64
             " entry=%" PRIu64 "/%" PRIu64 " module=%s\n",
             old_hash, new_hash, entry->hash, entry->serial, _cache_debug_module_name());
    dt_pthread_mutex_unlock(&cache->lock);
    return 1;
  }
 
  if(stolen_value != entry)
  {
    _cache_print(DT_DEBUG_PIPECACHE,
             "[pixelpipe_cache] rekey stolen-entry mismatch old=%" PRIu64 " new=%" PRIu64
             " expected=%" PRIu64 "/%" PRIu64 " got=%" PRIu64 "/%" PRIu64 " module=%s\n",
             old_hash, new_hash, entry->hash, entry->serial,
             ((dt_pixel_cache_entry_t *)stolen_value)->hash, ((dt_pixel_cache_entry_t *)stolen_value)->serial,
             _cache_debug_module_name());
    g_hash_table_insert(cache->entries, stolen_key, stolen_value);
    dt_pthread_mutex_unlock(&cache->lock);
    return 1;
  }
 
  /* Explicit rekeying also changes cacheline ownership. The OpenCL payload cache is only valid for the
   * previous hash, so do not let the new hash inherit stale device-side state. If some GPU code is still
   * borrowing one of these payloads, refuse the rekey instead of publishing an ambiguous cache entry. */
  dt_pthread_mutex_lock(&entry->cl_mem_lock);
  for(GList *l = entry->cl_mem_list; l; l = g_list_next(l))
  {
    dt_cache_clmem_t *c = (dt_cache_clmem_t *)l->data;
    if(c && c->refs > 0)
    {
      dt_pthread_mutex_unlock(&entry->cl_mem_lock);
      g_hash_table_insert(cache->entries, stolen_key, stolen_value);
      dt_pthread_mutex_unlock(&cache->lock);
      return 1;
    }
  }
  dt_pthread_mutex_unlock(&entry->cl_mem_lock);
  dt_dev_pixelpipe_cache_flush_entry_clmem(entry);
 
  *(uint64_t *)stolen_key = new_hash;
  entry->hash = new_hash;
  g_hash_table_insert(cache->entries, stolen_key, stolen_value);
  _cache_print(DT_DEBUG_PIPECACHE,
           "[pixelpipe_cache] rekey old=%" PRIu64 " new=%" PRIu64 " entry=%" PRIu64 "/%" PRIu64
           " refs=%i auto=%i data=%p module=%s\n",
           old_hash, new_hash, entry->hash, entry->serial, dt_atomic_get_int(&entry->refcount),
           entry->auto_destroy, entry->data, _cache_debug_module_name());
 
  dt_pthread_mutex_unlock(&cache->lock);
 
  _observe_rekey(old_hash, new_hash);
  return 0;
}
 
 
void dt_dev_pixelpipe_cache_print(void)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(!_verbose) return;
 
  _cache_print(DT_DEBUG_PIPECACHE, "[pixelpipe] cache hit rate so far: %.3f%% - size: %" G_GSIZE_FORMAT " MiB over %" G_GSIZE_FORMAT " MiB - %i items\n", 
    100. * (cache->hits) / (float)cache->queries, cache->current_memory / (1024 * 1024), 
    cache->max_memory / (1024 * 1024),
    g_hash_table_size(cache->entries));
}
 
void dt_dev_pixelpipe_cache_get_usage(size_t *current, size_t *max)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  if(current) *current = 0;
  if(max) *max = 0;
  if(IS_NULL_PTR(cache)) return;
  dt_pthread_mutex_lock(&cache->lock);
  if(current) *current = cache->current_memory;
  // The budget allocations currently evict down to, so tiling plans against it -- and never
  // under what is held, since the callers compute `max - current` unsigned.
  if(max) *max = dt_pixelpipe_cache_pressure_reported(&cache->psi.budget, cache->current_memory);
  dt_pthread_mutex_unlock(&cache->lock);
}
 
GArray *dt_dev_pixelpipe_cache_get_entries_stats(void)
{
  dt_dev_pixelpipe_cache_t *cache = _pixelpipe_cache;
  GArray *out = g_array_new(FALSE, FALSE, sizeof(dt_pixel_cache_stats_entry_t));
  if(IS_NULL_PTR(cache)) return out;
 
  dt_pthread_mutex_lock(&cache->lock);
  GHashTableIter it;
  gpointer key, value;
  g_hash_table_iter_init(&it, cache->entries);
  while(g_hash_table_iter_next(&it, &key, &value))
  {
    dt_pixel_cache_entry_t *e = (dt_pixel_cache_entry_t *)value;
    if(IS_NULL_PTR(e)) continue;
    dt_pixel_cache_stats_entry_t s = { 0 };
    s.hash = e->hash;
    s.size = e->size;
    s.refcount = dt_atomic_get_int((dt_atomic_int *)&e->refcount);
    s.hits = e->hits;
    if(e->name) g_strlcpy(s.name, e->name, sizeof(s.name));
 
#ifdef HAVE_OPENCL
    // Account the OpenCL device buffers attached to this entry. trylock avoids
    // both deadlock and racing the list against concurrent clmem mutation.
    if(dt_pthread_mutex_trylock(&e->cl_mem_lock) == 0)
    {
      for(GList *l = e->cl_mem_list; l; l = g_list_next(l))
      {
        const dt_cache_clmem_t *c = (const dt_cache_clmem_t *)l->data;
        if(IS_NULL_PTR(c) || IS_NULL_PTR(c->mem)) continue;
        s.cl_count++;
        s.cl_bytes += dt_opencl_get_mem_object_size((cl_mem)c->mem);
      }
      dt_pthread_mutex_unlock(&e->cl_mem_lock);
    }
#endif
 
    g_array_append_val(out, s);
  }
  dt_pthread_mutex_unlock(&cache->lock);
  return out;
}
 
size_t dt_dev_pixelpipe_cache_get_vram_total(void)
{
#ifdef HAVE_OPENCL
  if(!dt_opencl_is_enabled()) return 0;
  size_t total = 0;
  for(int i = 0; i < dt_opencl_get_num_devices(); i++)
    total += dt_opencl_get_device_max_global_mem(i);
  return total;
#else
  return 0;
#endif
}
 
// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on