pixelpipe_cache.h

Go to the documentation of this file.

/*
    This file is part of darktable,
    Copyright (C) 2009-2010 johannes hanika.
    Copyright (C) 2010-2011 Henrik Andersson.
    Copyright (C) 2012 Richard Wonka.
    Copyright (C) 2012, 2014, 2016 Tobias Ellinghaus.
    Copyright (C) 2014 Ulrich Pegelow.
    Copyright (C) 2016 Roman Lebedev.
    Copyright (C) 2020 Pascal Obry.
    Copyright (C) 2020 Ralf Brown.
    Copyright (C) 2022 Hanno Schwalm.
    Copyright (C) 2022 Martin Bařinka.
    Copyright (C) 2023, 2025-2026 Aurélien PIERRE.
    Copyright (C) 2026 Guillaume Stutin.
    
    darktable is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    darktable is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with darktable.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#ifndef DT_CACHES_PIXELPIPE_CACHE_H
#define DT_CACHES_PIXELPIPE_CACHE_H
 
#include "system/memory_arena.h"
#include "system/atomic.h"
#include "pixel/format.h"
#include <inttypes.h>
#include <glib.h>
#include <stddef.h>
 
/* Consumed by C++ translation units (iop/lens.cc, iop/bilateral.cc, ...) through
 * develop/pixelpipe_cache_alloc.h. Without this guard those TUs mangle the declarations
 * as C++ and fail to link against the C definitions -- invisible on ELF, which permits
 * undefined symbols in shared objects, but fatal on Mach-O. */
#ifdef __cplusplus
extern "C" {
#endif
 
struct dt_dev_pixelpipe_t;
struct dt_iop_module_t;
struct dt_iop_roi_t;
 
#define DT_PIXELPIPE_CACHE_HASH_INVALID ((uint64_t)-1)
 
 
/* Opaque, and there is no accessor: no function below takes a cache handle. */
typedef struct dt_dev_pixelpipe_cache_t dt_dev_pixelpipe_cache_t;
 
typedef enum dt_dev_pixelpipe_cache_writable_status_t
{
  DT_DEV_PIXELPIPE_CACHE_WRITABLE_ERROR = -1,
  DT_DEV_PIXELPIPE_CACHE_WRITABLE_EXACT_HIT = -2,
  DT_DEV_PIXELPIPE_CACHE_WRITABLE_CREATED = 1,
  DT_DEV_PIXELPIPE_CACHE_WRITABLE_REKEYED = 2
} dt_dev_pixelpipe_cache_writable_status_t;
 
gboolean dt_dev_pixelpipe_cache_init(size_t max_memory, const gboolean verbose,
                                     const gboolean verbose_detail);
 
gboolean dt_dev_pixelpipe_cache_is_ready(void);
 
void dt_dev_pixelpipe_cache_cleanup(void);
 
// What one pipeline-cache entry holds on one OpenCL device, for the GUI memory view.
typedef struct dt_pixel_cache_stats_device_t
{
  int cl_count;    // OpenCL buffers of the entry on the device
  size_t cl_bytes; // their vRAM bytes
} dt_pixel_cache_stats_device_t;
 
// One pipeline-cache entry, for the GUI memory view.
typedef struct dt_pixel_cache_stats_entry_t
{
  uint64_t hash;     // an allocation's is its address: no event names it
  size_t size;       // bytes counted against the cache budget, whether a host buffer holds them or not
  /* Not a line but a buffer of dt_pixelpipe_cache_alloc_align_*(): a module's working buffer for one
   * run, or what something keeps there for longer (a lookup table, a state struct). */
  gboolean allocation;
  int64_t age_us;    // since the cache last touched it: a line's last use, an allocation's allocation
  size_t host_bytes; // bytes of the host (RAM) buffer: 0 for a line kept on the device only
  int host_stale;    // the host buffer still holds the pixels of a hash the line no longer answers to
  int refcount;
  int hits;
  int cl_count;      // number of OpenCL device buffers attached to this entry
  size_t cl_bytes;   // their total vRAM bytes (0 without OpenCL)
  // The same, device by device: dt_opencl_get_num_devices() of them, or NULL without any.
  dt_pixel_cache_stats_device_t *devices;
  char name[64];
} dt_pixel_cache_stats_entry_t;
 
// Current/max bytes used by the pipeline cache (host RAM).
void dt_dev_pixelpipe_cache_get_usage(size_t *current, size_t *max);
 
// The budget the cache was created with. Kernel memory pressure lowers the budget allocations
// evict down to, which dt_dev_pixelpipe_cache_get_usage() reports; this one never moves.
size_t dt_dev_pixelpipe_cache_get_max_memory(void);
 
size_t dt_dev_pixelpipe_cache_get_available(void);
 
/* Largest contiguous free run in the arena (bytes): the real upper bound on
 * what an allocation — and, transitively, a tiled module's working set — can
 * get, as opposed to the byte headroom max - current. */
size_t dt_pixelpipe_cache_get_largest_free_run(void);
 
// Snapshot of all live entries, the lines then the allocations (newly-allocated GArray of
// dt_pixel_cache_stats_entry_t; free with g_array_free(), which frees each entry's `devices` too).
GArray *dt_dev_pixelpipe_cache_get_entries_stats(void);
 
/* Public for by-value snapshots in pipeline pieces (for example realtime
 * output cacheline reuse/rekey). Ownership still belongs to pixelpipe_cache.
 * External code must treat this as metadata only and never free internals. */
/* PUBLIC, and staying that way: develop/pixelpipe_hb.h embeds one BY VALUE in
 * dt_dev_pixelpipe_iop_t, as a snapshot of the last reusable cacheline's metadata. That makes
 * it a value type the pipeline carries, like dt_mipmap_buffer_t -- not the cache's internal
 * bookkeeping, which is dt_cache_entry_t and is private to this module. */
typedef struct dt_pixel_cache_entry_t
{
  uint64_t hash;            // unique identifier of the entry
  uint64_t serial;          // stable identity across rekeys, changes on fresh allocations
  void *data;               // buffer holding pixels... or anything else
  size_t size;              // size of the data buffer
  /* MRU timestamp, in `g_get_monotonic_time()` microseconds. Set when the entry is created AND
   * refreshed on every hit, so the eviction sweeps age an entry by its LAST USE rather than by
   * its creation -- otherwise a cacheline consumed on every frame looks like the oldest thing in
   * the cache and gets evicted first. Written from paths that hold no cache lock (finalize, the
   * OpenCL payload reuse under `cl_mem_lock` only) and read by the sweeps under `cache->lock`,
   * hence atomic. Touched exclusively through `_pixel_cache_touch()` in pixelpipe_cache.c. */
  dt_atomic_uint64 age;
  char *name;               // name of the cache entry, for debugging
  int id;                   // id of the pipeline owning this entry. Used when flushing, a pipe can only flush its own.
  uint64_t producer_node_key; // stable identity of the pipeline node that produced this output
                              // (dt_supervisor_node_key(pipe_type, op, multi_priority)), or INVALID.
                              // Travels with DT_SIGNAL_CACHELINE_READY so GUI waiters can match by
                              // producer node even when the exact output hash drifted. See
                              // doc/pipeline-cache.md §8.
  dt_atomic_int refcount;   // reference count for the cache entry, to avoid freeing it while still in use
  dt_pthread_rwlock_t lock; // read/write lock to avoid threads conflicts
  gboolean auto_destroy;    // disposable: the release of its last reference removes it. See dt_dev_pixelpipe_cache_flag_auto_destroy().
  gboolean external_alloc;  // TRUE for external buffers tracked in the cache
  /* Number of ASYNCHRONOUS REUSES of this entry: how many times a lookup found content someone
   * else had already published, in another run or another pipe. The intra-run producer->consumer
   * handoff is NOT counted -- a module publishing its output with a reference reserved for the
   * next module, which then reopens and releases it, is plain pipeline passthrough, not a reuse.
   * Written only under `cache->lock`, alongside the cache-wide `hits`/`queries` rate. Distinct
   * from `age`, which every use refreshes: see `_pixel_cache_touch()` in pixelpipe_cache.c. */
  int hits;
  dt_dev_pixelpipe_cache_t *cache; // reference to parent cache object
  GList *cl_mem_list;       // reusable OpenCL pinned buffers tied to this entry
  dt_pthread_mutex_t cl_mem_lock;
  /* TRUE while `data` holds the pixels of a hash this entry no longer answers to: set when a
   * writable acquisition rekeys the entry in place, cleared once the host buffer is rewritten for
   * the new hash (dt_dev_pixelpipe_cache_flag_host_written(), or a device->host copy). */
  dt_atomic_int host_stale;
} dt_pixel_cache_entry_t;
 
const char *dt_pixelpipe_cache_set_current_module(const char *module);
 
const char *dt_pixelpipe_cache_set_current_image(const char *image);
 
uint32_t dt_pixelpipe_cache_get_alloc_refusals(void);
 
struct dt_pixel_cache_entry_t *dt_dev_pixelpipe_cache_get_entry(const uint64_t hash);
 
/*
 * @brief Find a cache entry that holds the exact data buffer pointer `data`.
 *
 * This searches both regular and external cache tables under the cache mutex.
 * It does not change refcounts or locks on the returned entry; the caller must
 * manage lifetime if needed.
 */
struct dt_pixel_cache_entry_t *dt_dev_pixelpipe_cache_get_entry_by_data(void *data);
 
 
int dt_dev_pixelpipe_cache_get(const uint64_t hash, const size_t size,
                               const char *name, const int id, const gboolean alloc,
                               void **data,
                               struct dt_pixel_cache_entry_t **entry);
 
dt_dev_pixelpipe_cache_writable_status_t
dt_dev_pixelpipe_cache_get_writable(const uint64_t hash,
                                    const size_t size, const char *name, const int id,
                                    const gboolean alloc, const gboolean allow_rekey_reuse,
                                    const struct dt_pixel_cache_entry_t *reuse_hint,
                                    const struct dt_pixel_cache_entry_t *reuse_hint_prev,
                                    void **data,
                                    struct dt_pixel_cache_entry_t **entry);
 
void *dt_dev_pixelpipe_cache_borrow_cl_payload(struct dt_pixel_cache_entry_t *entry,
                                               int devid, int width, int height, int bpp);
 
void dt_dev_pixelpipe_cache_return_cl_payload(struct dt_pixel_cache_entry_t *entry, void *mem);
 
void dt_dev_pixelpipe_cache_flush_entry_clmem(struct dt_pixel_cache_entry_t *entry);
 
gboolean dt_dev_pixelpipe_cache_restore_host_payload(struct dt_pixel_cache_entry_t *entry,
                                                     int preferred_devid, void **data);
 
void *dt_dev_pixelpipe_cache_get_pinned_image(void *host_ptr,
                                              struct dt_pixel_cache_entry_t *entry_hint, int devid,
                                              int width, int height, int bpp, int flags,
                                              gboolean *out_reused);
 
void dt_dev_pixelpipe_cache_put_pinned_image(void *host_ptr,
                                             struct dt_pixel_cache_entry_t *entry_hint, void **mem);
 
gboolean dt_dev_pixelpipe_cache_flush_host_pinned_image(void *host_ptr,
                                                        struct dt_pixel_cache_entry_t *entry_hint, int devid);
 
void *dt_dev_pixelpipe_cache_get_cl_buffer(int devid, void *host_ptr, const struct dt_iop_roi_t *roi,
                                           size_t bpp, struct dt_iop_module_t *module,
                                           const char *message, struct dt_pixel_cache_entry_t *entry,
                                           gboolean *out_reused, void *keep);
 
void *dt_dev_pixelpipe_cache_alloc_cl_device_buffer(int devid, const struct dt_iop_roi_t *roi, size_t bpp,
                                                    const struct dt_iop_module_t *module,
                                                    const char *message, void *keep);
 
void dt_dev_pixelpipe_cache_release_cl_buffer(void **cl_mem_buffer, struct dt_pixel_cache_entry_t *entry,
                                              void *host_ptr, gboolean cache_device);
 
int dt_dev_pixelpipe_cache_sync_cl_buffer(int devid, void *host_ptr, void *cl_mem_buffer,
                                          const struct dt_iop_roi_t *roi, int cl_mode, size_t bpp,
                                          struct dt_iop_module_t *module, const char *message);
 
float *dt_dev_pixelpipe_cache_restore_cl_buffer(struct dt_dev_pixelpipe_t *pipe, float *input,
                                                void *cl_mem_input, const struct dt_iop_roi_t *roi_in,
                                                struct dt_iop_module_t *module, size_t in_bpp,
                                                struct dt_pixel_cache_entry_t *input_entry,
                                                const char *message);
 
int dt_dev_pixelpipe_cache_prepare_cl_input(struct dt_dev_pixelpipe_t *pipe,
                                            struct dt_iop_module_t *module,
                                            float *input, void **cl_mem_input,
                                            const struct dt_iop_roi_t *roi_in, size_t in_bpp,
                                            struct dt_pixel_cache_entry_t *input_entry,
                                            struct dt_pixel_cache_entry_t **locked_input_entry,
                                            void *keep);
 
struct dt_pixel_cache_entry_t *dt_dev_pixelpipe_cache_ref_entry_for_host_ptr(void *host_ptr);
 
gboolean dt_dev_pixelpipe_cache_ref_entry_by_hash(const uint64_t hash,
                                                  void **data,
                                                  struct dt_pixel_cache_entry_t **entry);
 
gboolean dt_dev_pixelpipe_cache_ref_host_entry_by_hash(const uint64_t hash,
                                                       void **data,
                                                       struct dt_pixel_cache_entry_t **entry);
 
void *dt_pixel_cache_entry_get_data(struct dt_pixel_cache_entry_t *entry);
 
void *dt_pixel_cache_entry_get_buffer(struct dt_pixel_cache_entry_t *entry);
 
void dt_dev_pixelpipe_cache_flag_host_written(struct dt_pixel_cache_entry_t *entry);
 
size_t dt_pixel_cache_entry_get_size(const struct dt_pixel_cache_entry_t *entry);
 
                      
void *dt_pixel_cache_alloc(struct dt_pixel_cache_entry_t *entry);
 
void *dt_pixelpipe_cache_alloc_align_cache_impl(size_t size, int id,
                                                const char *name);
 
void dt_pixelpipe_cache_free_align_cache(void **mem, const char *message);
 
 
 
gboolean dt_dev_pixelpipe_cache_peek(const uint64_t hash, void **data,
                                     struct dt_pixel_cache_entry_t **entry, const int preferred_devid,
                                     void **cl_mem_output);
 
void dt_dev_pixelpipe_cache_flush(const int id);
 
int dt_dev_pixelpipe_cache_invalidate_hashes(const uint64_t *hashes,
                                             const size_t count);
 
void dt_dev_pixelpipe_cache_flush_clmem(const int devid);
 
void dt_dev_pixelpipe_cache_flush_clmem_for_pipe(const int devid);
 
 
void dt_dev_pixelpipe_cache_print(void);
 
int dt_dev_pixel_pipe_cache_remove_lru(void);
 
void dt_dev_pixelpipe_cache_ref_count_entry(gboolean lock,
                                            struct dt_pixel_cache_entry_t *entry);
 
void dt_dev_pixelpipe_cache_wrlock_entry(gboolean lock,
                                         struct dt_pixel_cache_entry_t *entry);
 
 
void dt_dev_pixelpipe_cache_rdlock_entry(gboolean lock,
                                         struct dt_pixel_cache_entry_t *entry);
 
 
void dt_dev_pixelpipe_cache_flag_auto_destroy(struct dt_pixel_cache_entry_t *entry);
 
void dt_dev_pixelpipe_cache_unref_hash(const uint64_t hash);
 
void dt_dev_pixelpipe_cache_unref_entry(struct dt_pixel_cache_entry_t *entry);
 
int dt_dev_pixelpipe_cache_rekey(const uint64_t old_hash,
                                 const uint64_t new_hash, struct dt_pixel_cache_entry_t *entry);
 
/* --- Telling the rest of the application things ---------------------------
 *
 * The cache has three reasons to speak upward: alert the user that an allocation could not be
 * served (the cache is full, or the system itself runs out of memory), announce that a cacheline
 * became ready so a waiter can stop waiting, and feed the supervisor its bookkeeping.
 * Every one of those used to be a direct call -- dt_control_log(), a raised
 * DT_SIGNAL_CACHELINE_READY, dt_supervisor_*() -- which put control/ and develop/ headers in a
 * module that is otherwise pure storage, and made the cache depend on the application rather
 * than the other way round.
 *
 * They are handlers now, installed once by the orchestrator. Unset handlers are simply not
 * called, so the cache works in a build or a test that installs none. Same shape as
 * dt_colorspaces_set_profile_changed_handler().
 */
 
typedef void (*dt_pixelpipe_cache_warn_handler_t)(const char *message);
 
typedef void (*dt_pixelpipe_cache_alert_handler_t)(const char *message, const char *item);
 
void dt_dev_pixelpipe_cache_set_alert_handler(dt_pixelpipe_cache_alert_handler_t alert);
 
typedef void (*dt_pixelpipe_cache_ready_handler_t)(uint64_t hash, uint64_t producer_node_key);
 
typedef struct dt_pixelpipe_cache_observer_t
{
  gboolean (*active)(void);
  void (*cacheline_read)(uint64_t hash, size_t size);
  void (*cacheline_delete)(uint64_t hash, size_t size, int owner_pipe_id, const char *name);
  void (*rekey)(uint64_t old_hash, uint64_t new_hash);
} dt_pixelpipe_cache_observer_t;
 
void dt_dev_pixelpipe_cache_set_handlers(dt_pixelpipe_cache_warn_handler_t warn,
                                         dt_pixelpipe_cache_ready_handler_t ready,
                                         const dt_pixelpipe_cache_observer_t *observer);
 
 
#ifdef __cplusplus
}
#endif
 
#endif // DT_CACHES_PIXELPIPE_CACHE_H
 
// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on