pixelpipe_cache.h

Go to the documentation of this file.

/*
    This file is part of darktable,
    Copyright (C) 2009-2010 johannes hanika.
    Copyright (C) 2010-2011 Henrik Andersson.
    Copyright (C) 2012 Richard Wonka.
    Copyright (C) 2012, 2014, 2016 Tobias Ellinghaus.
    Copyright (C) 2014 Ulrich Pegelow.
    Copyright (C) 2016 Roman Lebedev.
    Copyright (C) 2020 Pascal Obry.
    Copyright (C) 2020 Ralf Brown.
    Copyright (C) 2022 Hanno Schwalm.
    Copyright (C) 2022 Martin Bařinka.
    Copyright (C) 2023, 2025-2026 Aurélien PIERRE.
    
    darktable is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    darktable is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with darktable.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#ifndef DT_CACHES_PIXELPIPE_CACHE_H
#define DT_CACHES_PIXELPIPE_CACHE_H
 
#include "system/memory_arena.h"
#include "system/atomic.h"
#include "pixel/format.h"
#include <inttypes.h>
#include <glib.h>
#include <stddef.h>
 
/* Consumed by C++ translation units (iop/lens.cc, iop/bilateral.cc, ...) through
 * develop/pixelpipe_cache_alloc.h. Without this guard those TUs mangle the declarations
 * as C++ and fail to link against the C definitions -- invisible on ELF, which permits
 * undefined symbols in shared objects, but fatal on Mach-O. */
#ifdef __cplusplus
extern "C" {
#endif
 
struct dt_dev_pixelpipe_t;
struct dt_iop_module_t;
struct dt_iop_roi_t;
 
#define DT_PIXELPIPE_CACHE_HASH_INVALID ((uint64_t)-1)
 
 
/* Opaque, and there is no accessor: no function below takes a cache handle. */
typedef struct dt_dev_pixelpipe_cache_t dt_dev_pixelpipe_cache_t;
 
typedef enum dt_dev_pixelpipe_cache_writable_status_t
{
  DT_DEV_PIXELPIPE_CACHE_WRITABLE_ERROR = -1,
  DT_DEV_PIXELPIPE_CACHE_WRITABLE_EXACT_HIT = -2,
  DT_DEV_PIXELPIPE_CACHE_WRITABLE_CREATED = 1,
  DT_DEV_PIXELPIPE_CACHE_WRITABLE_REKEYED = 2
} dt_dev_pixelpipe_cache_writable_status_t;
 
gboolean dt_dev_pixelpipe_cache_init(size_t max_memory, const gboolean verbose,
                                     const gboolean verbose_detail);
 
gboolean dt_dev_pixelpipe_cache_is_ready(void);
void dt_dev_pixelpipe_cache_cleanup(void);
 
// One pipeline-cache entry, for the GUI memory view.
typedef struct dt_pixel_cache_stats_entry_t
{
  uint64_t hash;
  size_t size;       // host (RAM) bytes
  int refcount;
  int hits;
  int cl_count;      // number of OpenCL device buffers attached to this entry
  size_t cl_bytes;   // their total vRAM bytes (0 without OpenCL)
  char name[64];
} dt_pixel_cache_stats_entry_t;
 
// Current/max bytes used by the pipeline cache (host RAM).
void dt_dev_pixelpipe_cache_get_usage(size_t *current, size_t *max);
 
/* Largest contiguous free run in the arena (bytes): the real upper bound on
 * what an allocation — and, transitively, a tiled module's working set — can
 * get, as opposed to the byte headroom max - current. */
size_t dt_pixelpipe_cache_get_largest_free_run(void);
 
// Total device memory across enabled OpenCL devices (0 if OpenCL is off/absent),
// for the vRAM usage bar's denominator.
size_t dt_dev_pixelpipe_cache_get_vram_total(void);
 
// Snapshot of all live entries (newly-allocated GArray of
// dt_pixel_cache_stats_entry_t; free with g_array_free()).
GArray *dt_dev_pixelpipe_cache_get_entries_stats(void);
 
/* Public for by-value snapshots in pipeline pieces (for example realtime
 * output cacheline reuse/rekey). Ownership still belongs to pixelpipe_cache.
 * External code must treat this as metadata only and never free internals. */
/* PUBLIC, and staying that way: develop/pixelpipe_hb.h embeds one BY VALUE in
 * dt_dev_pixelpipe_iop_t, as a snapshot of the last reusable cacheline's metadata. That makes
 * it a value type the pipeline carries, like dt_mipmap_buffer_t -- not the cache's internal
 * bookkeeping, which is dt_cache_entry_t and is private to this module. */
typedef struct dt_pixel_cache_entry_t
{
  uint64_t hash;            // unique identifier of the entry
  uint64_t serial;          // stable identity across rekeys, changes on fresh allocations
  void *data;               // buffer holding pixels... or anything else
  size_t size;              // size of the data buffer
  /* MRU timestamp, in `g_get_monotonic_time()` microseconds. Set when the entry is created AND
   * refreshed on every hit, so the eviction sweeps age an entry by its LAST USE rather than by
   * its creation -- otherwise a cacheline consumed on every frame looks like the oldest thing in
   * the cache and gets evicted first. Written from paths that hold no cache lock (finalize, the
   * OpenCL payload reuse under `cl_mem_lock` only) and read by the sweeps under `cache->lock`,
   * hence atomic. Touched exclusively through `_pixel_cache_touch()` in pixelpipe_cache.c. */
  dt_atomic_uint64 age;
  char *name;               // name of the cache entry, for debugging
  int id;                   // id of the pipeline owning this entry. Used when flushing, a pipe can only flush its own.
  uint64_t producer_node_key; // stable identity of the pipeline node that produced this output
                              // (dt_supervisor_node_key(pipe_type, op, multi_priority)), or INVALID.
                              // Travels with DT_SIGNAL_CACHELINE_READY so GUI waiters can match by
                              // producer node even when the exact output hash drifted. See
                              // doc/pipeline-cache.md §8.
  dt_atomic_int refcount;   // reference count for the cache entry, to avoid freeing it while still in use
  dt_pthread_rwlock_t lock; // read/write lock to avoid threads conflicts
  gboolean auto_destroy;    // TRUE for auto-destruction the next time it's used. Used for short-lived entries (transient states).
  gboolean external_alloc;  // TRUE for external buffers tracked in the cache
  /* Number of ASYNCHRONOUS REUSES of this entry: how many times a lookup found content someone
   * else had already published, in another run or another pipe. The intra-run producer->consumer
   * handoff is NOT counted -- a module publishing its output with a reference reserved for the
   * next module, which then reopens and releases it, is plain pipeline passthrough, not a reuse.
   * Written only under `cache->lock`, alongside the cache-wide `hits`/`queries` rate. Distinct
   * from `age`, which every use refreshes: see `_pixel_cache_touch()` in pixelpipe_cache.c. */
  int hits;
  dt_dev_pixelpipe_cache_t *cache; // reference to parent cache object
  GList *cl_mem_list;       // reusable OpenCL pinned buffers tied to this entry
  dt_pthread_mutex_t cl_mem_lock;
} dt_pixel_cache_entry_t;
 
const char *dt_pixelpipe_cache_set_current_module(const char *module);
 
struct dt_pixel_cache_entry_t *dt_dev_pixelpipe_cache_get_entry(const uint64_t hash);
 
/*
 * @brief Find a cache entry that holds the exact data buffer pointer `data`.
 *
 * This searches both regular and external cache tables under the cache mutex.
 * It does not change refcounts or locks on the returned entry; the caller must
 * manage lifetime if needed.
 */
struct dt_pixel_cache_entry_t *dt_dev_pixelpipe_cache_get_entry_by_data(void *data);
 
 
int dt_dev_pixelpipe_cache_get(const uint64_t hash, const size_t size,
                               const char *name, const int id, const gboolean alloc,
                               void **data,
                               struct dt_pixel_cache_entry_t **entry);
 
dt_dev_pixelpipe_cache_writable_status_t
dt_dev_pixelpipe_cache_get_writable(const uint64_t hash,
                                    const size_t size, const char *name, const int id,
                                    const gboolean alloc, const gboolean allow_rekey_reuse,
                                    const struct dt_pixel_cache_entry_t *reuse_hint,
                                    void **data,
                                    struct dt_pixel_cache_entry_t **entry);
 
void *dt_dev_pixelpipe_cache_borrow_cl_payload(struct dt_pixel_cache_entry_t *entry,
                                               int devid, int width, int height, int bpp);
 
void dt_dev_pixelpipe_cache_return_cl_payload(struct dt_pixel_cache_entry_t *entry, void *mem);
 
void dt_dev_pixelpipe_cache_flush_entry_clmem(struct dt_pixel_cache_entry_t *entry);
 
gboolean dt_dev_pixelpipe_cache_restore_host_payload(struct dt_pixel_cache_entry_t *entry,
                                                     int preferred_devid, void **data);
 
void *dt_dev_pixelpipe_cache_get_pinned_image(void *host_ptr,
                                              struct dt_pixel_cache_entry_t *entry_hint, int devid,
                                              int width, int height, int bpp, int flags,
                                              gboolean *out_reused);
 
void dt_dev_pixelpipe_cache_put_pinned_image(void *host_ptr,
                                             struct dt_pixel_cache_entry_t *entry_hint, void **mem);
 
gboolean dt_dev_pixelpipe_cache_flush_host_pinned_image(void *host_ptr,
                                                        struct dt_pixel_cache_entry_t *entry_hint, int devid);
 
void *dt_dev_pixelpipe_cache_get_cl_buffer(int devid, void *host_ptr, const struct dt_iop_roi_t *roi,
                                           size_t bpp, struct dt_iop_module_t *module,
                                           const char *message, struct dt_pixel_cache_entry_t *entry,
                                           gboolean *out_reused, void *keep);
 
void *dt_dev_pixelpipe_cache_alloc_cl_device_buffer(int devid, const struct dt_iop_roi_t *roi, size_t bpp,
                                                    const struct dt_iop_module_t *module,
                                                    const char *message, void *keep);
 
void dt_dev_pixelpipe_cache_release_cl_buffer(void **cl_mem_buffer, struct dt_pixel_cache_entry_t *entry,
                                              void *host_ptr, gboolean cache_device);
 
int dt_dev_pixelpipe_cache_sync_cl_buffer(int devid, void *host_ptr, void *cl_mem_buffer,
                                          const struct dt_iop_roi_t *roi, int cl_mode, size_t bpp,
                                          struct dt_iop_module_t *module, const char *message);
 
float *dt_dev_pixelpipe_cache_restore_cl_buffer(struct dt_dev_pixelpipe_t *pipe, float *input,
                                                void *cl_mem_input, const struct dt_iop_roi_t *roi_in,
                                                struct dt_iop_module_t *module, size_t in_bpp,
                                                struct dt_pixel_cache_entry_t *input_entry,
                                                const char *message);
 
int dt_dev_pixelpipe_cache_prepare_cl_input(struct dt_dev_pixelpipe_t *pipe,
                                            struct dt_iop_module_t *module,
                                            float *input, void **cl_mem_input,
                                            const struct dt_iop_roi_t *roi_in, size_t in_bpp,
                                            struct dt_pixel_cache_entry_t *input_entry,
                                            struct dt_pixel_cache_entry_t **locked_input_entry,
                                            void *keep);
 
struct dt_pixel_cache_entry_t *dt_dev_pixelpipe_cache_ref_entry_for_host_ptr(void *host_ptr);
 
gboolean dt_dev_pixelpipe_cache_ref_entry_by_hash(const uint64_t hash,
                                                  void **data,
                                                  struct dt_pixel_cache_entry_t **entry);
 
gboolean dt_dev_pixelpipe_cache_ref_host_entry_by_hash(const uint64_t hash,
                                                       void **data,
                                                       struct dt_pixel_cache_entry_t **entry);
 
void *dt_pixel_cache_entry_get_data(struct dt_pixel_cache_entry_t *entry);
 
size_t dt_pixel_cache_entry_get_size(struct dt_pixel_cache_entry_t *entry);
 
                      
void *dt_pixel_cache_alloc(struct dt_pixel_cache_entry_t *entry);
 
void *dt_pixelpipe_cache_alloc_align_cache_impl(size_t size, int id,
                                                const char *name);
 
void dt_pixelpipe_cache_free_align_cache(void **mem, const char *message);
 
 
 
gboolean dt_dev_pixelpipe_cache_peek(const uint64_t hash, void **data,
                                     struct dt_pixel_cache_entry_t **entry, const int preferred_devid,
                                     void **cl_mem_output);
 
void dt_dev_pixelpipe_cache_flush(const int id);
 
int dt_dev_pixelpipe_cache_invalidate_hashes(const uint64_t *hashes,
                                             const size_t count);
 
void dt_dev_pixelpipe_cache_flush_clmem(const int devid);
 
void dt_dev_pixelpipe_cache_flush_clmem_for_pipe(const int devid);
 
 
int dt_dev_pixelpipe_cache_remove(const gboolean force,
                                  struct dt_pixel_cache_entry_t *entry);
 
 
void dt_dev_pixelpipe_cache_print(void);
 
int dt_dev_pixel_pipe_cache_remove_lru(void);
 
void dt_dev_pixelpipe_cache_ref_count_entry(gboolean lock,
                                            struct dt_pixel_cache_entry_t *entry);
 
void dt_dev_pixelpipe_cache_wrlock_entry(gboolean lock,
                                         struct dt_pixel_cache_entry_t *entry);
 
 
void dt_dev_pixelpipe_cache_rdlock_entry(gboolean lock,
                                         struct dt_pixel_cache_entry_t *entry);
 
 
void dt_dev_pixelpipe_cache_flag_auto_destroy(struct dt_pixel_cache_entry_t *entry);
 
void dt_dev_pixelpipe_cache_auto_destroy_apply(struct dt_pixel_cache_entry_t *entry);
 
void dt_dev_pixelpipe_cache_unref_hash(const uint64_t hash);
 
int dt_dev_pixelpipe_cache_rekey(const uint64_t old_hash,
                                 const uint64_t new_hash, struct dt_pixel_cache_entry_t *entry);
 
/* --- Telling the rest of the application things ---------------------------
 *
 * The cache has three reasons to speak upward: warn the user that it is full, announce that a
 * cacheline became ready so a waiter can stop waiting, and feed the supervisor its bookkeeping.
 * Every one of those used to be a direct call -- dt_control_log(), a raised
 * DT_SIGNAL_CACHELINE_READY, dt_supervisor_*() -- which put control/ and develop/ headers in a
 * module that is otherwise pure storage, and made the cache depend on the application rather
 * than the other way round.
 *
 * They are handlers now, installed once by the orchestrator. Unset handlers are simply not
 * called, so the cache works in a build or a test that installs none. Same shape as
 * dt_colorspaces_set_profile_changed_handler().
 */
 
typedef void (*dt_pixelpipe_cache_warn_handler_t)(const char *message);
 
typedef void (*dt_pixelpipe_cache_ready_handler_t)(uint64_t hash, uint64_t producer_node_key);
 
typedef struct dt_pixelpipe_cache_observer_t
{
  gboolean (*active)(void);
  void (*cacheline_read)(uint64_t hash, size_t size);
  void (*cacheline_delete)(uint64_t hash, size_t size, int owner_pipe_id, const char *name);
  void (*rekey)(uint64_t old_hash, uint64_t new_hash);
} dt_pixelpipe_cache_observer_t;
 
void dt_dev_pixelpipe_cache_set_handlers(dt_pixelpipe_cache_warn_handler_t warn,
                                         dt_pixelpipe_cache_ready_handler_t ready,
                                         const dt_pixelpipe_cache_observer_t *observer);
 
 
#ifdef __cplusplus
}
#endif
 
#endif // DT_CACHES_PIXELPIPE_CACHE_H
 
// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on