#include <inttypes.h>
#include <glib.h>
#include <stdlib.h>
#include <signal.h>
#include <string.h>
#include "system/sys_resources.h"
#include "caches/pixelpipe_cache_pressure.h"
#include "develop/pixelpipe_hb.h"
#include "caches/pixelpipe_cache.h"
#include "common/opencl.h"
#include "pixel/format.h"
#include "develop/imageop.h"

Include dependency graph for pixelpipe_cache.c:

Data Structures
struct	dt_dev_pixelpipe_cache_t

struct	dt_cache_clmem_t

struct	_cache_lru_t

struct	_cache_invalidate_t

Macros
#define	_cache_print(channel, ...)

#define	DT_PIXELPIPE_CACHE_PRESSURE_EXEMPT_SIZE ((size_t)1024 * 1024)

#define	DT_PIXELPIPE_CACHE_PRESSURE_ALERT_PERIOD_US ((gint64)10 * G_USEC_PER_SEC)

Typedefs
typedef struct dt_dev_pixelpipe_cache_t	dt_dev_pixelpipe_cache_t

typedef struct dt_cache_clmem_t	dt_cache_clmem_t

typedef enum dt_pixel_cache_materialize_source_rank_t	dt_pixel_cache_materialize_source_rank_t

typedef struct _cache_lru_t	_cache_lru_t

typedef struct _cache_invalidate_t	_cache_invalidate_t

Enumerations
enum	dt_pixel_cache_materialize_source_rank_t { DT_PIXEL_CACHE_MATERIALIZE_SOURCE_NONE = 0 , DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_ANY = 1 , DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_PREFERRED = 2 , DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_ANY = 3 , DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_PREFERRED = 4 }

Functions
gboolean	dt_dev_pixelpipe_cache_is_ready (void)
	Has the pixelpipe cache been initialised? Callers that run before dt_dev_pixelpipe_cache_init() succeeds, or after its cleanup, ask this rather than testing a handle they should not hold.

void	dt_dev_pixelpipe_cache_set_handlers (dt_pixelpipe_cache_warn_handler_t warn, dt_pixelpipe_cache_ready_handler_t ready, const dt_pixelpipe_cache_observer_t *observer)
	Install the handlers. Call once, from the orchestrator, before any pipe runs.

void	dt_dev_pixelpipe_cache_set_alert_handler (dt_pixelpipe_cache_alert_handler_t alert)
	Install the alert handler. The GUI installs it, once it exists, apart from the others: without a GUI there is nothing to alert with, and an alert is then printed on stderr, the item on the line under the message.

static void	_alert_user (const char message, const char item)

static void	_warn_user (const char *message)

static gchar *	_cache_full_sentence (const char name, const char module)

static void	_alert_cache_refused (const char name, const char module)

static gboolean	_observed (void)

static void	_observe_read (uint64_t hash, size_t size)

static void	_observe_delete (uint64_t hash, size_t size, int owner_pipe_id, const char *name)

static void	_observe_rekey (uint64_t old_hash, uint64_t new_hash)

static dt_pixel_cache_entry_t *	_non_threadsafe_cache_get_entry (dt_dev_pixelpipe_cache_t cache, GHashTable table, const uint64_t key)

static const char *	_cache_debug_module_name (void)

static void	_trace_exact_hit (const char phase, const uint64_t hash, dt_pixel_cache_entry_t cache_entry, void data, void cl_mem_output, const int preferred_devid, const gboolean verbose)

const char *	dt_pixelpipe_cache_set_current_module (const char *module)
	Set the current module name for cache diagnostics (thread-local).

const char *	dt_pixelpipe_cache_set_current_image (const char *image)
	Set the image the current module runs on, for the message that reports a refused allocation (thread-local).

uint32_t	dt_pixelpipe_cache_get_alloc_refusals (void)
	Count the allocations the cache refused to the calling thread for lack of memory (thread-local).

void	_non_thread_safe_cache_ref_count_entry (dt_dev_pixelpipe_cache_t cache, gboolean lock, dt_pixel_cache_entry_t cache_entry)

static void	_free_cache_entry (dt_pixel_cache_entry_t *cache_entry)

static void	_pixelpipe_cache_finalize_entry (dt_pixel_cache_entry_t cache_entry, void data, const char message)

int	_non_thread_safe_cache_remove (dt_dev_pixelpipe_cache_t cache, const gboolean force, dt_pixel_cache_entry_t cache_entry, GHashTable *table)

static dt_pixel_cache_entry_t *	_pixelpipe_cache_create_entry_locked (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, const size_t size, const char name, const int id)

static dt_pixel_cache_entry_t *	dt_pixel_cache_new_entry (const uint64_t hash, const size_t size, const char name, const int id, dt_dev_pixelpipe_cache_t cache, gboolean alloc, GHashTable *table)

static gboolean	_cache_entry_clmem_flush_device (dt_pixel_cache_entry_t *entry, const int devid)

static gboolean	_cache_entry_materialize_host_data_locked (dt_pixel_cache_entry_t *entry, int preferred_devid, gboolean prefer_device_payload)

static int	dt_dev_pixelpipe_cache_flush_old (dt_dev_pixelpipe_cache_t *cache)

static int	_memory_pressure_shedder (dt_dev_pixelpipe_cache_t *cache)

static gboolean	_cache_entry_clmem_flush_host_pinned_locked (dt_pixel_cache_entry_t entry, void host_ptr, int devid)

static void	_pixel_cache_touch (dt_pixel_cache_entry_t *cache_entry)

static int64_t	_pixel_cache_get_age (dt_pixel_cache_entry_t *cache_entry)

static dt_pixel_cache_entry_t *	_cache_entry_for_host_ptr_locked (dt_dev_pixelpipe_cache_t cache, void host_ptr)

dt_pixel_cache_entry_t *	dt_dev_pixelpipe_cache_get_entry (const uint64_t hash)
	Get an internal reference to the cache entry matching hash. If you are going to access this entry more than once, keeping the reference and using it instead of hashes will prevent redundant lookups.

dt_pixel_cache_entry_t *	dt_dev_pixelpipe_cache_get_entry_by_data (void *data)

static size_t	_pixel_cache_get_size (dt_pixel_cache_entry_t *cache_entry)

static void	_pixel_cache_message (dt_pixel_cache_entry_t cache_entry, const char message, gboolean verbose)

gboolean	dt_dev_pixelpipe_cache_ref_entry_by_hash (const uint64_t hash, void data, dt_pixel_cache_entry_t entry)
	Resolve and retain an existing cache entry by hash.

gboolean	dt_dev_pixelpipe_cache_ref_host_entry_by_hash (const uint64_t hash, void data, dt_pixel_cache_entry_t entry)
	Resolve and retain an existing cacheline that already holds HOST pixels.

static gboolean	_cache_entry_materialize_host_data (dt_dev_pixelpipe_cache_t cache, int preferred_devid, dt_pixel_cache_entry_t entry)

static gboolean	_cache_entry_clmem_has_host_pinned_locked (dt_pixel_cache_entry_t entry, void host_ptr, int devid)

void	dt_dev_pixelpipe_cache_flush_clmem (const int devid)
	Release cached OpenCL buffers for a single device.

void	dt_dev_pixelpipe_cache_flush_clmem_for_pipe (const int devid)
	Like dt_dev_pixelpipe_cache_flush_clmem(), for callers that do not hold the device lock (`dt_opencl_reserve_device_by_id()`) (e.g. a pipe's own cleanup after dt_dev_pixelpipe_process() already released it). Takes the lock itself, then delegates. No-op if devid < 0 or OpenCL isn't available.

static gboolean	_entry_evictable (dt_pixel_cache_entry_t cache_entry, gboolean used)

static void	_cache_get_oldest (gpointer key, gpointer value, gpointer user_data)

static void	_cache_sum_evictable (__attribute__((unused)) gpointer key, gpointer value, gpointer user_data)

static void	_print_cache_lines (gpointer key, gpointer value, gpointer user_data)

static int	_non_thread_safe_pixel_pipe_cache_remove_lru (dt_dev_pixelpipe_cache_t *cache, const gboolean host_only)

int	dt_dev_pixel_pipe_cache_remove_lru (void)

static void *	_pixel_cache_clmem_get (dt_pixel_cache_entry_t entry, void host_ptr, int devid, int width, int height, int bpp, int flags)

void *	dt_dev_pixelpipe_cache_borrow_cl_payload (dt_pixel_cache_entry_t *entry, int devid, int width, int height, int bpp)
	Borrow a cached OpenCL payload attached to a cache entry.

void	dt_dev_pixelpipe_cache_return_cl_payload (dt_pixel_cache_entry_t entry, void mem)
	Return a borrowed cached OpenCL payload to its cache entry.

static int	_pixel_cache_clmem_put (dt_pixel_cache_entry_t entry, void host_ptr, void *mem)

static void	_pixel_cache_clmem_remove (dt_pixel_cache_entry_t entry, void mem)

void	dt_dev_pixelpipe_cache_flush_entry_clmem (dt_pixel_cache_entry_t *entry)
	Flush all reusable OpenCL payloads cached on one cache entry.

void *	dt_dev_pixelpipe_cache_get_pinned_image (void host_ptr, dt_pixel_cache_entry_t entry_hint, int devid, int width, int height, int bpp, int flags, gboolean *out_reused)
	Acquire a pinned OpenCL image for a host buffer tracked by the pixelpipe cache.

void	dt_dev_pixelpipe_cache_put_pinned_image (void host_ptr, dt_pixel_cache_entry_t entry_hint, void **mem)
	Release or cache a pinned OpenCL image acquired with `dt_dev_pixelpipe_cache_get_pinned_image()`.

gboolean	dt_dev_pixelpipe_cache_flush_host_pinned_image (void host_ptr, dt_pixel_cache_entry_t entry_hint, int devid)
	Drop cached pinned OpenCL images associated with a given host buffer.

static gboolean	_is_gamma_rgba8_output (const dt_iop_module_t module, const size_t bpp, const char message)

void *	dt_dev_pixelpipe_cache_alloc_cl_device_buffer (int devid, const dt_iop_roi_t roi, const size_t bpp, const dt_iop_module_t module, const char message, void keep)

void *	dt_dev_pixelpipe_cache_get_cl_buffer (int devid, void const host_ptr, const dt_iop_roi_t roi, const size_t bpp, dt_iop_module_t module, const char message, dt_pixel_cache_entry_t cache_entry, gboolean out_reused, void *keep)

void	dt_dev_pixelpipe_cache_release_cl_buffer (void *cl_mem_buffer, dt_pixel_cache_entry_t cache_entry, void *host_ptr, const gboolean cache_device)
	Release or cache an OpenCL image associated with a host cache line.

int	dt_dev_pixelpipe_cache_sync_cl_buffer (const int devid, void host_ptr, void cl_mem_buffer, const dt_iop_roi_t roi, int cl_mode, size_t bpp, dt_iop_module_t module, const char *message)
	Synchronize between host memory and a pinned OpenCL image.

float *	dt_dev_pixelpipe_cache_restore_cl_buffer (dt_dev_pixelpipe_t pipe, float input, void cl_mem_input, const dt_iop_roi_t roi_in, dt_iop_module_t module, const size_t in_bpp, dt_pixel_cache_entry_t input_entry, const char *message)
	Force device → host resynchronization of the pixelpipe input cache line.

int	dt_dev_pixelpipe_cache_prepare_cl_input (dt_dev_pixelpipe_t pipe, dt_iop_module_t module, float input, void cl_mem_input, const dt_iop_roi_t roi_in, const size_t in_bpp, dt_pixel_cache_entry_t input_entry, dt_pixel_cache_entry_t locked_input_entry, void keep)
	Prepare/obtain the OpenCL input image for a module.

dt_pixel_cache_entry_t *	dt_dev_pixelpipe_cache_ref_entry_for_host_ptr (void *host_ptr)
	Resolve and retain the cache entry owning a host pointer.

static gboolean	_pressure_alert_expired (gpointer key __attribute__((unused)), gpointer value, gpointer user_data)

static gboolean	_system_memory_pressure_valve (dt_dev_pixelpipe_cache_t *cache, size_t request_size)

static void *	_arena_alloc_with_defrag (dt_dev_pixelpipe_cache_t cache, size_t request_size, size_t actual_size, gboolean *system_refused)

static void	_arena_stats_bytes (dt_dev_pixelpipe_cache_t cache, uint32_t total_pages, uint32_t largest_pages, size_t total_bytes, size_t *largest_bytes)

size_t	dt_pixelpipe_cache_get_largest_free_run (void)

static void	_log_arena_allocation_failure (dt_dev_pixelpipe_cache_t cache, size_t request_size, const char entry_name, const char *module, uint64_t hash, gboolean name_is_file, gboolean system_refused)

void *	dt_pixel_cache_alloc (dt_pixel_cache_entry_t *cache_entry)
	Actually allocate the memory buffer attached to the cache entry once you create it with `dt_dev_pixelpipe_cache_get()`. Sizes and everything are already saved in the entry, and the cache will have the needed space reserved.

void *	dt_pixel_cache_entry_get_data (dt_pixel_cache_entry_t *entry)
	Peek the host pixels a cache entry holds for its current hash, without allocating.

void *	dt_pixel_cache_entry_get_buffer (dt_pixel_cache_entry_t *entry)
	The host buffer of a cache entry, whatever hash its content describes.

void	dt_dev_pixelpipe_cache_flag_host_written (dt_pixel_cache_entry_t *entry)
	Record that the host buffer of `entry` now holds the pixels of its current hash.

size_t	dt_pixel_cache_entry_get_size (const dt_pixel_cache_entry_t *entry)

static size_t	_pressure_held (void *user)

static size_t	_pressure_shed (void user, const size_t target, size_t given_back)

static dt_pixelpipe_cache_pressure_sink_t	_pressure_sink (dt_dev_pixelpipe_cache_t *cache)

static void	_pressure_wake (void *user)

static size_t	_cache_budget_locked (const dt_dev_pixelpipe_cache_t *cache)

static void	_pressure_react_locked (dt_dev_pixelpipe_cache_t *cache)

static int	_free_space_to_alloc (dt_dev_pixelpipe_cache_t cache, const size_t size, const uint64_t hash, const char name)

void *	dt_pixelpipe_cache_alloc_align_cache_impl (size_t size, int id, const char *name)
	Allocate aligned memory tracked by the pixelpipe cache. This allows LRU cache entries to be evicted if needed to make room.

void	dt_pixelpipe_cache_free_align_cache (void *mem, const char message)
	Free aligned memory allocated with dt_pixelpipe_cache_alloc_align_cache.

gboolean	dt_dev_pixelpipe_cache_init (size_t max_memory, const gboolean verbose, const gboolean verbose_detail)
	Reserve the cache's arena and start it.

void	dt_dev_pixelpipe_cache_cleanup (void)
	Release everything dt_dev_pixelpipe_cache_init() created, the instance included. Only after a successful init; dt_dev_pixelpipe_cache_is_ready() answers FALSE afterwards.

static dt_pixel_cache_entry_t *	_cache_try_rekey_reuse_locked (dt_dev_pixelpipe_cache_t cache, const uint64_t new_hash, const size_t size, const dt_pixel_cache_entry_t reuse_hint)

int	dt_dev_pixelpipe_cache_get (const uint64_t hash, const size_t size, const char name, const int id, const gboolean alloc, void data, dt_pixel_cache_entry_t *entry)
	Get a cache line from the cache.

dt_dev_pixelpipe_cache_writable_status_t	dt_dev_pixelpipe_cache_get_writable (const uint64_t hash, const size_t size, const char name, const int id, const gboolean alloc, const gboolean allow_rekey_reuse, const dt_pixel_cache_entry_t reuse_hint, const dt_pixel_cache_entry_t reuse_hint_prev, void data, dt_pixel_cache_entry_t *entry)

static dt_pixel_cache_entry_t *	_cache_lookup_existing (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, void *data)

static gboolean	_cache_try_restore_device_payload (dt_pixel_cache_entry_t cache_entry, const int preferred_devid, void *cl_mem_output)

gboolean	dt_dev_pixelpipe_cache_restore_host_payload (dt_pixel_cache_entry_t cache_entry, const int preferred_devid, void *data)
	Materialize a host payload for a live cache entry from its cached device payload.

gboolean	dt_dev_pixelpipe_cache_peek (const uint64_t hash, void data, dt_pixel_cache_entry_t entry, const int preferred_devid, void **cl_mem_output)
	Non-owning lookup of an existing cache line.

static gboolean	_for_each_remove (gpointer key, gpointer value, gpointer user_data)

void	dt_dev_pixelpipe_cache_flush (const int id)
	Remove cache lines matching id. Entries locked in read/write or having reference count greater than 0 are not removed.

int	dt_dev_pixelpipe_cache_invalidate_hashes (const uint64_t *hashes, const size_t count)
	Invalidate cache lines matching an explicit list of hashes.

static void	_cache_get_newest (gpointer key, gpointer value, gpointer user_data)

static gboolean	_for_each_remove_old (gpointer key, gpointer value, gpointer user_data)

void	dt_dev_pixelpipe_cache_ref_count_entry (gboolean lock, dt_pixel_cache_entry_t *cache_entry)
	Take or release one reference on a cache line. A referenced line is never evicted, rekeyed or removed.

void	dt_dev_pixelpipe_cache_wrlock_entry (gboolean lock, dt_pixel_cache_entry_t *cache_entry)
	Lock or release the write lock on the entry.

void	dt_dev_pixelpipe_cache_rdlock_entry (gboolean lock, dt_pixel_cache_entry_t *cache_entry)
	Lock or release the read lock on the entry.

void	dt_dev_pixelpipe_cache_flag_auto_destroy (dt_pixel_cache_entry_t *cache_entry)
	Flag a cache line as disposable (`auto_destroy`), for short-lived lines nothing will need again: the release of its last reference removes it, out of the LRU's age-based collection.

void	dt_dev_pixelpipe_cache_unref_entry (dt_pixel_cache_entry_t *entry)
	Release one reference held on `entry`, by POINTER.

void	dt_dev_pixelpipe_cache_unref_hash (const uint64_t hash)
	Find the entry matching hash, and decrease its ref_count if found.

int	dt_dev_pixelpipe_cache_rekey (const uint64_t old_hash, const uint64_t new_hash, dt_pixel_cache_entry_t *entry)
	Change the hash/key of an existing cache line in place, without freeing, reallocating or invalidating the underlying entry.

void	dt_dev_pixelpipe_cache_print (void)

void	dt_dev_pixelpipe_cache_get_usage (size_t current, size_t max)

size_t	dt_dev_pixelpipe_cache_get_available (void)

size_t	dt_dev_pixelpipe_cache_get_max_memory (void)

static void	_stats_entry_clear (gpointer data)

static void	_non_threadsafe_stats_append (GArray out, GHashTable table, const gboolean allocations, const int64_t now, const int num_devices)

GArray *	dt_dev_pixelpipe_cache_get_entries_stats (void)

Variables
static gboolean	_verbose = FALSE

static gboolean	_verbose_detail = FALSE

static dt_dev_pixelpipe_cache_t *	_pixelpipe_cache = NULL

static dt_pixelpipe_cache_warn_handler_t	_warn_handler = NULL

static dt_pixelpipe_cache_alert_handler_t	_alert_handler = NULL

static dt_pixelpipe_cache_ready_handler_t	_ready_handler = NULL

static const dt_pixelpipe_cache_observer_t *	_observer = NULL

static __thread const char *	dt_pixelpipe_cache_current_module = NULL

static __thread const char *	dt_pixelpipe_cache_current_image = NULL

static __thread uint32_t	dt_pixelpipe_cache_alloc_refusals = 0

static int	garbage_collection = 0

static int	pressure_shedding = 0

Macro Definition Documentation

◆ _cache_print

#define _cache_print	(	channel,
		...
	)

Value:

  do {                                                \
    if(_verbose) dt_print((channel), __VA_ARGS__);     \
  } while(0)

Definition at line 59 of file pixelpipe_cache.c.

◆ DT_PIXELPIPE_CACHE_PRESSURE_ALERT_PERIOD_US

#define DT_PIXELPIPE_CACHE_PRESSURE_ALERT_PERIOD_US ((gint64)10 * G_USEC_PER_SEC)

Definition at line 1749 of file pixelpipe_cache.c.

◆ DT_PIXELPIPE_CACHE_PRESSURE_EXEMPT_SIZE

#define DT_PIXELPIPE_CACHE_PRESSURE_EXEMPT_SIZE ((size_t)1024 * 1024)

Definition at line 1746 of file pixelpipe_cache.c.

Typedef Documentation

◆ _cache_invalidate_t

typedef struct _cache_invalidate_t _cache_invalidate_t

◆ _cache_lru_t

typedef struct _cache_lru_t _cache_lru_t

◆ dt_cache_clmem_t

typedef struct dt_cache_clmem_t dt_cache_clmem_t

◆ dt_dev_pixelpipe_cache_t

typedef struct dt_dev_pixelpipe_cache_t dt_dev_pixelpipe_cache_t

◆ dt_pixel_cache_materialize_source_rank_t

typedef enum dt_pixel_cache_materialize_source_rank_t dt_pixel_cache_materialize_source_rank_t

Enumeration Type Documentation

◆ dt_pixel_cache_materialize_source_rank_t

enum dt_pixel_cache_materialize_source_rank_t

Enumerator
DT_PIXEL_CACHE_MATERIALIZE_SOURCE_NONE
DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_ANY
DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_PREFERRED
DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_ANY
DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_PREFERRED

Definition at line 281 of file pixelpipe_cache.c.

Function Documentation

◆ _alert_cache_refused()

static void _alert_cache_refused	(	const char *	name,
		const char *	module
	)

static

Definition at line 178 of file pixelpipe_cache.c.

References _alert_user(), dt_free, IS_NULL_PTR, and name.

Referenced by _free_space_to_alloc(), and _log_arena_allocation_failure().

◆ _alert_user()

static void _alert_user	(	const char *	message,
		const char *	item
	)

static

Definition at line 134 of file pixelpipe_cache.c.

References _alert_handler, and IS_NULL_PTR.

Referenced by _alert_cache_refused(), and _system_memory_pressure_valve().

◆ _arena_alloc_with_defrag()

static void * _arena_alloc_with_defrag	(	dt_dev_pixelpipe_cache_t *	cache,
		size_t	request_size,
		size_t *	actual_size,
		gboolean *	system_refused
	)

inlinestatic

Definition at line 1942 of file pixelpipe_cache.c.

References _non_thread_safe_pixel_pipe_cache_remove_lru(), _system_memory_pressure_valve(), dt_dev_pixelpipe_cache_t::arena, dt_cache_arena_alloc(), dt_cache_arena_calc(), dt_cache_arena_stats(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, FALSE, IS_NULL_PTR, and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_pixel_cache_alloc(), and dt_pixelpipe_cache_alloc_align_cache_impl().

◆ _arena_stats_bytes()

static void _arena_stats_bytes	(	dt_dev_pixelpipe_cache_t *	cache,
		uint32_t *	total_pages,
		uint32_t *	largest_pages,
		size_t *	total_bytes,
		size_t *	largest_bytes
	)

inlinestatic

Definition at line 1971 of file pixelpipe_cache.c.

References dt_dev_pixelpipe_cache_t::arena, dt_cache_arena_stats(), and dt_cache_arena_t::page_size.

Referenced by _log_arena_allocation_failure(), and dt_pixelpipe_cache_get_largest_free_run().

◆ _cache_budget_locked()

static size_t _cache_budget_locked ( const dt_dev_pixelpipe_cache_t * cache )

inlinestatic

Definition at line 2180 of file pixelpipe_cache.c.

References dt_pixelpipe_cache_pressure_monitor_t::budget, dt_pixelpipe_cache_pressure_budget(), and dt_dev_pixelpipe_cache_t::psi.

Referenced by _free_space_to_alloc().

◆ _cache_debug_module_name()

static const char * _cache_debug_module_name ( void )

inlinestatic

Definition at line 234 of file pixelpipe_cache.c.

References dt_pixelpipe_cache_current_module.

Referenced by _cache_try_rekey_reuse_locked(), _pixel_cache_message(), _trace_exact_hit(), and dt_dev_pixelpipe_cache_rekey().

◆ _cache_entry_clmem_flush_device()

static gboolean _cache_entry_clmem_flush_device	(	dt_pixel_cache_entry_t *	entry,
		const int	devid
	)

static

Definition at line 2038 of file pixelpipe_cache.c.

References _cache_print, _verbose, _verbose_detail, dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, DT_DEBUG_OPENCL, dt_free, dt_opencl_get_mem_context_id(), dt_opencl_release_mem_object(), dt_pixel_cache_entry_get_data(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_pixel_cache_entry_t::hash, and IS_NULL_PTR.

Referenced by dt_dev_pixelpipe_cache_flush_clmem().

◆ _cache_entry_clmem_flush_host_pinned_locked()

static gboolean _cache_entry_clmem_flush_host_pinned_locked	(	dt_pixel_cache_entry_t *	entry,
		void *	host_ptr,
		int	devid
	)

static

Definition at line 700 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_free, dt_opencl_get_mem_context_id(), dt_opencl_release_mem_object(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), FALSE, IS_NULL_PTR, and TRUE.

Referenced by dt_dev_pixelpipe_cache_flush_host_pinned_image().

◆ _cache_entry_clmem_has_host_pinned_locked()

static gboolean _cache_entry_clmem_has_host_pinned_locked	(	dt_pixel_cache_entry_t *	entry,
		void *	host_ptr,
		int	devid
	)

static

Definition at line 678 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_opencl_get_mem_context_id(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), FALSE, IS_NULL_PTR, and TRUE.

Referenced by dt_dev_pixelpipe_cache_flush_host_pinned_image().

◆ _cache_entry_for_host_ptr_locked()

static dt_pixel_cache_entry_t * _cache_entry_for_host_ptr_locked	(	dt_dev_pixelpipe_cache_t *	cache,
		void *	host_ptr
	)

static

Definition at line 339 of file pixelpipe_cache.c.

References _non_threadsafe_cache_get_entry(), dt_pixel_cache_entry_t::data, dt_pixel_cache_entry_t::external_alloc, dt_dev_pixelpipe_cache_t::external_entries, hash, and IS_NULL_PTR.

Referenced by dt_dev_pixelpipe_cache_flush_host_pinned_image(), dt_dev_pixelpipe_cache_get_pinned_image(), and dt_dev_pixelpipe_cache_ref_entry_for_host_ptr().

◆ _cache_entry_materialize_host_data()

static gboolean _cache_entry_materialize_host_data	(	dt_dev_pixelpipe_cache_t *	cache,
		int	preferred_devid,
		dt_pixel_cache_entry_t *	entry
	)

static

Definition at line 658 of file pixelpipe_cache.c.

References _cache_entry_materialize_host_data_locked(), dt_dev_pixelpipe_cache_wrlock_entry(), dt_pixel_cache_alloc(), dt_pixel_cache_entry_get_data(), FALSE, IS_NULL_PTR, and TRUE.

Referenced by dt_dev_pixelpipe_cache_restore_host_payload().

◆ _cache_entry_materialize_host_data_locked()

static gboolean _cache_entry_materialize_host_data_locked	(	dt_pixel_cache_entry_t *	entry,
		int	preferred_devid,
		gboolean	prefer_device_payload
	)

static

Definition at line 559 of file pixelpipe_cache.c.

References bpp, dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_pixel_cache_entry_t::data, dt_atomic_set_int(), dt_opencl_get_image_element_size(), dt_opencl_get_image_height(), dt_opencl_get_image_width(), dt_opencl_get_mem_context_id(), dt_opencl_is_pinned_memory(), dt_opencl_map_image(), dt_opencl_read_host_from_device(), dt_opencl_unmap_mem_object(), DT_PIXEL_CACHE_MATERIALIZE_SOURCE_NONE, DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_ANY, DT_PIXEL_CACHE_MATERIALIZE_SOURCE_PRIMARY_PREFERRED, DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_ANY, DT_PIXEL_CACHE_MATERIALIZE_SOURCE_SECONDARY_PREFERRED, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), FALSE, height, dt_cache_clmem_t::host_ptr, dt_pixel_cache_entry_t::host_stale, IS_NULL_PTR, dt_cache_clmem_t::mem, TRUE, and width.

Referenced by _cache_entry_materialize_host_data().

◆ _cache_full_sentence()

static gchar * _cache_full_sentence	(	const char *	name,
		const char *	module
	)

static

Definition at line 152 of file pixelpipe_cache.c.

References IS_NULL_PTR, and name.

Referenced by _free_space_to_alloc().

◆ _cache_get_newest()

static void _cache_get_newest	(	gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

static

Definition at line 3056 of file pixelpipe_cache.c.

References _pixel_cache_get_age(), and value.

Referenced by dt_dev_pixelpipe_cache_flush_old().

◆ _cache_get_oldest()

static void _cache_get_oldest	(	gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

static

Definition at line 853 of file pixelpipe_cache.c.

References _entry_evictable(), _pixel_cache_get_age(), _pixel_cache_message(), _cache_lru_t::cache_entry, dt_pixel_cache_entry_t::data, FALSE, _cache_lru_t::hash, dt_pixel_cache_entry_t::hash, _cache_lru_t::host_only, IS_NULL_PTR, _cache_lru_t::max_age, TRUE, and value.

Referenced by _non_thread_safe_pixel_pipe_cache_remove_lru().

◆ _cache_lookup_existing()

static dt_pixel_cache_entry_t * _cache_lookup_existing	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		void **	data
	)

static

Definition at line 2816 of file pixelpipe_cache.c.

References _non_threadsafe_cache_get_entry(), _observe_read(), _pixelpipe_cache_finalize_entry(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, hash, dt_dev_pixelpipe_cache_t::hits, dt_pixel_cache_entry_t::hits, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, dt_dev_pixelpipe_cache_t::queries, and dt_pixel_cache_entry_t::size.

Referenced by dt_dev_pixelpipe_cache_peek().

◆ _cache_sum_evictable()

static void _cache_sum_evictable	(	__attribute__((unused)) gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

static

Definition at line 883 of file pixelpipe_cache.c.

References _entry_evictable(), dt_pixel_cache_entry_t::data, FALSE, IS_NULL_PTR, dt_pixel_cache_entry_t::size, and value.

Referenced by dt_dev_pixelpipe_cache_get_available().

◆ _cache_try_rekey_reuse_locked()

static dt_pixel_cache_entry_t * _cache_try_rekey_reuse_locked	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	new_hash,
		const size_t	size,
		const dt_pixel_cache_entry_t *	reuse_hint
	)

static

◆ _cache_try_restore_device_payload()

static gboolean _cache_try_restore_device_payload	(	dt_pixel_cache_entry_t *	cache_entry,
		const int	preferred_devid,
		void **	cl_mem_output
	)

static

Definition at line 2840 of file pixelpipe_cache.c.

References _pixel_cache_touch(), dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_free, dt_opencl_get_mem_context_id(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), FALSE, and IS_NULL_PTR.

Referenced by dt_dev_pixelpipe_cache_peek().

◆ _entry_evictable()

static gboolean _entry_evictable	(	dt_pixel_cache_entry_t *	cache_entry,
		gboolean *	used
	)

static

Definition at line 843 of file pixelpipe_cache.c.

References dt_atomic_get_int(), dt_pthread_rwlock_trywrlock(), dt_pthread_rwlock_unlock(), FALSE, dt_pixel_cache_entry_t::lock, and dt_pixel_cache_entry_t::refcount.

Referenced by _cache_get_oldest(), and _cache_sum_evictable().

◆ _for_each_remove()

static gboolean _for_each_remove	(	gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

static

Definition at line 2998 of file pixelpipe_cache.c.

References dt_atomic_get_int(), dt_pthread_rwlock_trywrlock(), dt_pthread_rwlock_unlock(), dt_pixel_cache_entry_t::id, dt_pixel_cache_entry_t::lock, dt_pixel_cache_entry_t::refcount, and value.

Referenced by dt_dev_pixelpipe_cache_flush().

◆ _for_each_remove_old()

static gboolean _for_each_remove_old	(	gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

static

Definition at line 3066 of file pixelpipe_cache.c.

References _pixel_cache_get_age(), dt_atomic_get_int(), dt_pthread_rwlock_trywrlock(), dt_pthread_rwlock_unlock(), dt_pixel_cache_entry_t::hits, dt_pixel_cache_entry_t::lock, dt_pixel_cache_entry_t::refcount, and value.

Referenced by dt_dev_pixelpipe_cache_flush_old().

◆ _free_cache_entry()

static void _free_cache_entry ( dt_pixel_cache_entry_t * cache_entry )

static

Definition at line 2389 of file pixelpipe_cache.c.

Referenced by dt_dev_pixelpipe_cache_init().

◆ _free_space_to_alloc()

static int _free_space_to_alloc	(	dt_dev_pixelpipe_cache_t *	cache,
		const size_t	size,
		const uint64_t	hash,
		const char *	name
	)

static

◆ _is_gamma_rgba8_output()

static gboolean _is_gamma_rgba8_output	(	const dt_iop_module_t *	module,
		const size_t	bpp,
		const char *	message
	)

inlinestatic

Definition at line 1289 of file pixelpipe_cache.c.

Referenced by dt_dev_pixelpipe_cache_alloc_cl_device_buffer(), and dt_dev_pixelpipe_cache_get_cl_buffer().

◆ _log_arena_allocation_failure()

static void _log_arena_allocation_failure	(	dt_dev_pixelpipe_cache_t *	cache,
		size_t	request_size,
		const char *	entry_name,
		const char *	module,
		uint64_t	hash,
		gboolean	name_is_file,
		gboolean	system_refused
	)

inlinestatic

Definition at line 1997 of file pixelpipe_cache.c.

References _alert_cache_refused(), _arena_stats_bytes(), dt_dev_pixelpipe_cache_t::current_memory, dt_pixelpipe_cache_alloc_refusals, hash, dt_dev_pixelpipe_cache_t::max_memory, and void().

Referenced by dt_pixel_cache_alloc(), and dt_pixelpipe_cache_alloc_align_cache_impl().

◆ _memory_pressure_shedder()

static int _memory_pressure_shedder ( dt_dev_pixelpipe_cache_t * cache )

static

◆ _non_thread_safe_cache_ref_count_entry()

void _non_thread_safe_cache_ref_count_entry	(	dt_dev_pixelpipe_cache_t *	cache,
		gboolean	lock,
		dt_pixel_cache_entry_t *	cache_entry
	)

◆ _non_thread_safe_cache_remove()

int _non_thread_safe_cache_remove	(	dt_dev_pixelpipe_cache_t *	cache,
		const gboolean	force,
		dt_pixel_cache_entry_t *	cache_entry,
		GHashTable *	table
	)

Definition at line 525 of file pixelpipe_cache.c.

References _cache_print, _pixel_cache_message(), dt_atomic_get_int(), DT_DEBUG_PIPECACHE, dt_pthread_rwlock_trywrlock(), dt_pthread_rwlock_unlock(), dt_pixel_cache_entry_t::hash, IS_NULL_PTR, dt_pixel_cache_entry_t::lock, dt_pixel_cache_entry_t::refcount, and TRUE.

Referenced by _non_thread_safe_pixel_pipe_cache_remove_lru(), dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_cache_get_writable(), dt_dev_pixelpipe_cache_invalidate_hashes(), dt_dev_pixelpipe_cache_peek(), and dt_dev_pixelpipe_cache_ref_count_entry().

◆ _non_thread_safe_pixel_pipe_cache_remove_lru()

static int _non_thread_safe_pixel_pipe_cache_remove_lru	(	dt_dev_pixelpipe_cache_t *	cache,
		const gboolean	host_only
	)

static

◆ _non_threadsafe_cache_get_entry()

dt_pixel_cache_entry_t * _non_threadsafe_cache_get_entry	(	dt_dev_pixelpipe_cache_t *	cache,
		GHashTable *	table,
		const uint64_t	key
	)

static

Definition at line 349 of file pixelpipe_cache.c.

References key.

Referenced by _cache_entry_for_host_ptr_locked(), _cache_lookup_existing(), _cache_try_rekey_reuse_locked(), dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_cache_get_entry(), dt_dev_pixelpipe_cache_get_writable(), dt_dev_pixelpipe_cache_invalidate_hashes(), dt_dev_pixelpipe_cache_peek(), dt_dev_pixelpipe_cache_ref_entry_by_hash(), dt_dev_pixelpipe_cache_ref_host_entry_by_hash(), dt_dev_pixelpipe_cache_rekey(), dt_dev_pixelpipe_cache_unref_hash(), and dt_pixelpipe_cache_free_align_cache().

◆ _non_threadsafe_stats_append()

static void _non_threadsafe_stats_append	(	GArray *	out,
		GHashTable *	table,
		const gboolean	allocations,
		const int64_t	now,
		const int	num_devices
	)

static

Definition at line 3463 of file pixelpipe_cache.c.

Referenced by dt_dev_pixelpipe_cache_get_entries_stats().

◆ _observe_delete()

static void _observe_delete	(	uint64_t	hash,
		size_t	size,
		int	owner_pipe_id,
		const char *	name
	)

inlinestatic

Definition at line 210 of file pixelpipe_cache.c.

References _observed(), _observer, dt_pixelpipe_cache_observer_t::cacheline_delete, hash, name, and size.

Referenced by _free_cache_entry().

◆ _observe_read()

static void _observe_read	(	uint64_t	hash,
		size_t	size
	)

inlinestatic

Definition at line 206 of file pixelpipe_cache.c.

References _observed(), _observer, dt_pixelpipe_cache_observer_t::cacheline_read, hash, and size.

Referenced by _cache_lookup_existing(), dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_cache_ref_entry_by_hash(), and dt_dev_pixelpipe_cache_ref_host_entry_by_hash().

◆ _observe_rekey()

static void _observe_rekey	(	uint64_t	old_hash,
		uint64_t	new_hash
	)

inlinestatic

Definition at line 214 of file pixelpipe_cache.c.

References _observed(), _observer, and dt_pixelpipe_cache_observer_t::rekey.

Referenced by _cache_try_rekey_reuse_locked(), and dt_dev_pixelpipe_cache_rekey().

◆ _observed()

static gboolean _observed ( void )

inlinestatic

Definition at line 199 of file pixelpipe_cache.c.

References _observer, and dt_pixelpipe_cache_observer_t::active.

Referenced by _observe_delete(), _observe_read(), and _observe_rekey().

◆ _pixel_cache_clmem_get()

static void * _pixel_cache_clmem_get	(	dt_pixel_cache_entry_t *	entry,
		void *	host_ptr,
		int	devid,
		int	width,
		int	height,
		int	bpp,
		int	flags
	)

static

◆ _pixel_cache_clmem_put()

static int _pixel_cache_clmem_put	(	dt_pixel_cache_entry_t *	entry,
		void *	host_ptr,
		void *	mem
	)

static

Parameters

entry
host_ptr
mem

Returns: int 3 if the memory was already cached (no-op), 2 if we updated a previous cacheline (same width/height/bpp/devid) with a new buffer, 0 if we failed to create a new cacheline, 1 if we created a new cacheline.

Definition at line 1057 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_opencl_get_mem_context_id(), dt_opencl_release_mem_object(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), and IS_NULL_PTR.

Referenced by dt_dev_pixelpipe_cache_put_pinned_image(), and dt_dev_pixelpipe_cache_release_cl_buffer().

◆ _pixel_cache_clmem_remove()

static void _pixel_cache_clmem_remove	(	dt_pixel_cache_entry_t *	entry,
		void *	mem
	)

static

Definition at line 1101 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_free, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), and IS_NULL_PTR.

Referenced by dt_dev_pixelpipe_cache_get_pinned_image(), and dt_dev_pixelpipe_cache_release_cl_buffer().

◆ _pixel_cache_get_age()

static int64_t _pixel_cache_get_age ( dt_pixel_cache_entry_t * cache_entry )

inlinestatic

Definition at line 333 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::age, and dt_atomic_get_uint64().

Referenced by _cache_get_newest(), _cache_get_oldest(), _for_each_remove_old(), _non_threadsafe_stats_append(), and _pixel_cache_message().

◆ _pixel_cache_get_size()

static size_t _pixel_cache_get_size ( dt_pixel_cache_entry_t * cache_entry )

static

Definition at line 413 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::size.

Referenced by _pixel_cache_message().

◆ _pixel_cache_message()

static void _pixel_cache_message	(	dt_pixel_cache_entry_t *	cache_entry,
		const char *	message,
		gboolean	verbose
	)

static

Definition at line 419 of file pixelpipe_cache.c.

Referenced by _cache_get_oldest(), _free_cache_entry(), _non_thread_safe_cache_ref_count_entry(), _non_thread_safe_cache_remove(), _pixelpipe_cache_finalize_entry(), _print_cache_lines(), dt_dev_pixelpipe_cache_flag_auto_destroy(), dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_cache_get_writable(), dt_dev_pixelpipe_cache_rdlock_entry(), and dt_dev_pixelpipe_cache_wrlock_entry().

◆ _pixel_cache_touch()

static void _pixel_cache_touch ( dt_pixel_cache_entry_t * cache_entry )

inlinestatic

Definition at line 327 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::age, dt_atomic_set_uint64(), and IS_NULL_PTR.

Referenced by _cache_try_restore_device_payload(), _non_thread_safe_cache_ref_count_entry(), _pixel_cache_clmem_get(), _pixelpipe_cache_finalize_entry(), dt_dev_pixelpipe_cache_borrow_cl_payload(), dt_dev_pixelpipe_cache_get_entry(), dt_dev_pixelpipe_cache_get_entry_by_data(), dt_dev_pixelpipe_cache_get_writable(), dt_dev_pixelpipe_cache_restore_host_payload(), and dt_pixelpipe_cache_alloc_align_cache_impl().

◆ _pixelpipe_cache_create_entry_locked()

static dt_pixel_cache_entry_t * _pixelpipe_cache_create_entry_locked	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		const size_t	size,
		const char *	name,
		const int	id
	)

static

Definition at line 2554 of file pixelpipe_cache.c.

References _non_thread_safe_cache_ref_count_entry(), dt_dev_pixelpipe_cache_wrlock_entry(), dt_pixel_cache_new_entry(), dt_dev_pixelpipe_cache_t::entries, FALSE, hash, IS_NULL_PTR, name, size, and TRUE.

Referenced by dt_dev_pixelpipe_cache_get(), and dt_dev_pixelpipe_cache_get_writable().

◆ _pixelpipe_cache_finalize_entry()

static void _pixelpipe_cache_finalize_entry	(	dt_pixel_cache_entry_t *	cache_entry,
		void **	data,
		const char *	message
	)

static

Definition at line 434 of file pixelpipe_cache.c.

References _pixel_cache_message(), _pixel_cache_touch(), DT_CACHELINE_BYTES, dt_pixel_cache_entry_get_data(), FALSE, and IS_NULL_PTR.

Referenced by _cache_lookup_existing(), dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_cache_get_writable(), dt_dev_pixelpipe_cache_ref_entry_by_hash(), and dt_dev_pixelpipe_cache_ref_host_entry_by_hash().

◆ _pressure_alert_expired()

static gboolean _pressure_alert_expired	(	gpointer key	__attribute__(unused),
		gpointer	value,
		gpointer	user_data
	)

static

Definition at line 1751 of file pixelpipe_cache.c.

References DT_PIXELPIPE_CACHE_PRESSURE_ALERT_PERIOD_US, and value.

Referenced by _system_memory_pressure_valve().

◆ _pressure_held()

static size_t _pressure_held ( void * user )

static

Definition at line 2141 of file pixelpipe_cache.c.

Referenced by _pressure_sink().

◆ _pressure_react_locked()

static void _pressure_react_locked ( dt_dev_pixelpipe_cache_t * cache )

static

Definition at line 2186 of file pixelpipe_cache.c.

References _pressure_sink(), dt_pixelpipe_cache_pressure_react(), and dt_dev_pixelpipe_cache_t::psi.

Referenced by _free_space_to_alloc(), and _memory_pressure_shedder().

◆ _pressure_shed()

static size_t _pressure_shed	(	void *	user,
		const size_t	target,
		size_t *	given_back
	)

static

Definition at line 2146 of file pixelpipe_cache.c.

References _non_thread_safe_pixel_pipe_cache_remove_lru(), dt_dev_pixelpipe_cache_t::arena, dt_dev_pixelpipe_cache_t::current_memory, dt_cache_arena_trim(), dt_invalidate_system_available_mem(), dt_dev_pixelpipe_cache_t::entries, FALSE, and dt_dev_pixelpipe_cache_t::sys_probe_time_us.

Referenced by _pressure_sink().

◆ _pressure_sink()

static dt_pixelpipe_cache_pressure_sink_t _pressure_sink ( dt_dev_pixelpipe_cache_t * cache )

static

Definition at line 2161 of file pixelpipe_cache.c.

References _pressure_held(), _pressure_shed(), and dt_pixelpipe_cache_pressure_sink_t::held.

Referenced by _pressure_react_locked(), and _pressure_wake().

◆ _pressure_wake()

static void _pressure_wake ( void * user )

static

Definition at line 2169 of file pixelpipe_cache.c.

References _pressure_sink(), dt_pixelpipe_cache_pressure_triggered(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::lock, and dt_dev_pixelpipe_cache_t::psi.

Referenced by dt_dev_pixelpipe_cache_init().

◆ _print_cache_lines()

static void _print_cache_lines	(	gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

static

Definition at line 894 of file pixelpipe_cache.c.

References _pixel_cache_message(), FALSE, and value.

Referenced by _non_thread_safe_pixel_pipe_cache_remove_lru().

◆ _stats_entry_clear()

static void _stats_entry_clear ( gpointer data )

static

Definition at line 3456 of file pixelpipe_cache.c.

References dt_pixel_cache_stats_entry_t::devices.

Referenced by dt_dev_pixelpipe_cache_get_entries_stats().

◆ _system_memory_pressure_valve()

static gboolean _system_memory_pressure_valve	(	dt_dev_pixelpipe_cache_t *	cache,
		size_t	request_size
	)

static

Definition at line 1778 of file pixelpipe_cache.c.

Referenced by _arena_alloc_with_defrag().

◆ _trace_exact_hit()

static void _trace_exact_hit	(	const char *	phase,
		const uint64_t	hash,
		dt_pixel_cache_entry_t *	cache_entry,
		void *	data,
		void *	cl_mem_output,
		const int	preferred_devid,
		const gboolean	verbose
	)

static

Definition at line 239 of file pixelpipe_cache.c.

References _cache_debug_module_name(), _cache_print, _verbose, _verbose_detail, dt_pixel_cache_entry_t::auto_destroy, dt_atomic_get_int(), DT_DEBUG_PIPECACHE, DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pixel_cache_entry_t::hash, hash, dt_pixel_cache_entry_t::name, dt_pixel_cache_entry_t::refcount, and dt_pixel_cache_entry_t::serial.

Referenced by dt_dev_pixelpipe_cache_peek().

◆ _warn_user()

static void _warn_user ( const char * message )

static

Definition at line 145 of file pixelpipe_cache.c.

References _warn_handler, and IS_NULL_PTR.

Referenced by _free_space_to_alloc().

◆ dt_dev_pixel_pipe_cache_remove_lru()

int dt_dev_pixel_pipe_cache_remove_lru ( void )

remove the least used cache entry

Returns: 0 on success, 1 on error

Definition at line 935 of file pixelpipe_cache.c.

References _non_thread_safe_pixel_pipe_cache_remove_lru(), _pixelpipe_cache, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), error(), FALSE, and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_tiling_piece_fits_host_memory().

◆ dt_dev_pixelpipe_cache_alloc_cl_device_buffer()

void * dt_dev_pixelpipe_cache_alloc_cl_device_buffer	(	int	devid,
		const dt_iop_roi_t *	roi,
		const size_t	bpp,
		const dt_iop_module_t *	module,
		const char *	message,
		void *	keep
	)

Definition at line 1296 of file pixelpipe_cache.c.

References _is_gamma_rgba8_output(), bpp, dt_opencl_alloc_device(), DT_OPENCL_BPP_ENCODE_RGBA8, dt_iop_roi_t::height, and dt_iop_roi_t::width.

Referenced by dt_dev_pixelpipe_cache_get_cl_buffer(), and pixelpipe_process_on_GPU().

◆ dt_dev_pixelpipe_cache_borrow_cl_payload()

void * dt_dev_pixelpipe_cache_borrow_cl_payload	(	struct dt_pixel_cache_entry_t *	entry,
		int	devid,
		int	width,
		int	height,
		int	bpp
	)

Borrow a cached OpenCL payload attached to a cache entry.

This reopens a cached cl_mem image already tracked by the cache entry and increments its internal borrow counter so cache flush/eviction paths will keep it alive until the caller returns it with dt_dev_pixelpipe_cache_return_cl_payload().

Parameters

entry	Cache entry owning the payload.
host_ptr	Host pointer key for pinned payloads, or NULL for device-only payloads.
devid	OpenCL device id.
width	Image width.
height	Image height.
bpp	Bytes per pixel.
flags	Tracked OpenCL flags used when the payload was cached.

Returns: void* Borrowed cl_mem, or NULL if no matching cached payload exists.

Definition at line 994 of file pixelpipe_cache.c.

References _pixel_cache_touch(), bpp, dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_opencl_get_image_element_size(), dt_opencl_get_image_height(), dt_opencl_get_image_width(), dt_opencl_get_mem_context_id(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), height, and width.

Referenced by pixelpipe_process_on_GPU().

◆ dt_dev_pixelpipe_cache_cleanup()

void dt_dev_pixelpipe_cache_cleanup ( void )

Release everything dt_dev_pixelpipe_cache_init() created, the instance included. Only after a successful init; dt_dev_pixelpipe_cache_is_ready() answers FALSE afterwards.

Definition at line 2522 of file pixelpipe_cache.c.

References _pixelpipe_cache, dt_dev_pixelpipe_cache_t::arena, dt_cache_arena_cleanup(), dt_free, dt_pixelpipe_cache_pressure_watch_stop(), dt_pthread_mutex_destroy(), dt_dev_pixelpipe_cache_t::entries, dt_dev_pixelpipe_cache_t::external_entries, garbage_collection, dt_dev_pixelpipe_cache_t::lock, dt_dev_pixelpipe_cache_t::pressure_alerts, pressure_shedding, and dt_dev_pixelpipe_cache_t::psi.

Referenced by _teardown(), _teardown(), _teardown(), _teardown(), _teardown(), and dt_cleanup().

◆ dt_dev_pixelpipe_cache_flag_auto_destroy()

void dt_dev_pixelpipe_cache_flag_auto_destroy ( struct dt_pixel_cache_entry_t * entry )

Flag a cache line as disposable (auto_destroy), for short-lived lines nothing will need again: the release of its last reference removes it, out of the LRU's age-based collection.

This is how a holder drops a line: flag it while still holding it, then release it with dt_dev_pixelpipe_cache_ref_count_entry(). A line still held by someone else is removed by the last of them; one still locked when released is left to the LRU.

Parameters

entry The cache line, held by the caller. NULL is a no-op.

Definition at line 3273 of file pixelpipe_cache.c.

References _pixel_cache_message(), _pixelpipe_cache, dt_pixel_cache_entry_t::auto_destroy, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, and TRUE.

Referenced by _abort_module_shutdown_cleanup(), _develop_blend_release_mask_entry(), _flagged_line_goes_with_its_last_holder(), _flagged_line_goes_with_its_release(), _group_publish_prefix(), dt_dev_pixelpipe_cleanup(), dt_dev_pixelpipe_process_rec(), process(), process_cl(), and rt_scaled_mask_publish().

◆ dt_dev_pixelpipe_cache_flag_host_written()

void dt_dev_pixelpipe_cache_flag_host_written ( struct dt_pixel_cache_entry_t * entry )

Record that the host buffer of entry now holds the pixels of its current hash.

Called by the producer before it releases the write lock, when it wrote the host buffer or copied its device output back into it. Without it, an entry rekeyed in place keeps reporting no host pixels (dt_pixel_cache_entry_get_data() returns NULL).

Definition at line 2125 of file pixelpipe_cache.c.

References dt_atomic_set_int(), FALSE, and dt_pixel_cache_entry_t::host_stale.

Referenced by _gpu_init_input(), _produce(), dt_dev_pixelpipe_cache_restore_cl_buffer(), and dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixelpipe_cache_flush()

void dt_dev_pixelpipe_cache_flush ( const int id )

Remove cache lines matching id. Entries locked in read/write or having reference count greater than 0 are not removed.

Parameters

cache
id	ID of the pipeline owning the cache line, or -1 to remove all lines.

Definition at line 3017 of file pixelpipe_cache.c.

References _for_each_remove(), _pixelpipe_cache, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, and dt_dev_pixelpipe_cache_t::lock.

Referenced by _held_line_survives_the_flush(), _released_line_goes_with_the_flush(), _released_masks_go_with_the_flush(), and dt_dev_pixelpipe_reset_all().

◆ dt_dev_pixelpipe_cache_flush_clmem()

void dt_dev_pixelpipe_cache_flush_clmem ( const int devid )

Release cached OpenCL buffers for a single device.

This is intentionally a lightweight VRAM-pressure/retry helper: it drops cached cl_mem objects without taking per-entry write locks. Realtime paths rely on it to free scratch OpenCL buffers without stalling in-flight renders.

Parameters

devid Device to flush, or a negative value for a no-op (e.g. a pipe that never used OpenCL). The caller must already hold the device lock (dt_opencl_reserve_device_by_id()) – this is the case for the pixelpipe currently running on that device. Callers that don't hold it (e.g. cleaning up a pipe after it finished) must use dt_dev_pixelpipe_cache_flush_clmem_for_pipe() instead.

Definition at line 741 of file pixelpipe_cache.c.

References _cache_entry_clmem_flush_device(), _cache_print, _pixelpipe_cache, _verbose, _verbose_detail, dt_atomic_get_int(), DT_DEBUG_OPENCL, dt_opencl_events_wait_for(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_pthread_rwlock_trywrlock(), dt_pthread_rwlock_unlock(), dt_dev_pixelpipe_cache_t::entries, dt_pixel_cache_entry_t::hash, key, dt_dev_pixelpipe_cache_t::lock, dt_pixel_cache_entry_t::lock, dt_pixel_cache_entry_t::refcount, and value.

Referenced by _dt_opencl_alloc_image2d(), dt_dev_pixelpipe_cache_flush_clmem_for_pipe(), dt_opencl_alloc_device_buffer_with_flags(), guided_filter_cl(), and pixelpipe_process_on_GPU().

◆ dt_dev_pixelpipe_cache_flush_clmem_for_pipe()

void dt_dev_pixelpipe_cache_flush_clmem_for_pipe ( const int devid )

Like dt_dev_pixelpipe_cache_flush_clmem(), for callers that do not hold the device lock (dt_opencl_reserve_device_by_id()) (e.g. a pipe's own cleanup after dt_dev_pixelpipe_process() already released it). Takes the lock itself, then delegates. No-op if devid < 0 or OpenCL isn't available.

Definition at line 811 of file pixelpipe_cache.c.

References dt_dev_pixelpipe_cache_flush_clmem(), dt_opencl_is_inited(), dt_opencl_release_device(), and dt_opencl_reserve_device_by_id().

Referenced by _studio_dev_teardown(), dt_dev_pixelpipe_cleanup(), and leave().

◆ dt_dev_pixelpipe_cache_flush_entry_clmem()

void dt_dev_pixelpipe_cache_flush_entry_clmem ( struct dt_pixel_cache_entry_t * entry )

Flush all reusable OpenCL payloads cached on one cache entry.

Borrowed payloads are preserved until all borrowers returned them.

Parameters

entry Cache entry whose cached cl_mem payloads should be flushed.

Definition at line 1121 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_free, dt_opencl_release_mem_object(), dt_pthread_mutex_lock(), and dt_pthread_mutex_unlock().

Referenced by _free_cache_entry(), dt_dev_pixelpipe_cache_rekey(), and dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixelpipe_cache_flush_host_pinned_image()

gboolean dt_dev_pixelpipe_cache_flush_host_pinned_image	(	void *	host_ptr,
		struct dt_pixel_cache_entry_t *	entry_hint,
		int	devid
	)

Drop cached pinned OpenCL images associated with a given host buffer.

This is meant for host buffers that remain alive and continue to be mutated by the CPU (for example, privately owned working patches). Reusing a stale CL_MEM_USE_HOST_PTR image for such buffers can hide later host-side edits on some drivers. Flushing the cached pinned images forces the next use to rebind the host storage.

Parameters

cache	Pixelpipe cache.
host_ptr	Host-backed image data.
entry_hint	Optional owning cache entry for regular cache lines, or NULL.
devid	Device id to flush, or -1 for all cached devices for that host buffer.

Returns: gboolean TRUE if at least one pinned image was flushed, FALSE otherwise.

Definition at line 1236 of file pixelpipe_cache.c.

References _cache_entry_clmem_flush_host_pinned_locked(), _cache_entry_clmem_has_host_pinned_locked(), _cache_entry_for_host_ptr_locked(), _pixelpipe_cache, dt_dev_pixelpipe_cache_ref_count_entry(), dt_opencl_events_wait_for(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), FALSE, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, and TRUE.

Referenced by _rasterize_pending_dab_batch(), dt_dev_pixelpipe_gpu_flush_host_pinned_images(), and dt_drawlayer_rekey_shared_base_patch().

◆ dt_dev_pixelpipe_cache_flush_old()

static int dt_dev_pixelpipe_cache_flush_old ( dt_dev_pixelpipe_cache_t * cache )

static

Definition at line 3086 of file pixelpipe_cache.c.

References _cache_get_newest(), _for_each_remove_old(), dt_dev_pixelpipe_cache_t::arena, dt_cache_arena_trim(), dt_get_memory_pressure_floor(), dt_get_system_available_mem(), dt_pthread_mutex_trylock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_dev_pixelpipe_cache_init().

◆ dt_dev_pixelpipe_cache_get()

int dt_dev_pixelpipe_cache_get	(	const uint64_t	hash,
		const size_t	size,
		const char *	name,
		const int	id,
		const gboolean	alloc,
		void **	data,
		struct dt_pixel_cache_entry_t **	entry
	)

Get a cache line from the cache.

WARNING: This internally increases the reference count, so you have to manually decrease it using dt_dev_pixelpipe_ref_count_entry() once the cache line content has been consumed or it will never be freed.

WARNING: if the cache line was newly allocated, a write lock is put on straight away. You will have to release it from the same calling thread next, to avoid dead locks.

Parameters

cache
hash	State checksum of the cache line.
size	Buffer size in bytes.
name	Name of the cache line (for debugging).
id	ID of the pipeline owning the cache line.
data	Pointer to the buffer pointer (returned).
alloc	Whether or not we should actually alloc the buffer, or simply reserve it. If FALSE use `dt_pixel_cache_alloc()` when you actually need the buffer.
cache_entry	a reference to the cache entry, to be reused later. Can be NULL. The caller doesn't own the data and shouldn't free it.

Returns: int 1 if the cache line was freshly allocated, 0 if it was found in the cache.

Definition at line 2662 of file pixelpipe_cache.c.

References _cache_print, _non_thread_safe_cache_ref_count_entry(), _non_thread_safe_cache_remove(), _non_threadsafe_cache_get_entry(), _observe_read(), _pixel_cache_message(), _pixelpipe_cache, _pixelpipe_cache_create_entry_locked(), _pixelpipe_cache_finalize_entry(), dt_pixel_cache_entry_t::auto_destroy, dt_pixel_cache_entry_t::data, DT_DEBUG_PIPECACHE, dt_dev_pixelpipe_cache_wrlock_entry(), dt_pixel_cache_alloc(), DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, FALSE, hash, dt_dev_pixelpipe_cache_t::hits, dt_pixel_cache_entry_t::hits, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, name, dt_dev_pixelpipe_cache_t::queries, size, dt_pixel_cache_entry_t::size, and TRUE.

Referenced by _create(), _create_held(), _group_publish_prefix(), _publish(), dt_develop_blend_process(), dt_develop_blend_process_cl(), dt_drawlayer_cache_patch_alloc_shared(), process(), process_cl(), rt_scaled_mask_publish(), and rt_shape_box().

◆ dt_dev_pixelpipe_cache_get_available()

size_t dt_dev_pixelpipe_cache_get_available ( void )

What an allocation made now could get, in bytes, by the rules that grant or refuse it: the room the plan and the system's available RAM over the pressure floor leave, counting what eviction can take back – lines nobody holds, and of the system's share only their host RAM. Tile planners size working sets against it.

Definition at line 3426 of file pixelpipe_cache.c.

References _cache_sum_evictable(), _pixelpipe_cache, dt_dev_pixelpipe_cache_t::current_memory, dt_get_memory_pressure_floor(), dt_get_system_available_mem(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, dt_dev_pixelpipe_cache_t::lock, dt_dev_pixelpipe_cache_t::max_memory, and MIN.

Referenced by _host_tile_buffer_size(), and dt_tiling_piece_fits_host_memory().

◆ dt_dev_pixelpipe_cache_get_cl_buffer()

void * dt_dev_pixelpipe_cache_get_cl_buffer	(	int	devid,
		void *const	host_ptr,
		const dt_iop_roi_t *	roi,
		const size_t	bpp,
		dt_iop_module_t *	module,
		const char *	message,
		dt_pixel_cache_entry_t *	cache_entry,
		gboolean *	out_reused,
		void *	keep
	)

◆ dt_dev_pixelpipe_cache_get_entries_stats()

GArray * dt_dev_pixelpipe_cache_get_entries_stats ( void )

Definition at line 3526 of file pixelpipe_cache.c.

References _non_threadsafe_stats_append(), _pixelpipe_cache, _stats_entry_clear(), dt_opencl_get_num_devices(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, dt_dev_pixelpipe_cache_t::external_entries, FALSE, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, out, and TRUE.

Referenced by _rebuild_memory().

◆ dt_dev_pixelpipe_cache_get_entry()

dt_pixel_cache_entry_t * dt_dev_pixelpipe_cache_get_entry ( const uint64_t hash )

Get an internal reference to the cache entry matching hash. If you are going to access this entry more than once, keeping the reference and using it instead of hashes will prevent redundant lookups.

Parameters

cache
hash

Returns: struct dt_pixel_cache_entry_t*

Definition at line 357 of file pixelpipe_cache.c.

References _non_threadsafe_cache_get_entry(), _pixel_cache_touch(), _pixelpipe_cache, DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, hash, and dt_dev_pixelpipe_cache_t::lock.

Referenced by _cleanup_nodes_releases_the_masks(), _detail_output_with_its_mask_stands(), _detail_output_without_its_mask_goes(), _flagged_line_goes_with_its_last_holder(), _flagged_line_goes_with_its_release(), _held_line_survives_the_flush(), _other_output_stands(), _rekeyed_line_publishes_no_host_pixels(), _released_line_goes_with_the_flush(), _released_masks_go_with_the_flush(), and dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixelpipe_cache_get_entry_by_data()

dt_pixel_cache_entry_t * dt_dev_pixelpipe_cache_get_entry_by_data ( void * data )

Definition at line 372 of file pixelpipe_cache.c.

References _pixel_cache_touch(), _pixelpipe_cache, dt_pixel_cache_entry_t::data, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, dt_dev_pixelpipe_cache_t::external_entries, key, dt_dev_pixelpipe_cache_t::lock, and value.

◆ dt_dev_pixelpipe_cache_get_max_memory()

size_t dt_dev_pixelpipe_cache_get_max_memory ( void )

Definition at line 3449 of file pixelpipe_cache.c.

References _pixelpipe_cache, IS_NULL_PTR, and dt_dev_pixelpipe_cache_t::max_memory.

Referenced by _rebuild_memory().

◆ dt_dev_pixelpipe_cache_get_pinned_image()

void * dt_dev_pixelpipe_cache_get_pinned_image	(	void *	host_ptr,
		struct dt_pixel_cache_entry_t *	entry_hint,
		int	devid,
		int	width,
		int	height,
		int	bpp,
		int	flags,
		gboolean *	out_reused
	)

Acquire a pinned OpenCL image for a host buffer tracked by the pixelpipe cache.

This is the public helper for modules that manage their own host buffers but still want the pixelpipe cache to own the reusable pinned cl_mem images. The helper accepts an optional cache-entry hint for normal cache lines. If that hint is NULL, it will try to resolve the host pointer against the cache's privately-owned external allocations.

On reuse, the helper synchronizes host memory to the OpenCL image before returning.

Parameters

	cache	Pixelpipe cache.
	host_ptr	Host-backed image data.
	entry_hint	Optional owning cache entry for regular cache lines, or NULL.
	devid	OpenCL device id.
	width	Image width.
	height	Image height.
	bpp	Bytes per pixel.
	flags	OpenCL allocation flags (must include `CL_MEM_USE_HOST_PTR`).
[out]	out_reused	Optional flag set TRUE when an existing pinned image was reused.

Returns: void* OpenCL image (cl_mem) or NULL on failure.

Definition at line 1143 of file pixelpipe_cache.c.

References _cache_entry_for_host_ptr_locked(), _cache_print, _pixel_cache_clmem_get(), _pixel_cache_clmem_remove(), _pixelpipe_cache, bpp, DT_DEBUG_OPENCL, dt_opencl_alloc_device_use_host_pointer(), dt_opencl_is_pinned_memory(), dt_opencl_map_image(), dt_opencl_release_mem_object(), dt_opencl_unmap_mem_object(), dt_opencl_use_pinned_memory(), dt_opencl_write_host_to_device(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), FALSE, flags, height, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, TRUE, and width.

Referenced by _blend_layer_over_input_cl(), and _drawlayer_acquire_source_image().

◆ dt_dev_pixelpipe_cache_get_usage()

void dt_dev_pixelpipe_cache_get_usage	(	size_t *	current,
		size_t *	max
	)

Definition at line 3412 of file pixelpipe_cache.c.

References _pixelpipe_cache, dt_pixelpipe_cache_pressure_monitor_t::budget, dt_dev_pixelpipe_cache_t::current_memory, dt_pixelpipe_cache_pressure_reported(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, max, and dt_dev_pixelpipe_cache_t::psi.

Referenced by _rebuild_memory(), and dt_get_available_mem().

◆ dt_dev_pixelpipe_cache_get_writable()

dt_dev_pixelpipe_cache_writable_status_t dt_dev_pixelpipe_cache_get_writable	(	const uint64_t	hash,
		const size_t	size,
		const char *	name,
		const int	id,
		const gboolean	alloc,
		const gboolean	allow_rekey_reuse,
		const dt_pixel_cache_entry_t *	reuse_hint,
		const dt_pixel_cache_entry_t *	reuse_hint_prev,
		void **	data,
		dt_pixel_cache_entry_t **	entry
	)

Definition at line 2736 of file pixelpipe_cache.c.

References _cache_try_rekey_reuse_locked(), _non_thread_safe_cache_ref_count_entry(), _non_thread_safe_cache_remove(), _non_threadsafe_cache_get_entry(), _pixel_cache_message(), _pixel_cache_touch(), _pixelpipe_cache, _pixelpipe_cache_create_entry_locked(), _pixelpipe_cache_finalize_entry(), dt_pixel_cache_entry_t::auto_destroy, dt_pixel_cache_entry_t::data, DT_DEV_PIXELPIPE_CACHE_WRITABLE_CREATED, DT_DEV_PIXELPIPE_CACHE_WRITABLE_ERROR, DT_DEV_PIXELPIPE_CACHE_WRITABLE_EXACT_HIT, DT_DEV_PIXELPIPE_CACHE_WRITABLE_REKEYED, dt_pixel_cache_alloc(), DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, FALSE, hash, dt_dev_pixelpipe_cache_t::hits, dt_pixel_cache_entry_t::hits, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, name, dt_dev_pixelpipe_cache_t::queries, size, and TRUE.

Referenced by _budget_spent_by_a_line_in_use_is_counted_once(), _produce(), _without_rekey_the_previous_output_stays(), and dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixelpipe_cache_init()

gboolean dt_dev_pixelpipe_cache_init	(	size_t	max_memory,
		const gboolean	verbose,
		const gboolean	verbose_detail
	)

Reserve the cache's arena and start it.

constructs a new cache with given cache line count (entries) and float buffer entry size in bytes.

Parameters

[out]	returns	0 if fail to allocate mem cache.
	max_memory	arena size in bytes. A session constant: the orchestrator decides it from the resource plan and retries smaller on failure, which is resource policy.
	verbose	whether the cache traces at all (was `-d pipecache`).
	verbose_detail	whether it traces in detail (was `-d verbose`). Ignored when `verbose` is FALSE, matching the old two-level gate.

Both flags are read ONCE, here, from the session's debug flags by the orchestrator. The cache does not consult them at runtime: a cache that changes what it does halfway through a session because a global moved is harder to reason about than one told at startup, and it keeps the debug machinery out of a storage module.

Definition at line 2466 of file pixelpipe_cache.c.

Referenced by _setup(), _setup(), _setup(), _setup(), _setup(), and dt_init().

◆ dt_dev_pixelpipe_cache_invalidate_hashes()

int dt_dev_pixelpipe_cache_invalidate_hashes	(	const uint64_t *	hashes,
		const size_t	count
	)

Invalidate cache lines matching an explicit list of hashes.

Entries which are not in use are removed immediately. Entries still referenced or locked are left untouched because they may be published backbuffers owned by another pipeline.

This targets shared cache states independently of the pipeline which created them, while preserving every cache line not named by hashes.

Parameters

cache	Pixelpipe cache to update.
hashes	Cache keys to invalidate.
count	Number of keys in `hashes`.

Returns: int Number of matching entries which could not be removed.

Definition at line 3025 of file pixelpipe_cache.c.

References _non_thread_safe_cache_remove(), _non_threadsafe_cache_get_entry(), _pixelpipe_cache, DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, FALSE, IS_NULL_PTR, k, and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_dev_pixelpipe_process(), and dt_dev_pixelpipe_rawdetail_mask_check_hit().

◆ dt_dev_pixelpipe_cache_is_ready()

gboolean dt_dev_pixelpipe_cache_is_ready ( void )

Has the pixelpipe cache been initialised? Callers that run before dt_dev_pixelpipe_cache_init() succeeds, or after its cleanup, ask this rather than testing a handle they should not hold.

The application-wide pixelpipe cache singleton. DECLARED here because it is this module's object; BOUND by the orchestrator (darktable.c), so this header never needs to see the application struct.

Definition at line 104 of file pixelpipe_cache.c.

References _pixelpipe_cache, and IS_NULL_PTR.

Referenced by dt_cleanup(), and dt_init().

◆ dt_dev_pixelpipe_cache_peek()

gboolean dt_dev_pixelpipe_cache_peek	(	const uint64_t	hash,
		void **	data,
		struct dt_pixel_cache_entry_t **	entry,
		const int	preferred_devid,
		void **	cl_mem_output
	)

Non-owning lookup of an existing cache line.

This does not create a new cache line and does not change reference counts or entry locks. Callers that need lifetime guarantees must retain the entry explicitly with dt_dev_pixelpipe_cache_ref_count_entry() and/or dt_dev_pixelpipe_cache_rdlock_entry().

If !IS_NULL_PTR(cl_mem_output), the lookup becomes authoritative for exact-hit consumers:

write-locked / auto-destroy entries are rejected,
host data is restored from cached device state when possible,
device data is restored into cl_mem_output when requested,
broken entries with neither authoritative RAM nor vRAM payload are removed.

Definition at line 2896 of file pixelpipe_cache.c.

References _cache_lookup_existing(), _cache_print, _cache_try_restore_device_payload(), _non_thread_safe_cache_remove(), _non_threadsafe_cache_get_entry(), _pixelpipe_cache, _trace_exact_hit(), dt_pixel_cache_entry_t::auto_destroy, DT_DEBUG_PIPECACHE, dt_dev_pixelpipe_cache_restore_host_payload(), dt_pixel_cache_entry_get_data(), DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_pthread_rwlock_tryrdlock(), dt_pthread_rwlock_unlock(), dt_dev_pixelpipe_cache_t::entries, FALSE, hash, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, dt_pixel_cache_entry_t::lock, and TRUE.

Referenced by _peek_gui(), _rekeyed_line_publishes_no_host_pixels(), and dt_dev_retrieve_rawdetail_mask().

◆ dt_dev_pixelpipe_cache_prepare_cl_input()

int dt_dev_pixelpipe_cache_prepare_cl_input	(	dt_dev_pixelpipe_t *	pipe,
		dt_iop_module_t *	module,
		float *	input,
		void **	cl_mem_input,
		const dt_iop_roi_t *	roi_in,
		const size_t	in_bpp,
		dt_pixel_cache_entry_t *	input_entry,
		dt_pixel_cache_entry_t **	locked_input_entry,
		void *	keep
	)

Prepare/obtain the OpenCL input image for a module.

Parameters

	pipe	Current pixelpipe (provides device id + global settings).
	module	Module being processed (for debug logs).
	input	Host input pointer (may be NULL on GPU-only paths).
[in,out]	cl_mem_input	OpenCL input image; may already be set by the previous module.
	roi_in	ROI for the input buffer.
	in_bpp	Input bytes-per-pixel.
	input_entry	Pixelpipe cache entry corresponding to the input hash.
[out]	locked_input_entry	If non-NULL on return, the caller must unlock it after GPU work completed.
	keep	OpenCL buffer to keep alive while flushing device scratch allocations.

Returns: 0 on success, 1 on failure.

There are two major cases:

1) !IS_NULL_PTR(*cl_mem_input): The previous module already produced an OpenCL buffer and we are continuing on GPU. We may still need to keep the cache entry write-locked if it is a true zero-copy pinned image, because in-place OpenCL colorspace transforms can mutate the host-backed buffer before the current module runs.

2) IS_NULL_PTR(*cl_mem_input): We start from a host cache buffer (input). We allocate (or reuse) a pinned image backed by that host buffer, and if it is not true zero-copy we push host→device once before running kernels.

Definition at line 1555 of file pixelpipe_cache.c.

References _cache_print, dt_dev_pixelpipe_t::devid, DT_DEBUG_OPENCL, dt_dev_pixelpipe_cache_get_cl_buffer(), dt_dev_pixelpipe_cache_rdlock_entry(), dt_opencl_events_wait_for(), dt_opencl_is_pinned_memory(), dt_opencl_write_host_to_device(), FALSE, dt_iop_roi_t::height, IS_NULL_PTR, dt_iop_module_t::op, pipe, TRUE, and dt_iop_roi_t::width.

Referenced by pixelpipe_process_on_GPU().

◆ dt_dev_pixelpipe_cache_print()

void dt_dev_pixelpipe_cache_print ( void )

print out cache lines/hashes (debug).

Definition at line 3401 of file pixelpipe_cache.c.

References _cache_print, _pixelpipe_cache, _verbose, dt_dev_pixelpipe_cache_t::current_memory, DT_DEBUG_PIPECACHE, dt_dev_pixelpipe_cache_t::entries, dt_dev_pixelpipe_cache_t::hits, dt_dev_pixelpipe_cache_t::max_memory, and dt_dev_pixelpipe_cache_t::queries.

Referenced by dt_dev_pixelpipe_process().

◆ dt_dev_pixelpipe_cache_put_pinned_image()

void dt_dev_pixelpipe_cache_put_pinned_image	(	void *	host_ptr,
		struct dt_pixel_cache_entry_t *	entry_hint,
		void **	mem
	)

Release or cache a pinned OpenCL image acquired with dt_dev_pixelpipe_cache_get_pinned_image().

If the image is still a host-backed pinned allocation and an owning cache entry can be resolved, it is returned to that cache entry for reuse. Otherwise it is released.

Parameters

	cache	Pixelpipe cache.
	host_ptr	Host-backed image data.
	entry_hint	Optional owning cache entry for regular cache lines, or NULL.
[in,out]	mem	Pointer to the `cl_mem` handle (cleared on return).

Todo:: : is it safe to cache non-pinned vRAM buffers (aka no CL_MEM_USE_HOST_PTR in flags) ?

Definition at line 1217 of file pixelpipe_cache.c.

References _cache_print, _pixel_cache_clmem_put(), _verbose, _verbose_detail, DT_DEBUG_OPENCL, IS_NULL_PTR, and state.

Referenced by _blend_layer_over_input_cl().

◆ dt_dev_pixelpipe_cache_rdlock_entry()

void dt_dev_pixelpipe_cache_rdlock_entry	(	gboolean	lock,
		struct dt_pixel_cache_entry_t *	entry
	)

Lock or release the read lock on the entry.

Parameters

cache
lock	TRUE to lock, FALSE to release
entry	The cache entry object to lock.

Definition at line 3257 of file pixelpipe_cache.c.

References _pixel_cache_message(), dt_pthread_rwlock_rdlock(), dt_pthread_rwlock_unlock(), dt_pixel_cache_entry_t::lock, lock, and TRUE.

Referenced by _build_preview_fallback_surface(), _draw_preview_fallback(), _group_get_mask_roi(), _lib_navigation_compose(), _pixelpipe_pick_from_image(), _process_histogram(), _process_vectorscope(), _refresh_preview_cursor_sample(), _refresh_preview_module_histogram_for_hash(), _sample_picker_from_cache(), color_picker_apply(), dt_dev_get_raster_mask(), dt_dev_pixelpipe_cache_prepare_cl_input(), dt_dev_pixelpipe_process_rec(), dt_dev_render_locked_surface(), dt_dev_resync_mipmap_cache(), dt_drawlayer_cache_patch_rdlock(), dt_drawlayer_cache_patch_rdunlock(), dt_imageio_export_with_flags(), dt_iop_autoset_advance(), gui_post_expose(), mouse_moved(), pixelpipe_process_on_CPU(), pixelpipe_process_on_GPU(), rt_release_shape(), rt_scaled_mask_publish(), rt_scaled_mask_take_memo(), rt_shape_box(), scrolled(), and update_histogram().

◆ dt_dev_pixelpipe_cache_ref_count_entry()

void dt_dev_pixelpipe_cache_ref_count_entry	(	gboolean	lock,
		struct dt_pixel_cache_entry_t *	entry
	)

Take or release one reference on a cache line. A referenced line is never evicted, rekeyed or removed.

Releasing the last reference of a line flagged with dt_dev_pixelpipe_cache_flag_auto_destroy() removes it, in the same hold of the cache lock. After its release, the caller may not name the entry again: any thread's eviction can free it from then on.

Parameters

lock	TRUE to take a reference, FALSE to release one
entry	The cache line

Definition at line 3212 of file pixelpipe_cache.c.

References _cache_print, _non_thread_safe_cache_ref_count_entry(), _non_thread_safe_cache_remove(), _pixelpipe_cache, dt_pixel_cache_entry_t::auto_destroy, DT_DEBUG_PIPECACHE, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, FALSE, dt_pixel_cache_entry_t::hash, hash, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, and lock.

Referenced by _abort_module_shutdown_cleanup(), _blend_layer_over_input_cl(), _budget_spent_by_a_line_in_use_is_counted_once(), _develop_blend_release_mask_entry(), _develop_cacheline_ready_callback(), _develop_history_resync_callback(), _flagged_line_goes_with_its_last_holder(), _flagged_line_goes_with_its_release(), _group_get_mask_roi(), _group_publish_prefix(), _locked_bind_entry(), _peek_gui(), _produce(), _refresh_global_histogram_backbuf_for_hash(), _refresh_preview_cursor_sample(), _refresh_preview_module_histogram_for_hash(), _rekeyed_line_publishes_no_host_pixels(), _retain_base_patch_stroke_ref(), _rewritten_line_publishes_its_host_pixels(), _sample_picker_from_cache(), _unflagged_line_outlives_its_release(), _without_rekey_the_previous_output_stays(), color_picker_apply(), dt_dev_backbuf_take_keepalive(), dt_dev_get_raster_mask(), dt_dev_pixelpipe_cache_flush_host_pinned_image(), dt_dev_pixelpipe_cache_unref_entry(), dt_dev_pixelpipe_cache_unref_hash(), dt_dev_pixelpipe_process_rec(), dt_dev_resync_mipmap_cache(), dt_drawlayer_cache_patch_alloc_shared(), dt_drawlayer_cache_patch_clear(), dt_drawlayer_release_all_base_patch_extra_refs(), dt_drawlayer_retain_base_patch_loaded_ref(), dt_imageio_export_with_flags(), dt_iop_autoset_advance(), gui_cleanup(), gui_focus(), gui_post_expose(), invalidate_luminance_cache(), mouse_moved(), process(), process_cl(), rt_release_shape(), rt_scaled_mask_publish(), rt_scaled_mask_take_memo(), rt_shape_box(), scrolled(), and update_histogram().

◆ dt_dev_pixelpipe_cache_ref_entry_by_hash()

gboolean dt_dev_pixelpipe_cache_ref_entry_by_hash	(	const uint64_t	hash,
		void **	data,
		struct dt_pixel_cache_entry_t **	entry
	)

Resolve and retain an existing cache entry by hash.

This is the race-free counterpart to dt_dev_pixelpipe_cache_peek() for GUI code that needs to keep a cacheline past the lookup call. The cache lock is held while the entry is found and its refcount is incremented, so normal cache eviction cannot destroy the returned entry before the caller takes its own read lock.

The function never creates or allocates a cache entry. Release a successful result with dt_dev_pixelpipe_cache_ref_count_entry(cache, FALSE, entry).

Parameters

cache	Pixelpipe cache.
hash	Cacheline hash to resolve.
data	Returned current host buffer pointer, if requested.
entry	Returned retained cache entry, if requested.

Returns: TRUE when an existing non-auto-destroy entry was retained.

Definition at line 446 of file pixelpipe_cache.c.

References _non_thread_safe_cache_ref_count_entry(), _non_threadsafe_cache_get_entry(), _observe_read(), _pixelpipe_cache, _pixelpipe_cache_finalize_entry(), dt_pixel_cache_entry_t::auto_destroy, DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, FALSE, hash, dt_dev_pixelpipe_cache_t::hits, dt_pixel_cache_entry_t::hits, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, dt_dev_pixelpipe_cache_t::queries, dt_pixel_cache_entry_t::size, and TRUE.

Referenced by _develop_cacheline_ready_callback(), _develop_history_resync_callback(), _flagged_line_goes_with_its_last_holder(), _rekeyed_line_publishes_no_host_pixels(), _unflagged_line_outlives_its_release(), dt_dev_get_raster_mask(), dt_dev_pixelpipe_process(), dt_dev_pixelpipe_process_rec(), dt_dev_pixelpipe_rawdetail_mask_check_hit(), dt_dev_resync_mipmap_cache(), dt_dev_retrieve_rawdetail_mask(), dt_imageio_export_with_flags(), gui_focus(), and rt_scaled_mask_take_memo().

◆ dt_dev_pixelpipe_cache_ref_entry_for_host_ptr()

dt_pixel_cache_entry_t * dt_dev_pixelpipe_cache_ref_entry_for_host_ptr ( void * host_ptr )

Resolve and retain the cache entry owning a host pointer.

This helper searches both regular and external pixelpipe cache tables for the entry whose host buffer pointer exactly matches host_ptr.

On success it increments the entry refcount before returning, so callers can safely use the entry across asynchronous OpenCL operations. Release it with: dt_dev_pixelpipe_cache_ref_count_entry(cache, FALSE, entry).

Parameters

cache	Pixelpipe cache.
host_ptr	Host buffer pointer to resolve.

Returns: dt_pixel_cache_entry_t* Owning cache entry with retained refcount, or NULL.

Definition at line 1728 of file pixelpipe_cache.c.

References _cache_entry_for_host_ptr_locked(), _non_thread_safe_cache_ref_count_entry(), _pixelpipe_cache, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, and TRUE.

Referenced by _blend_layer_over_input_cl(), dt_drawlayer_cache_ensure_mask_buffer(), and dt_drawlayer_cache_ensure_process_patch_buffer().

◆ dt_dev_pixelpipe_cache_ref_host_entry_by_hash()

gboolean dt_dev_pixelpipe_cache_ref_host_entry_by_hash	(	const uint64_t	hash,
		void **	data,
		struct dt_pixel_cache_entry_t **	entry
	)

Resolve and retain an existing cacheline that already holds HOST pixels.

The race-free lookup for GUI consumers that sample an intermediate module cacheline – color picker, module histograms, autoset, colorequal. Those cachelines sit at refcount 0 between renders, so the pipeline is free to evict them at any moment: looking one up and THEN taking a reference leaves a window in which it is freed, and the sampling pass then reads a dangling buffer. Here the lookup and the reference happen under one hold of the cache lock, so an entry handed back cannot have been evicted in between.

Unlike dt_dev_pixelpipe_cache_ref_entry_by_hash(), this one additionally requires host pixels to be present (GUI threads own no OpenCL device and must never materialize a device-only payload) and refuses an entry that is currently write-locked. It never allocates and never waits on a lock.

Release a successful result with dt_dev_pixelpipe_cache_ref_count_entry(FALSE, entry).

Parameters

hash	Cacheline hash to resolve.
data	Returned host buffer pointer, if requested. Never NULL on success.
entry	Returned retained cache entry, if requested.

Returns: TRUE when a host-resident, unlocked, non-auto-destroy entry was retained.

Definition at line 478 of file pixelpipe_cache.c.

References _non_thread_safe_cache_ref_count_entry(), _non_threadsafe_cache_get_entry(), _observe_read(), _pixelpipe_cache, _pixelpipe_cache_finalize_entry(), dt_pixel_cache_entry_t::auto_destroy, dt_pixel_cache_entry_get_data(), DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_pthread_rwlock_tryrdlock(), dt_pthread_rwlock_unlock(), dt_dev_pixelpipe_cache_t::entries, FALSE, hash, dt_dev_pixelpipe_cache_t::hits, dt_pixel_cache_entry_t::hits, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, dt_pixel_cache_entry_t::lock, dt_dev_pixelpipe_cache_t::queries, dt_pixel_cache_entry_t::size, and TRUE.

Referenced by _group_get_mask_roi(), _peek_gui(), _rekeyed_line_publishes_no_host_pixels(), _rewritten_line_publishes_its_host_pixels(), _without_rekey_the_previous_output_stays(), and dt_dev_pixelpipe_process().

◆ dt_dev_pixelpipe_cache_rekey()

int dt_dev_pixelpipe_cache_rekey	(	const uint64_t	old_hash,
		const uint64_t	new_hash,
		struct dt_pixel_cache_entry_t *	entry
	)

Change the hash/key of an existing cache line in place, without freeing, reallocating or invalidating the underlying entry.

This is useful when a cache line remains the authoritative buffer but the caller needs its identity to follow a new logical snapshot (for example a new module-parameter hash). The entry contents, locks, refcount and storage are preserved; only the hash-table key and entry->hash are updated.

The operation fails if:

the source entry cannot be found,
or another different entry already exists at new_hash.

Parameters

cache
old_hash	Current key of the entry.
new_hash	Desired new key.
entry	Optional direct entry reference. May be NULL.

Returns: int 0 on success, 1 on error.

Definition at line 3311 of file pixelpipe_cache.c.

References _cache_debug_module_name(), _cache_print, _non_threadsafe_cache_get_entry(), _observe_rekey(), _pixelpipe_cache, dt_pixel_cache_entry_t::auto_destroy, dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_pixel_cache_entry_t::data, dt_atomic_get_int(), DT_DEBUG_PIPECACHE, dt_dev_pixelpipe_cache_flush_entry_clmem(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, dt_pixel_cache_entry_t::hash, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, dt_pixel_cache_entry_t::refcount, and dt_pixel_cache_entry_t::serial.

Referenced by dt_drawlayer_rekey_shared_base_patch().

◆ dt_dev_pixelpipe_cache_release_cl_buffer()

void dt_dev_pixelpipe_cache_release_cl_buffer	(	void **	cl_mem_buffer,
		dt_pixel_cache_entry_t *	cache_entry,
		void *	host_ptr,
		const gboolean	cache_device
	)

Release or cache an OpenCL image associated with a host cache line.

Release or cache an OpenCL image associated with one cache entry.

Parameters

[in,out]	cl_mem_buffer	Pointer to a `cl_mem` stored as `void*`.
	cache_entry	Pixelpipe cache entry the host pointer belongs to (may be NULL).
	host_ptr	Host pointer backing the OpenCL image (may be NULL).
	cache_device	Allow caching pure device-side buffers for scratch reuse.

This helper is a single point of truth for OpenCL image lifetime management in the pixelpipe:

If the image is host-backed (CL_MEM_USE_HOST_PTR) and we have both cache_entry and host_ptr, we put it in the cache entry's cl_mem_list for reuse.
Otherwise, we release it immediately.
Pure device allocations may also be cached for scratch-pad reuse. When those cached cl_mem objects are later evicted, the cache layer is responsible for materializing host RAM first if no authoritative host buffer exists yet.

Additionally, when we release an image, we must ensure there is no stale pointer in cl_mem_list (for example, if some earlier path cached it and we are now deciding to free it). We call _pixel_cache_clmem_remove() before releasing to keep the cache bookkeeping coherent.

Definition at line 1406 of file pixelpipe_cache.c.

References _pixel_cache_clmem_put(), _pixel_cache_clmem_remove(), dt_opencl_release_mem_object(), and IS_NULL_PTR.

Referenced by _abort_module_shutdown_cleanup(), _blend_layer_over_input_cl(), _drawlayer_acquire_source_image(), _gpu_early_cpu_fallback_if_unsupported(), and pixelpipe_process_on_GPU().

◆ dt_dev_pixelpipe_cache_restore_cl_buffer()

float * dt_dev_pixelpipe_cache_restore_cl_buffer	(	dt_dev_pixelpipe_t *	pipe,
		float *	input,
		void *	cl_mem_input,
		const dt_iop_roi_t *	roi_in,
		dt_iop_module_t *	module,
		const size_t	in_bpp,
		dt_pixel_cache_entry_t *	input_entry,
		const char *	message
	)

Force device → host resynchronization of the pixelpipe input cache line.

This is used when we are about to switch from GPU processing to CPU processing for a given module. In that scenario, the most recent correct pixels may only exist in cl_mem_input (GPU-only intermediate), while input (host pointer) is either NULL or stale.

The function:

write-locks the cache entry (we are modifying host memory),
performs a device→host copy (map/unmap if possible, explicit copy otherwise),
updates the buffer descriptor colorspace tag.

Definition at line 1513 of file pixelpipe_cache.c.

References dt_dev_pixelpipe_t::devid, dt_dev_pixelpipe_cache_flag_host_written(), dt_dev_pixelpipe_cache_sync_cl_buffer(), dt_dev_pixelpipe_cache_wrlock_entry(), FALSE, IS_NULL_PTR, pipe, and TRUE.

Referenced by _gpu_early_cpu_fallback_if_unsupported().

◆ dt_dev_pixelpipe_cache_restore_host_payload()

gboolean dt_dev_pixelpipe_cache_restore_host_payload	(	struct dt_pixel_cache_entry_t *	entry,
		int	preferred_devid,
		void **	data
	)

Materialize a host payload for a live cache entry from its cached device payload.

Recursive pixelpipe stages reopen transient upstream cache entries directly, before those entries become valid exact-hits. When such an entry only carries a GPU payload, the caller may still need a host pointer for CPU fallback or for a non-OpenCL module. This helper keeps the recovery local to the cache layer:

allocate host RAM for entry when needed,
copy the most relevant cached device payload back to host,
return the restored host pointer through data.

The function leaves the cache entry owned by the caller. It does not change refcounts.

Parameters

	cache	Pixelpipe cache.
	entry	Live cache entry to restore.
	preferred_devid	Preferred OpenCL device id, or `-1` for any.
[out]	data	Restored host pointer, or NULL on failure.

Returns: gboolean TRUE when host data is available after the call, FALSE otherwise.

Definition at line 2874 of file pixelpipe_cache.c.

References _cache_entry_materialize_host_data(), _pixel_cache_touch(), _pixelpipe_cache, dt_pixel_cache_entry_get_data(), FALSE, IS_NULL_PTR, and TRUE.

Referenced by dt_dev_pixelpipe_cache_peek(), dt_dev_pixelpipe_process(), and dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixelpipe_cache_return_cl_payload()

void dt_dev_pixelpipe_cache_return_cl_payload	(	struct dt_pixel_cache_entry_t *	entry,
		void *	mem
	)

Return a borrowed cached OpenCL payload to its cache entry.

Parameters

entry	Cache entry owning the payload.
mem	Borrowed `cl_mem` to release back to cache bookkeeping.

Definition at line 1023 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), IS_NULL_PTR, and void().

Referenced by _gpu_early_cpu_fallback_if_unsupported(), and pixelpipe_process_on_GPU().

◆ dt_dev_pixelpipe_cache_set_alert_handler()

void dt_dev_pixelpipe_cache_set_alert_handler ( dt_pixelpipe_cache_alert_handler_t alert )

Install the alert handler. The GUI installs it, once it exists, apart from the others: without a GUI there is nothing to alert with, and an alert is then printed on stderr, the item on the line under the message.

Parameters

alert may be NULL, which restores that fallback.

Definition at line 125 of file pixelpipe_cache.c.

References _alert_handler.

Referenced by dt_gui_gtk_init().

◆ dt_dev_pixelpipe_cache_set_handlers()

void dt_dev_pixelpipe_cache_set_handlers	(	dt_pixelpipe_cache_warn_handler_t	warn,
		dt_pixelpipe_cache_ready_handler_t	ready,
		const dt_pixelpipe_cache_observer_t *	observer
	)

Install the handlers. Call once, from the orchestrator, before any pipe runs.

Parameters

warn	may be NULL – the warning is then dropped, not printed somewhere else.
ready	may be NULL.
observer	BORROWED, not copied: it must outlive the cache. Pass NULL to detach.

Definition at line 116 of file pixelpipe_cache.c.

References _observer, _ready_handler, and _warn_handler.

Referenced by dt_init().

◆ dt_dev_pixelpipe_cache_sync_cl_buffer()

int dt_dev_pixelpipe_cache_sync_cl_buffer	(	const int	devid,
		void *	host_ptr,
		void *	cl_mem_buffer,
		const dt_iop_roi_t *	roi,
		int	cl_mode,
		size_t	bpp,
		dt_iop_module_t *	module,
		const char *	message
	)

Synchronize between host memory and a pinned OpenCL image.

Parameters

devid	OpenCL device index.
host_ptr	Host pointer to read from / write to.
cl_mem_buffer	OpenCL image.
roi	Image dimensions.
cl_mode	`CL_MAP_WRITE` for host→device, `CL_MAP_READ` for device→host.
bpp	Bytes per pixel.
module	Module for debug logs (may be NULL).
message	Context string for debug logs.

Returns: 0 on success, 1 on failure.

This function intentionally tries a hierarchy of synchronization mechanisms:

For CL_MEM_USE_HOST_PTR images, we attempt a map/unmap cycle. If the mapped pointer equals host_ptr, we treat it as true zero-copy and the map/unmap acts as a synchronization barrier (fast, avoids extra copies).
Otherwise, we fall back to explicit blocking transfers (dt_opencl_write_host_to_device / dt_opencl_read_host_from_device), which already guarantee that the copied pixels are visible on the host/device side when the helper returns.

The map/unmap approach is used as a synchronization barrier because on many drivers it will:

flush CPU caches / invalidate as needed,
ensure GPU work touching that memory is completed (for blocking map),
and potentially avoid a full copy when true zero-copy is supported.

Definition at line 1454 of file pixelpipe_cache.c.

References _cache_print, bpp, DT_DEBUG_OPENCL, dt_opencl_is_pinned_memory(), dt_opencl_map_image(), dt_opencl_read_host_from_device(), dt_opencl_unmap_mem_object(), dt_opencl_write_host_to_device(), dt_iop_roi_t::height, IS_NULL_PTR, dt_iop_module_t::op, TRUE, and dt_iop_roi_t::width.

Referenced by _gpu_init_input(), dt_dev_pixelpipe_cache_restore_cl_buffer(), and pixelpipe_process_on_GPU().

◆ dt_dev_pixelpipe_cache_unref_entry()

void dt_dev_pixelpipe_cache_unref_entry ( struct dt_pixel_cache_entry_t * entry )

Release one reference held on entry, by POINTER.

Use this wherever the reference was TAKEN by pointer (dt_dev_pixelpipe_cache_ref_count_entry(TRUE, entry)), which is every long-lived claim in the tree. dt_dev_pixelpipe_cache_unref_hash() resolves the entry by looking its hash up in the cache, so the two are only interchangeable while an entry's hash never changes under its holder – and it does: _cache_try_rekey_reuse_locked() moves a live entry to a new hash in place, which is the whole point of rekey reuse. Released by the old hash afterwards, the lookup finds nothing, the reference is never dropped, and the entry becomes permanently unevictable while its refcount stops meaning anything.

A NULL entry is a no-op, so a holder can release unconditionally.

Definition at line 3289 of file pixelpipe_cache.c.

References dt_dev_pixelpipe_cache_ref_count_entry(), FALSE, and IS_NULL_PTR.

Referenced by _detail_output_with_its_mask_stands(), _detailmask_output(), _held_line_survives_the_flush(), _refresh_global_histogram_backbuf_for_hash(), _released_line_goes_with_the_flush(), dt_dev_backbuf_release_keepalive(), and dt_dev_pixelpipe_process().

◆ dt_dev_pixelpipe_cache_unref_hash()

void dt_dev_pixelpipe_cache_unref_hash ( const uint64_t hash )

Find the entry matching hash, and decrease its ref_count if found.

Parameters

cache
hash

Definition at line 3297 of file pixelpipe_cache.c.

References _non_threadsafe_cache_get_entry(), _pixelpipe_cache, dt_dev_pixelpipe_cache_ref_count_entry(), DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, FALSE, hash, dt_dev_pixelpipe_cache_t::lock, and dt_dev_pixelpipe_cache_t::queries.

Referenced by dt_dev_clear_rawdetail_mask(), dt_dev_pixelpipe_cleanup_nodes(), and dt_dev_pixelpipe_process().

◆ dt_dev_pixelpipe_cache_wrlock_entry()

void dt_dev_pixelpipe_cache_wrlock_entry	(	gboolean	lock,
		struct dt_pixel_cache_entry_t *	entry
	)

Lock or release the write lock on the entry.

Parameters

cache
lock	TRUE to lock, FALSE to release

Definition at line 3234 of file pixelpipe_cache.c.

References _pixel_cache_message(), _ready_handler, DT_PIXELPIPE_CACHE_HASH_INVALID, dt_pthread_rwlock_unlock(), dt_pthread_rwlock_wrlock(), dt_pixel_cache_entry_t::hash, dt_pixel_cache_entry_t::lock, lock, dt_pixel_cache_entry_t::producer_node_key, and TRUE.

Referenced by _budget_spent_by_a_line_in_use_is_counted_once(), _cache_entry_materialize_host_data(), _cache_try_rekey_reuse_locked(), _create_held(), _develop_blend_release_mask_entry(), _flagged_line_goes_with_its_last_holder(), _flagged_line_goes_with_its_release(), _gpu_early_cpu_fallback_if_unsupported(), _gpu_init_input(), _group_publish_prefix(), _pixelpipe_cache_create_entry_locked(), _produce(), _publish(), _unflagged_line_outlives_its_release(), dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_cache_restore_cl_buffer(), dt_dev_pixelpipe_process_rec(), dt_develop_blend_process(), dt_develop_blend_process_cl(), dt_drawlayer_cache_patch_alloc_shared(), dt_drawlayer_cache_patch_wrlock(), dt_drawlayer_cache_patch_wrunlock(), process(), process_cl(), rt_scaled_mask_publish(), and rt_shape_box().

◆ dt_pixel_cache_alloc()

void * dt_pixel_cache_alloc ( struct dt_pixel_cache_entry_t * entry )

Actually allocate the memory buffer attached to the cache entry once you create it with dt_dev_pixelpipe_cache_get(). Sizes and everything are already saved in the entry, and the cache will have the needed space reserved.

Parameters

cache
entry	the cache entry

Returns: void* Pointer to the allocated data buffer.

Definition at line 2095 of file pixelpipe_cache.c.

References _arena_alloc_with_defrag(), _log_arena_allocation_failure(), _pixelpipe_cache, dt_pixel_cache_entry_t::data, FALSE, dt_pixel_cache_entry_t::hash, IS_NULL_PTR, dt_pixel_cache_entry_t::name, and dt_pixel_cache_entry_t::size.

Referenced by _cache_entry_materialize_host_data(), _gpu_early_cpu_fallback_if_unsupported(), _gpu_init_input(), _produce(), dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_cache_get_writable(), dt_pixel_cache_new_entry(), pixelpipe_process_on_CPU(), and pixelpipe_process_on_GPU().

◆ dt_pixel_cache_entry_get_buffer()

void * dt_pixel_cache_entry_get_buffer ( struct dt_pixel_cache_entry_t * entry )

The host buffer of a cache entry, whatever hash its content describes.

For the producer holding the write lock from dt_dev_pixelpipe_cache_get_writable() only: a rekeyed entry's buffer is about to be overwritten, and it still backs the pinned OpenCL image of an output that is not copied back to RAM. Everyone else reads pixels through dt_pixel_cache_entry_get_data().

Definition at line 2120 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::data.

Referenced by _rekeyed_line_publishes_no_host_pixels(), and pixelpipe_process_on_GPU().

◆ dt_pixel_cache_entry_get_data()

void * dt_pixel_cache_entry_get_data ( struct dt_pixel_cache_entry_t * entry )

Peek the host pixels a cache entry holds for its current hash, without allocating.

Returns: NULL when the entry has no host buffer, and also when its host buffer still holds the pixels of the hash it was rekeyed from (see dt_pixel_cache_entry_t.host_stale). A consumer treats both alike: the pixels, if any, are on the device and must be copied back first.

Definition at line 2115 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::data, dt_atomic_get_int(), dt_pixel_cache_entry_t::host_stale, and IS_NULL_PTR.

Referenced by _cache_entry_clmem_flush_device(), _cache_entry_materialize_host_data(), _lib_navigation_compose(), _pixelpipe_cache_finalize_entry(), _rekeyed_line_publishes_no_host_pixels(), color_picker_apply(), dt_dev_lock_pipe_surface(), dt_dev_pixelpipe_cache_gpu_device_buffer(), dt_dev_pixelpipe_cache_peek(), dt_dev_pixelpipe_cache_ref_host_entry_by_hash(), dt_dev_pixelpipe_cache_restore_host_payload(), dt_dev_pixelpipe_process_rec(), gui_post_expose(), mouse_moved(), pixelpipe_process_on_CPU(), pixelpipe_process_on_GPU(), scrolled(), and update_histogram().

◆ dt_pixel_cache_entry_get_size()

size_t dt_pixel_cache_entry_get_size ( const dt_pixel_cache_entry_t * entry )

Definition at line 2130 of file pixelpipe_cache.c.

References dt_pixel_cache_entry_t::size.

Referenced by _group_get_mask_roi(), _lib_navigation_compose(), _refresh_global_histogram_backbuf_for_hash(), _refresh_preview_module_histogram_for_hash(), _update_backbuf_cache_reference(), dt_dev_lock_pipe_surface(), dt_dev_pixelpipe_process_rec(), luminance_entry_fits(), and rt_scaled_mask_memo_holds().

◆ dt_pixel_cache_new_entry()

static dt_pixel_cache_entry_t * dt_pixel_cache_new_entry	(	const uint64_t	hash,
		const size_t	size,
		const char *	name,
		const int	id,
		dt_dev_pixelpipe_cache_t *	cache,
		gboolean	alloc,
		GHashTable *	table
	)

static

Definition at line 2317 of file pixelpipe_cache.c.

Referenced by _pixelpipe_cache_create_entry_locked(), and dt_pixelpipe_cache_alloc_align_cache_impl().

◆ dt_pixelpipe_cache_alloc_align_cache_impl()

void * dt_pixelpipe_cache_alloc_align_cache_impl	(	size_t	size,
		int	id,
		const char *	name
	)

Allocate aligned memory tracked by the pixelpipe cache. This allows LRU cache entries to be evicted if needed to make room.

Parameters

cache	Pixelpipe cache to manage.
size	Buffer size in bytes.
id	Pipeline type owning the buffer.
name	Human-readable name.

Returns: void* Pointer to the allocated buffer, or NULL on failure.

Definition at line 2242 of file pixelpipe_cache.c.

References _arena_alloc_with_defrag(), _free_space_to_alloc(), _log_arena_allocation_failure(), _non_thread_safe_cache_ref_count_entry(), _pixel_cache_touch(), _pixelpipe_cache, dt_dev_pixelpipe_cache_t::arena, dt_pixel_cache_entry_t::data, dt_cache_arena_free(), DT_CACHELINE_BYTES, dt_pixel_cache_new_entry(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), error(), dt_pixel_cache_entry_t::external_alloc, dt_dev_pixelpipe_cache_t::external_entries, FALSE, hash, IS_NULL_PTR, dt_dev_pixelpipe_cache_t::lock, name, size, and TRUE.

Referenced by _budget_spent_by_a_line_in_use_is_counted_once(), _granted_allocation_is_not_counted(), _refuse_on_this_thread(), _request_larger_than_the_cache_is_counted_once(), dt_drawlayer_cache_alloc_temp_buffer(), and dt_pixelpipe_cache_alloc_perthread_impl().

◆ dt_pixelpipe_cache_free_align_cache()

void dt_pixelpipe_cache_free_align_cache	(	void **	mem,
		const char *	message
	)

Free aligned memory allocated with dt_pixelpipe_cache_alloc_align_cache.

Parameters

cache	Pixelpipe cache to manage.
mem	Pointer to the buffer pointer. Set to NULL on successful free.

Definition at line 2292 of file pixelpipe_cache.c.

References _non_thread_safe_cache_ref_count_entry(), _non_threadsafe_cache_get_entry(), _pixelpipe_cache, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_pixel_cache_entry_t::external_alloc, dt_dev_pixelpipe_cache_t::external_entries, FALSE, dt_pixel_cache_entry_t::hash, hash, IS_NULL_PTR, and dt_dev_pixelpipe_cache_t::lock.

Referenced by _granted_allocation_is_not_counted(), dt_drawlayer_cache_free_temp_buffer(), and dt_drawlayer_cache_patch_clear().

◆ dt_pixelpipe_cache_get_alloc_refusals()

uint32_t dt_pixelpipe_cache_get_alloc_refusals ( void )

Count the allocations the cache refused to the calling thread for lack of memory (thread-local).

A refusal is a NULL returned because memory ran short: the cache's budget was spent with every line in use, the arena had no free run long enough, or the system had no RAM left. The count only goes up. A caller that reads it before and after a pixelpipe run learns whether that run was refused memory: the pipe runs on its caller's thread. Allocations made on other threads, such as OpenMP workers, are not counted here.

Returns: uint32_t Refusals so far on this thread.

Definition at line 269 of file pixelpipe_cache.c.

References dt_pixelpipe_cache_alloc_refusals.

Referenced by _budget_spent_by_a_line_in_use_is_counted_once(), _granted_allocation_is_not_counted(), _refusal_on_another_thread_is_not_counted_here(), _refuse_on_this_thread(), _request_larger_than_the_cache_is_counted_once(), and dt_imageio_export_with_flags().

◆ dt_pixelpipe_cache_get_largest_free_run()

size_t dt_pixelpipe_cache_get_largest_free_run ( void )

Definition at line 1986 of file pixelpipe_cache.c.

References _arena_stats_bytes(), _pixelpipe_cache, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), and dt_dev_pixelpipe_cache_t::lock.

Referenced by _host_tile_buffer_size(), and dt_tiling_piece_fits_host_memory().

◆ dt_pixelpipe_cache_set_current_image()

const char * dt_pixelpipe_cache_set_current_image ( const char * image )

Set the image the current module runs on, for the message that reports a refused allocation (thread-local).

The image comes as text its caller wrote, as the module comes as its name: the cache prints it, and knows nothing of images. The pointer is kept, not copied.

Parameters

image The image as it should read, which must outlive the setting, or NULL to clear.

Returns: const char* Previous image.

Definition at line 262 of file pixelpipe_cache.c.

References dt_pixelpipe_cache_current_image.

Referenced by dt_dev_pixelpipe_get_roi_in(), and dt_dev_pixelpipe_process_rec().

◆ dt_pixelpipe_cache_set_current_module()

const char * dt_pixelpipe_cache_set_current_module ( const char * module )

Set the current module name for cache diagnostics (thread-local).

Parameters

module Module op name or NULL to clear.

Returns: const char* Previous module name.

Definition at line 255 of file pixelpipe_cache.c.

References dt_pixelpipe_cache_current_module.

Referenced by dt_dev_pixelpipe_get_roi_in(), and dt_dev_pixelpipe_process_rec().

Variable Documentation

◆ _alert_handler

dt_pixelpipe_cache_alert_handler_t _alert_handler = NULL

static

Definition at line 112 of file pixelpipe_cache.c.

Referenced by _alert_user(), and dt_dev_pixelpipe_cache_set_alert_handler().

◆ _observer

const dt_pixelpipe_cache_observer_t* _observer = NULL

static

Definition at line 114 of file pixelpipe_cache.c.

Referenced by _observe_delete(), _observe_read(), _observe_rekey(), _observed(), and dt_dev_pixelpipe_cache_set_handlers().

◆ _pixelpipe_cache

dt_dev_pixelpipe_cache_t* _pixelpipe_cache = NULL

static

Definition at line 102 of file pixelpipe_cache.c.

◆ _ready_handler

dt_pixelpipe_cache_ready_handler_t _ready_handler = NULL

static

Definition at line 113 of file pixelpipe_cache.c.

Referenced by dt_dev_pixelpipe_cache_set_handlers(), and dt_dev_pixelpipe_cache_wrlock_entry().

◆ _verbose

gboolean _verbose = FALSE

static

Definition at line 55 of file pixelpipe_cache.c.

Referenced by _cache_entry_clmem_flush_device(), _pixel_cache_clmem_get(), _pixel_cache_message(), _trace_exact_hit(), dt_dev_pixelpipe_cache_flush_clmem(), dt_dev_pixelpipe_cache_get_cl_buffer(), dt_dev_pixelpipe_cache_init(), dt_dev_pixelpipe_cache_print(), and dt_dev_pixelpipe_cache_put_pinned_image().

◆ _verbose_detail

gboolean _verbose_detail = FALSE

static

Definition at line 56 of file pixelpipe_cache.c.

Referenced by _cache_entry_clmem_flush_device(), _pixel_cache_clmem_get(), _pixel_cache_message(), _trace_exact_hit(), dt_dev_pixelpipe_cache_flush_clmem(), dt_dev_pixelpipe_cache_get_cl_buffer(), dt_dev_pixelpipe_cache_init(), and dt_dev_pixelpipe_cache_put_pinned_image().

◆ _warn_handler

dt_pixelpipe_cache_warn_handler_t _warn_handler = NULL

static

Definition at line 111 of file pixelpipe_cache.c.

Referenced by _warn_user(), and dt_dev_pixelpipe_cache_set_handlers().

◆ dt_pixelpipe_cache_alloc_refusals

__thread uint32_t dt_pixelpipe_cache_alloc_refusals = 0

static

Definition at line 229 of file pixelpipe_cache.c.

Referenced by _free_space_to_alloc(), _log_arena_allocation_failure(), and dt_pixelpipe_cache_get_alloc_refusals().

◆ dt_pixelpipe_cache_current_image

__thread const char* dt_pixelpipe_cache_current_image = NULL

static

Definition at line 223 of file pixelpipe_cache.c.

Referenced by _system_memory_pressure_valve(), and dt_pixelpipe_cache_set_current_image().

◆ dt_pixelpipe_cache_current_module

__thread const char* dt_pixelpipe_cache_current_module = NULL

static

Definition at line 220 of file pixelpipe_cache.c.

Referenced by _cache_debug_module_name(), and dt_pixelpipe_cache_set_current_module().

◆ garbage_collection

int garbage_collection = 0

static

Definition at line 2463 of file pixelpipe_cache.c.

Referenced by dt_dev_pixelpipe_cache_cleanup(), and dt_dev_pixelpipe_cache_init().

◆ pressure_shedding

int pressure_shedding = 0

static

Definition at line 2464 of file pixelpipe_cache.c.

Referenced by dt_dev_pixelpipe_cache_cleanup(), and dt_dev_pixelpipe_cache_init().

Data Structures

Macros

Typedefs

Enumerations

Functions

Variables

Macro Definition Documentation

◆ _cache_print

◆ DT_PIXELPIPE_CACHE_PRESSURE_ALERT_PERIOD_US

◆ DT_PIXELPIPE_CACHE_PRESSURE_EXEMPT_SIZE

Typedef Documentation

◆ _cache_invalidate_t

◆ _cache_lru_t

◆ dt_cache_clmem_t

◆ dt_dev_pixelpipe_cache_t

◆ dt_pixel_cache_materialize_source_rank_t

Enumeration Type Documentation

◆ dt_pixel_cache_materialize_source_rank_t

Function Documentation

◆ _alert_cache_refused()

◆ _alert_user()

◆ _arena_alloc_with_defrag()

◆ _arena_stats_bytes()

◆ _cache_budget_locked()

◆ _cache_debug_module_name()

◆ _cache_entry_clmem_flush_device()

◆ _cache_entry_clmem_flush_host_pinned_locked()

◆ _cache_entry_clmem_has_host_pinned_locked()

◆ _cache_entry_for_host_ptr_locked()

◆ _cache_entry_materialize_host_data()

◆ _cache_entry_materialize_host_data_locked()

◆ _cache_full_sentence()

◆ _cache_get_newest()

◆ _cache_get_oldest()

◆ _cache_lookup_existing()

◆ _cache_sum_evictable()

◆ _cache_try_rekey_reuse_locked()

◆ _cache_try_restore_device_payload()

◆ _entry_evictable()

◆ _for_each_remove()

◆ _for_each_remove_old()

◆ _free_cache_entry()

◆ _free_space_to_alloc()

◆ _is_gamma_rgba8_output()

◆ _log_arena_allocation_failure()

◆ _memory_pressure_shedder()

◆ _non_thread_safe_cache_ref_count_entry()

◆ _non_thread_safe_cache_remove()

◆ _non_thread_safe_pixel_pipe_cache_remove_lru()

◆ _non_threadsafe_cache_get_entry()

◆ _non_threadsafe_stats_append()

◆ _observe_delete()

◆ _observe_read()

◆ _observe_rekey()

◆ _observed()

◆ _pixel_cache_clmem_get()

◆ _pixel_cache_clmem_put()

◆ _pixel_cache_clmem_remove()

◆ _pixel_cache_get_age()

◆ _pixel_cache_get_size()

◆ _pixel_cache_message()

◆ _pixel_cache_touch()

◆ _pixelpipe_cache_create_entry_locked()

◆ _pixelpipe_cache_finalize_entry()

◆ _pressure_alert_expired()

◆ _pressure_held()

◆ _pressure_react_locked()

◆ _pressure_shed()

◆ _pressure_sink()

◆ _pressure_wake()

◆ _print_cache_lines()

◆ _stats_entry_clear()

◆ _system_memory_pressure_valve()

◆ _trace_exact_hit()

◆ _warn_user()

◆ dt_dev_pixel_pipe_cache_remove_lru()

◆ dt_dev_pixelpipe_cache_alloc_cl_device_buffer()

◆ dt_dev_pixelpipe_cache_borrow_cl_payload()

◆ dt_dev_pixelpipe_cache_cleanup()

◆ dt_dev_pixelpipe_cache_flag_auto_destroy()