#include <inttypes.h>
#include <glib.h>
#include <errno.h>
#include <string.h>
#include <stdlib.h>
#include <signal.h>
#include <sys/mman.h>
#include "control/control.h"
#include "develop/pixelpipe_cache.h"
#include "common/darktable.h"
#include "common/debug.h"
#include "common/opencl.h"
#include "develop/format.h"
#include "develop/pixelpipe_hb.h"

Include dependency graph for pixelpipe_cache.c:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	dt_pixel_cache_entry_t

struct	dt_free_run_t

struct	dt_cache_clmem_t

struct	_cache_lru_t

struct	_cache_invalidate_t

Typedefs
typedef struct dt_pixel_cache_entry_t	dt_pixel_cache_entry_t

typedef struct dt_free_run_t	dt_free_run_t

typedef struct dt_cache_clmem_t	dt_cache_clmem_t

typedef struct _cache_lru_t	_cache_lru_t

typedef struct _cache_invalidate_t	_cache_invalidate_t

Functions
const char *	dt_pixelpipe_cache_set_current_module (const char *module)
	Set the current module name for cache diagnostics (thread-local).

void	_non_thread_safe_cache_ref_count_entry (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, gboolean lock, dt_pixel_cache_entry_t cache_entry)

static dt_pixel_cache_entry_t *	dt_pixel_cache_new_entry (const uint64_t hash, const size_t size, const dt_iop_buffer_dsc_t dsc, const char name, const int id, dt_dev_pixelpipe_cache_t cache, gboolean alloc)

static void	_cache_entry_clmem_flush_device (dt_pixel_cache_entry_t *entry, const int devid)

uint64_t	_non_thread_safe_cache_get_hash_data (dt_dev_pixelpipe_cache_t cache, void data, dt_pixel_cache_entry_t **entry)

dt_pixel_cache_entry_t *	_non_threadsafe_cache_get_entry (dt_dev_pixelpipe_cache_t *cache, const uint64_t hash)

dt_pixel_cache_entry_t *	dt_dev_pixelpipe_cache_get_entry (dt_dev_pixelpipe_cache_t *cache, const uint64_t hash)
	Get an internal reference to the cache entry matching hash. If you are going to access this entry more than once, keeping the reference and using it instead of hashes will prevent redundant lookups.

size_t	dt_pixel_cache_get_size (dt_pixel_cache_entry_t *cache_entry)

void	dt_pixel_cache_message (dt_pixel_cache_entry_t cache_entry, const char message, gboolean verbose)

int	_non_thread_safe_cache_remove (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, const gboolean force, dt_pixel_cache_entry_t cache_entry)

int	dt_dev_pixelpipe_cache_remove (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, const gboolean force, dt_pixel_cache_entry_t cache_entry)
	Arbitrarily remove the cache entry matching hash. Entries having a reference count > 0 (inter-thread locked) or being having their read/write lock locked will be ignored. If force is TRUE, we ignore reference count, but not locks.

void	dt_dev_pixelpipe_cache_flush_clmem (dt_dev_pixelpipe_cache_t *cache, const int devid)

void	_cache_get_oldest (gpointer key, gpointer value, gpointer user_data)

void	_print_cache_lines (gpointer key, gpointer value, gpointer user_data)

static int	_non_thread_safe_pixel_pipe_cache_remove_lru (dt_dev_pixelpipe_cache_t *cache)

int	dt_dev_pixel_pipe_cache_remove_lru (dt_dev_pixelpipe_cache_t *cache)

static size_t	align_up (size_t v, size_t a)

static gboolean	dt_cache_arena_calc (const dt_cache_arena_t a, size_t size, uint32_t out_pages, size_t *out_size)

static void *	dt_cache_arena_alloc (dt_cache_arena_t a, size_t size, size_t out_size)

static void	dt_cache_arena_free (dt_cache_arena_t a, void ptr, size_t size)

void *	dt_pixel_cache_clmem_get (dt_pixel_cache_entry_t entry, void host_ptr, int devid, int width, int height, int bpp, int flags, int *out_cst)

void	dt_pixel_cache_clmem_put (dt_pixel_cache_entry_t entry, void host_ptr, int devid, int width, int height, int bpp, int flags, int cst, void *mem)

void	dt_pixel_cache_clmem_flush (dt_pixel_cache_entry_t *entry)

void *	dt_pixel_cache_alloc (dt_dev_pixelpipe_cache_t cache, dt_pixel_cache_entry_t cache_entry)
	Actually allocate the memory buffer attached to the cache entry once you create it with `dt_dev_pixelpipe_cache_get()`. Sizes and everything are already saved in the entry, and the cache will have the needed space reserved.

static int	_free_space_to_alloc (dt_dev_pixelpipe_cache_t cache, const size_t size, const uint64_t hash, const char name)

void *	dt_pixelpipe_cache_alloc_align_cache_impl (dt_dev_pixelpipe_cache_t cache, size_t size, int id, const char name)
	Allocate aligned memory tracked by the pixelpipe cache. This allows LRU cache entries to be evicted if needed to make room.

void	dt_pixelpipe_cache_free_align_cache (dt_dev_pixelpipe_cache_t cache, void mem, const char message)
	Free aligned memory allocated with dt_pixelpipe_cache_alloc_align_cache.

static void	_free_cache_entry (dt_pixel_cache_entry_t *cache_entry)

static void	dt_cache_arena_cleanup (dt_cache_arena_t *a)

static int	dt_cache_arena_init (dt_cache_arena_t *a, size_t total_size)

dt_dev_pixelpipe_cache_t *	dt_dev_pixelpipe_cache_init (size_t max_memory)

void	dt_dev_pixelpipe_cache_cleanup (dt_dev_pixelpipe_cache_t *cache)

int	dt_dev_pixelpipe_cache_get (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, const size_t size, const char name, const int id, void data, dt_iop_buffer_dsc_t dsc, dt_pixel_cache_entry_t **entry, const gboolean alloc)
	Get a cache line from the cache.

int	dt_dev_pixelpipe_cache_get_existing (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, void data, dt_iop_buffer_dsc_t dsc, dt_pixel_cache_entry_t *entry)
	Get an existing cache line from the cache. This is similar to `dt_dev_pixelpipe_cache_get`, but it does not create a new cache line if it is not found.

gboolean	_for_each_remove (gpointer key, gpointer value, gpointer user_data)

void	dt_dev_pixelpipe_cache_flush (dt_dev_pixelpipe_cache_t *cache, const int id)
	Remove cache lines matching id. Entries locked in read/write or having reference count greater than 0 are not removed.

gboolean	_for_each_remove_old (gpointer key, gpointer value, gpointer user_data)

int	dt_dev_pixelpipe_cache_flush_old (dt_dev_pixelpipe_cache_t *cache)
	Free cache entries older than 3 min, that are not locked and have been used 3 times or less.

uint64_t	dt_dev_pixelpipe_cache_get_hash_data (dt_dev_pixelpipe_cache_t cache, void data, dt_pixel_cache_entry_t **entry)
	Find the hash of the cache entry holding the buffer data.

void	dt_dev_pixelpipe_cache_ref_count_entry (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, gboolean lock, dt_pixel_cache_entry_t cache_entry)
	Increase/Decrease the reference count on the cache line as to prevent LRU item removal. This function should be called within a read/write lock-protected section to avoid changing an entry while or after it is deleted in parallel.

void	dt_dev_pixelpipe_cache_wrlock_entry (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, gboolean lock, dt_pixel_cache_entry_t cache_entry)
	Lock or release the write lock on the entry.

void	dt_dev_pixelpipe_cache_rdlock_entry (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, gboolean lock, dt_pixel_cache_entry_t cache_entry)
	Lock or release the read lock on the entry.

void	dt_dev_pixelpipe_cache_flag_auto_destroy (dt_dev_pixelpipe_cache_t cache, uint64_t hash, dt_pixel_cache_entry_t cache_entry)
	Flag the cache entry matching hash as "auto_destroy". This is useful for short-lived/disposable cache entries, that won't be needed in the future. These will be freed out of the typical LRU, aged-based garbage collection. The thread that tagged this entry as "auto_destroy" is responsible for freeing it as soon as it is done with it, using `dt_dev_pixelpipe_cache_auto_destroy_apply()`. If not manually freed this way, the entry will be caught using the generic LRU garbage collection.

void	dt_dev_pixel_pipe_cache_auto_destroy_apply (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, dt_pixel_cache_entry_t cache_entry)

void *	dt_dev_pixelpipe_cache_get_read_only (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, dt_pixel_cache_entry_t cache_entry, dt_develop_t dev, dt_dev_pixelpipe_t *pipe)
	Find the existing cache entry linked to hash if any, lock it in read mode and increase its ref_count all at once.

void	dt_dev_pixelpipe_cache_close_read_only (dt_dev_pixelpipe_cache_t cache, const uint64_t hash, dt_pixel_cache_entry_t cache_entry)
	Decrease the ref_count and release the read lock over cache_entry all at once.

void	dt_dev_pixelpipe_cache_unref_hash (dt_dev_pixelpipe_cache_t *cache, const uint64_t hash)
	Find the entry matching hash, and decrease its ref_count if found.

void	dt_dev_pixelpipe_cache_print (dt_dev_pixelpipe_cache_t *cache)

Variables
static __thread const char *	dt_pixelpipe_cache_current_module = NULL

static int	garbage_collection = 0

Typedef Documentation

◆ _cache_invalidate_t

typedef struct _cache_invalidate_t _cache_invalidate_t

◆ _cache_lru_t

typedef struct _cache_lru_t _cache_lru_t

◆ dt_cache_clmem_t

typedef struct dt_cache_clmem_t dt_cache_clmem_t

◆ dt_free_run_t

typedef struct dt_free_run_t dt_free_run_t

◆ dt_pixel_cache_entry_t

typedef struct dt_pixel_cache_entry_t dt_pixel_cache_entry_t

Function Documentation

◆ _cache_entry_clmem_flush_device()

static void _cache_entry_clmem_flush_device	(	dt_pixel_cache_entry_t *	entry,
		const int	devid
	)

static

References dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_opencl_release_mem_object(), dt_pthread_mutex_lock(), and dt_pthread_mutex_unlock().

Referenced by dt_dev_pixelpipe_cache_flush_clmem().

◆ _cache_get_oldest()

void _cache_get_oldest	(	gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

References dt_pixel_cache_entry_t::age, _cache_lru_t::cache_entry, dt_atomic_get_int(), dt_pixel_cache_message(), dt_pthread_rwlock_trywrlock, dt_pthread_rwlock_unlock, dt_pixel_cache_entry_t::hash, _cache_lru_t::hash, dt_pixel_cache_entry_t::lock, _cache_lru_t::max_age, dt_pixel_cache_entry_t::refcount, and TRUE.

Referenced by _non_thread_safe_pixel_pipe_cache_remove_lru().

◆ _for_each_remove()

gboolean _for_each_remove	(	gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

References dt_pthread_rwlock_trywrlock, dt_pthread_rwlock_unlock, dt_pixel_cache_entry_t::id, and dt_pixel_cache_entry_t::lock.

Referenced by dt_dev_pixelpipe_cache_flush().

◆ _for_each_remove_old()

gboolean _for_each_remove_old	(	gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

References dt_pixel_cache_entry_t::age, dt_atomic_get_int(), dt_pthread_rwlock_trywrlock, dt_pthread_rwlock_unlock, dt_pixel_cache_entry_t::hits, dt_pixel_cache_entry_t::lock, and dt_pixel_cache_entry_t::refcount.

Referenced by dt_dev_pixelpipe_cache_flush_old().

◆ _free_cache_entry()

static void _free_cache_entry ( dt_pixel_cache_entry_t * cache_entry )

static

References dt_dev_pixelpipe_cache_t::arena, dt_pixel_cache_entry_t::cache, dt_pixel_cache_entry_t::cl_mem_lock, dt_dev_pixelpipe_cache_t::current_memory, dt_pixel_cache_entry_t::data, dt_cache_arena_free(), dt_pixel_cache_clmem_flush(), dt_pixel_cache_message(), dt_pthread_mutex_destroy(), dt_pthread_rwlock_destroy, FALSE, dt_pixel_cache_entry_t::lock, dt_pixel_cache_entry_t::name, and dt_pixel_cache_entry_t::size.

Referenced by dt_dev_pixelpipe_cache_init().

◆ _free_space_to_alloc()

static int _free_space_to_alloc	(	dt_dev_pixelpipe_cache_t *	cache,
		const size_t	size,
		const uint64_t	hash,
		const char *	name
	)

static

References _non_thread_safe_pixel_pipe_cache_remove_lru(), dt_dev_pixelpipe_cache_t::current_memory, dt_control_log(), dt_dev_pixelpipe_cache_t::entries, error(), dt_dev_pixelpipe_cache_t::max_memory, name, and size.

Referenced by dt_pixel_cache_new_entry(), and dt_pixelpipe_cache_alloc_align_cache_impl().

◆ _non_thread_safe_cache_get_hash_data()

uint64_t _non_thread_safe_cache_get_hash_data	(	dt_dev_pixelpipe_cache_t *	cache,
		void *	data,
		dt_pixel_cache_entry_t **	entry
	)

References dt_pixel_cache_entry_t::data, dt_dev_pixelpipe_cache_t::entries, dt_pixel_cache_entry_t::hash, dt_pixel_cache_entry_t::hits, and key.

Referenced by dt_dev_pixelpipe_cache_get_hash_data(), and dt_pixelpipe_cache_free_align_cache().

◆ _non_thread_safe_cache_ref_count_entry()

void _non_thread_safe_cache_ref_count_entry	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		gboolean	lock,
		dt_pixel_cache_entry_t *	cache_entry
	)

References _non_threadsafe_cache_get_entry(), dt_atomic_add_int(), dt_atomic_sub_int(), dt_pixel_cache_message(), dt_pixel_cache_entry_t::refcount, and TRUE.

Referenced by dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_cache_ref_count_entry(), dt_pixelpipe_cache_alloc_align_cache_impl(), and dt_pixelpipe_cache_free_align_cache().

◆ _non_thread_safe_cache_remove()

int _non_thread_safe_cache_remove	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		const gboolean	force,
		dt_pixel_cache_entry_t *	cache_entry
	)

References _non_threadsafe_cache_get_entry(), dt_atomic_get_int(), DT_DEBUG_CACHE, dt_pixel_cache_message(), dt_print(), dt_pthread_rwlock_trywrlock, dt_pthread_rwlock_unlock, dt_dev_pixelpipe_cache_t::entries, dt_pixel_cache_entry_t::hash, key, dt_pixel_cache_entry_t::lock, dt_pixel_cache_entry_t::refcount, and TRUE.

Referenced by _non_thread_safe_pixel_pipe_cache_remove_lru(), and dt_dev_pixelpipe_cache_remove().

◆ _non_thread_safe_pixel_pipe_cache_remove_lru()

static int _non_thread_safe_pixel_pipe_cache_remove_lru ( dt_dev_pixelpipe_cache_t * cache )

static

References _cache_get_oldest(), _non_thread_safe_cache_remove(), _print_cache_lines(), _cache_lru_t::cache_entry, dt_dev_pixelpipe_cache_t::current_memory, DT_DEBUG_CACHE, dt_print(), dt_dev_pixelpipe_cache_t::entries, error(), FALSE, _cache_lru_t::hash, and _cache_lru_t::max_age.

Referenced by _free_space_to_alloc(), and dt_dev_pixel_pipe_cache_remove_lru().

◆ _non_threadsafe_cache_get_entry()

dt_pixel_cache_entry_t * _non_threadsafe_cache_get_entry	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash
	)

References dt_dev_pixelpipe_cache_t::entries, dt_pixel_cache_entry_t::hits, and key.

Referenced by _non_thread_safe_cache_ref_count_entry(), _non_thread_safe_cache_remove(), dt_dev_pixel_pipe_cache_auto_destroy_apply(), dt_dev_pixelpipe_cache_flag_auto_destroy(), dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_cache_get_entry(), dt_dev_pixelpipe_cache_get_existing(), dt_dev_pixelpipe_cache_unref_hash(), and dt_pixelpipe_cache_free_align_cache().

◆ _print_cache_lines()

void _print_cache_lines	(	gpointer	key,
		gpointer	value,
		gpointer	user_data
	)

References dt_pixel_cache_message(), and FALSE.

Referenced by _non_thread_safe_pixel_pipe_cache_remove_lru().

◆ align_up()

static size_t align_up	(	size_t	v,
		size_t	a
	)

inlinestatic

◆ dt_cache_arena_alloc()

static void * dt_cache_arena_alloc	(	dt_cache_arena_t *	a,
		size_t	size,
		size_t *	out_size
	)

static

References dt_cache_arena_t::base, dt_cache_arena_calc(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_cache_arena_t::free_runs, dt_cache_arena_t::lock, dt_cache_arena_t::page_size, and size.

Referenced by dt_pixel_cache_alloc(), and dt_pixelpipe_cache_alloc_align_cache_impl().

◆ dt_cache_arena_calc()

static gboolean dt_cache_arena_calc	(	const dt_cache_arena_t *	a,
		size_t	size,
		uint32_t *	out_pages,
		size_t *	out_size
	)

inlinestatic

References dt_cache_arena_t::base, FALSE, dt_cache_arena_t::num_pages, dt_cache_arena_t::page_size, size, and TRUE.

Referenced by dt_cache_arena_alloc(), dt_cache_arena_free(), and dt_pixel_cache_new_entry().

◆ dt_cache_arena_cleanup()

static void dt_cache_arena_cleanup ( dt_cache_arena_t * a )

static

References dt_cache_arena_t::base, dt_pthread_mutex_destroy(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_cache_arena_t::free_runs, dt_cache_arena_t::lock, dt_cache_arena_t::num_pages, dt_cache_arena_t::page_size, dt_cache_arena_t::size, and TRUE.

Referenced by dt_dev_pixelpipe_cache_cleanup().

◆ dt_cache_arena_free()

static void dt_cache_arena_free	(	dt_cache_arena_t *	a,
		void *	ptr,
		size_t	size
	)

static

References dt_cache_arena_t::base, dt_cache_arena_calc(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_cache_arena_t::free_runs, dt_free_run_t::length, dt_cache_arena_t::lock, dt_cache_arena_t::num_pages, dt_cache_arena_t::page_size, size, dt_cache_arena_t::size, and dt_free_run_t::start.

Referenced by _free_cache_entry(), and dt_pixelpipe_cache_alloc_align_cache_impl().

◆ dt_cache_arena_init()

static int dt_cache_arena_init	(	dt_cache_arena_t *	a,
		size_t	total_size
	)

static

References dt_cache_arena_t::base, dt_pthread_mutex_init(), FALSE, dt_cache_arena_t::free_runs, dt_cache_arena_t::lock, dt_cache_arena_t::num_pages, dt_cache_arena_t::page_size, dt_cache_arena_t::size, and dt_free_run_t::start.

Referenced by dt_dev_pixelpipe_cache_init().

◆ dt_dev_pixel_pipe_cache_auto_destroy_apply()

void dt_dev_pixel_pipe_cache_auto_destroy_apply	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		dt_pixel_cache_entry_t *	cache_entry
	)

References _non_threadsafe_cache_get_entry(), dt_pixel_cache_entry_t::auto_destroy, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, key, and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixel_pipe_cache_remove_lru()

int dt_dev_pixel_pipe_cache_remove_lru ( dt_dev_pixelpipe_cache_t * cache )

remove the least used cache entry

Returns: 0 on success, 1 on error

References _non_thread_safe_pixel_pipe_cache_remove_lru(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), error(), and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_tiling_piece_fits_host_memory().

◆ dt_dev_pixelpipe_cache_cleanup()

void dt_dev_pixelpipe_cache_cleanup ( dt_dev_pixelpipe_cache_t * cache )

References dt_dev_pixelpipe_cache_t::arena, dt_cache_arena_cleanup(), dt_pthread_mutex_destroy(), dt_dev_pixelpipe_cache_t::entries, garbage_collection, and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_cleanup().

◆ dt_dev_pixelpipe_cache_close_read_only()

void dt_dev_pixelpipe_cache_close_read_only	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		struct dt_pixel_cache_entry_t *	cache_entry
	)

Decrease the ref_count and release the read lock over cache_entry all at once.

Parameters

cache
hash
cache_entry

References dt_dev_pixelpipe_cache_rdlock_entry(), dt_dev_pixelpipe_cache_ref_count_entry(), and FALSE.

Referenced by _get_surface(), _lib_navigation_draw_callback(), _pixelpipe_pick_from_image(), _process_histogram(), _process_vectorscope(), and _process_waveform().

◆ dt_dev_pixelpipe_cache_flag_auto_destroy()

void dt_dev_pixelpipe_cache_flag_auto_destroy	(	dt_dev_pixelpipe_cache_t *	cache,
		uint64_t	hash,
		struct dt_pixel_cache_entry_t *	entry
	)

Flag the cache entry matching hash as "auto_destroy". This is useful for short-lived/disposable cache entries, that won't be needed in the future. These will be freed out of the typical LRU, aged-based garbage collection. The thread that tagged this entry as "auto_destroy" is responsible for freeing it as soon as it is done with it, using dt_dev_pixelpipe_cache_auto_destroy_apply(). If not manually freed this way, the entry will be caught using the generic LRU garbage collection.

Parameters

cache
hash

References _non_threadsafe_cache_get_entry(), dt_pixel_cache_entry_t::auto_destroy, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::lock, and TRUE.

Referenced by dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixelpipe_cache_flush()

void dt_dev_pixelpipe_cache_flush	(	dt_dev_pixelpipe_cache_t *	cache,
		const int	id
	)

Remove cache lines matching id. Entries locked in read/write or having reference count greater than 0 are not removed.

Parameters

cache
id	ID of the pipeline owning the cache line, or -1 to remove all lines.

References _for_each_remove(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_dev_darkroom_pipeline(), and dt_dev_pixelpipe_reset_all().

◆ dt_dev_pixelpipe_cache_flush_clmem()

void dt_dev_pixelpipe_cache_flush_clmem	(	dt_dev_pixelpipe_cache_t *	cache,
		const int	devid
	)

Release cached OpenCL pinned buffers for a device (-1 for all).

References _cache_entry_clmem_flush_device(), dt_opencl_events_wait_for(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, key, and dt_dev_pixelpipe_cache_t::lock.

◆ dt_dev_pixelpipe_cache_flush_old()

int dt_dev_pixelpipe_cache_flush_old ( dt_dev_pixelpipe_cache_t * cache )

Free cache entries older than 3 min, that are not locked and have been used 3 times or less.

Parameters

cache

References _for_each_remove_old(), dt_pthread_mutex_trylock(), dt_pthread_mutex_unlock(), dt_dev_pixelpipe_cache_t::entries, and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_dev_pixelpipe_cache_init().

◆ dt_dev_pixelpipe_cache_get()

int dt_dev_pixelpipe_cache_get	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		const size_t	size,
		const char *	name,
		const int	id,
		void **	data,
		struct dt_iop_buffer_dsc_t **	dsc,
		struct dt_pixel_cache_entry_t **	entry,
		const gboolean	alloc
	)

Get a cache line from the cache.

WARNING: This internally increases the reference count, so you have to manually decrease it using dt_dev_pixelpipe_ref_count_entry() once the cache line content has been consumed or it will never be freed.

WARNING: if the cache line was newly allocated, a write lock is put on straight away. You will have to release it from the same calling thread next, to avoid dead locks.

Parameters

cache
hash	State checksum of the cache line.
size	Buffer size in bytes.
name	Name of the cache line (for debugging).
id	ID of the pipeline owning the cache line.
data	Pointer to the buffer pointer (returned).
dsc	Pointer to the buffer descriptor (returned).
alloc	Whether or not we should actually alloc the buffer, or simply reserve it. If FALSE use `dt_pixel_cache_alloc()` when you actually need the buffer.
cache_entry	a reference to the cache entry, to be reused later. Can be NULL. The caller doesn't own the data and shouldn't free it.

Returns: int 1 if the cache line was freshly allocated, 0 if it was found in the cache.

References _non_thread_safe_cache_ref_count_entry(), _non_threadsafe_cache_get_entry(), dt_pixel_cache_entry_t::age, dt_pixel_cache_entry_t::data, dt_pixel_cache_entry_t::dsc, DT_CACHELINE_BYTES, DT_DEBUG_CACHE, dt_dev_pixelpipe_cache_wrlock_entry(), dt_pixel_cache_alloc(), dt_pixel_cache_message(), dt_pixel_cache_new_entry(), dt_print(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), FALSE, dt_dev_pixelpipe_cache_t::hits, dt_dev_pixelpipe_cache_t::lock, name, dt_dev_pixelpipe_cache_t::queries, size, and TRUE.

Referenced by _init_base_buffer(), and dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixelpipe_cache_get_entry()

dt_pixel_cache_entry_t * dt_dev_pixelpipe_cache_get_entry	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash
	)

Get an internal reference to the cache entry matching hash. If you are going to access this entry more than once, keeping the reference and using it instead of hashes will prevent redundant lookups.

Parameters

cache
hash

Returns: struct dt_pixel_cache_entry_t*

References _non_threadsafe_cache_get_entry(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_dev_pixelpipe_cache_rdlock_entry(), and dt_dev_pixelpipe_cache_wrlock_entry().

◆ dt_dev_pixelpipe_cache_get_existing()

int dt_dev_pixelpipe_cache_get_existing	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		void **	data,
		struct dt_iop_buffer_dsc_t **	dsc,
		struct dt_pixel_cache_entry_t **	entry
	)

Get an existing cache line from the cache. This is similar to dt_dev_pixelpipe_cache_get, but it does not create a new cache line if it is not found.

This internally increases the reference count, so you have to manually decrease it using dt_dev_pixelpipe_ref_count_entry() once the cache line content has been consumed or it will never be freed.

Parameters

cache
hash
data
dsc

Returns: int TRUE if found, FALSE if not found.

References _non_threadsafe_cache_get_entry(), dt_pixel_cache_entry_t::age, dt_pixel_cache_entry_t::data, dt_pixel_cache_entry_t::dsc, DT_CACHELINE_BYTES, dt_pixel_cache_message(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), FALSE, dt_dev_pixelpipe_cache_t::hits, dt_dev_pixelpipe_cache_t::lock, and dt_dev_pixelpipe_cache_t::queries.

Referenced by _resync_global_histograms(), dt_dev_pixelpipe_cache_get_read_only(), dt_dev_pixelpipe_process(), dt_dev_pixelpipe_process_rec(), dt_imageio_export_with_flags(), and pixelpipe_get_histogram_backbuf().

◆ dt_dev_pixelpipe_cache_get_hash_data()

uint64_t dt_dev_pixelpipe_cache_get_hash_data	(	dt_dev_pixelpipe_cache_t *	cache,
		void *	data,
		struct dt_pixel_cache_entry_t **	entry
	)

Find the hash of the cache entry holding the buffer data.

Parameters

cache
data
cache_entry	a reference to the cache entry, to be reused later. Can be NULL. The caller doesn't own the data and shouldn't free it.

Returns: uint64_t defaults to 0 if nothing was found.

References _non_thread_safe_cache_get_hash_data(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_dev_pixelpipe_process(), and dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixelpipe_cache_get_read_only()

void * dt_dev_pixelpipe_cache_get_read_only	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		struct dt_pixel_cache_entry_t **	cache_entry,
		struct dt_develop_t *	dev,
		struct dt_dev_pixelpipe_t *	pipe
	)

Find the existing cache entry linked to hash if any, lock it in read mode and increase its ref_count all at once.

Parameters

cache
hash
cache_entry	Found cache entry if any, this is written by the function
pipe	Pixelpipe to recompute if we fail to find the cacheline associated to the hash

Returns: void* Data buffer associated with the cache entry, or NULL.

References DT_CACHELINE_BYTES, dt_dev_pixelpipe_cache_get_existing(), dt_dev_pixelpipe_cache_ref_count_entry(), dt_dev_process(), dt_pthread_rwlock_tryrdlock, and TRUE.

Referenced by _get_surface(), _lib_navigation_draw_callback(), _pixelpipe_pick_from_image(), _process_histogram(), _process_vectorscope(), and _process_waveform().

◆ dt_dev_pixelpipe_cache_init()

dt_dev_pixelpipe_cache_t * dt_dev_pixelpipe_cache_init ( size_t max_memory )

constructs a new cache with given cache line count (entries) and float buffer entry size in bytes.

Parameters

[out] returns 0 if fail to allocate mem cache.

References _free_cache_entry(), dt_dev_pixelpipe_cache_t::arena, dt_dev_pixelpipe_cache_t::current_memory, dt_cache_arena_init(), dt_dev_pixelpipe_cache_flush_old(), dt_pthread_mutex_destroy(), dt_pthread_mutex_init(), dt_dev_pixelpipe_cache_t::entries, garbage_collection, dt_dev_pixelpipe_cache_t::hits, dt_dev_pixelpipe_cache_t::lock, dt_dev_pixelpipe_cache_t::max_memory, and dt_dev_pixelpipe_cache_t::queries.

Referenced by dt_init().

◆ dt_dev_pixelpipe_cache_print()

void dt_dev_pixelpipe_cache_print ( dt_dev_pixelpipe_cache_t * cache )

print out cache lines/hashes (debug).

References dt_dev_pixelpipe_cache_t::current_memory, darktable, DT_DEBUG_CACHE, dt_print(), dt_dev_pixelpipe_cache_t::entries, dt_dev_pixelpipe_cache_t::hits, dt_dev_pixelpipe_cache_t::max_memory, dt_dev_pixelpipe_cache_t::queries, and darktable_t::unmuted.

Referenced by dt_dev_pixelpipe_process().

◆ dt_dev_pixelpipe_cache_rdlock_entry()

void dt_dev_pixelpipe_cache_rdlock_entry	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		gboolean	lock,
		struct dt_pixel_cache_entry_t *	entry
	)

Lock or release the read lock on the entry.

Parameters

cache
hash	checksum of the cache entry to fetch. Optional if `entry` is not NULL.
lock	TRUE to lock, FALSE to release
entry	The cache entry object to lock, if a reference is already known. Can be NULL, but then you need to pass a hash.

References dt_dev_pixelpipe_cache_get_entry(), dt_pixel_cache_message(), dt_pthread_rwlock_rdlock, dt_pthread_rwlock_unlock, dt_pixel_cache_entry_t::lock, and TRUE.

Referenced by _sample_gui(), dt_dev_pixelpipe_cache_close_read_only(), dt_dev_pixelpipe_process_rec(), dt_imageio_export_with_flags(), and pixelpipe_process_on_CPU().

◆ dt_dev_pixelpipe_cache_ref_count_entry()

void dt_dev_pixelpipe_cache_ref_count_entry	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		gboolean	lock,
		struct dt_pixel_cache_entry_t *	entry
	)

Increase/Decrease the reference count on the cache line as to prevent LRU item removal. This function should be called within a read/write lock-protected section to avoid changing an entry while or after it is deleted in parallel.

WARNING: cache entries whose reference count is greater than 0 will never be deleted from cache.

Parameters

cache
hash
lock	TRUE to lock, FALSE to unlock

References _non_thread_safe_cache_ref_count_entry(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_dev_pixelpipe_cache_close_read_only(), dt_dev_pixelpipe_cache_get_read_only(), dt_dev_pixelpipe_cache_unref_hash(), dt_dev_pixelpipe_process(), dt_dev_pixelpipe_process_rec(), dt_imageio_export_with_flags(), and pixelpipe_get_histogram_backbuf().

◆ dt_dev_pixelpipe_cache_remove()

int dt_dev_pixelpipe_cache_remove	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		const gboolean	force,
		struct dt_pixel_cache_entry_t *	entry
	)

Arbitrarily remove the cache entry matching hash. Entries having a reference count > 0 (inter-thread locked) or being having their read/write lock locked will be ignored. If force is TRUE, we ignore reference count, but not locks.

Parameters

cache
hash
force

References _non_thread_safe_cache_remove(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), error(), and dt_dev_pixelpipe_cache_t::lock.

Referenced by dt_dev_pixelpipe_process_rec().

◆ dt_dev_pixelpipe_cache_unref_hash()

void dt_dev_pixelpipe_cache_unref_hash	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash
	)

Find the entry matching hash, and decrease its ref_count if found.

Parameters

cache
hash

References _non_threadsafe_cache_get_entry(), dt_dev_pixelpipe_cache_ref_count_entry(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), FALSE, dt_dev_pixelpipe_cache_t::lock, and dt_dev_pixelpipe_cache_t::queries.

Referenced by dt_dev_pixelpipe_process(), and leave().

◆ dt_dev_pixelpipe_cache_wrlock_entry()

void dt_dev_pixelpipe_cache_wrlock_entry	(	dt_dev_pixelpipe_cache_t *	cache,
		const uint64_t	hash,
		gboolean	lock,
		struct dt_pixel_cache_entry_t *	entry
	)

Lock or release the write lock on the entry.

Parameters

cache
hash
lock	TRUE to lock, FALSE to release

References dt_dev_pixelpipe_cache_get_entry(), dt_pixel_cache_message(), dt_pthread_rwlock_unlock, dt_pthread_rwlock_wrlock, dt_pixel_cache_entry_t::lock, and TRUE.

Referenced by _init_base_buffer(), dt_dev_pixelpipe_cache_get(), dt_dev_pixelpipe_process_rec(), and pixelpipe_process_on_CPU().

◆ dt_pixel_cache_alloc()

void * dt_pixel_cache_alloc	(	dt_dev_pixelpipe_cache_t *	cache,
		struct dt_pixel_cache_entry_t *	entry
	)

Actually allocate the memory buffer attached to the cache entry once you create it with dt_dev_pixelpipe_cache_get(). Sizes and everything are already saved in the entry, and the cache will have the needed space reserved.

Parameters

cache
entry	the cache entry

Returns: void* Pointer to the allocated data buffer.

References dt_dev_pixelpipe_cache_t::arena, dt_pixel_cache_entry_t::data, dt_cache_arena_alloc(), and dt_pixel_cache_entry_t::size.

Referenced by dt_dev_pixelpipe_cache_get(), and dt_pixel_cache_new_entry().

◆ dt_pixel_cache_clmem_flush()

void dt_pixel_cache_clmem_flush ( dt_pixel_cache_entry_t * entry )

References dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_opencl_release_mem_object(), dt_pthread_mutex_lock(), and dt_pthread_mutex_unlock().

Referenced by _free_cache_entry().

◆ dt_pixel_cache_clmem_get()

void * dt_pixel_cache_clmem_get	(	struct dt_pixel_cache_entry_t *	entry,
		void *	host_ptr,
		int	devid,
		int	width,
		int	height,
		int	bpp,
		int	flags,
		int *	out_cst
	)

OpenCL pinned buffer reuse tied to cache entries.

References bpp, dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), flags, height, and width.

◆ dt_pixel_cache_clmem_put()

void dt_pixel_cache_clmem_put	(	dt_pixel_cache_entry_t *	entry,
		void *	host_ptr,
		int	devid,
		int	width,
		int	height,
		int	bpp,
		int	flags,
		int	cst,
		void *	mem
	)

References bpp, dt_pixel_cache_entry_t::cl_mem_list, dt_pixel_cache_entry_t::cl_mem_lock, dt_opencl_release_mem_object(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), flags, height, and width.

◆ dt_pixel_cache_get_size()

size_t dt_pixel_cache_get_size ( dt_pixel_cache_entry_t * cache_entry )

References dt_pixel_cache_entry_t::size.

Referenced by dt_pixel_cache_message().

◆ dt_pixel_cache_message()

void dt_pixel_cache_message	(	dt_pixel_cache_entry_t *	cache_entry,
		const char *	message,
		gboolean	verbose
	)

◆ dt_pixel_cache_new_entry()

static dt_pixel_cache_entry_t * dt_pixel_cache_new_entry	(	const uint64_t	hash,
		const size_t	size,
		const dt_iop_buffer_dsc_t	dsc,
		const char *	name,
		const int	id,
		dt_dev_pixelpipe_cache_t *	cache,
		gboolean	alloc
	)

static

Referenced by dt_dev_pixelpipe_cache_get(), and dt_pixelpipe_cache_alloc_align_cache_impl().

◆ dt_pixelpipe_cache_alloc_align_cache_impl()

void * dt_pixelpipe_cache_alloc_align_cache_impl	(	dt_dev_pixelpipe_cache_t *	cache,
		size_t	size,
		int	id,
		const char *	name
	)

Allocate aligned memory tracked by the pixelpipe cache. This allows LRU cache entries to be evicted if needed to make room.

Parameters

cache	Pixelpipe cache to manage.
size	Buffer size in bytes.
id	Pipeline type owning the buffer.
name	Human-readable name.

Returns: void* Pointer to the allocated buffer, or NULL on failure.

References _free_space_to_alloc(), _non_thread_safe_cache_ref_count_entry(), dt_pixel_cache_entry_t::age, dt_dev_pixelpipe_cache_t::arena, dt_pixel_cache_entry_t::data, dt_cache_arena_alloc(), dt_cache_arena_free(), DT_CACHELINE_BYTES, dt_hash(), dt_pixel_cache_new_entry(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_pthread_rwlock_wrlock, error(), dt_pixel_cache_entry_t::external_alloc, FALSE, dt_pixel_cache_entry_t::lock, dt_dev_pixelpipe_cache_t::lock, name, size, and TRUE.

Referenced by dt_pixelpipe_cache_alloc_perthread_impl().

◆ dt_pixelpipe_cache_free_align_cache()

void dt_pixelpipe_cache_free_align_cache	(	dt_dev_pixelpipe_cache_t *	cache,
		void **	mem,
		const char *	message
	)

Free aligned memory allocated with dt_pixelpipe_cache_alloc_align_cache.

Parameters

cache	Pixelpipe cache to manage.
mem	Pointer to the buffer pointer. Set to NULL on successful free.

References _non_thread_safe_cache_get_hash_data(), _non_thread_safe_cache_ref_count_entry(), _non_threadsafe_cache_get_entry(), dt_dev_pixelpipe_cache_t::arena, dt_cache_arena_t::base, dt_pixel_cache_entry_t::data, dt_atomic_get_int(), DT_DEBUG_CACHE, dt_hash(), dt_print(), dt_pthread_mutex_lock(), dt_pthread_mutex_unlock(), dt_pthread_rwlock_unlock, dt_dev_pixelpipe_cache_t::entries, dt_pixel_cache_entry_t::external_alloc, FALSE, dt_pixel_cache_entry_t::hash, key, dt_pixel_cache_entry_t::lock, dt_dev_pixelpipe_cache_t::lock, dt_pixel_cache_entry_t::name, dt_pixel_cache_entry_t::refcount, dt_pixel_cache_entry_t::size, and dt_cache_arena_t::size.

◆ dt_pixelpipe_cache_set_current_module()

const char * dt_pixelpipe_cache_set_current_module ( const char * module )

Set the current module name for cache diagnostics (thread-local).

Parameters

module Module op name or NULL to clear.

Returns: const char* Previous module name.

References dt_pixelpipe_cache_current_module.

Referenced by dt_dev_pixelpipe_process_rec().

Variable Documentation

◆ dt_pixelpipe_cache_current_module

__thread const char* dt_pixelpipe_cache_current_module = NULL

static

Referenced by dt_pixelpipe_cache_set_current_module().

◆ garbage_collection

int garbage_collection = 0

static

Referenced by dt_dev_pixelpipe_cache_cleanup(), and dt_dev_pixelpipe_cache_init().

Data Structures

Typedefs

Functions

Variables

Typedef Documentation

◆ _cache_invalidate_t

◆ _cache_lru_t

◆ dt_cache_clmem_t

◆ dt_free_run_t

◆ dt_pixel_cache_entry_t

Function Documentation

◆ _cache_entry_clmem_flush_device()

◆ _cache_get_oldest()

◆ _for_each_remove()

◆ _for_each_remove_old()

◆ _free_cache_entry()

◆ _free_space_to_alloc()

◆ _non_thread_safe_cache_get_hash_data()

◆ _non_thread_safe_cache_ref_count_entry()

◆ _non_thread_safe_cache_remove()

◆ _non_thread_safe_pixel_pipe_cache_remove_lru()

◆ _non_threadsafe_cache_get_entry()

◆ _print_cache_lines()

◆ align_up()

◆ dt_cache_arena_alloc()

◆ dt_cache_arena_calc()

◆ dt_cache_arena_cleanup()

◆ dt_cache_arena_free()

◆ dt_cache_arena_init()

◆ dt_dev_pixel_pipe_cache_auto_destroy_apply()

◆ dt_dev_pixel_pipe_cache_remove_lru()

◆ dt_dev_pixelpipe_cache_cleanup()

◆ dt_dev_pixelpipe_cache_close_read_only()

◆ dt_dev_pixelpipe_cache_flag_auto_destroy()

◆ dt_dev_pixelpipe_cache_flush()

◆ dt_dev_pixelpipe_cache_flush_clmem()

◆ dt_dev_pixelpipe_cache_flush_old()

◆ dt_dev_pixelpipe_cache_get()

◆ dt_dev_pixelpipe_cache_get_entry()

◆ dt_dev_pixelpipe_cache_get_existing()

◆ dt_dev_pixelpipe_cache_get_hash_data()

◆ dt_dev_pixelpipe_cache_get_read_only()

◆ dt_dev_pixelpipe_cache_init()

◆ dt_dev_pixelpipe_cache_print()

◆ dt_dev_pixelpipe_cache_rdlock_entry()

◆ dt_dev_pixelpipe_cache_ref_count_entry()

◆ dt_dev_pixelpipe_cache_remove()

◆ dt_dev_pixelpipe_cache_unref_hash()

◆ dt_dev_pixelpipe_cache_wrlock_entry()

◆ dt_pixel_cache_alloc()

◆ dt_pixel_cache_clmem_flush()

◆ dt_pixel_cache_clmem_get()

◆ dt_pixel_cache_clmem_put()

◆ dt_pixel_cache_get_size()

◆ dt_pixel_cache_message()

◆ dt_pixel_cache_new_entry()

◆ dt_pixelpipe_cache_alloc_align_cache_impl()

◆ dt_pixelpipe_cache_free_align_cache()

◆ dt_pixelpipe_cache_set_current_module()

Variable Documentation

◆ dt_pixelpipe_cache_current_module

◆ garbage_collection