mipmap_cache.c

Go to the documentation of this file.

/*
    This file is part of darktable,
    Copyright (C) 2011-2012, 2015 Edouard Gomez.
    Copyright (C) 2011-2012 Henrik Andersson.
    Copyright (C) 2011-2017 johannes hanika.
    Copyright (C) 2011-2012 José Carlos García Sogo.
    Copyright (C) 2012 Christian Tellefsen.
    Copyright (C) 2012, 2014 Jérémy Rosen.
    Copyright (C) 2012 Richard Wonka.
    Copyright (C) 2012-2017, 2019-2020 Tobias Ellinghaus.
    Copyright (C) 2012, 2014 Ulrich Pegelow.
    Copyright (C) 2013-2014, 2019-2021 Pascal Obry.
    Copyright (C) 2013 Simon Spannagel.
    Copyright (C) 2014, 2016 Dan Torop.
    Copyright (C) 2014-2015 parafin.
    Copyright (C) 2014-2016 Pedro Côrte-Real.
    Copyright (C) 2014-2016 Roman Lebedev.
    Copyright (C) 2015 Matthias Gehre.
    Copyright (C) 2016-2017 Peter Budai.
    Copyright (C) 2017 luzpaz.
    Copyright (C) 2019-2021 Aldric Renaudin.
    Copyright (C) 2019, 2022-2023, 2025-2026 Aurélien PIERRE.
    Copyright (C) 2019-2020 Heiko Bauke.
    Copyright (C) 2019 jakubfi.
    Copyright (C) 2019-2020 Philippe Weyland.
    Copyright (C) 2020-2022 Hanno Schwalm.
    Copyright (C) 2020-2021 Hubert Kowalski.
    Copyright (C) 2021 Ralf Brown.
    Copyright (C) 2022 Martin Bařinka.
    Copyright (C) 2022 Miloš Komarčević.
    Copyright (C) 2023 Ricky Moon.
    Copyright (C) 2024 Alynx Zhou.
    Copyright (C) 2025-2026 Guillaume Stutin.
    
    darktable is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    darktable is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with darktable.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#include "caches/mipmap_cache.h"
#include "common/paths.h"   // DT_PATH_MAX
#include "system/sys_resources.h"
#include "caches/pixelpipe_cache_alloc.h"
#include "common/file_location.h"
#include "common/grealpath.h"
#include "common/image_extensions.h"
#include "caches/image_cache.h"
#include "develop/pixelpipe_hb.h"
#include "develop/supervisor.h"
#include "imageio/imageio_core.h"
#include "imageio/imageio_jpeg.h"
#include "imageio/imageio_module.h"
#include "control/jobs.h"
#include "develop/imageop_math.h"
 
#include <assert.h>
#include <glib.h>
#include <glib/gstdio.h>
#include <inttypes.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include "database/database.h"
 
/* Set once by dt_mipmap_cache_init(); see the header. */
static gboolean _verbose = FALSE;
 
 
/* Statement-safe: a bare `if(_verbose) dt_print(...)` swallows a following `else`. */
#define _cache_print(channel, ...)                    \
  do {                                                \
    if(_verbose) dt_print((channel), __VA_ARGS__);     \
  } while(0)
 
 
/* PRIVATE. Nothing outside this file reads a field of it -- the header exposes the
 * type as an opaque handle so that it cannot. */
typedef struct dt_mipmap_cache_one_t
{
  // one cache per mipmap scale!
  dt_cache_t cache;
 
  // a few stats on usage in this run.
  // long int to give 32-bits on old archs, so __sync* calls will work.
  long int stats_requests;   // number of total requests
  long int stats_near_match; // served with smaller mip res
  long int stats_misses;     // nothing returned at all.
  long int stats_fetches;    // texture was fetched (either as a stand-in or as per request)
  long int stats_standin;    // texture used as stand-in
} dt_mipmap_cache_one_t;
 
 
/* PRIVATE. Nothing outside this file reads a field of it -- the header exposes the
 * type as an opaque handle so that it cannot. */
typedef struct dt_mipmap_cache_t
{
  // real width and height are stored per element
  // (could be smaller than the max for this mip level,
  // due to aspect ratio)
  size_t max_width[DT_MIPMAP_NONE];
  size_t max_height[DT_MIPMAP_NONE];
  // size of an element inside buf
  size_t buffer_size[DT_MIPMAP_NONE];
 
  // one cache per mipmap level
  dt_mipmap_cache_one_t mip_thumbs;
  dt_mipmap_cache_one_t mip_f;
  dt_mipmap_cache_one_t mip_full;
  char cachedir[DT_PATH_MAX]; // cached sha1sum filename for faster access
} dt_mipmap_cache_t;
 
 
#if !defined(_WIN32)
#include <sys/statvfs.h>
#else
//statvfs does not exist in Windows, providing implementation
#include "win/statvfs.h"
#endif
 
/* The cache instance, owned HERE. It used to be calloc'd by darktable.c and hung off the
 * application struct, so the accessor lived in the orchestrator and the struct was reachable
 * from anything that included darktable.h. */
static dt_mipmap_cache_t *_mipmap_cache = NULL;
 
/* The user's choices, told to us by the application. Guarded because the GUI thread can
 * replace them while worker threads are mid-decode, and four fields read one at a time can be
 * a mix of old and new -- the same torn-read hazard the colour module documents. */
static dt_mipmap_cache_settings_t _settings = { 0 };
static dt_pthread_mutex_t _settings_lock;
static gsize _settings_lock_inited = 0;
 
static inline void _settings_lock_ensure(void)
{
  if(g_once_init_enter(&_settings_lock_inited))
  {
    dt_pthread_mutex_init(&_settings_lock, NULL);
    g_once_init_leave(&_settings_lock_inited, 1);
  }
}
 
static inline dt_mipmap_cache_settings_t _settings_get(void)
{
  _settings_lock_ensure();
  dt_pthread_mutex_lock(&_settings_lock);
  const dt_mipmap_cache_settings_t s = _settings;
  dt_pthread_mutex_unlock(&_settings_lock);
  return s;
}
 
void dt_mipmap_cache_get_settings(dt_mipmap_cache_settings_t *settings)
{
  if(!IS_NULL_PTR(settings)) *settings = _settings_get();
}
 
void dt_mipmap_cache_set_settings(const dt_mipmap_cache_settings_t *settings)
{
  if(IS_NULL_PTR(settings)) return;
  _settings_lock_ensure();
  dt_pthread_mutex_lock(&_settings_lock);
  _settings = *settings;
  dt_pthread_mutex_unlock(&_settings_lock);
 
  /* The LRU quota is a live field and the limit is soft: lowering it makes the next
   * insertions evict harder rather than freeing anything now. */
  if(!IS_NULL_PTR(_mipmap_cache))
    _mipmap_cache->mip_thumbs.cache.cost_quota = settings->max_memory;
}
 
 
#define DT_MIPMAP_CACHE_FILE_MAGIC 0xD71337
#define DT_MIPMAP_CACHE_FILE_VERSION 23
#define DT_MIPMAP_CACHE_DEFAULT_FILE_NAME "mipmaps"
 
typedef enum dt_mipmap_buffer_dsc_flags
{
  DT_MIPMAP_BUFFER_DSC_FLAG_NONE = 0,
  DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE = 1 << 0,
  DT_MIPMAP_BUFFER_DSC_FLAG_INVALIDATE = 1 << 1,
  DT_MIPMAP_BUFFER_DSC_FLAG_ON_DISK = 1 << 2, // the pixels are those of the disk-cache file, as read
  /* One generation attempt has already failed for this entry. Read together with GENERATE it gives
   * the three states a failed thumbnail can be in: GENERATE|RETRIED is "failed once, try again on
   * the next request", RETRIED alone is "failed twice, this is a skull and we have stopped", and
   * neither is a thumbnail that generated normally. */
  DT_MIPMAP_BUFFER_DSC_FLAG_RETRIED = 1 << 3
} dt_mipmap_buffer_dsc_flags;
 
// the embedded Exif data to tag thumbnails as sRGB or AdobeRGB
static const uint8_t dt_mipmap_cache_exif_data_srgb[] = {
  0x45, 0x78, 0x69, 0x66, 0x00, 0x00, 0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x69,
  0x87, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
  0x01, 0xa0, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
static const uint8_t dt_mipmap_cache_exif_data_adobergb[] = {
  0x45, 0x78, 0x69, 0x66, 0x00, 0x00, 0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x69,
  0x87, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
  0x01, 0xa0, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
static const int dt_mipmap_cache_exif_data_srgb_length
                      = sizeof(dt_mipmap_cache_exif_data_srgb) / sizeof(*dt_mipmap_cache_exif_data_srgb);
static const int dt_mipmap_cache_exif_data_adobergb_length
                      = sizeof(dt_mipmap_cache_exif_data_adobergb) / sizeof(*dt_mipmap_cache_exif_data_adobergb);
 
struct dt_mipmap_buffer_dsc
{
  uint32_t width;
  uint32_t height;
  float iscale;
  size_t size;
  dt_mipmap_buffer_dsc_flags flags;
  dt_colorspaces_color_profile_type_t color_space;
  // why width and height are 0, kept so the GUI can say it over the skull. Never written to
  // the disk cache, which only ever receives real thumbnails.
  dt_imageio_retval_t status;
 
#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
  // do not touch!
  // must be the last element.
  // must be no less than 16bytes
  char redzone[16];
#endif
 
  /* NB: sizeof must be a multiple of 4*sizeof(float) */
} __attribute__((packed, aligned(DT_CACHELINE_BYTES)));
 
#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
static const size_t dt_mipmap_buffer_dsc_size __attribute__((unused))
= sizeof(struct dt_mipmap_buffer_dsc) - sizeof(((struct dt_mipmap_buffer_dsc *)0)->redzone);
#else
static const size_t dt_mipmap_buffer_dsc_size __attribute__((unused)) = sizeof(struct dt_mipmap_buffer_dsc);
#endif
 
static uint8_t * _get_buffer_from_dsc(struct dt_mipmap_buffer_dsc *dsc);
 
// The 8x8 skull a thumbnail shows when it holds no image, in DT_COLORSPACE_DISPLAY.
#define X 0xffffffffu
#define o 0u
static const uint32_t _skull_8[64] DT_ALIGNED_ARRAY
    = { o, o, o, o, o, o, o, o,
        o, o, X, X, X, X, o, o,
        o, X, o, X, X, o, X, o,
        o, X, X, X, X, X, X, o,
        o, o, X, o, o, X, o, o,
        o, o, o, o, o, o, o, o,
        o, o, X, X, X, X, o, o,
        o, o, o, o, o, o, o, o };
#undef o
#undef X
 
static inline void *dead_image_8(struct dt_mipmap_buffer_dsc *dsc)
{
  dsc->width = dsc->height = 8;
  dsc->iscale = 1.0f;
  dsc->color_space = DT_COLORSPACE_DISPLAY;
  assert(dsc->size > sizeof(_skull_8));
  memcpy(_get_buffer_from_dsc(dsc), _skull_8, sizeof(_skull_8));
  return _get_buffer_from_dsc(dsc);
}
 
static inline int32_t get_key(const int32_t imgid, const dt_mipmap_size_t size)
{
  // imgid can't be >= 2^28 (~250 million images)
  return (((int32_t)size) << 28) | (imgid - 1);
}
 
static inline uint32_t get_imgid(const uint32_t key)
{
  return (key & 0xfffffff) + 1;
}
 
static inline dt_mipmap_size_t get_size(const uint32_t key)
{
  return (dt_mipmap_size_t)(key >> 28);
}
 
void dt_mipmap_get_cache_dir(char path[DT_PATH_MAX], dt_mipmap_size_t mip)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  g_snprintf(path, sizeof(char) * DT_PATH_MAX, "%s.d" G_DIR_SEPARATOR_S "%d",
             cache->cachedir, (int)mip);
}
 
void dt_mipmap_get_cache_filename(char path[DT_PATH_MAX], dt_mipmap_size_t mip, const int32_t imgid)
{
  gchar cache_path[DT_PATH_MAX];
  dt_mipmap_get_cache_dir(cache_path, mip);
 
  gchar *file = g_strdup_printf("%u.jpg", imgid);
  dt_concat_path_file(path, cache_path, file);
  dt_free(file);
}
 
static int dt_mipmap_cache_get_filename(gchar *mipmapfilename, size_t size)
{
  int r = -1;
  char *abspath = NULL;
 
  // Directory
  char cachedir[DT_PATH_MAX] = { 0 };
  dt_loc_get_user_cache_dir(cachedir, sizeof(cachedir));
 
  // Build the mipmap filename fram hashing the path of the library DB
  const gchar *dbfilename = dt_database_get_path();
 
  if(!strcmp(dbfilename, ":memory:"))
  {
    mipmapfilename[0] = '\0';
    r = 0;
    goto exit;
  }
 
  abspath = g_realpath(dbfilename);
  if(IS_NULL_PTR(abspath)) abspath = g_strdup(dbfilename);
 
  GChecksum *chk = g_checksum_new(G_CHECKSUM_SHA1);
  g_checksum_update(chk, (guchar *)abspath, strlen(abspath));
  const gchar *filename = g_checksum_get_string(chk);
 
  if(!filename || filename[0] == '\0')
    snprintf(mipmapfilename, size, "%s/%s", cachedir, DT_MIPMAP_CACHE_DEFAULT_FILE_NAME);
  else
    snprintf(mipmapfilename, size, "%s/%s-%s", cachedir, DT_MIPMAP_CACHE_DEFAULT_FILE_NAME, filename);
 
  g_checksum_free(chk);
  r = 0;
 
exit:
  dt_free(abspath);
 
  return r;
}
 
static void _get_thumbnail_input(const int32_t imgid, char *filename, char *ext, gboolean *input_exists,
                                 gboolean *is_jpg_input, gboolean *altered, gboolean *may_be_raw)
{
  const dt_image_t *img = dt_image_cache_get(imgid, 'r');
  const dt_image_path_source_t source = dt_image_choose_input_path(img, filename, DT_PATH_MAX, FALSE);
  if(source == DT_IMAGE_PATH_NONE) filename[0] = '\0';
  *input_exists = (source != DT_IMAGE_PATH_NONE);
 
  const char *_ext = filename + strlen(filename);
  while(*_ext != '.' && _ext > filename) _ext--;
  g_strlcpy(ext, _ext, 6);
  *is_jpg_input = !strcasecmp(_ext, ".jpg") || !strcasecmp(_ext, ".jpeg");
 
  *altered = !IS_NULL_PTR(img) && img->history_items > 0;
 
  // Before the first decode, an image is neither LDR nor HDR when its extension leaves that open,
  // but the extension itself settles the question asked here: a .tif or a .heif is RGB, whether
  // LDR or HDR; only a raw extension, like .dng, may be a raw.
  *may_be_raw = IS_NULL_PTR(img) || dt_image_needs_rawprepare(img)
                || (!dt_image_is_ldr(img) && !dt_image_is_hdr(img) && dt_image_ext_is_raw(_ext + 1));
 
  if(img) dt_image_cache_read_release(img);
}
 
 
static void _init_f(dt_mipmap_buffer_t *mipmap_buf, float *buf, uint32_t *width, uint32_t *height, float *iscale,
                    const int32_t imgid);
static dt_imageio_retval_t _init_8(struct dt_mipmap_buffer_dsc *dsc, const int32_t imgid,
                                   const dt_mipmap_size_t size, const int embedded_jpg_mode,
                                   dt_atomic_int *shutdown);
 
// Gets the address of the `buf->buf` pixel buffer, from the cache entry.
static uint8_t *_get_buffer_from_dsc(struct dt_mipmap_buffer_dsc *dsc)
{
  if(dsc)
    return (uint8_t *)(dsc + 1);
  else
    return NULL;
}
 
 
struct dt_mipmap_buffer_dsc *_get_dsc_from_entry(dt_cache_entry_t *entry)
{
  return (struct dt_mipmap_buffer_dsc *)entry->data;
}
 
// update buffer params with dsc
static void _sync_dsc_to_buf(dt_mipmap_buffer_t *buf, struct dt_mipmap_buffer_dsc *dsc, const int32_t imgid,
                             const dt_mipmap_size_t mip)
{
  buf->width = dsc->width;
  buf->height = dsc->height;
  buf->iscale = dsc->iscale;
  buf->color_space = dsc->color_space;
  buf->status = dsc->status;
  buf->imgid = imgid;
  buf->size = mip;
  buf->buf = _get_buffer_from_dsc(dsc);
}
 
 
// flag a buffer as invalid and set it NULL
// This doesn't paint skulls
static void _invalidate_buffer(dt_mipmap_buffer_t *buf)
{
  buf->width = buf->height = 0;
  buf->iscale = 0.0f;
  buf->buf = NULL;
  buf->status = DT_IMAGEIO_OK;
}
 
 
static size_t _get_entry_size(const size_t buffer_size)
{
  return buffer_size + dt_mipmap_buffer_dsc_size;
}
 
void dt_mipmap_cache_update_buffer_addresses(dt_cache_entry_t *entry, struct dt_mipmap_buffer_dsc **dsc,
                                             const size_t width, const size_t height,
                                             const size_t buffer_size)
{
  if(IS_NULL_PTR(entry->data))
  {
    entry->data_size = 0;
    *dsc = NULL;
    return;
  }
 
  // Update the entry datasize
  entry->data_size = _get_entry_size(buffer_size);
 
  // Update the dsc parameters
  *dsc = _get_dsc_from_entry(entry);
  (*dsc)->width = width;
  (*dsc)->height = height;
  (*dsc)->iscale = 1.0f;
  (*dsc)->color_space = DT_COLORSPACE_NONE;
  (*dsc)->status = DT_IMAGEIO_OK;
  (*dsc)->flags = DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE;
  (*dsc)->size = _get_entry_size(buffer_size);
}
 
 
// callback for the imageio core to allocate memory.
// only needed for _FULL buffers, as they change size
// with the input image. will allocate img->width*img->height*img->bpp bytes.
void *dt_mipmap_cache_alloc(dt_mipmap_buffer_t *buf, const dt_image_t *img)
{
  if(IS_NULL_PTR(buf)) return NULL;
 
  if(buf->size != DT_MIPMAP_FULL)
  {
    fprintf(stderr, "trying to alloc a wrong mipmap size for %s: %i (should be: %i)\n", img->filename, buf->size, DT_MIPMAP_FULL);
    return NULL;
  }
 
  dt_cache_entry_t *entry = buf->cache_entry;
 
  if(IS_NULL_PTR(entry))
  {
    fprintf(stderr, "trying to alloc a buffer entry that has no back-reference to cache entry\n");
    return NULL;
  }
 
  /* Free and reset everything. ASan poisoning applies to live cache payloads only: after freeing
   * the old allocation there is no valid user region left, and poisoning a NULL reset pointer makes
   * the ASan runtime abort before it can report the real caller context. */
  if(!IS_NULL_PTR(entry->data))
    ASAN_POISON_MEMORY_REGION(entry->data, entry->data_size);
  dt_free_align(entry->data);
  entry->data = NULL;
 
  // Get a new allocation
  const int wd = img->width;
  const int ht = img->height;
  const size_t bpp = img->dsc.bpp;
  const size_t buffer_size = wd * ht * bpp;
  const size_t min_buffer_size = 64 * 4 * sizeof(float);
  entry->data = dt_alloc_align(_get_entry_size(MAX(buffer_size, min_buffer_size)));
  if(IS_NULL_PTR(entry->data))
  {
    /* The old payload is already gone, and every reader of a cache entry takes entry->data for a
     * valid descriptor: _generate_blocking() writes dsc->status as soon as the loader returns
     * DT_IMAGEIO_CACHE_FULL, so leaving NULL here crashes there. Fall back to the empty 0x0
     * payload a full-size entry is created with (dt_mipmap_cache_allocate_dynamic()), so the
     * load fails instead. */
    entry->data = dt_alloc_align(_get_entry_size(min_buffer_size));
    struct dt_mipmap_buffer_dsc *empty = NULL;
    dt_mipmap_cache_update_buffer_addresses(entry, &empty, 0, 0, min_buffer_size);
    return NULL;
  }
 
  // Update the references
  struct dt_mipmap_buffer_dsc *dsc = NULL;
  dt_mipmap_cache_update_buffer_addresses(entry, &dsc, wd, ht, buffer_size);
 
  // Unpoison the pixel buffer region
  ASAN_UNPOISON_MEMORY_REGION(buf->buf, dsc->size - dt_mipmap_buffer_dsc_size);
 
  // Unpoison the dsc region
  ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
 
  // So the padding region between dsc and buf->buf should be poisoned still
  assert(entry->data == dsc);
 
  if(dsc)
  {
    assert(entry->data_size == dsc->size);
  }
 
  // return pointer to start of payload
  return _get_buffer_from_dsc(dsc);
}
 
// callback for the cache backend to initialize payload pointers
// It's actually not dynamic at all, fixed size only.
void dt_mipmap_cache_allocate_dynamic(void *data, dt_cache_entry_t *entry)
{
  dt_mipmap_cache_t *cache = (dt_mipmap_cache_t *)data;
  const dt_mipmap_size_t mip = get_size(entry->key);
  const int32_t imgid = get_imgid(entry->key);
 
  assert(mip < DT_MIPMAP_NONE);
 
  // Free and reset everything
  dt_free_align(entry->data);
  entry->data = NULL;
 
  // Get a new allocation
  const size_t buffer_size = (mip <= DT_MIPMAP_F) ? cache->buffer_size[mip] : _get_entry_size(sizeof(float) * 4 * 64);
  entry->data = dt_alloc_align(buffer_size);
  if(IS_NULL_PTR(entry->data)) return;
 
  // Update the references
  struct dt_mipmap_buffer_dsc *dsc = NULL;
  if(mip <= DT_MIPMAP_F)
    dt_mipmap_cache_update_buffer_addresses(entry, &dsc, cache->max_width[mip], cache->max_height[mip], buffer_size);
  else
    dt_mipmap_cache_update_buffer_addresses(entry, &dsc, 0, 0, buffer_size);
 
  if(IS_NULL_PTR(dsc)) return;
 
  // Register the mipmap object, and the image cache object it links to. The image
  // is usually already cached (so its allocate callback never fired), hence we
  // register it here from the live dt_image_t; dt_supervisor_image() promotes the
  // first sighting to a `create` so the link resolves.
  if(dt_supervisor_active())
  {
    const dt_image_t *img = dt_image_cache_get(imgid, 'r');
    if(img) dt_supervisor_image(DT_SV_UPDATE, imgid, img);
    dt_supervisor_mipmap(DT_SV_CREATE, imgid, mip);
    if(img) dt_image_cache_read_release(img);
  }
 
  if(cache->cachedir[0] && _settings_get().disk_backend && mip < DT_MIPMAP_F)
  {
    // try and load from disk, if successful set flag
    char filename[DT_PATH_MAX] = {0};
    dt_mipmap_get_cache_filename(filename, mip, get_imgid(entry->key));
 
    gboolean io_error = FALSE;
    gchar *error = NULL;
    uint8_t *blob = NULL;
    dt_colorspaces_color_profile_type_t color_space = DT_COLORSPACE_DISPLAY;
    dt_imageio_jpeg_t jpg;
    FILE *f = NULL;
 
    f = g_fopen(filename, "rb");
    if(IS_NULL_PTR(f))
    {
      _cache_print(DT_DEBUG_CACHE,
               "[mipmap_cache] cached file for image %d at mip size %i does not exist\n",
               imgid, mip);
      goto finish; // file doesn't exist
    }
 
    fseek(f, 0, SEEK_END);
    long len = ftell(f);
    if(len <= 0)
    {
      error = "empty file";
      io_error = TRUE;
      goto finish;
    }
 
    blob = (uint8_t *)dt_alloc_align(len);
    if(IS_NULL_PTR(blob))
    {
      error = "out of memory";
      io_error = TRUE;
      goto finish;
    }
 
    fseek(f, 0, SEEK_SET);
    const size_t rd = fread(blob, sizeof(uint8_t), len, f);
    if(rd != len)
    {
      error = "corrupted file";
      io_error = TRUE;
      goto finish;
    }
 
    if(dt_imageio_jpeg_decompress_header(blob, len, &jpg))
    {
      error = "couldn't decompress header";
      io_error = TRUE;
      goto finish;
    }
 
    color_space = dt_imageio_jpeg_read_color_space(&jpg);
 
    if(dt_imageio_jpeg_decompress(&jpg, _get_buffer_from_dsc(dsc)))
    {
      error = "couldn't decompress JPEG";
      io_error = TRUE;
      goto finish;
    }
 
    dsc->width = jpg.width;
    dsc->height = jpg.height;
    dsc->iscale = 1.0f;
    dsc->color_space = color_space;
    dsc->flags = DT_MIPMAP_BUFFER_DSC_FLAG_ON_DISK;
 
    _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d at mip size %d (%ix%i) loaded from disk cache\n",
              imgid, mip, jpg.width, jpg.height);
 
finish:
    dt_free_align(blob);
 
    // Closed before the unlink: Windows cannot delete a file the CRT still holds open.
    if(f)
    {
      fclose(f);
      f = NULL;
    }
 
    if(io_error)
    {
      // Delete the file, we will regenerate it
      g_unlink(filename);
      fprintf(stderr, "[mipmap_cache] failed to open thumbnail for image %" PRIu32 " from `%s'. Reason: %s\n",
              imgid, filename, error);
    }
  }
 
  // cost is just flat one for the buffer, as the buffers might have different sizes,
  // to make sure quota is meaningful.
  if(mip >= DT_MIPMAP_F)
    entry->cost = 1;
  else
    entry->cost = cache->buffer_size[mip];
}
 
static void dt_mipmap_cache_unlink_ondisk_thumbnail(void *data, int32_t imgid, dt_mipmap_size_t mip)
{
  dt_mipmap_cache_t *cache = (dt_mipmap_cache_t *)data;
 
  // also remove jpg backing (always try to do that, in case user just temporarily switched it off,
  // to avoid inconsistencies.
  // if(_settings_get().disk_backend)
  if(cache->cachedir[0])
  {
    char filename[DT_PATH_MAX] = { 0 };
    dt_mipmap_get_cache_filename(filename, mip, imgid);
    g_unlink(filename);
    _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] image %i for size %i was deleted from disk cache\n", imgid, mip);
  }
}
 
static gboolean _disk_has_room_for_a_thumbnail(const char *dirname)
{
  struct statvfs vfsbuf;
  if(statvfs(dirname, &vfsbuf)) return TRUE;
 
  const int64_t free_mb = (((int64_t)vfsbuf.f_frsize * (int64_t)vfsbuf.f_bavail) >> 20);
  if(free_mb < 100)
  {
    fprintf(stderr, "[mipmap_cache] not writing to the disk cache: only %" PRId64 " MB free on `%s'\n",
            free_mb, dirname);
    return FALSE;
  }
  return TRUE;
}
 
static void _write_thumbnail_to_disk_cache(struct dt_mipmap_buffer_dsc *dsc, const int32_t imgid,
                                           const dt_mipmap_size_t mip)
{
  gchar cache_path[DT_PATH_MAX] = { 0 };
  dt_mipmap_get_cache_dir(cache_path, mip);
  if(g_mkdir_with_parents(cache_path, 0750)) return;
 
  if(!_disk_has_room_for_a_thumbnail(cache_path)) return;
 
  char filename[DT_PATH_MAX] = { 0 };
  dt_mipmap_get_cache_filename(filename, mip, imgid);
 
  const uint8_t *exif = NULL;
  int exif_len = 0;
  if(dsc->color_space == DT_COLORSPACE_SRGB)
  {
    exif = dt_mipmap_cache_exif_data_srgb;
    exif_len = dt_mipmap_cache_exif_data_srgb_length;
  }
  else if(dsc->color_space == DT_COLORSPACE_ADOBERGB)
  {
    exif = dt_mipmap_cache_exif_data_adobergb;
    exif_len = dt_mipmap_cache_exif_data_adobergb_length;
  }
 
  // dt_imageio_jpeg_write() opens and closes the file itself: no other handle on it, or Windows
  // cannot unlink it below.
  const int cache_quality = _settings_get().cache_quality;
  if(dt_imageio_jpeg_write(filename, _get_buffer_from_dsc(dsc), dsc->width, dsc->height,
                           MIN(100, MAX(10, cache_quality)), exif, exif_len))
  {
    g_unlink(filename);
    fprintf(stderr, "[mipmap_cache] failed to write image %" PRId32 " at size %d to the disk cache at `%s'\n",
            imgid, mip, filename);
    return;
  }
 
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] image %" PRId32 " for size %d was written to cache at %s\n", imgid,
               mip, filename);
}
 
void dt_mipmap_cache_deallocate_dynamic(void *data, dt_cache_entry_t *entry)
{
  dt_mipmap_cache_t *cache = (dt_mipmap_cache_t *)data;
  const dt_mipmap_size_t mip = get_size(entry->key);
 
  if(dt_supervisor_active())
    dt_supervisor_mipmap(DT_SV_DELETE, get_imgid(entry->key), (int)mip);
 
  if(mip < DT_MIPMAP_F)
  {
    const int32_t imgid = get_imgid(entry->key);
    struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
 
    if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_INVALIDATE)
    {
      // The pixels no longer describe the image: the file goes, and nothing may replace it.
      dt_mipmap_cache_unlink_ondisk_thumbnail(data, imgid, mip);
    }
    // Not an entry read from its file and unchanged since: writing it would only re-encode it, at a loss.
    else if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_ON_DISK)
      _cache_print(DT_DEBUG_CACHE,
                   "[mipmap_cache] image %" PRId32 " at mip size %d is not written to the disk cache: read from it\n",
                   imgid, mip);
    // Neither a render that never ran (GENERATE, over an uninitialised buffer) nor a skull (8x8).
    else if(!(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE) && dsc->width > 8 && dsc->height > 8)
    {
      if(cache->cachedir[0] && _settings_get().disk_backend) _write_thumbnail_to_disk_cache(dsc, imgid, mip);
    }
    else
    {
      // Logged, so that a real session says how often an aborted render leaves an entry behind.
      _cache_print(DT_DEBUG_CACHE,
                   "[mipmap_cache] image %" PRId32 " at mip size %d is not written to the disk cache: %s\n",
                   imgid, mip,
                   (dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE) ? "never generated" : "a skull");
    }
  }
 
  dt_free_align(entry->data);
  entry->data = NULL;
}
 
void dt_mipmap_cache_init(const dt_mipmap_cache_settings_t *settings, const gboolean verbose)
{
  _verbose = verbose;
  if(!IS_NULL_PTR(settings)) dt_mipmap_cache_set_settings(settings);
  if(_mipmap_cache) return;
  _mipmap_cache = (dt_mipmap_cache_t *)calloc(1, sizeof(dt_mipmap_cache_t));
  if(IS_NULL_PTR(_mipmap_cache)) return;
  dt_mipmap_cache_t *cache = _mipmap_cache;
  dt_mipmap_cache_get_filename(cache->cachedir, sizeof(cache->cachedir));
 
  // Fixed sizes for the thumbnail mip levels, selected for coverage of most screen sizes
  // Starting at 4K, we use 3:2 ratio of camera sensor instead of 16:9/16:10 of display,
  // in order to start caching full-resolution JPEG for 100% zoom in lighttable
  size_t mipsizes[DT_MIPMAP_F][2] = {
    { 360, 225 },             // mip0 - 1/2 size previous one
    { 720, 450 },             // mip1 - 1/2 size previous one
    { 1440, 900 },            // mip2 - covers HD, WXGA+
    { 1920, 1200 },           // mip3 - covers 1080p and 1600x1200
    { 2560, 1600 },           // mip4 - covers 2560x1440
    { 3840, 2560 },           // mip5 - covers 4K and UHD
    { 5120, 3414 },           // mip6 - covers 5K
    { 6144, 4096 },           // mip7 - covers 6K
    { 7680, 5120 },           // mip8 - covers 8K
  };
  // Set mipf for preview pipe to 1440x900
  cache->max_width[DT_MIPMAP_F] = mipsizes[DT_MIPMAP_2][0];
  cache->max_height[DT_MIPMAP_F] = mipsizes[DT_MIPMAP_2][1];
  for(int k = DT_MIPMAP_F-1; k >= 0; k--)
  {
    cache->max_width[k]  = mipsizes[k][0];
    cache->max_height[k] = mipsizes[k][1];
  }
    // header + buffer
  for(int k = DT_MIPMAP_F-1; k >= 0; k--)
    cache->buffer_size[k] = _get_entry_size(cache->max_width[k] * cache->max_height[k] * 4);
 
  // clear stats:
  cache->mip_thumbs.stats_requests = 0;
  cache->mip_thumbs.stats_near_match = 0;
  cache->mip_thumbs.stats_misses = 0;
  cache->mip_thumbs.stats_fetches = 0;
  cache->mip_thumbs.stats_standin = 0;
  cache->mip_f.stats_requests = 0;
  cache->mip_f.stats_near_match = 0;
  cache->mip_f.stats_misses = 0;
  cache->mip_f.stats_fetches = 0;
  cache->mip_f.stats_standin = 0;
  cache->mip_full.stats_requests = 0;
  cache->mip_full.stats_near_match = 0;
  cache->mip_full.stats_misses = 0;
  cache->mip_full.stats_fetches = 0;
  cache->mip_full.stats_standin = 0;
 
  dt_cache_init(&cache->mip_thumbs.cache, 0, _settings_get().max_memory);
  dt_cache_set_allocate_callback(&cache->mip_thumbs.cache, dt_mipmap_cache_allocate_dynamic, cache);
  dt_cache_set_cleanup_callback(&cache->mip_thumbs.cache, dt_mipmap_cache_deallocate_dynamic, cache);
 
  dt_cache_init(&cache->mip_full.cache, 0, dt_worker_threads() + DT_CTL_WORKER_RESERVED);
  dt_cache_set_allocate_callback(&cache->mip_full.cache, dt_mipmap_cache_allocate_dynamic, cache);
  dt_cache_set_cleanup_callback(&cache->mip_full.cache, dt_mipmap_cache_deallocate_dynamic, cache);
  cache->buffer_size[DT_MIPMAP_FULL] = 0;
 
  dt_cache_init(&cache->mip_f.cache, 0, dt_worker_threads() + DT_CTL_WORKER_RESERVED);
  dt_cache_set_allocate_callback(&cache->mip_f.cache, dt_mipmap_cache_allocate_dynamic, cache);
  dt_cache_set_cleanup_callback(&cache->mip_f.cache, dt_mipmap_cache_deallocate_dynamic, cache);
  cache->buffer_size[DT_MIPMAP_F]
      = _get_entry_size(4 * sizeof(float) * cache->max_width[DT_MIPMAP_F] * cache->max_height[DT_MIPMAP_F]);
}
 
void dt_mipmap_cache_flush_to_disk(void)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  if(IS_NULL_PTR(cache)) return;
 
  /* A fill ratio of 0 evicts every entry no other thread holds. The eviction walk is reused because
   * it already skips held entries and runs the cleanup callback, which writes the thumbnail. Only
   * mip_thumbs: DT_MIPMAP_F and DT_MIPMAP_FULL are never written to disk. */
  const size_t reclaimed = dt_cache_gc(&cache->mip_thumbs.cache, 0.0f);
 
  // What left RAM, not what was written: an entry read from its file is dropped unwritten. The
  // callback logs each write on its own.
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] flushed to disk: %.2f MB evicted from RAM\n",
               reclaimed / (1024.0 * 1024.0));
}
 
void dt_mipmap_cache_cleanup(void)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  if(IS_NULL_PTR(cache)) return;
  dt_mipmap_cache_print();
  dt_cache_cleanup(&cache->mip_thumbs.cache);
  dt_cache_cleanup(&cache->mip_full.cache);
  dt_cache_cleanup(&cache->mip_f.cache);
 
  dt_free(_mipmap_cache);
  _mipmap_cache = NULL;
}
 
void dt_mipmap_cache_print(void)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] thumbs fill %.2f/%.2f MB (%.2f%%)\n",
         cache->mip_thumbs.cache.cost / (1024.0 * 1024.0),
         cache->mip_thumbs.cache.cost_quota / (1024.0 * 1024.0),
         100.0f * (float)cache->mip_thumbs.cache.cost / (float)cache->mip_thumbs.cache.cost_quota);
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] float fill %"PRIu32"/%"PRIu32" slots (%.2f%%)\n",
         (uint32_t)cache->mip_f.cache.cost, (uint32_t)cache->mip_f.cache.cost_quota,
         100.0f * (float)cache->mip_f.cache.cost / (float)cache->mip_f.cache.cost_quota);
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] full  fill %"PRIu32"/%"PRIu32" slots (%.2f%%)\n",
         (uint32_t)cache->mip_full.cache.cost, (uint32_t)cache->mip_full.cache.cost_quota,
         100.0f * (float)cache->mip_full.cache.cost / (float)cache->mip_full.cache.cost_quota);
 
  uint64_t sum = 0;
  uint64_t sum_fetches = 0;
  uint64_t sum_standins = 0;
  sum += cache->mip_thumbs.stats_requests;
  sum_fetches += cache->mip_thumbs.stats_fetches;
  sum_standins += cache->mip_thumbs.stats_standin;
  sum += cache->mip_f.stats_requests;
  sum_fetches += cache->mip_f.stats_fetches;
  sum_standins += cache->mip_f.stats_standin;
  sum += cache->mip_full.stats_requests;
  sum_fetches += cache->mip_full.stats_fetches;
  sum_standins += cache->mip_full.stats_standin;
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] level | near match | miss | stand-in | fetches | total rq\n");
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] thumb | %6.2f%% | %6.2f%% | %6.2f%%  | %6.2f%% | %6.2f%%\n",
         100.0 * cache->mip_thumbs.stats_near_match / (float)cache->mip_thumbs.stats_requests,
         100.0 * cache->mip_thumbs.stats_misses / (float)cache->mip_thumbs.stats_requests,
         100.0 * cache->mip_thumbs.stats_standin / (float)sum_standins,
         100.0 * cache->mip_thumbs.stats_fetches / (float)sum_fetches,
         100.0 * cache->mip_thumbs.stats_requests / (float)sum);
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] float | %6.2f%% | %6.2f%% | %6.2f%%  | %6.2f%% | %6.2f%%\n",
         100.0 * cache->mip_f.stats_near_match / (float)cache->mip_f.stats_requests,
         100.0 * cache->mip_f.stats_misses / (float)cache->mip_f.stats_requests,
         100.0 * cache->mip_f.stats_standin / (float)sum_standins,
         100.0 * cache->mip_f.stats_fetches / (float)sum_fetches,
         100.0 * cache->mip_f.stats_requests / (float)sum);
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] full  | %6.2f%% | %6.2f%% | %6.2f%%  | %6.2f%% | %6.2f%%\n",
         100.0 * cache->mip_full.stats_near_match / (float)cache->mip_full.stats_requests,
         100.0 * cache->mip_full.stats_misses / (float)cache->mip_full.stats_requests,
         100.0 * cache->mip_full.stats_standin / (float)sum_standins,
         100.0 * cache->mip_full.stats_fetches / (float)sum_fetches,
         100.0 * cache->mip_full.stats_requests / (float)sum);
  _cache_print(DT_DEBUG_CACHE, "\n\n");
}
 
// Collect one sub-cache's live entries (imgid + mip + real bytes) under its lock.
static void _mipmap_subcache_collect(dt_cache_t *c, GArray *out)
{
  dt_pthread_mutex_lock(&c->lock);
  GHashTableIter it;
  gpointer key, value;
  g_hash_table_iter_init(&it, c->hashtable);
  while(g_hash_table_iter_next(&it, &key, &value))
  {
    const dt_cache_entry_t *e = (const dt_cache_entry_t *)value;
    if(!e) continue;
    dt_mipmap_cache_stats_entry_t s = { 0 };
    s.imgid = get_imgid(e->key);
    s.mip = (int)get_size(e->key);
    s.size = e->data_size; // real allocation (cost is byte-based for thumbs, slot-based for f/full)
    g_array_append_val(out, s);
  }
  dt_pthread_mutex_unlock(&c->lock);
}
 
void dt_mipmap_cache_get_usage(size_t *current, size_t *max)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  if(current) *current = 0;
  if(max) *max = 0;
  if(IS_NULL_PTR(cache)) return;
 
  // Real bytes currently held = sum of entry allocations across sub-caches.
  GArray *entries = dt_mipmap_cache_get_entries_stats();
  size_t used = 0;
  for(guint i = 0; i < entries->len; i++)
    used += g_array_index(entries, dt_mipmap_cache_stats_entry_t, i).size;
  g_array_free(entries, TRUE);
 
  if(current) *current = used;
  if(max) *max = _settings_get().max_memory; // user-specified mipmap RAM budget
}
 
GArray *dt_mipmap_cache_get_entries_stats(void)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  GArray *out = g_array_new(FALSE, FALSE, sizeof(dt_mipmap_cache_stats_entry_t));
  if(IS_NULL_PTR(cache)) return out;
  _mipmap_subcache_collect(&cache->mip_thumbs.cache, out);
  _mipmap_subcache_collect(&cache->mip_f.cache, out);
  _mipmap_subcache_collect(&cache->mip_full.cache, out);
  return out;
}
 
static dt_mipmap_cache_one_t *_get_cache(dt_mipmap_cache_t *cache, const dt_mipmap_size_t mip)
{
  switch(mip)
  {
    case DT_MIPMAP_FULL:
      return &cache->mip_full;
    case DT_MIPMAP_F:
      return &cache->mip_f;
    default:
      return &cache->mip_thumbs;
  }
}
 
// if we get a zero-sized image, paint skulls to signal a missing image
static void _paint_skulls(dt_mipmap_buffer_t *buf, struct dt_mipmap_buffer_dsc *dsc, const int32_t imgid, const dt_mipmap_size_t mip)
{
  if(dsc->width == 0 || dsc->height == 0)
  {
    _cache_print(DT_DEBUG_CACHE, "[mipmap cache get] got a zero-sized image for img %u mip %d!\n", imgid, mip);
    if(mip < DT_MIPMAP_F)
      buf->buf = dead_image_8(dsc);
    else
      buf->buf = NULL; // full images with NULL buffer have to be handled, indicates `missing image', but still return locked slot
  }
}
 
static void _validate_buffer(dt_mipmap_buffer_t *buf, struct dt_mipmap_buffer_dsc *dsc, const int32_t imgid,
                             const dt_mipmap_size_t mip)
{
  // Unpoison the dsc region
  ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
 
  // May update buf->buf:
  _sync_dsc_to_buf(buf, dsc, imgid, mip);
 
  // Unpoison the pixel buffer region
  ASAN_UNPOISON_MEMORY_REGION(buf->buf, dsc->size - dt_mipmap_buffer_dsc_size);
}
 
// Grab our own local copy of the image structure
static gboolean _get_image_copy(const int32_t imgid, dt_image_t *buffered_image)
{
  // load the image:
  // make sure we access the r/w lock as shortly as possible!
  gboolean no_buffer = TRUE;
  const dt_image_t *cimg = dt_image_cache_get(imgid, 'r');
 
  if(cimg)
  {
    *buffered_image = *cimg;
    no_buffer = FALSE;
  }
 
  dt_image_cache_read_release(cimg);
 
  return no_buffer;
}
 
// Actually do the work: produce an image
static void _generate_blocking(dt_cache_entry_t *entry, dt_mipmap_buffer_t *buf,
                               const int32_t imgid, const dt_mipmap_size_t mip, dt_atomic_int *shutdown)
{
  struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
  // Unpoison the descriptor before reading any field in this function.
  ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
  // _generate_blocking() can write directly into the cache pixel payload
  // (notably through _init_8() -> _write_image()), so unpoison it up-front.
  ASAN_UNPOISON_MEMORY_REGION(_get_buffer_from_dsc(dsc), dsc->size - dt_mipmap_buffer_dsc_size);
  const uint32_t original_width = dsc->width;
  const uint32_t original_height = dsc->height;
  const float original_iscale = dsc->iscale;
  const dt_colorspaces_color_profile_type_t original_color_space = dsc->color_space;
  const dt_imageio_retval_t original_status = dsc->status;
  gboolean mode_changed = FALSE;
 
  /* Retrying after a failure. The failed attempt left the skull's 8x8 in the descriptor, and
   * _init_8() opens with `if(size >= DT_MIPMAP_F || dsc->width < 16 || dsc->height < 16) return
   * DT_IMAGEIO_OK;` -- so re-entering with those dimensions does nothing at all, silently, and the
   * skull would be kept as a thumbnail that generated. Put back what the allocator asked for
   * (dt_mipmap_cache_update_buffer_addresses() sets exactly these). */
  if((dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE)
     && (dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_RETRIED)
     && mip <= DT_MIPMAP_F
     && !IS_NULL_PTR(_mipmap_cache))
  {
    dsc->width = _mipmap_cache->max_width[mip];
    dsc->height = _mipmap_cache->max_height[mip];
    dsc->iscale = 1.0f;
    dsc->color_space = DT_COLORSPACE_NONE;
  }
 
  if(!(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE))
  {
    // Already in cache, no I/O needed
    _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d at mip size %i (%ix%i) will skip disk I/O, found in RAM cache.\n", imgid, mip,
             dsc->width, dsc->height);
    _validate_buffer(buf, dsc, imgid, mip);
    return;
  }
 
  if(mip == DT_MIPMAP_FULL)
  {
    // Get the input file path, possibly from our local cache of copied images
    dt_image_t buffered_image;
    char filename[DT_PATH_MAX] = { 0 };
    /* Nothing to read: tell the caller why, but leave the entry flagged for generation, since a
     * file missing now (unplugged drive) may be back by the next request. */
    if(_get_image_copy(imgid, &buffered_image)
       || dt_image_choose_input_path(&buffered_image, filename, sizeof(filename), FALSE) == DT_IMAGE_PATH_NONE)
    {
      _invalidate_buffer(buf);
      buf->status = DT_IMAGEIO_FILE_NOT_FOUND;
      return;
    }
 
    _cache_print(DT_DEBUG_CACHE,
      "[mipmap_cache] fetch image %i at mip size %d float32 (%ix%i) from original file I/O\n",
      imgid, mip, dsc->width, dsc->height);
 
    /* dt_mipmap_cache_alloc() resets the flags to GENERATE alone, which would forget that this is
     * already the retry: a file that never loads would then be read again on every request. */
    const dt_mipmap_buffer_dsc_flags retried = dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_RETRIED;
 
    // will call dt_mipmap_cache_alloc() internally:
    dt_imageio_retval_t ret = dt_imageio_open(&buffered_image, filename, buf);
 
    // We need to update dsc because dt_imageio_open re-alloc entry->data
    dsc = _get_dsc_from_entry(entry);
    dsc->status = ret;
    dsc->flags |= retried;
 
    if(ret == DT_IMAGEIO_OK)
    {
      // swap back new image data, may contain updated EXIF & colorspace
      dt_image_t *img = dt_image_cache_get(imgid, 'w');
      // dt_image_cache_get() returns NULL if the image vanished from the DB/cache while this
      // full-res I/O was in flight (e.g. removed from the library concurrently) -- nothing left
      // to swap the freshly-read data back into.
      if(!IS_NULL_PTR(img))
      {
        *img = buffered_image;
        dt_image_cache_write_release(img, DT_IMAGE_CACHE_RELAXED);
      }
    }
    else
    {
      dsc->width = dsc->height = 0;
      dsc->iscale = 0.f;
    }
  }
  else if(mip == DT_MIPMAP_F)
  {
    _cache_print(DT_DEBUG_CACHE,
             "[mipmap_cache] compute mip size %d float32 for image %i (%ix%i) from original file \n", mip,
             imgid, dsc->width, dsc->height);
    _init_f(buf, (float *)_get_buffer_from_dsc(dsc), &dsc->width, &dsc->height, &dsc->iscale, imgid);
  }
  else
  {
    // 8-bit thumbs
    _cache_print(DT_DEBUG_CACHE,
             "[mipmap_cache] compute mip size %d uint8 for image %i (%ix%i) from original file \n", mip,
             imgid, dsc->width, dsc->height);
    // The lighttable menu can change the embedded JPEG mode during the render, and its purge skips
    // this entry, write-locked until we return: render in one mode, and check it is still current.
    const int embedded_jpg_mode = _settings_get().embedded_jpg;
    dsc->status = _init_8(dsc, imgid, mip, embedded_jpg_mode, shutdown);
    const int current_mode = _settings_get().embedded_jpg;
    mode_changed = embedded_jpg_mode != current_mode;
    if(mode_changed)
      _cache_print(DT_DEBUG_CACHE,
                   "[mipmap_cache] image %d at mip size %d rendered in embedded JPEG mode %d, now %d: not kept\n",
                   imgid, mip, embedded_jpg_mode, current_mode);
  }
 
  if((shutdown && dt_atomic_get_int(shutdown)) || mode_changed)
  {
    /* A stale GUI request stopped its thumbnail/export pipe while this cache entry
     * was still being produced, or the thumbnail was rendered in an embedded JPEG
     * mode that is no longer the current one. Keep the entry marked as "generate"
     * so the next request renders it again, instead of reusing a poisoned empty
     * thumbnail, or one in the old mode, for the rest of the session. */
    dsc->width = original_width;
    dsc->height = original_height;
    dsc->iscale = original_iscale;
    dsc->color_space = original_color_space;
    dsc->status = original_status;
    _invalidate_buffer(buf);
    return;
  }
 
  /* _init_8() and _init_f() report a failure as a zero-sized image. Clearing GENERATE on it would
   * latch the failure: every later request takes the "already in cache" path and gets the skull
   * back, and dt_view_image_get_surface() reports an 8x8 buffer as DT_VIEW_SURFACE_OK, so the
   * widget commits it and never asks again. One transient fault -- a momentary database lock, a
   * network share reconnecting -- would cost an image its thumbnail for the whole session,
   * clearable only through "Purge selected thumbnails from cache".
   *
   * One retry is granted instead. Bounded on purpose: an image that is genuinely unreadable must not
   * re-run a full pipeline on every redraw, so it costs at most two attempts per cache entry. The
   * skull is still published for this request either way -- the caller needs something to draw. */
  const gboolean failed = (dsc->width == 0 || dsc->height == 0);
 
  if(failed && !(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_RETRIED))
  {
    dsc->flags |= DT_MIPMAP_BUFFER_DSC_FLAG_RETRIED;   // GENERATE deliberately left set
    _paint_skulls(buf, dsc, imgid, mip);
    _validate_buffer(buf, dsc, imgid, mip);
    _cache_print(DT_DEBUG_CACHE,
                 "[mipmap_cache] image %d at mip size %d failed to generate; the next request will try"
                 " again\n",
                 imgid, mip);
    return;
  }
 
  dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE;
  if(!failed) dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_RETRIED;
  _paint_skulls(buf, dsc, imgid, mip);
  _validate_buffer(buf, dsc, imgid, mip);
 
  if(failed)
    _cache_print(DT_DEBUG_CACHE,
                 "[mipmap_cache] image %d at mip size %d failed to generate twice; keeping the skull\n",
                 imgid, mip);
 
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d at mip size %d got a new cache entry (%ix%i / %ix%i) at %p\n", imgid, mip, 
    buf->width, buf->height, dsc->width, dsc->height, buf->buf);
}
 
 
void dt_mipmap_cache_get_with_caller(dt_mipmap_buffer_t *buf, const int32_t imgid,
                                     const dt_mipmap_size_t mip, const dt_mipmap_get_flags_t flags,
                                     const char mode, const char *file, int line)
{
  dt_mipmap_cache_get_with_caller_and_shutdown(buf, imgid, mip, flags, mode, NULL, file, line);
}
 
void dt_mipmap_cache_get_with_caller_and_shutdown(dt_mipmap_buffer_t *buf,
                                                  const int32_t imgid, const dt_mipmap_size_t mip,
                                                  const dt_mipmap_get_flags_t flags, const char mode,
                                                  dt_atomic_int *shutdown, const char *file, int line)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  assert(mip <= DT_MIPMAP_NONE && mip >= DT_MIPMAP_0);
 
  const int32_t key = get_key(imgid, mip);
 
  // Allocation functions will check those if we run them
  buf->imgid = imgid;
  buf->size = mip;
 
  if(flags == DT_MIPMAP_TESTLOCK)
  {
    // simple case: only get and lock if it's there.
    dt_cache_entry_t *entry = dt_cache_testget(&_get_cache(cache, mip)->cache, key, mode);
    if(entry)
    {
      struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
      ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
      /* Not there yet: a render aborted mid-way leaves the entry flagged, over a buffer holding nothing.
       * Nor is a render that failed: its buffer is the 8x8 skull, and the _init_8() loop would
       * downsample it into a smaller size as if it were that image's thumbnail. */
      if((dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE) || dsc->status != DT_IMAGEIO_OK)
      {
        dt_cache_release(&_get_cache(cache, mip)->cache, entry);
        entry = NULL;
      }
    }
    buf->cache_entry = entry;
 
    if(entry)
      _validate_buffer(buf, _get_dsc_from_entry(entry), imgid, mip);
    else
      _invalidate_buffer(buf);
  }
  else if(flags == DT_MIPMAP_BLOCKING)
  {
    // simple case: blocking get
    dt_cache_entry_t *entry =  dt_cache_get_with_caller(&_get_cache(cache, mip)->cache, key, mode, file, line);
 
    /* The payload could not be allocated: CACHE_FULL for the callers that pass the status on (the
     * export), to whom an OK status with no buffer reads as a success. A thumbnail still shows a
     * skull, the shared one since no entry holds a copy: read-only, and no entry to release. */
    if(IS_NULL_PTR(entry))
    {
      buf->cache_entry = NULL;
      _invalidate_buffer(buf);
      buf->status = DT_IMAGEIO_CACHE_FULL;
      if(mip < DT_MIPMAP_F)
      {
        buf->width = 8;
        buf->height = 8;
        buf->iscale = 1.0f;
        buf->color_space = DT_COLORSPACE_DISPLAY;
        buf->buf = (uint8_t *)_skull_8;
      }
      fprintf(stderr, "[mipmap_cache] could not allocate a cache entry for image %" PRId32 " at mip size %d\n",
              imgid, mip);
      return;
    }
 
    /* An entry already in the cache comes back read-locked on a 'r' get. One still flagged for
     * generation -- a failed render, retried -- would then be rendered into under that read lock,
     * by several readers at once. Take it again for writing; _generate_blocking() re-reads the flags,
     * so a writer that got there first leaves nothing to do. */
    if(mode == 'r')
    {
      struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
      ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
      if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE)
      {
        entry->_lock_demoting = 1;
        dt_cache_release(&_get_cache(cache, mip)->cache, entry);
        entry = dt_cache_get_with_caller(&_get_cache(cache, mip)->cache, key, 'w', file, line);
        entry->_lock_demoting = 0;
      }
    }
 
    buf->cache_entry = entry;
    __sync_fetch_and_add(&(_get_cache(cache, mip)->stats_fetches), 1);
    _generate_blocking(entry, buf, imgid, mip, shutdown);
 
    // image cache is leaving the write lock in place in case the image has been newly allocated.
    // this leads to a slight increase in thread contention, so we opt for dropping the write lock
    // and acquiring a read lock immediately after. since this opens a small window for other threads
    // to get in between, we need to take some care to re-init cache entries and dsc.
    // note that concurrencykit has rw locks that can be demoted from w->r without losing the lock in between.
    if(mode == 'r')
    {
      entry->_lock_demoting = 1;
      // drop the write lock
      dt_cache_release(&_get_cache(cache, mip)->cache, entry);
      // get a read lock
      entry = dt_cache_get(&_get_cache(cache, mip)->cache, key, mode);
      entry->_lock_demoting = 0;
    }
 
    buf->cache_entry = entry;
 
  }
  else
  {
    _invalidate_buffer(buf);
  }
}
 
void dt_mipmap_cache_write_get_with_caller(dt_mipmap_buffer_t *buf, const int32_t imgid, const int mip, const char *file, int line)
{
  dt_mipmap_cache_get_with_caller(buf, imgid, mip, DT_MIPMAP_BLOCKING, 'w', file, line);
}
 
void dt_mipmap_cache_release_with_caller(dt_mipmap_buffer_t *buf, const char *file,
                                         int line)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  if(buf->size == DT_MIPMAP_NONE || IS_NULL_PTR(buf->cache_entry)) return;
  assert(buf->imgid > 0);
  assert(buf->size >= DT_MIPMAP_0);
  assert(buf->size < DT_MIPMAP_NONE);
  dt_cache_release_with_caller(&_get_cache(cache, buf->size)->cache, buf->cache_entry, file, line);
  buf->size = DT_MIPMAP_NONE;
  buf->buf = NULL;
}
 
 
// return index dt_mipmap_size_t having at least width & height requested instead of minimum combined diff
// please note that the requested size is in pixels not dots.
dt_mipmap_size_t dt_mipmap_cache_get_matching_size(const int32_t width,
                                                   const int32_t height, const uint32_t imgid)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  for(int k = DT_MIPMAP_0; k < DT_MIPMAP_F; k++)
  {
    // We assume a "fit" situation, typically rectangle within square
    // so we don't need both dimensions to be greater than requested
    if((cache->max_width[k] >= width) || (cache->max_height[k] >= height))
    {
      _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d will load a mip size %i (%" G_GSIZE_FORMAT "x%" G_GSIZE_FORMAT ")\n", imgid, k, cache->max_width[k], cache->max_height[k]);
      return k;
    }
  }
  return DT_MIPMAP_F - 1;
}
 
dt_mipmap_size_t dt_mipmap_cache_get_fitting_size(const int32_t width,
                                                   const int32_t height, const uint32_t imgid)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  for(int k = DT_MIPMAP_F - 1; k >= DT_MIPMAP_0; k--)
  {
    if((cache->max_width[k] <= width) && (cache->max_height[k] <= height))
    {
      _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d will fit a mip size %i (%" G_GSIZE_FORMAT "x%" G_GSIZE_FORMAT ")\n", imgid, k, cache->max_width[k], cache->max_height[k]);
      return k;
    }
  }
  return DT_MIPMAP_0;
}
 
void dt_mipmap_cache_swap_at_size(const int32_t imgid, const dt_mipmap_size_t mip, const uint8_t *const in,
  const int32_t width, const int32_t height, dt_colorspaces_color_profile_type_t profile)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  if(mip >= DT_MIPMAP_F || mip < DT_MIPMAP_0) return;
 
  const uint32_t key = get_key(imgid, mip);
  dt_cache_entry_t *entry = dt_cache_get_with_caller(&_get_cache(cache, mip)->cache, key, 'w', __FILE__, __LINE__);
  if(entry)
  {
    struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
    // Unpoison descriptor and pixel payload before reading/writing either.
    ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
    ASAN_UNPOISON_MEMORY_REGION(_get_buffer_from_dsc(dsc), dsc->size - dt_mipmap_buffer_dsc_size);
    _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d is synchronized from pipeline at size %i (%ix%i->%ix%i)\n", 
      imgid, mip, width, height, dsc->width, dsc->height);
 
    // Downscale
    dsc->iscale = 1.f;
    const int32_t wd = dsc->width;
    const int32_t ht = dsc->height;
    uint8_t *buf =(uint8_t *)_get_buffer_from_dsc(dsc);
    dt_iop_flip_and_zoom_8(in, width, height, buf, wd, ht,
                           ORIENTATION_NONE, &dsc->width, &dsc->height);
 
    /* Thumbnails are stored in AdobeRGB. In place: the conversion also swaps BGRA back
     * to RGBA, which is what happens even when no transform could be built. */
    dt_colorprofiles_bgra8_to_adobergb_rgba8(buf, buf, dsc->width, dsc->height, profile);
 
    dsc->color_space = DT_COLORSPACE_ADOBERGB;
    dsc->status = DT_IMAGEIO_OK;
    // New pixels: the file, if there is one, no longer holds them, and no failure is left to retry.
    dsc->flags &= ~(DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE | DT_MIPMAP_BUFFER_DSC_FLAG_ON_DISK
                    | DT_MIPMAP_BUFFER_DSC_FLAG_RETRIED);
    dt_cache_release(&_get_cache(cache, mip)->cache, entry);
  }
}
 
 
void dt_mipmap_cache_remove_at_size(const int32_t imgid, const dt_mipmap_size_t mip, const gboolean flush_disk)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  if(mip >= DT_MIPMAP_F || mip < DT_MIPMAP_0) return;
 
  // get rid of all ldr thumbnails:
  const uint32_t key = get_key(imgid, mip);
  dt_cache_entry_t *entry = dt_cache_testget(&_get_cache(cache, mip)->cache, key, 'w');
  if(entry)
  {
    struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
    ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
    if(flush_disk) dsc->flags |= DT_MIPMAP_BUFFER_DSC_FLAG_INVALIDATE;
    dt_cache_release(&_get_cache(cache, mip)->cache, entry);
    dt_cache_remove(&_get_cache(cache, mip)->cache, key);
  }
  if(flush_disk)
  {
    // directly remove the file on disk cache even if we don't have a memory entry
    dt_mipmap_cache_unlink_ondisk_thumbnail(cache, imgid, mip);
  }
}
 
// get rid of all ldr thumbnails, and of a full-size load that failed:
void dt_mipmap_cache_remove(const int32_t imgid, const gboolean flush_disk)
{
  for(dt_mipmap_size_t k = DT_MIPMAP_0; k < DT_MIPMAP_F; k++)
    dt_mipmap_cache_remove_at_size(imgid, k, flush_disk);
 
  /* The decoded raw stays: a development change does not invalidate it. A failed load holds no
   * raw, and the thumbnails generated next would read its failure back instead of trying the file
   * again, which may have been repaired, or may now fit in memory. */
  dt_cache_t *full = &_get_cache(_mipmap_cache, DT_MIPMAP_FULL)->cache;
  const uint32_t key = get_key(imgid, DT_MIPMAP_FULL);
  dt_cache_entry_t *entry = dt_cache_testget(full, key, 'r');
  if(IS_NULL_PTR(entry)) return;
 
  struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
  ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
  const gboolean failed = dsc->status != DT_IMAGEIO_OK;
  dt_cache_release(full, entry);
  if(failed) dt_cache_remove(full, key);
}
 
/* Same as _remove_at_size(), minus the "thumbnails only" guard, so the two input sizes can
 * be dropped by the one caller entitled to: the image is going away. DT_MIPMAP_F and
 * DT_MIPMAP_FULL are RAM-only -- every disk write is gated on mip < DT_MIPMAP_F -- so there
 * is no file to unlink for them and flush_disk has nothing to do. */
static void _remove_one_size(const int32_t imgid, const dt_mipmap_size_t mip)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  const uint32_t key = get_key(imgid, mip);
  dt_cache_entry_t *entry = dt_cache_testget(&_get_cache(cache, mip)->cache, key, 'w');
  if(entry)
  {
    struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
    ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
    dsc->flags |= DT_MIPMAP_BUFFER_DSC_FLAG_INVALIDATE;
    dt_cache_release(&_get_cache(cache, mip)->cache, entry);
    dt_cache_remove(&_get_cache(cache, mip)->cache, key);
  }
}
 
void dt_mipmap_cache_remove_all_sizes(const int32_t imgid, const gboolean flush_disk)
{
  dt_mipmap_cache_remove(imgid, flush_disk);
  _remove_one_size(imgid, DT_MIPMAP_F);
  _remove_one_size(imgid, DT_MIPMAP_FULL);
}
 
// write thumbnail to disc if not existing there
void dt_mimap_cache_evict(const int32_t imgid)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  for(dt_mipmap_size_t k = DT_MIPMAP_0; k < DT_MIPMAP_F; k++)
    dt_cache_remove(&_get_cache(cache, k)->cache, get_key(imgid, k));
}
 
static void _init_f(dt_mipmap_buffer_t *mipmap_buf, float *out, uint32_t *width, uint32_t *height, float *iscale,
                    const int32_t imgid)
{
  const uint32_t wd = *width, ht = *height;
 
  /* do not even try to process file if it isn't available */
  char filename[DT_PATH_MAX] = { 0 };
  dt_image_path_source_t source = DT_IMAGE_PATH_NONE;
  {
    const dt_image_t *img = dt_image_cache_get(imgid, 'r');
    if(!IS_NULL_PTR(img))
    {
      source = dt_image_choose_input_path(img, filename, sizeof(filename), FALSE);
      dt_image_cache_read_release(img);
    }
  }
  if(source == DT_IMAGE_PATH_NONE)
  {
    *width = *height = 0;
    *iscale = 0.0f;
    return;
  }
 
  dt_mipmap_buffer_t buf;
  dt_mipmap_cache_get(&buf, imgid, DT_MIPMAP_FULL, DT_MIPMAP_BLOCKING, 'r');
 
  // lock image after we have the buffer, we might need to lock the image struct for
  // writing during raw loading, to write to width/height.
  const dt_image_t *image = dt_image_cache_get(imgid, 'r');
  if(IS_NULL_PTR(image) || IS_NULL_PTR(buf.buf) || buf.width == 0 || buf.height == 0)
  {
    dt_image_cache_read_release(image);
    dt_mipmap_cache_release(&buf);
    *width = *height = 0;
    *iscale = 0.0f;
    return;
  }
 
  dt_iop_roi_t roi_in, roi_out;
  roi_in.x = roi_in.y = 0;
  roi_in.width = image->width;
  roi_in.height = image->height;
  roi_in.scale = 1.0f;
 
  roi_out.x = roi_out.y = 0;
 
  // now let's figure out the scaling...
 
  // MIP_F is 4 channels, and we do not demosaic here
  roi_out.scale = fminf(((float)wd) / (float)image->width, ((float)ht) / (float)image->height);
  roi_out.width = roi_out.scale * roi_in.width;
  roi_out.height = roi_out.scale * roi_in.height;
 
  mipmap_buf->color_space = DT_COLORSPACE_NONE; // TODO: do we need that information in this buffer?
 
  if(image->dsc.filters)
  {
    if(image->dsc.filters != 9u && image->dsc.datatype == TYPE_FLOAT)
    {
      dt_iop_clip_and_zoom_mosaic_half_size_f((float *const)out, (const float *const)buf.buf, &roi_out, &roi_in,
                                              roi_out.width, roi_in.width, image->dsc.filters);
    }
    else if(image->dsc.filters != 9u && image->dsc.datatype == TYPE_UINT16)
    {
      dt_iop_clip_and_zoom_mosaic_half_size((uint16_t * const)out, (const uint16_t *)buf.buf, &roi_out, &roi_in,
                                            roi_out.width, roi_in.width, image->dsc.filters);
    }
    else if(image->dsc.filters == 9u && image->dsc.datatype == TYPE_UINT16)
    {
      dt_iop_clip_and_zoom_mosaic_third_size_xtrans((uint16_t * const)out, (const uint16_t *)buf.buf, &roi_out,
                                                    &roi_in, roi_out.width, roi_in.width, image->dsc.xtrans);
    }
    else if(image->dsc.filters == 9u && image->dsc.datatype == TYPE_FLOAT)
    {
      dt_iop_clip_and_zoom_mosaic_third_size_xtrans_f(out, (const float *)buf.buf, &roi_out, &roi_in,
                                                      roi_out.width, roi_in.width, image->dsc.xtrans);
    }
    else
    {
      dt_unreachable_codepath();
    }
  }
  else
  {
    // downsample
    dt_iop_clip_and_zoom(out, (const float *)buf.buf, &roi_out, &roi_in, roi_out.width, roi_in.width);
  }
 
  dt_mipmap_cache_release(&buf);
 
  *width = roi_out.width;
  *height = roi_out.height;
  *iscale = (float)image->width / (float)roi_out.width;
 
  dt_image_cache_read_release(image);
}
 
 
// dummy functions for `export' to mipmap buffers:
typedef struct _dummy_data_t
{
  dt_imageio_module_data_t head;
  uint8_t *buf;
} _dummy_data_t;
 
static int _levels(dt_imageio_module_data_t *data)
{
  return IMAGEIO_RGB | IMAGEIO_INT8;
}
 
static int _bpp(dt_imageio_module_data_t *data)
{
  return 8;
}
 
static int _write_image(dt_imageio_module_data_t *data, const char *filename, const void *in,
                        dt_colorspaces_color_profile_type_t over_type, const char *over_filename,
                        void *exif, int exif_len, int32_t imgid, int num, int total, dt_dev_pixelpipe_t *pipe,
                        const gboolean export_masks)
{
  _dummy_data_t *d = (_dummy_data_t *)data;
  memcpy(d->buf, in, sizeof(uint32_t) * data->width * data->height);
  return 0;
}
 
static int _load_jpg(const char *filename, const int32_t imgid, const uint32_t wd, const uint32_t ht,
                     const dt_mipmap_size_t size, const dt_image_orientation_t orientation, uint8_t *buf,
                     uint32_t *width, uint32_t *height, dt_colorspaces_color_profile_type_t *color_space)
{
  int res = 1;
  dt_imageio_jpeg_t jpg;
  if(!dt_imageio_jpeg_read_header(filename, &jpg))
  {
    *color_space = dt_imageio_jpeg_read_color_space(&jpg);
    uint8_t *tmp = (uint8_t *)dt_alloc_align(sizeof(uint8_t) * jpg.width * jpg.height * 4);
    if(tmp && !dt_imageio_jpeg_read(&jpg, tmp))
    {
      // scale to fit
      _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] generate mip size %d for image %d from jpeg\n", size, imgid);
      dt_iop_flip_and_zoom_8(tmp, jpg.width, jpg.height, buf, wd, ht, orientation, width, height);
      res = 0;
    }
    dt_free_align(tmp);
  }
  return res;
}
 
static int _find_sidecar_jpg(const char *filename, const char *ext, char *sidecar)
{
  const size_t filename_len = strlen(filename) - strlen(ext);
  const char *exts[4] = { ".jpg", ".JPG", ".jpeg", ".JPEG" };
 
  for(int i = 0; i < 4; i++)
  {
    // Damage control. Should never happen.
    if(filename_len + strlen(exts[i]) >= DT_PATH_MAX)
      continue;
 
    // Construct the sidecar filename
    const size_t str_copy = g_snprintf(sidecar, DT_PATH_MAX, "%.*s%s", (int)filename_len, filename, exts[i]);
 
    // Check if the filename was too long or if the copy failed.
    if (str_copy == 0 || str_copy >= DT_PATH_MAX)
      continue;
 
    if(g_file_test(sidecar, G_FILE_TEST_EXISTS))
      return 1;
  }
 
  return 0;
}
 
static dt_imageio_retval_t _init_8(struct dt_mipmap_buffer_dsc *dsc, const int32_t imgid,
                                   const dt_mipmap_size_t size, const int embedded_jpg_mode,
                                   dt_atomic_int *shutdown)
{
  if(size >= DT_MIPMAP_F || dsc->width < 16 || dsc->height < 16) return DT_IMAGEIO_OK;
 
  uint8_t *buf = _get_buffer_from_dsc(dsc);
  dsc->iscale = 1.0f;
  const uint32_t wd = dsc->width;
  const uint32_t ht = dsc->height;
 
  char filename[DT_PATH_MAX] = { 0 };
  char ext[6] = { 0 };
  gboolean input_exists = FALSE;
  gboolean is_jpg_input = FALSE;
  gboolean altered = FALSE;
  gboolean may_be_raw = TRUE;
  _get_thumbnail_input(imgid, filename, ext, &input_exists, &is_jpg_input, &altered, &may_be_raw);
 
  // embedded JPG mode:
  // 0 = never use embedded thumbnail
  // 1 = only on unedited pics,
  // 2 = always use embedded thumbnail
  // so an edited image uses it only in mode 2, an unedited one in modes 1 and 2.
  const gboolean use_embedded_jpg = altered ? embedded_jpg_mode == 2 : embedded_jpg_mode >= 1;
 
  /* do not even try to process file if it isn't available */
  if(!input_exists)
  {
    dsc->width = 0;
    dsc->height = 0;
    dsc->iscale = 0.0f;
    dsc->color_space = DT_COLORSPACE_NONE;
    return DT_IMAGEIO_FILE_NOT_FOUND;
  }
 
  int res = 1;
 
  if(!use_embedded_jpg)
  {
    // try to generate mip from larger mip
    // This expects that invalid mips will be flushed, so the assumption is:
    // if mip then it's valid (with regard to current history)
    for(dt_mipmap_size_t k = size + 1; k < DT_MIPMAP_F; k++)
    {
      dt_mipmap_buffer_t tmp;
      dt_mipmap_cache_get(&tmp, imgid, k, DT_MIPMAP_TESTLOCK, 'r');
      if(IS_NULL_PTR(tmp.buf)) continue;
 
      dsc->color_space = tmp.color_space;
      // downsample
      dt_iop_flip_and_zoom_8(tmp.buf, tmp.width, tmp.height, buf, wd, ht, ORIENTATION_NONE, &dsc->width,
                             &dsc->height);
      _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] generate mip size %d for image %d from mip size %d (%ix%i->%ix%i)\n",
        size, imgid, k, tmp.width, tmp.height, dsc->width, dsc->height);
 
      dt_mipmap_cache_release(&tmp);
      return DT_IMAGEIO_OK;
    }
  }
  else
  {
    // Orientation and camera framing are only needed when loading embedded JPEGs.
    dt_image_orientation_t orientation = ORIENTATION_NONE;
    dt_boundingbox_t usercrop = { 0.f, 0.f, 1.f, 1.f };
    const dt_image_t *img = dt_image_cache_get(imgid, 'r');
    if(img)
    {
      orientation = img->orientation;
      dt_image_cache_read_release(img);
    }
    if(orientation == ORIENTATION_NULL) orientation = ORIENTATION_NONE;
 
    // Resolve outside the read lock: this path never decodes the raw, so the framing may still
    // have to be read from the file, and storing the answer needs the write lock.
    dt_image_resolve_usercrop(imgid, usercrop);
 
    char sidecar_filename[DT_PATH_MAX] = { 0 };
 
    if(is_jpg_input)
    {
      // Input file is a JPEG
      res = _load_jpg(filename, imgid, wd, ht, size, orientation, buf, &dsc->width, &dsc->height,
                      &dsc->color_space);
    }
    else if(_find_sidecar_jpg(filename, ext, sidecar_filename))
    {
      // input file is a RAW but we have a companion JPEG file in the same folder:
      // use it in priority (it may be higher resolution/quality than embedded JPEG).
      res = _load_jpg(sidecar_filename, imgid, wd, ht, size, orientation, buf, &dsc->width, &dsc->height,
                      &dsc->color_space);
    }
    else
    {
      // input file is a RAW without companion JPEG:
      // try to load the embedded thumbnail. Might not be large enough though.
      uint8_t *tmp = NULL;
      int32_t thumb_width, thumb_height;
      res = dt_imageio_large_thumbnail(filename, &tmp, &thumb_width, &thumb_height, &dsc->color_space,
                                       dsc->width, dsc->height);
      if(!res)
      {
        // We take the thumbnail no matter its size. It might be too small for the requested dimension,
        // and end up blurry. But it's less bad than the following scenario:
        // 1. user displays collections in grid of 5 columns (small thumbnail -> fetch embedded JPEG for performance, all good),
        // 2. user zooms in the grid, to 2 or 3 columns,
        // 3. suddently, embedded JPEG is too small so we ditch it for a full pipe recompute,
        // 4. but then, color/contrast/appearance unexpectedly changes and user doesn't understand WTF just happened.
        // Blurry is less bad than randomly inconsistent, plus user has a GUI way in lighttable to
        // change how thumbs are processed at runtime.
        _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] generate mip size %d for image %d from embedded jpeg\n", size, imgid);
        // The camera renders its embedded previews from the full default-crop rectangle even when
        // it recorded a narrower framing, so trim them to what the photographer actually framed.
        // Only here: the two branches above read a separate JPEG file, which the camera already
        // wrote cropped. This runs before the rotation below, hence the un-oriented box.
        dt_imageio_crop_thumbnail(usercrop, tmp, &thumb_width, &thumb_height);
        dt_iop_flip_and_zoom_8(tmp, thumb_width, thumb_height, buf, wd, ht, orientation, &dsc->width,
                               &dsc->height);
        dt_pixelpipe_cache_free_align(tmp);
      }
    }
  }
 
  if(!res) return DT_IMAGEIO_OK;
 
  // Mode 2 spares the cost of decoding a raw. A PNG, a TIFF or any other RGB file usually has
  // no embedded preview, and is itself the cheap picture that preview stands in for.
  if(embedded_jpg_mode == 2 && may_be_raw)
  {
    _cache_print(DT_DEBUG_CACHE,
             "[mipmap_cache] embedded JPEG mode forbids raw processing for image %d at mip %d\n",
             imgid, size);
    dsc->width = 0;
    dsc->height = 0;
    dsc->iscale = 0.0f;
    dsc->color_space = DT_COLORSPACE_NONE;
    return DT_IMAGEIO_NO_EMBEDDED_THUMBNAIL;
  }
 
  // What an embedded preview shows of a raw is the picture before any edit. An RGB file that has
  // none shows the same, so the embedded modes render it without its history.
  const gboolean ignore_history = use_embedded_jpg && !may_be_raw;
 
  // try the real thing: rawspeed + pixelpipe
  dt_imageio_module_format_t format;
  _dummy_data_t dat;
  format.bpp = _bpp;
  format.write_image = _write_image;
  format.levels = _levels;
  dat.head.max_width = wd;
  dat.head.max_height = ht;
  dat.buf = buf;
  // export with flags: ignore exif (don't load from disk), don't swap byte order, don't do hq processing,
  // no upscaling and signal we want thumbnail export
  const dt_imageio_retval_t status
      = dt_imageio_export_with_flags(imgid, "unused", &format, (dt_imageio_module_data_t *)&dat, TRUE, FALSE,
                                     FALSE, FALSE, TRUE, ignore_history, NULL, FALSE, FALSE,
                                     DT_COLORSPACE_ADOBERGB, NULL,
                                     DT_INTENT_LAST, NULL, NULL, 1, 1, NULL, shutdown);
  if(status != DT_IMAGEIO_OK)
  {
    g_printerr("[mipmap_cache] could not process thumbnail for image %" PRId32 " (%s): error %d\n", imgid,
               filename, status);
    dsc->width = 0;
    dsc->height = 0;
    dsc->iscale = 0.0f;
    dsc->color_space = DT_COLORSPACE_NONE;
    return status;
  }
 
  _cache_print(DT_DEBUG_CACHE, "[mipmap_cache] generated mip %d for image %d from scratch\n", size, imgid);
  // might be smaller, or have a different aspect than what we got as input.
  dsc->width = dat.head.width;
  dsc->height = dat.head.height;
  dsc->iscale = 1.0f;
  dsc->color_space = DT_COLORSPACE_ADOBERGB;
  return DT_IMAGEIO_OK;
}
 
void dt_mipmap_cache_copy_thumbnails(const uint32_t dst_imgid, const uint32_t src_imgid)
{
  dt_mipmap_cache_t *cache = _mipmap_cache;
  if(cache->cachedir[0] && _settings_get().disk_backend)
  {
    for(dt_mipmap_size_t mip = DT_MIPMAP_0; mip < DT_MIPMAP_F; mip++)
    {
      // try and load from disk, if successful set flag
      char srcpath[DT_PATH_MAX] = {0};
      char dstpath[DT_PATH_MAX] = {0};
      dt_mipmap_get_cache_filename(srcpath, mip, src_imgid);
      dt_mipmap_get_cache_filename(dstpath, mip, dst_imgid);
      GFile *src = g_file_new_for_path(srcpath);
      GFile *dst = g_file_new_for_path(dstpath);
      GError *gerror = NULL;
      g_file_copy(src, dst, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
      // ignore errors, we tried what we could.
      g_object_unref(dst);
      g_object_unref(src);
      g_clear_error(&gerror);
    }
  }
}
 
// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on