mipmap_cache.c

Go to the documentation of this file.

/*
    This file is part of darktable,
    Copyright (C) 2011-2012, 2015 Edouard Gomez.
    Copyright (C) 2011-2012 Henrik Andersson.
    Copyright (C) 2011-2017 johannes hanika.
    Copyright (C) 2011-2012 José Carlos García Sogo.
    Copyright (C) 2012 Christian Tellefsen.
    Copyright (C) 2012, 2014 Jérémy Rosen.
    Copyright (C) 2012 Richard Wonka.
    Copyright (C) 2012-2017, 2019-2020 Tobias Ellinghaus.
    Copyright (C) 2012, 2014 Ulrich Pegelow.
    Copyright (C) 2013-2014, 2019-2021 Pascal Obry.
    Copyright (C) 2013 Simon Spannagel.
    Copyright (C) 2014, 2016 Dan Torop.
    Copyright (C) 2014-2015 parafin.
    Copyright (C) 2014-2016 Pedro Côrte-Real.
    Copyright (C) 2014-2016 Roman Lebedev.
    Copyright (C) 2015 Matthias Gehre.
    Copyright (C) 2016-2017 Peter Budai.
    Copyright (C) 2017 luzpaz.
    Copyright (C) 2019-2021 Aldric Renaudin.
    Copyright (C) 2019, 2022-2023, 2025-2026 Aurélien PIERRE.
    Copyright (C) 2019-2020 Heiko Bauke.
    Copyright (C) 2019 jakubfi.
    Copyright (C) 2019-2020 Philippe Weyland.
    Copyright (C) 2020-2022 Hanno Schwalm.
    Copyright (C) 2020-2021 Hubert Kowalski.
    Copyright (C) 2021 Ralf Brown.
    Copyright (C) 2022 Martin Bařinka.
    Copyright (C) 2022 Miloš Komarčević.
    Copyright (C) 2023 Ricky Moon.
    Copyright (C) 2024 Alynx Zhou.
    Copyright (C) 2025 Guillaume Stutin.
    
    darktable is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    darktable is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with darktable.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#include "common/mipmap_cache.h"
#include "common/darktable.h"
#include "common/debug.h"
#include "common/exif.h"
#include "common/file_location.h"
#include "common/grealpath.h"
#include "common/image_cache.h"
#include "develop/supervisor.h"
#include "common/history.h"
#include "common/imageio.h"
#include "common/imageio_jpeg.h"
#include "common/imageio_module.h"
#include "control/conf.h"
#include "control/jobs.h"
#include "develop/imageop_math.h"
#include "gui/gtk.h"
 
#include <assert.h>
#include <errno.h>
#include <fcntl.h>
#include <glib.h>
#include <glib/gstdio.h>
#include <inttypes.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <unistd.h>
 
#if !defined(_WIN32)
#include <sys/statvfs.h>
#else
//statvfs does not exist in Windows, providing implementation
#include "win/statvfs.h"
#endif
 
#define DT_MIPMAP_CACHE_FILE_MAGIC 0xD71337
#define DT_MIPMAP_CACHE_FILE_VERSION 23
#define DT_MIPMAP_CACHE_DEFAULT_FILE_NAME "mipmaps"
 
typedef enum dt_mipmap_buffer_dsc_flags
{
  DT_MIPMAP_BUFFER_DSC_FLAG_NONE = 0,
  DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE = 1 << 0,
  DT_MIPMAP_BUFFER_DSC_FLAG_INVALIDATE = 1 << 1
} dt_mipmap_buffer_dsc_flags;
 
// the embedded Exif data to tag thumbnails as sRGB or AdobeRGB
static const uint8_t dt_mipmap_cache_exif_data_srgb[] = {
  0x45, 0x78, 0x69, 0x66, 0x00, 0x00, 0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x69,
  0x87, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
  0x01, 0xa0, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
static const uint8_t dt_mipmap_cache_exif_data_adobergb[] = {
  0x45, 0x78, 0x69, 0x66, 0x00, 0x00, 0x49, 0x49, 0x2a, 0x00, 0x08, 0x00, 0x00, 0x00, 0x01, 0x00, 0x69,
  0x87, 0x04, 0x00, 0x01, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
  0x01, 0xa0, 0x03, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
};
static const int dt_mipmap_cache_exif_data_srgb_length
                      = sizeof(dt_mipmap_cache_exif_data_srgb) / sizeof(*dt_mipmap_cache_exif_data_srgb);
static const int dt_mipmap_cache_exif_data_adobergb_length
                      = sizeof(dt_mipmap_cache_exif_data_adobergb) / sizeof(*dt_mipmap_cache_exif_data_adobergb);
 
struct dt_mipmap_buffer_dsc
{
  uint32_t width;
  uint32_t height;
  float iscale;
  size_t size;
  dt_mipmap_buffer_dsc_flags flags;
  dt_colorspaces_color_profile_type_t color_space;
 
#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
  // do not touch!
  // must be the last element.
  // must be no less than 16bytes
  char redzone[16];
#endif
 
  /* NB: sizeof must be a multiple of 4*sizeof(float) */
} __attribute__((packed, aligned(DT_CACHELINE_BYTES)));
 
#if __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__)
static const size_t dt_mipmap_buffer_dsc_size __attribute__((unused))
= sizeof(struct dt_mipmap_buffer_dsc) - sizeof(((struct dt_mipmap_buffer_dsc *)0)->redzone);
#else
static const size_t dt_mipmap_buffer_dsc_size __attribute__((unused)) = sizeof(struct dt_mipmap_buffer_dsc);
#endif
 
static uint8_t * _get_buffer_from_dsc(struct dt_mipmap_buffer_dsc *dsc);
 
static inline void *dead_image_8(struct dt_mipmap_buffer_dsc *dsc)
{
  dsc->width = dsc->height = 8;
  dsc->iscale = 1.0f;
  dsc->color_space = DT_COLORSPACE_DISPLAY;
  assert(dsc->size > 64 * sizeof(uint32_t));
  const uint32_t X = 0xffffffffu;
  const uint32_t o = 0u;
  const uint32_t image[]
      = { o, o, o, o, o, o, o, o, 
          o, o, X, X, X, X, o, o, 
          o, X, o, X, X, o, X, o, 
          o, X, X, X, X, X, X, o,
          o, o, X, o, o, X, o, o, 
          o, o, o, o, o, o, o, o, 
          o, o, X, X, X, X, o, o, 
          o, o, o, o, o, o, o, o };
  memcpy(_get_buffer_from_dsc(dsc), image, sizeof(uint32_t) * 64);
  return _get_buffer_from_dsc(dsc);
}
 
static inline int32_t get_key(const int32_t imgid, const dt_mipmap_size_t size)
{
  // imgid can't be >= 2^28 (~250 million images)
  return (((int32_t)size) << 28) | (imgid - 1);
}
 
static inline uint32_t get_imgid(const uint32_t key)
{
  return (key & 0xfffffff) + 1;
}
 
static inline dt_mipmap_size_t get_size(const uint32_t key)
{
  return (dt_mipmap_size_t)(key >> 28);
}
 
void dt_mipmap_get_cache_dir(char path[PATH_MAX], const dt_mipmap_cache_t *cache, dt_mipmap_size_t mip)
{
  g_snprintf(path, sizeof(char) * PATH_MAX, "%s.d" G_DIR_SEPARATOR_S "%d",
             cache->cachedir, (int)mip);
}
 
void dt_mipmap_get_cache_filename(char path[PATH_MAX], const dt_mipmap_cache_t *cache, dt_mipmap_size_t mip, const int32_t imgid)
{
  gchar cache_path[PATH_MAX];
  dt_mipmap_get_cache_dir(cache_path, cache, mip);
 
  gchar *file = g_strdup_printf("%u.jpg", imgid);
  dt_concat_path_file(path, cache_path, file);
  dt_free(file);
}
 
static int dt_mipmap_cache_get_filename(gchar *mipmapfilename, size_t size)
{
  int r = -1;
  char *abspath = NULL;
 
  // Directory
  char cachedir[PATH_MAX] = { 0 };
  dt_loc_get_user_cache_dir(cachedir, sizeof(cachedir));
 
  // Build the mipmap filename fram hashing the path of the library DB
  const gchar *dbfilename = dt_database_get_path(darktable.db);
 
  if(!strcmp(dbfilename, ":memory:"))
  {
    mipmapfilename[0] = '\0';
    r = 0;
    goto exit;
  }
 
  abspath = g_realpath(dbfilename);
  if(IS_NULL_PTR(abspath)) abspath = g_strdup(dbfilename);
 
  GChecksum *chk = g_checksum_new(G_CHECKSUM_SHA1);
  g_checksum_update(chk, (guchar *)abspath, strlen(abspath));
  const gchar *filename = g_checksum_get_string(chk);
 
  if(!filename || filename[0] == '\0')
    snprintf(mipmapfilename, size, "%s/%s", cachedir, DT_MIPMAP_CACHE_DEFAULT_FILE_NAME);
  else
    snprintf(mipmapfilename, size, "%s/%s-%s", cachedir, DT_MIPMAP_CACHE_DEFAULT_FILE_NAME, filename);
 
  g_checksum_free(chk);
  r = 0;
 
exit:
  dt_free(abspath);
 
  return r;
}
 
static void _write_mipmap_to_disk(const int32_t imgid, char *filename, char *ext, gboolean *input_exists,
                                  gboolean *is_jpg_input, gboolean *use_embedded_jpg, gboolean *write_to_disk)
{
  // Get file name
  char _filename[PATH_MAX] = { 0 };
  const dt_image_t *img = dt_image_cache_get(darktable.image_cache, imgid, 'r');
  if(filename || ext || input_exists || is_jpg_input)
  {
    const dt_image_path_source_t source = dt_image_choose_input_path(img, _filename, sizeof(_filename), FALSE);
    if(source == DT_IMAGE_PATH_NONE) _filename[0] = '\0';
    if(filename) strncpy(filename, _filename, PATH_MAX);
    if(input_exists) *input_exists = (source != DT_IMAGE_PATH_NONE);
  }
 
  // Get the file extension
  if(ext || is_jpg_input)
  {
    char *_ext = _filename + strlen(_filename);
    while(*_ext != '.' && _ext > _filename) _ext--;
    if(ext) strncpy(ext, _ext, 5);
    if(is_jpg_input) *is_jpg_input = !strcasecmp(_ext, ".jpg") || !strcasecmp(_ext, ".jpeg");
  }
 
  // embedded JPG mode:
  // 0 = never use embedded thumbnail
  // 1 = only on unedited pics,
  // 2 = always use embedded thumbnail
  int mode = dt_conf_get_int("lighttable/embedded_jpg");
  gboolean altered = FALSE;
  if(mode == 1)
  {
    if(img) altered = img->history_items > 0;
  }
  const gboolean _use_embedded_jpg
      = (mode == 2                           // always use embedded
         || (mode == 1 && !altered));        // use embedded only on unaltered images
  if(use_embedded_jpg) *use_embedded_jpg = _use_embedded_jpg;
 
  // Save to cache only if the option is enabled by user and the thumbnail is not the embedded thumbnail
  // The rationale is getting the un-processed JPEG thumbnail is cheap and not worth saving on disk.
  // This allows fast toggling between JPEG and processed RAW thumbnail from GUI.
  if(write_to_disk)
  {
    *write_to_disk = dt_conf_get_bool("cache_disk_backend");
  }
 
  if(img)
    dt_image_cache_read_release(darktable.image_cache, img);
}
 
 
static void _init_f(dt_mipmap_buffer_t *mipmap_buf, float *buf, uint32_t *width, uint32_t *height, float *iscale,
                    const int32_t imgid);
static void _init_8(uint8_t *buf, uint32_t *width, uint32_t *height, float *iscale,
                    dt_colorspaces_color_profile_type_t *color_space, const int32_t imgid,
                    const dt_mipmap_size_t size, dt_atomic_int *shutdown);
 
// Gets the address of the `buf->buf` pixel buffer, from the cache entry.
static uint8_t *_get_buffer_from_dsc(struct dt_mipmap_buffer_dsc *dsc)
{
  if(dsc)
    return (uint8_t *)(dsc + 1);
  else
    return NULL;
}
 
 
struct dt_mipmap_buffer_dsc *_get_dsc_from_entry(dt_cache_entry_t *entry)
{
  return (struct dt_mipmap_buffer_dsc *)entry->data;
}
 
// update buffer params with dsc
static void _sync_dsc_to_buf(dt_mipmap_buffer_t *buf, struct dt_mipmap_buffer_dsc *dsc, const int32_t imgid,
                             const dt_mipmap_size_t mip)
{
  buf->width = dsc->width;
  buf->height = dsc->height;
  buf->iscale = dsc->iscale;
  buf->color_space = dsc->color_space;
  buf->imgid = imgid;
  buf->size = mip;
  buf->buf = _get_buffer_from_dsc(dsc);
}
 
 
// flag a buffer as invalid and set it NULL
// This doesn't paint skulls
static void _invalidate_buffer(dt_mipmap_buffer_t *buf)
{
  buf->width = buf->height = 0;
  buf->iscale = 0.0f;
  buf->buf = NULL;
}
 
 
static size_t _get_entry_size(const size_t buffer_size)
{
  return buffer_size + dt_mipmap_buffer_dsc_size;
}
 
void dt_mipmap_cache_update_buffer_addresses(dt_cache_entry_t *entry, struct dt_mipmap_buffer_dsc **dsc,
                                             const size_t width, const size_t height,
                                             const size_t buffer_size)
{
  if(IS_NULL_PTR(entry->data))
  {
    entry->data_size = 0;
    *dsc = NULL;
    return;
  }
 
  // Update the entry datasize
  entry->data_size = _get_entry_size(buffer_size);
 
  // Update the dsc parameters
  *dsc = _get_dsc_from_entry(entry);
  (*dsc)->width = width;
  (*dsc)->height = height;
  (*dsc)->iscale = 1.0f;
  (*dsc)->color_space = DT_COLORSPACE_NONE;
  (*dsc)->flags = DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE;
  (*dsc)->size = _get_entry_size(buffer_size);
}
 
 
// callback for the imageio core to allocate memory.
// only needed for _FULL buffers, as they change size
// with the input image. will allocate img->width*img->height*img->bpp bytes.
void *dt_mipmap_cache_alloc(dt_mipmap_buffer_t *buf, const dt_image_t *img)
{
  if(IS_NULL_PTR(buf)) return NULL;
 
  if(buf->size != DT_MIPMAP_FULL)
  {
    fprintf(stderr, "trying to alloc a wrong mipmap size for %s: %i (should be: %i)\n", img->filename, buf->size, DT_MIPMAP_FULL);
    return NULL;
  }
 
  dt_cache_entry_t *entry = buf->cache_entry;
 
  if(IS_NULL_PTR(entry))
  {
    fprintf(stderr, "trying to alloc a buffer entry that has no back-reference to cache entry\n");
    return NULL;
  }
 
  /* Free and reset everything. ASan poisoning applies to live cache payloads only: after freeing
   * the old allocation there is no valid user region left, and poisoning a NULL reset pointer makes
   * the ASan runtime abort before it can report the real caller context. */
  if(!IS_NULL_PTR(entry->data))
    ASAN_POISON_MEMORY_REGION(entry->data, entry->data_size);
  dt_free_align(entry->data);
  entry->data = NULL;
 
  // Get a new allocation
  const int wd = img->width;
  const int ht = img->height;
  const size_t bpp = img->dsc.bpp;
  const size_t buffer_size = wd * ht * bpp;
  const size_t min_buffer_size = 64 * 4 * sizeof(float);
  entry->data = dt_alloc_align(_get_entry_size(MAX(buffer_size, min_buffer_size)));
  if(IS_NULL_PTR(entry->data)) return NULL;
 
  // Update the references
  struct dt_mipmap_buffer_dsc *dsc = NULL;
  dt_mipmap_cache_update_buffer_addresses(entry, &dsc, wd, ht, buffer_size);
 
  // Unpoison the pixel buffer region
  ASAN_UNPOISON_MEMORY_REGION(buf->buf, dsc->size - dt_mipmap_buffer_dsc_size);
 
  // Unpoison the dsc region
  ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
 
  // So the padding region between dsc and buf->buf should be poisoned still
  assert(entry->data == dsc);
 
  if(dsc)
  {
    assert(entry->data_size == dsc->size);
  }
 
  // return pointer to start of payload
  return _get_buffer_from_dsc(dsc);
}
 
// callback for the cache backend to initialize payload pointers
// It's actually not dynamic at all, fixed size only.
void dt_mipmap_cache_allocate_dynamic(void *data, dt_cache_entry_t *entry)
{
  dt_mipmap_cache_t *cache = (dt_mipmap_cache_t *)data;
  const dt_mipmap_size_t mip = get_size(entry->key);
  const int32_t imgid = get_imgid(entry->key);
 
  assert(mip < DT_MIPMAP_NONE);
 
  // Free and reset everything
  dt_free_align(entry->data);
  entry->data = NULL;
 
  // Get a new allocation
  const size_t buffer_size = (mip <= DT_MIPMAP_F) ? cache->buffer_size[mip] : _get_entry_size(sizeof(float) * 4 * 64);
  entry->data = dt_alloc_align(buffer_size);
  if(IS_NULL_PTR(entry->data)) return;
 
  // Update the references
  struct dt_mipmap_buffer_dsc *dsc = NULL;
  if(mip <= DT_MIPMAP_F)
    dt_mipmap_cache_update_buffer_addresses(entry, &dsc, cache->max_width[mip], cache->max_height[mip], buffer_size);
  else
    dt_mipmap_cache_update_buffer_addresses(entry, &dsc, 0, 0, buffer_size);
 
  if(IS_NULL_PTR(dsc)) return;
 
  // Register the mipmap object, and the image cache object it links to. The image
  // is usually already cached (so its allocate callback never fired), hence we
  // register it here from the live dt_image_t; dt_supervisor_image() promotes the
  // first sighting to a `create` so the link resolves.
  if(dt_supervisor_active())
  {
    const dt_image_t *img = dt_image_cache_get(darktable.image_cache, imgid, 'r');
    if(img) dt_supervisor_image(DT_SV_UPDATE, imgid, img);
    dt_supervisor_mipmap(DT_SV_CREATE, imgid, mip);
    if(img) dt_image_cache_read_release(darktable.image_cache, img);
  }
 
  gboolean write_to_disk;
  _write_mipmap_to_disk(imgid, NULL, NULL, NULL, NULL, NULL, &write_to_disk);
 
  if(cache->cachedir[0] && write_to_disk && mip < DT_MIPMAP_F)
  {
    // try and load from disk, if successful set flag
    char filename[PATH_MAX] = {0};
    dt_mipmap_get_cache_filename(filename, cache, mip, get_imgid(entry->key));
 
    gboolean io_error = FALSE;
    gchar *error = NULL;
    uint8_t *blob = NULL;
    dt_colorspaces_color_profile_type_t color_space = DT_COLORSPACE_DISPLAY;
    dt_imageio_jpeg_t jpg;
    FILE *f = NULL;
 
    f = g_fopen(filename, "rb");
    if(IS_NULL_PTR(f))
    {
      dt_print(DT_DEBUG_CACHE,
               "[mipmap_cache] cached file for image %d at mip size %i does not exist\n",
               imgid, mip);
      goto finish; // file doesn't exist
    }
 
    fseek(f, 0, SEEK_END);
    long len = ftell(f);
    if(len <= 0)
    {
      error = "empty file";
      io_error = TRUE;
      goto finish;
    }
 
    blob = (uint8_t *)dt_alloc_align(len);
    if(IS_NULL_PTR(blob))
    {
      error = "out of memory";
      io_error = TRUE;
      goto finish;
    }
 
    fseek(f, 0, SEEK_SET);
    const size_t rd = fread(blob, sizeof(uint8_t), len, f);
    if(rd != len)
    {
      error = "corrupted file";
      io_error = TRUE;
      goto finish;
    }
 
    if(dt_imageio_jpeg_decompress_header(blob, len, &jpg))
    {
      error = "couldn't decompress header";
      io_error = TRUE;
      goto finish;
    }
 
    color_space = dt_imageio_jpeg_read_color_space(&jpg);
 
    if(dt_imageio_jpeg_decompress(&jpg, _get_buffer_from_dsc(dsc)))
    {
      error = "couldn't decompress JPEG";
      io_error = TRUE;
      goto finish;
    }
 
    dsc->width = jpg.width;
    dsc->height = jpg.height;
    dsc->iscale = 1.0f;
    dsc->color_space = color_space;
    dsc->flags = 0;
 
    dt_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d at mip size %d (%ix%i) loaded from disk cache\n",
              imgid, mip, jpg.width, jpg.height);
 
finish:
    if(f && io_error)
    {
      // Delete the file, we will regenerate it
      g_unlink(filename);
      fprintf(stderr, "[mipmap_cache] failed to open thumbnail for image %" PRIu32 " from `%s'. Reason: %s\n",
              imgid, filename, error);
    }
 
    dt_free_align(blob);
    if(f) fclose(f);
  }
 
  // cost is just flat one for the buffer, as the buffers might have different sizes,
  // to make sure quota is meaningful.
  if(mip >= DT_MIPMAP_F)
    entry->cost = 1;
  else
    entry->cost = cache->buffer_size[mip];
}
 
static void dt_mipmap_cache_unlink_ondisk_thumbnail(void *data, int32_t imgid, dt_mipmap_size_t mip)
{
  dt_mipmap_cache_t *cache = (dt_mipmap_cache_t *)data;
 
  // also remove jpg backing (always try to do that, in case user just temporarily switched it off,
  // to avoid inconsistencies.
  // if(dt_conf_get_bool("cache_disk_backend"))
  if(cache->cachedir[0])
  {
    char filename[PATH_MAX] = { 0 };
    dt_mipmap_get_cache_filename(filename, cache, mip, imgid);
    g_unlink(filename);
    dt_print(DT_DEBUG_CACHE, "[mipmap_cache] image %i for size %i was deleted from disk cache\n", imgid, mip);
  }
}
 
void dt_mipmap_cache_deallocate_dynamic(void *data, dt_cache_entry_t *entry)
{
  dt_mipmap_cache_t *cache = (dt_mipmap_cache_t *)data;
  const dt_mipmap_size_t mip = get_size(entry->key);
 
  if(dt_supervisor_active())
    dt_supervisor_mipmap(DT_SV_DELETE, get_imgid(entry->key), (int)mip);
  if(mip < DT_MIPMAP_F)
  {
    const int32_t imgid = get_imgid(entry->key);
    gboolean write_to_disk;
    _write_mipmap_to_disk(imgid, NULL, NULL, NULL, NULL, NULL, &write_to_disk);
 
    struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
    // don't write skulls:
    if(dsc->width > 8 && dsc->height > 8)
    {
      if(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_INVALIDATE)
      {
        dt_mipmap_cache_unlink_ondisk_thumbnail(data, get_imgid(entry->key), mip);
      }
      if(cache->cachedir[0] && write_to_disk && mip < DT_MIPMAP_F)
      {
        // serialize to disk
        gchar cache_path[PATH_MAX];
        dt_mipmap_get_cache_dir(cache_path, cache, mip);
        const int mkd = g_mkdir_with_parents(cache_path, 0750);
 
        if(!mkd)
        {
          char filename[PATH_MAX] = {0};
          dt_mipmap_get_cache_filename(filename, cache, mip, get_imgid(entry->key));
          // Don't write existing files as both performance and quality (lossy jpg) suffer
          // FIXME: actually, yes, we write existing files too. See FIXME above.
          FILE *f = NULL;
          if((f = g_fopen(filename, "wb"))) // !g_file_test(filename, G_FILE_TEST_EXISTS)
          {
            // first check the disk isn't full
            struct statvfs vfsbuf;
            if (!statvfs(filename, &vfsbuf))
            {
              const int64_t free_mb = ((vfsbuf.f_frsize * vfsbuf.f_bavail) >> 20);
              if (free_mb < 100)
              {
                fprintf(stderr, "Aborting image write as only %" PRId64 " MB free to write %s\n", free_mb, filename);
                goto write_error;
              }
            }
            else
            {
              fprintf(stderr, "Aborting image write since couldn't determine free space available to write %s\n", filename);
              goto write_error;
            }
 
            const int cache_quality = dt_conf_get_int("database_cache_quality");
            const uint8_t *exif = NULL;
            int exif_len = 0;
            if(dsc->color_space == DT_COLORSPACE_SRGB)
            {
              exif = dt_mipmap_cache_exif_data_srgb;
              exif_len = dt_mipmap_cache_exif_data_srgb_length;
            }
            else if(dsc->color_space == DT_COLORSPACE_ADOBERGB)
            {
              exif = dt_mipmap_cache_exif_data_adobergb;
              exif_len = dt_mipmap_cache_exif_data_adobergb_length;
            }
            if(dt_imageio_jpeg_write(filename, _get_buffer_from_dsc(dsc), dsc->width, dsc->height,
                                     MIN(100, MAX(10, cache_quality)), exif, exif_len))
            {
write_error:
              g_unlink(filename);
            }
            else
            {
              dt_print(DT_DEBUG_CACHE, "[mipmap_cache] image %i for size %i was written to cache at %s\n", imgid, mip, filename);
            }
          }
          if(f) fclose(f);
        }
      }
    }
  }
  dt_free_align(entry->data);
  entry->data = NULL;
}
 
void dt_mipmap_cache_init(dt_mipmap_cache_t *cache)
{
  dt_mipmap_cache_get_filename(cache->cachedir, sizeof(cache->cachedir));
 
  // Fixed sizes for the thumbnail mip levels, selected for coverage of most screen sizes
  // Starting at 4K, we use 3:2 ratio of camera sensor instead of 16:9/16:10 of display,
  // in order to start caching full-resolution JPEG for 100% zoom in lighttable
  size_t mipsizes[DT_MIPMAP_F][2] = {
    { 360, 225 },             // mip0 - 1/2 size previous one
    { 720, 450 },             // mip1 - 1/2 size previous one
    { 1440, 900 },            // mip2 - covers HD, WXGA+
    { 1920, 1200 },           // mip3 - covers 1080p and 1600x1200
    { 2560, 1600 },           // mip4 - covers 2560x1440
    { 3840, 2560 },           // mip5 - covers 4K and UHD
    { 5120, 3414 },           // mip6 - covers 5K
    { 6144, 4096 },           // mip7 - covers 6K
    { 7680, 5120 },           // mip8 - covers 8K
  };
  // Set mipf for preview pipe to 1440x900
  cache->max_width[DT_MIPMAP_F] = mipsizes[DT_MIPMAP_2][0];
  cache->max_height[DT_MIPMAP_F] = mipsizes[DT_MIPMAP_2][1];
  for(int k = DT_MIPMAP_F-1; k >= 0; k--)
  {
    cache->max_width[k]  = mipsizes[k][0];
    cache->max_height[k] = mipsizes[k][1];
  }
    // header + buffer
  for(int k = DT_MIPMAP_F-1; k >= 0; k--)
    cache->buffer_size[k] = _get_entry_size(cache->max_width[k] * cache->max_height[k] * 4);
 
  // clear stats:
  cache->mip_thumbs.stats_requests = 0;
  cache->mip_thumbs.stats_near_match = 0;
  cache->mip_thumbs.stats_misses = 0;
  cache->mip_thumbs.stats_fetches = 0;
  cache->mip_thumbs.stats_standin = 0;
  cache->mip_f.stats_requests = 0;
  cache->mip_f.stats_near_match = 0;
  cache->mip_f.stats_misses = 0;
  cache->mip_f.stats_fetches = 0;
  cache->mip_f.stats_standin = 0;
  cache->mip_full.stats_requests = 0;
  cache->mip_full.stats_near_match = 0;
  cache->mip_full.stats_misses = 0;
  cache->mip_full.stats_fetches = 0;
  cache->mip_full.stats_standin = 0;
 
  dt_cache_init(&cache->mip_thumbs.cache, 0, dt_get_mipmap_mem());
  dt_cache_set_allocate_callback(&cache->mip_thumbs.cache, dt_mipmap_cache_allocate_dynamic, cache);
  dt_cache_set_cleanup_callback(&cache->mip_thumbs.cache, dt_mipmap_cache_deallocate_dynamic, cache);
 
  dt_cache_init(&cache->mip_full.cache, 0, dt_worker_threads() + DT_CTL_WORKER_RESERVED);
  dt_cache_set_allocate_callback(&cache->mip_full.cache, dt_mipmap_cache_allocate_dynamic, cache);
  dt_cache_set_cleanup_callback(&cache->mip_full.cache, dt_mipmap_cache_deallocate_dynamic, cache);
  cache->buffer_size[DT_MIPMAP_FULL] = 0;
 
  dt_cache_init(&cache->mip_f.cache, 0, dt_worker_threads() + DT_CTL_WORKER_RESERVED);
  dt_cache_set_allocate_callback(&cache->mip_f.cache, dt_mipmap_cache_allocate_dynamic, cache);
  dt_cache_set_cleanup_callback(&cache->mip_f.cache, dt_mipmap_cache_deallocate_dynamic, cache);
  cache->buffer_size[DT_MIPMAP_F]
      = _get_entry_size(4 * sizeof(float) * cache->max_width[DT_MIPMAP_F] * cache->max_height[DT_MIPMAP_F]);
}
 
void dt_mipmap_cache_cleanup(dt_mipmap_cache_t *cache)
{
  dt_mipmap_cache_print(cache);
  dt_cache_cleanup(&cache->mip_thumbs.cache);
  dt_cache_cleanup(&cache->mip_full.cache);
  dt_cache_cleanup(&cache->mip_f.cache);
}
 
void dt_mipmap_cache_print(dt_mipmap_cache_t *cache)
{
  dt_print(DT_DEBUG_CACHE, "[mipmap_cache] thumbs fill %.2f/%.2f MB (%.2f%%)\n",
         cache->mip_thumbs.cache.cost / (1024.0 * 1024.0),
         cache->mip_thumbs.cache.cost_quota / (1024.0 * 1024.0),
         100.0f * (float)cache->mip_thumbs.cache.cost / (float)cache->mip_thumbs.cache.cost_quota);
  dt_print(DT_DEBUG_CACHE, "[mipmap_cache] float fill %"PRIu32"/%"PRIu32" slots (%.2f%%)\n",
         (uint32_t)cache->mip_f.cache.cost, (uint32_t)cache->mip_f.cache.cost_quota,
         100.0f * (float)cache->mip_f.cache.cost / (float)cache->mip_f.cache.cost_quota);
  dt_print(DT_DEBUG_CACHE, "[mipmap_cache] full  fill %"PRIu32"/%"PRIu32" slots (%.2f%%)\n",
         (uint32_t)cache->mip_full.cache.cost, (uint32_t)cache->mip_full.cache.cost_quota,
         100.0f * (float)cache->mip_full.cache.cost / (float)cache->mip_full.cache.cost_quota);
 
  uint64_t sum = 0;
  uint64_t sum_fetches = 0;
  uint64_t sum_standins = 0;
  sum += cache->mip_thumbs.stats_requests;
  sum_fetches += cache->mip_thumbs.stats_fetches;
  sum_standins += cache->mip_thumbs.stats_standin;
  sum += cache->mip_f.stats_requests;
  sum_fetches += cache->mip_f.stats_fetches;
  sum_standins += cache->mip_f.stats_standin;
  sum += cache->mip_full.stats_requests;
  sum_fetches += cache->mip_full.stats_fetches;
  sum_standins += cache->mip_full.stats_standin;
  dt_print(DT_DEBUG_CACHE, "[mipmap_cache] level | near match | miss | stand-in | fetches | total rq\n");
  dt_print(DT_DEBUG_CACHE, "[mipmap_cache] thumb | %6.2f%% | %6.2f%% | %6.2f%%  | %6.2f%% | %6.2f%%\n",
         100.0 * cache->mip_thumbs.stats_near_match / (float)cache->mip_thumbs.stats_requests,
         100.0 * cache->mip_thumbs.stats_misses / (float)cache->mip_thumbs.stats_requests,
         100.0 * cache->mip_thumbs.stats_standin / (float)sum_standins,
         100.0 * cache->mip_thumbs.stats_fetches / (float)sum_fetches,
         100.0 * cache->mip_thumbs.stats_requests / (float)sum);
  dt_print(DT_DEBUG_CACHE, "[mipmap_cache] float | %6.2f%% | %6.2f%% | %6.2f%%  | %6.2f%% | %6.2f%%\n",
         100.0 * cache->mip_f.stats_near_match / (float)cache->mip_f.stats_requests,
         100.0 * cache->mip_f.stats_misses / (float)cache->mip_f.stats_requests,
         100.0 * cache->mip_f.stats_standin / (float)sum_standins,
         100.0 * cache->mip_f.stats_fetches / (float)sum_fetches,
         100.0 * cache->mip_f.stats_requests / (float)sum);
  dt_print(DT_DEBUG_CACHE, "[mipmap_cache] full  | %6.2f%% | %6.2f%% | %6.2f%%  | %6.2f%% | %6.2f%%\n",
         100.0 * cache->mip_full.stats_near_match / (float)cache->mip_full.stats_requests,
         100.0 * cache->mip_full.stats_misses / (float)cache->mip_full.stats_requests,
         100.0 * cache->mip_full.stats_standin / (float)sum_standins,
         100.0 * cache->mip_full.stats_fetches / (float)sum_fetches,
         100.0 * cache->mip_full.stats_requests / (float)sum);
  dt_print(DT_DEBUG_CACHE, "\n\n");
}
 
// Collect one sub-cache's live entries (imgid + mip + real bytes) under its lock.
static void _mipmap_subcache_collect(dt_cache_t *c, GArray *out)
{
  dt_pthread_mutex_lock(&c->lock);
  GHashTableIter it;
  gpointer key, value;
  g_hash_table_iter_init(&it, c->hashtable);
  while(g_hash_table_iter_next(&it, &key, &value))
  {
    const dt_cache_entry_t *e = (const dt_cache_entry_t *)value;
    if(!e) continue;
    dt_mipmap_cache_stats_entry_t s = { 0 };
    s.imgid = get_imgid(e->key);
    s.mip = (int)get_size(e->key);
    s.size = e->data_size; // real allocation (cost is byte-based for thumbs, slot-based for f/full)
    g_array_append_val(out, s);
  }
  dt_pthread_mutex_unlock(&c->lock);
}
 
void dt_mipmap_cache_get_usage(dt_mipmap_cache_t *cache, size_t *current, size_t *max)
{
  if(current) *current = 0;
  if(max) *max = 0;
  if(IS_NULL_PTR(cache)) return;
 
  // Real bytes currently held = sum of entry allocations across sub-caches.
  GArray *entries = dt_mipmap_cache_get_entries_stats(cache);
  size_t used = 0;
  for(guint i = 0; i < entries->len; i++)
    used += g_array_index(entries, dt_mipmap_cache_stats_entry_t, i).size;
  g_array_free(entries, TRUE);
 
  if(current) *current = used;
  if(max) *max = darktable.dtresources.mipmap_memory; // user-specified mipmap RAM budget
}
 
GArray *dt_mipmap_cache_get_entries_stats(dt_mipmap_cache_t *cache)
{
  GArray *out = g_array_new(FALSE, FALSE, sizeof(dt_mipmap_cache_stats_entry_t));
  if(IS_NULL_PTR(cache)) return out;
  _mipmap_subcache_collect(&cache->mip_thumbs.cache, out);
  _mipmap_subcache_collect(&cache->mip_f.cache, out);
  _mipmap_subcache_collect(&cache->mip_full.cache, out);
  return out;
}
 
static dt_mipmap_cache_one_t *_get_cache(dt_mipmap_cache_t *cache, const dt_mipmap_size_t mip)
{
  switch(mip)
  {
    case DT_MIPMAP_FULL:
      return &cache->mip_full;
    case DT_MIPMAP_F:
      return &cache->mip_f;
    default:
      return &cache->mip_thumbs;
  }
}
 
// if we get a zero-sized image, paint skulls to signal a missing image
static void _paint_skulls(dt_mipmap_buffer_t *buf, struct dt_mipmap_buffer_dsc *dsc, const int32_t imgid, const dt_mipmap_size_t mip)
{
  if(dsc->width == 0 || dsc->height == 0)
  {
    dt_print(DT_DEBUG_CACHE, "[mipmap cache get] got a zero-sized image for img %u mip %d!\n", imgid, mip);
    if(mip < DT_MIPMAP_F)
      buf->buf = dead_image_8(dsc);
    else
      buf->buf = NULL; // full images with NULL buffer have to be handled, indicates `missing image', but still return locked slot
  }
}
 
static void _validate_buffer(dt_mipmap_buffer_t *buf, struct dt_mipmap_buffer_dsc *dsc, const int32_t imgid,
                             const dt_mipmap_size_t mip)
{
  // Unpoison the dsc region
  ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
 
  // May update buf->buf:
  _sync_dsc_to_buf(buf, dsc, imgid, mip);
 
  // Unpoison the pixel buffer region
  ASAN_UNPOISON_MEMORY_REGION(buf->buf, dsc->size - dt_mipmap_buffer_dsc_size);
}
 
// Grab our own local copy of the image structure
static gboolean _get_image_copy(const int32_t imgid, dt_image_t *buffered_image)
{
  // load the image:
  // make sure we access the r/w lock as shortly as possible!
  gboolean no_buffer = TRUE;
  const dt_image_t *cimg = dt_image_cache_get(darktable.image_cache, imgid, 'r');
 
  if(cimg)
  {
    *buffered_image = *cimg;
    no_buffer = FALSE;
  }
 
  dt_image_cache_read_release(darktable.image_cache, cimg);
 
  return no_buffer;
}
 
// Actually do the work: produce an image
static void _generate_blocking(dt_cache_entry_t *entry, dt_mipmap_buffer_t *buf,
                               const int32_t imgid, const dt_mipmap_size_t mip, dt_atomic_int *shutdown)
{
  struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
  // Unpoison the descriptor before reading any field in this function.
  ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
  // _generate_blocking() can write directly into the cache pixel payload
  // (notably through _init_8() -> _write_image()), so unpoison it up-front.
  ASAN_UNPOISON_MEMORY_REGION(_get_buffer_from_dsc(dsc), dsc->size - dt_mipmap_buffer_dsc_size);
  const uint32_t original_width = dsc->width;
  const uint32_t original_height = dsc->height;
  const float original_iscale = dsc->iscale;
  const dt_colorspaces_color_profile_type_t original_color_space = dsc->color_space;
 
  if(!(dsc->flags & DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE))
  {
    // Already in cache, no I/O needed
    dt_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d at mip size %i (%ix%i) will skip disk I/O, found in RAM cache.\n", imgid, mip,
             dsc->width, dsc->height);
    _validate_buffer(buf, dsc, imgid, mip);
    return;
  }
 
  if(mip == DT_MIPMAP_FULL)
  {
    dt_image_t buffered_image;
    if(_get_image_copy(imgid, &buffered_image)) return;
 
    // Get the input file path, possibly from our local cache of copied images
    char filename[PATH_MAX] = { 0 };
    dt_image_path_source_t source = dt_image_choose_input_path(&buffered_image, filename, sizeof(filename), FALSE);
    if(source == DT_IMAGE_PATH_NONE) return;
 
    dt_print(DT_DEBUG_CACHE,
      "[mipmap_cache] fetch image %i at mip size %d float32 (%ix%i) from original file I/O\n",
      imgid, mip, dsc->width, dsc->height);
 
    // will call dt_mipmap_cache_alloc() internally:
    dt_imageio_retval_t ret = dt_imageio_open(&buffered_image, filename, buf);
 
    // We need to update dsc because dt_imageio_open re-alloc entry->data
    dsc = _get_dsc_from_entry(entry);
 
    if(ret == DT_IMAGEIO_OK)
    {
      // swap back new image data, may contain updated EXIF & colorspace
      dt_image_t *img = dt_image_cache_get(darktable.image_cache, imgid, 'w');
      // dt_image_cache_get() returns NULL if the image vanished from the DB/cache while this
      // full-res I/O was in flight (e.g. removed from the library concurrently) -- nothing left
      // to swap the freshly-read data back into.
      if(!IS_NULL_PTR(img))
      {
        *img = buffered_image;
        dt_image_cache_write_release(darktable.image_cache, img, DT_IMAGE_CACHE_RELAXED);
      }
    }
    else
    {
      dsc->width = dsc->height = 0;
      dsc->iscale = 0.f;
    }
  }
  else if(mip == DT_MIPMAP_F)
  {
    dt_print(DT_DEBUG_CACHE,
             "[mipmap_cache] compute mip size %d float32 for image %i (%ix%i) from original file \n", mip,
             imgid, dsc->width, dsc->height);
    _init_f(buf, (float *)_get_buffer_from_dsc(dsc), &dsc->width, &dsc->height, &dsc->iscale, imgid);
  }
  else
  {
    // 8-bit thumbs
    dt_print(DT_DEBUG_CACHE,
             "[mipmap_cache] compute mip size %d uint8 for image %i (%ix%i) from original file \n", mip,
             imgid, dsc->width, dsc->height);
    _init_8((uint8_t *)_get_buffer_from_dsc(dsc), &dsc->width, &dsc->height, &dsc->iscale, &dsc->color_space, imgid,
            mip, shutdown);
  }
 
  if(shutdown && dt_atomic_get_int(shutdown))
  {
    /* A stale GUI request stopped its thumbnail/export pipe while this cache entry
     * was still being produced. Keep the entry marked as "generate" so the next
     * request can restart from a clean state instead of reusing a poisoned empty
     * thumbnail for the rest of the session. */
    dsc->width = original_width;
    dsc->height = original_height;
    dsc->iscale = original_iscale;
    dsc->color_space = original_color_space;
    _invalidate_buffer(buf);
    return;
  }
 
  dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE;
  _paint_skulls(buf, dsc, imgid, mip);
  _validate_buffer(buf, dsc, imgid, mip);
 
  dt_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d at mip size %d got a new cache entry (%ix%i / %ix%i) at %p\n", imgid, mip, 
    buf->width, buf->height, dsc->width, dsc->height, buf->buf);
}
 
 
void dt_mipmap_cache_get_with_caller(dt_mipmap_cache_t *cache, dt_mipmap_buffer_t *buf, const int32_t imgid,
                                     const dt_mipmap_size_t mip, const dt_mipmap_get_flags_t flags,
                                     const char mode, const char *file, int line)
{
  dt_mipmap_cache_get_with_caller_and_shutdown(cache, buf, imgid, mip, flags, mode, NULL, file, line);
}
 
void dt_mipmap_cache_get_with_caller_and_shutdown(dt_mipmap_cache_t *cache, dt_mipmap_buffer_t *buf,
                                                  const int32_t imgid, const dt_mipmap_size_t mip,
                                                  const dt_mipmap_get_flags_t flags, const char mode,
                                                  dt_atomic_int *shutdown, const char *file, int line)
{
  assert(mip <= DT_MIPMAP_NONE && mip >= DT_MIPMAP_0);
 
  const int32_t key = get_key(imgid, mip);
 
  // Allocation functions will check those if we run them
  buf->imgid = imgid;
  buf->size = mip;
 
  if(flags == DT_MIPMAP_TESTLOCK)
  {
    // simple case: only get and lock if it's there.
    dt_cache_entry_t *entry = dt_cache_testget(&_get_cache(cache, mip)->cache, key, mode);
    buf->cache_entry = entry;
 
    if(entry)
      _validate_buffer(buf, _get_dsc_from_entry(entry), imgid, mip);
    else
      _invalidate_buffer(buf);
  }
  else if(flags == DT_MIPMAP_BLOCKING)
  {
    // simple case: blocking get
    dt_cache_entry_t *entry =  dt_cache_get_with_caller(&_get_cache(cache, mip)->cache, key, mode, file, line);
    buf->cache_entry = entry;
    __sync_fetch_and_add(&(_get_cache(cache, mip)->stats_fetches), 1);
    _generate_blocking(entry, buf, imgid, mip, shutdown);
 
    // image cache is leaving the write lock in place in case the image has been newly allocated.
    // this leads to a slight increase in thread contention, so we opt for dropping the write lock
    // and acquiring a read lock immediately after. since this opens a small window for other threads
    // to get in between, we need to take some care to re-init cache entries and dsc.
    // note that concurrencykit has rw locks that can be demoted from w->r without losing the lock in between.
    if(mode == 'r')
    {
      entry->_lock_demoting = 1;
      // drop the write lock
      dt_cache_release(&_get_cache(cache, mip)->cache, entry);
      // get a read lock
      entry = dt_cache_get(&_get_cache(cache, mip)->cache, key, mode);
      entry->_lock_demoting = 0;
    }
 
    buf->cache_entry = entry;
 
    // The cache can't be locked twice from the same thread,
    // because then it would never unlock.
#ifdef _DEBUG
    const pthread_t writer = dt_pthread_rwlock_get_writer(&(buf->cache_entry->lock));
    if(mode == 'w')
    {
      assert(pthread_equal(writer, pthread_self()));
    }
    else
    {
      assert(!pthread_equal(writer, pthread_self()));
    }
#endif
  }
  else
  {
    _invalidate_buffer(buf);
  }
}
 
void dt_mipmap_cache_write_get_with_caller(dt_mipmap_cache_t *cache, dt_mipmap_buffer_t *buf, const int32_t imgid, const int mip, const char *file, int line)
{
  dt_mipmap_cache_get_with_caller(cache, buf, imgid, mip, DT_MIPMAP_BLOCKING, 'w', file, line);
}
 
void dt_mipmap_cache_release_with_caller(dt_mipmap_cache_t *cache, dt_mipmap_buffer_t *buf, const char *file,
                                         int line)
{
  if(buf->size == DT_MIPMAP_NONE || IS_NULL_PTR(buf->cache_entry)) return;
  assert(buf->imgid > 0);
  assert(buf->size >= DT_MIPMAP_0);
  assert(buf->size < DT_MIPMAP_NONE);
  dt_cache_release_with_caller(&_get_cache(cache, buf->size)->cache, buf->cache_entry, file, line);
  buf->size = DT_MIPMAP_NONE;
  buf->buf = NULL;
}
 
 
// return index dt_mipmap_size_t having at least width & height requested instead of minimum combined diff
// please note that the requested size is in pixels not dots.
dt_mipmap_size_t dt_mipmap_cache_get_matching_size(const dt_mipmap_cache_t *cache, const int32_t width,
                                                   const int32_t height, const uint32_t imgid)
{
  for(int k = DT_MIPMAP_0; k < DT_MIPMAP_F; k++)
  {
    // We assume a "fit" situation, typically rectangle within square
    // so we don't need both dimensions to be greater than requested
    if((cache->max_width[k] >= width) || (cache->max_height[k] >= height))
    {
      dt_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d will load a mip size %i (%" G_GSIZE_FORMAT "x%" G_GSIZE_FORMAT ")\n", imgid, k, cache->max_width[k], cache->max_height[k]);
      return k;
    }
  }
  return DT_MIPMAP_F - 1;
}
 
dt_mipmap_size_t dt_mipmap_cache_get_fitting_size(const dt_mipmap_cache_t *cache, const int32_t width,
                                                   const int32_t height, const uint32_t imgid)
{
  for(int k = DT_MIPMAP_F - 1; k >= DT_MIPMAP_0; k--)
  {
    if((cache->max_width[k] <= width) && (cache->max_height[k] <= height))
    {
      dt_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d will fit a mip size %i (%" G_GSIZE_FORMAT "x%" G_GSIZE_FORMAT ")\n", imgid, k, cache->max_width[k], cache->max_height[k]);
      return k;
    }
  }
  return DT_MIPMAP_0;
}
 
void dt_mipmap_cache_swap_at_size(dt_mipmap_cache_t *cache, const int32_t imgid, const dt_mipmap_size_t mip, const uint8_t *const in,
  const int32_t width, const int32_t height, dt_colorspaces_color_profile_type_t profile)
{
  if(mip >= DT_MIPMAP_F || mip < DT_MIPMAP_0) return;
 
  const uint32_t key = get_key(imgid, mip);
  dt_cache_entry_t *entry = dt_cache_get_with_caller(&_get_cache(cache, mip)->cache, key, 'w', __FILE__, __LINE__);
  if(entry)
  {
    struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
    // Unpoison descriptor and pixel payload before reading/writing either.
    ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
    ASAN_UNPOISON_MEMORY_REGION(_get_buffer_from_dsc(dsc), dsc->size - dt_mipmap_buffer_dsc_size);
    dt_print(DT_DEBUG_CACHE, "[mipmap_cache] image %d is synchronized from pipeline at size %i (%ix%i->%ix%i)\n", 
      imgid, mip, width, height, dsc->width, dsc->height);
 
    // Downscale
    dsc->iscale = 1.f;
    const int32_t wd = dsc->width;
    const int32_t ht = dsc->height;
    uint8_t *buf =(uint8_t *)_get_buffer_from_dsc(dsc);
    dt_iop_flip_and_zoom_8(in, width, height, buf, wd, ht,
                           ORIENTATION_NONE, &dsc->width, &dsc->height);
 
    // Color convert
    cmsHTRANSFORM transform = NULL;
    pthread_rwlock_rdlock(&darktable.color_profiles->xprofile_lock);
    gboolean alloc = FALSE;
 
    if(profile == DT_COLORSPACE_DISPLAY)
    { 
      // Convert to whatever display space to save thumbnails into Adobe RGB
      transform = darktable.color_profiles->transform_display_to_adobe_rgb;
    }
    else
    {
      const dt_colorspaces_color_profile_t *const profile_in
          = dt_colorspaces_get_profile(profile, "", DT_PROFILE_DIRECTION_DISPLAY);
      const dt_colorspaces_color_profile_t *const profile_out
          = dt_colorspaces_get_profile(DT_COLORSPACE_ADOBERGB, "", DT_PROFILE_DIRECTION_DISPLAY);
      if(profile_in && profile_out)
      {
        alloc = TRUE;
        transform = cmsCreateTransform(profile_in->profile, TYPE_BGRA_8, profile_out->profile, TYPE_RGBA_8,
                                        INTENT_PERCEPTUAL, 0);
      }
    }
 
    // Need to save BGRA back to RGBA. The function name is misleading, 
    // it's still only swapping R <-> B.
    dt_colorspaces_transform_rgba8_to_bgra8(transform, buf, buf, dsc->width, dsc->height);
    if(alloc) cmsDeleteTransform(transform);
    pthread_rwlock_unlock(&darktable.color_profiles->xprofile_lock);
 
    dsc->color_space = DT_COLORSPACE_ADOBERGB;
    dsc->flags &= ~DT_MIPMAP_BUFFER_DSC_FLAG_GENERATE;
    dt_cache_release(&_get_cache(cache, mip)->cache, entry);
  }
}
 
 
void dt_mipmap_cache_remove_at_size(dt_mipmap_cache_t *cache, const int32_t imgid, const dt_mipmap_size_t mip, const gboolean flush_disk)
{
  if(mip >= DT_MIPMAP_F || mip < DT_MIPMAP_0) return;
 
  // get rid of all ldr thumbnails:
  const uint32_t key = get_key(imgid, mip);
  dt_cache_entry_t *entry = dt_cache_testget(&_get_cache(cache, mip)->cache, key, 'w');
  if(entry)
  {
    struct dt_mipmap_buffer_dsc *dsc = _get_dsc_from_entry(entry);
    ASAN_UNPOISON_MEMORY_REGION(dsc, dt_mipmap_buffer_dsc_size);
    if(flush_disk) dsc->flags |= DT_MIPMAP_BUFFER_DSC_FLAG_INVALIDATE;
    dt_cache_release(&_get_cache(cache, mip)->cache, entry);
    dt_cache_remove(&_get_cache(cache, mip)->cache, key);
  }
  if(flush_disk)
  {
    // directly remove the file on disk cache even if we don't have a memory entry
    dt_mipmap_cache_unlink_ondisk_thumbnail(cache, imgid, mip);
  }
}
 
// get rid of all ldr thumbnails:
void dt_mipmap_cache_remove(dt_mipmap_cache_t *cache, const int32_t imgid, const gboolean flush_disk)
{
  for(dt_mipmap_size_t k = DT_MIPMAP_0; k < DT_MIPMAP_F; k++)
    dt_mipmap_cache_remove_at_size(cache, imgid, k, flush_disk);
}
 
// write thumbnail to disc if not existing there
void dt_mimap_cache_evict(dt_mipmap_cache_t *cache, const int32_t imgid)
{
  for(dt_mipmap_size_t k = DT_MIPMAP_0; k < DT_MIPMAP_F; k++)
    dt_cache_remove(&_get_cache(cache, k)->cache, get_key(imgid, k));
}
 
static void _init_f(dt_mipmap_buffer_t *mipmap_buf, float *out, uint32_t *width, uint32_t *height, float *iscale,
                    const int32_t imgid)
{
  const uint32_t wd = *width, ht = *height;
 
  /* do not even try to process file if it isn't available */
  char filename[PATH_MAX] = { 0 };
  dt_image_path_source_t source = DT_IMAGE_PATH_NONE;
  {
    const dt_image_t *img = dt_image_cache_get(darktable.image_cache, imgid, 'r');
    if(img)
    {
      source = dt_image_choose_input_path(img, filename, sizeof(filename), FALSE);
      dt_image_cache_read_release(darktable.image_cache, img);
    }
  }
  if(source == DT_IMAGE_PATH_NONE)
  {
    *width = *height = 0;
    *iscale = 0.0f;
    return;
  }
 
  dt_mipmap_buffer_t buf;
  dt_mipmap_cache_get(darktable.mipmap_cache, &buf, imgid, DT_MIPMAP_FULL, DT_MIPMAP_BLOCKING, 'r');
 
  // lock image after we have the buffer, we might need to lock the image struct for
  // writing during raw loading, to write to width/height.
  const dt_image_t *image = dt_image_cache_get(darktable.image_cache, imgid, 'r');
 
  dt_iop_roi_t roi_in, roi_out;
  roi_in.x = roi_in.y = 0;
  roi_in.width = image->width;
  roi_in.height = image->height;
  roi_in.scale = 1.0f;
 
  roi_out.x = roi_out.y = 0;
 
  // now let's figure out the scaling...
 
  // MIP_F is 4 channels, and we do not demosaic here
  roi_out.scale = fminf(((float)wd) / (float)image->width, ((float)ht) / (float)image->height);
  roi_out.width = roi_out.scale * roi_in.width;
  roi_out.height = roi_out.scale * roi_in.height;
 
  if(IS_NULL_PTR(buf.buf) || buf.width == 0 || buf.height == 0)
  {
    dt_image_cache_read_release(darktable.image_cache, image);
    *width = *height = 0;
    *iscale = 0.0f;
    return;
  }
 
  mipmap_buf->color_space = DT_COLORSPACE_NONE; // TODO: do we need that information in this buffer?
 
  if(image->dsc.filters)
  {
    if(image->dsc.filters != 9u && image->dsc.datatype == TYPE_FLOAT)
    {
      dt_iop_clip_and_zoom_mosaic_half_size_f((float *const)out, (const float *const)buf.buf, &roi_out, &roi_in,
                                              roi_out.width, roi_in.width, image->dsc.filters);
    }
    else if(image->dsc.filters != 9u && image->dsc.datatype == TYPE_UINT16)
    {
      dt_iop_clip_and_zoom_mosaic_half_size((uint16_t * const)out, (const uint16_t *)buf.buf, &roi_out, &roi_in,
                                            roi_out.width, roi_in.width, image->dsc.filters);
    }
    else if(image->dsc.filters == 9u && image->dsc.datatype == TYPE_UINT16)
    {
      dt_iop_clip_and_zoom_mosaic_third_size_xtrans((uint16_t * const)out, (const uint16_t *)buf.buf, &roi_out,
                                                    &roi_in, roi_out.width, roi_in.width, image->dsc.xtrans);
    }
    else if(image->dsc.filters == 9u && image->dsc.datatype == TYPE_FLOAT)
    {
      dt_iop_clip_and_zoom_mosaic_third_size_xtrans_f(out, (const float *)buf.buf, &roi_out, &roi_in,
                                                      roi_out.width, roi_in.width, image->dsc.xtrans);
    }
    else
    {
      dt_unreachable_codepath();
    }
  }
  else
  {
    // downsample
    dt_iop_clip_and_zoom(out, (const float *)buf.buf, &roi_out, &roi_in, roi_out.width, roi_in.width);
  }
 
  dt_mipmap_cache_release(darktable.mipmap_cache, &buf);
 
  *width = roi_out.width;
  *height = roi_out.height;
  *iscale = (float)image->width / (float)roi_out.width;
 
  dt_image_cache_read_release(darktable.image_cache, image);
}
 
 
// dummy functions for `export' to mipmap buffers:
typedef struct _dummy_data_t
{
  dt_imageio_module_data_t head;
  uint8_t *buf;
} _dummy_data_t;
 
static int _levels(dt_imageio_module_data_t *data)
{
  return IMAGEIO_RGB | IMAGEIO_INT8;
}
 
static int _bpp(dt_imageio_module_data_t *data)
{
  return 8;
}
 
static int _write_image(dt_imageio_module_data_t *data, const char *filename, const void *in,
                        dt_colorspaces_color_profile_type_t over_type, const char *over_filename,
                        void *exif, int exif_len, int32_t imgid, int num, int total, dt_dev_pixelpipe_t *pipe,
                        const gboolean export_masks)
{
  _dummy_data_t *d = (_dummy_data_t *)data;
  memcpy(d->buf, in, sizeof(uint32_t) * data->width * data->height);
  return 0;
}
 
static int _load_jpg(const char *filename, const int32_t imgid, const uint32_t wd, const uint32_t ht,
                     const dt_mipmap_size_t size, const dt_image_orientation_t orientation, uint8_t *buf,
                     uint32_t *width, uint32_t *height, dt_colorspaces_color_profile_type_t *color_space)
{
  int res = 1;
  dt_imageio_jpeg_t jpg;
  if(!dt_imageio_jpeg_read_header(filename, &jpg))
  {
    *color_space = dt_imageio_jpeg_read_color_space(&jpg);
    uint8_t *tmp = (uint8_t *)dt_alloc_align(sizeof(uint8_t) * jpg.width * jpg.height * 4);
    if(tmp && !dt_imageio_jpeg_read(&jpg, tmp))
    {
      // scale to fit
      dt_print(DT_DEBUG_CACHE, "[mipmap_cache] generate mip size %d for image %d from jpeg\n", size, imgid);
      dt_iop_flip_and_zoom_8(tmp, jpg.width, jpg.height, buf, wd, ht, orientation, width, height);
      res = 0;
    }
    dt_free_align(tmp);
  }
  return res;
}
 
static int _find_sidecar_jpg(const char *filename, const char *ext, char *sidecar)
{
  const size_t filename_len = strlen(filename) - strlen(ext);
  const char *exts[4] = { ".jpg", ".JPG", ".jpeg", ".JPEG" };
 
  for(int i = 0; i < 4; i++)
  {
    // Damage control. Should never happen.
    if(filename_len + strlen(exts[i]) >= PATH_MAX)
      continue;
 
    // Construct the sidecar filename
    const size_t str_copy = g_snprintf(sidecar, PATH_MAX, "%.*s%s", (int)filename_len, filename, exts[i]);
 
    // Check if the filename was too long or if the copy failed.
    if (str_copy == 0 || str_copy >= PATH_MAX)
      continue;
 
    if(g_file_test(sidecar, G_FILE_TEST_EXISTS))
      return 1;
  }
 
  return 0;
}
 
static void _init_8(uint8_t *buf, uint32_t *width, uint32_t *height, float *iscale,
                    dt_colorspaces_color_profile_type_t *color_space, const int32_t imgid,
                    const dt_mipmap_size_t size, dt_atomic_int *shutdown)
{
  if(size >= DT_MIPMAP_F || *width < 16 || *height < 16) return;
 
  *iscale = 1.0f;
  const uint32_t wd = *width, ht = *height;
 
  char filename[PATH_MAX] = { 0 };
  char ext[6] = { 0 };
  gboolean input_exists, is_jpg_input, use_embedded_jpg;
  const int embedded_jpg_mode = dt_conf_get_int("lighttable/embedded_jpg");
  _write_mipmap_to_disk(imgid, filename, ext, &input_exists, &is_jpg_input, &use_embedded_jpg, NULL);
 
  /* do not even try to process file if it isn't available */
  if(!input_exists)
  {
    *width = *height = 0;
    *iscale = 0.0f;
    *color_space = DT_COLORSPACE_NONE;
    return;
  }
 
  int res = 1;
 
  // try to generate mip from larger mip
  // This expects that invalid mips will be flushed, so the assumption is:
  // if mip then it's valid (with regard to current history)
  if(res && !use_embedded_jpg && size < DT_MIPMAP_F - 1)
  {
    for(dt_mipmap_size_t k = size + 1; k < DT_MIPMAP_F; k++)
    {
      dt_mipmap_buffer_t tmp;
      dt_mipmap_cache_get(darktable.mipmap_cache, &tmp, imgid, k, DT_MIPMAP_TESTLOCK, 'r');
      if(IS_NULL_PTR(tmp.buf)) continue;
 
      *color_space = tmp.color_space;
      // downsample
      dt_iop_flip_and_zoom_8(tmp.buf, tmp.width, tmp.height, buf, wd, ht, ORIENTATION_NONE, width, height);
      dt_print(DT_DEBUG_CACHE, "[mipmap_cache] generate mip size %d for image %d from mip size %d (%ix%i->%ix%i)\n", 
        size, imgid, k, tmp.width, tmp.height, *width, *height);
 
      dt_mipmap_cache_release(darktable.mipmap_cache, &tmp);
      res = 0;
      break;
    }
  }
 
  // Orientation is only needed when loading embedded JPEGs.
  dt_image_orientation_t orientation = ORIENTATION_NONE;
  if(use_embedded_jpg)
  {
    const dt_image_t *img = dt_image_cache_get(darktable.image_cache, imgid, 'r');
    if(img)
    {
      orientation = (img->orientation != ORIENTATION_NULL) ? img->orientation : ORIENTATION_NONE;
      dt_image_cache_read_release(darktable.image_cache, img);
    }
  }
 
  if(res && use_embedded_jpg)
  {
    char sidecar_filename[PATH_MAX] = { 0 };
 
    if(is_jpg_input)
    {
      // Input file is a JPEG
      res = _load_jpg(filename, imgid, wd, ht, size, orientation, buf, width, height, color_space);
    }
    else if(_find_sidecar_jpg(filename, ext, sidecar_filename))
    {
      // input file is a RAW but we have a companion JPEG file in the same folder:
      // use it in priority (it may be higher resolution/quality than embedded JPEG).
      res = _load_jpg(sidecar_filename, imgid, wd, ht, size, orientation, buf, width, height, color_space);
    }
    else
    {
      // input file is a RAW without companion JPEG:
      // try to load the embedded thumbnail. Might not be large enough though.
      uint8_t *tmp = NULL;
      int32_t thumb_width, thumb_height;
      res = dt_imageio_large_thumbnail(filename, &tmp, &thumb_width, &thumb_height, color_space, *width, *height);
      if(!res)
      {
        // We take the thumbnail no matter its size. It might be too small for the requested dimension,
        // and end up blurry. But it's less bad than the following scenario:
        // 1. user displays collections in grid of 5 columns (small thumbnail -> fetch embedded JPEG for performance, all good),
        // 2. user zooms in the grid, to 2 or 3 columns,
        // 3. suddently, embedded JPEG is too small so we ditch it for a full pipe recompute,
        // 4. but then, color/contrast/appearance unexpectedly changes and user doesn't understand WTF just happened.
        // Blurry is less bad than randomly inconsistent, plus user has a GUI way in lighttable to
        // change how thumbs are processed at runtime.
        dt_print(DT_DEBUG_CACHE, "[mipmap_cache] generate mip size %d for image %d from embedded jpeg\n", size, imgid);
        dt_iop_flip_and_zoom_8(tmp, thumb_width, thumb_height, buf, wd, ht, orientation, width, height);
        dt_pixelpipe_cache_free_align(tmp);
      }
    }
  }
 
  if(res)
  {
    if(embedded_jpg_mode == 2)
    {
      dt_print(DT_DEBUG_CACHE,
               "[mipmap_cache] embedded JPEG mode forbids raw processing for image %d at mip %d\n",
               imgid, size);
      *width = *height = 0;
      *iscale = 0.0f;
      *color_space = DT_COLORSPACE_NONE;
      return;
    }
 
    // try the real thing: rawspeed + pixelpipe
    dt_imageio_module_format_t format;
    _dummy_data_t dat;
    format.bpp = _bpp;
    format.write_image = _write_image;
    format.levels = _levels;
    dat.head.max_width = wd;
    dat.head.max_height = ht;
    dat.buf = buf;
    // export with flags: ignore exif (don't load from disk), don't swap byte order, don't do hq processing,
    // no upscaling and signal we want thumbnail export
    res = dt_imageio_export_with_flags(imgid, "unused", &format, (dt_imageio_module_data_t *)&dat, TRUE, FALSE, FALSE,
                                       FALSE, TRUE, NULL, FALSE, FALSE, DT_COLORSPACE_ADOBERGB, NULL, DT_INTENT_LAST, NULL,
                                       NULL, 1, 1, NULL, shutdown);
    if(!res)
    {
      dt_print(DT_DEBUG_CACHE, "[mipmap_cache] generated mip %d for image %d from scratch\n", size, imgid);
      // might be smaller, or have a different aspect than what we got as input.
      *width = dat.head.width;
      *height = dat.head.height;
      *iscale = 1.0f;
      *color_space = DT_COLORSPACE_ADOBERGB;
    }
  }
 
  // any errors?
  if(res)
  {
    fprintf(stderr, "[mipmap_cache] could not process thumbnail!\n");
    *width = *height = 0;
    *iscale = 0.0f;
    *color_space = DT_COLORSPACE_NONE;
  }
}
 
void dt_mipmap_cache_copy_thumbnails(const dt_mipmap_cache_t *cache, const uint32_t dst_imgid, const uint32_t src_imgid)
{
  gboolean write_to_disk_src, write_to_disk_dst;
  _write_mipmap_to_disk(src_imgid, NULL, NULL, NULL, NULL, NULL, &write_to_disk_src);
  _write_mipmap_to_disk(dst_imgid, NULL, NULL, NULL, NULL, NULL, &write_to_disk_dst);
 
  if(cache->cachedir[0] && write_to_disk_src && write_to_disk_dst)
  {
    for(dt_mipmap_size_t mip = DT_MIPMAP_0; mip < DT_MIPMAP_F; mip++)
    {
      // try and load from disk, if successful set flag
      char srcpath[PATH_MAX] = {0};
      char dstpath[PATH_MAX] = {0};
      dt_mipmap_get_cache_filename(srcpath, cache, mip, src_imgid);
      dt_mipmap_get_cache_filename(dstpath, cache, mip, dst_imgid);
      GFile *src = g_file_new_for_path(srcpath);
      GFile *dst = g_file_new_for_path(dstpath);
      GError *gerror = NULL;
      g_file_copy(src, dst, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
      // ignore errors, we tried what we could.
      g_object_unref(dst);
      g_object_unref(src);
      g_clear_error(&gerror);
    }
  }
}
 
// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on