common/image.c

Go to the documentation of this file.

/*
  This file is part of darktable,
  Copyright (C) 2009-2014, 2016 johannes hanika.
  Copyright (C) 2010, 2015 Bruce Guenter.
  Copyright (C) 2010-2012 Henrik Andersson.
  Copyright (C) 2010-2012 Jérémy Rosen.
  Copyright (C) 2010 Stuart Henderson.
  Copyright (C) 2010-2018, 2020 Tobias Ellinghaus.
  Copyright (C) 2010 Zeus V Panchenko.
  Copyright (C) 2011 Brian Teague.
  Copyright (C) 2011, 2013 José Carlos García Sogo.
  Copyright (C) 2011 Robert Bieber.
  Copyright (C) 2012 Alexander Clausen.
  Copyright (C) 2012 Christian Tellefsen.
  Copyright (C) 2012-2013 Dennis Gnad.
  Copyright (C) 2012 James C. McPherson.
  Copyright (C) 2012 marcel.
  Copyright (C) 2012 Richard Wonka.
  Copyright (C) 2012-2014 Ulrich Pegelow.
  Copyright (C) 2013-2016, 2018-2022 Pascal Obry.
  Copyright (C) 2013-2016 Roman Lebedev.
  Copyright (C) 2014 Pascal de Bruijn.
  Copyright (C) 2014-2016 Pedro Côrte-Real.
  Copyright (C) 2016 Mark Oteiza.
  Copyright (C) 2016-2018 Peter Budai.
  Copyright (C) 2017, 2019 luzpaz.
  Copyright (C) 2017 Žilvinas Žaltiena.
  Copyright (C) 2018-2019 Edgardo Hoszowski.
  Copyright (C) 2018 Kelvie Wong.
  Copyright (C) 2018 Mario Lueder.
  Copyright (C) 2019-2022 Aldric Renaudin.
  Copyright (C) 2019 August Schwerdfeger.
  Copyright (C) 2019 Bill Ferguson.
  Copyright (C) 2019 fvollmer.
  Copyright (C) 2019-2022 Hanno Schwalm.
  Copyright (C) 2019 jakubfi.
  Copyright (C) 2019-2022 Philippe Weyland.
  Copyright (C) 2020-2026 Aurélien PIERRE.
  Copyright (C) 2020 Chris Elston.
  Copyright (C) 2020 Dan Torop.
  Copyright (C) 2020 GrahamByrnes.
  Copyright (C) 2020 hatsunearu.
  Copyright (C) 2020 Hubert Kowalski.
  Copyright (C) 2020 JP Verrue.
  Copyright (C) 2020 Marco.
  Copyright (C) 2020 U-DESKTOP-HQME86J\marco.
  Copyright (C) 2021 Daniel Vogelbacher.
  Copyright (C) 2021-2022 HansBull.
  Copyright (C) 2021 Marco Carrarini.
  Copyright (C) 2021-2022 Miloš Komarčević.
  Copyright (C) 2021 Ralf Brown.
  Copyright (C) 2021 wpferguson.
  Copyright (C) 2022 Martin Bařinka.
  Copyright (C) 2022 Nicolas Auffray.
  Copyright (C) 2022 paolodepetrillo.
  Copyright (C) 2022 Victor Forsiuk.
  Copyright (C) 2023 Ricky Moon.
  Copyright (C) 2024-2025 Guillaume Stutin.
  
  darktable is free software: you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation, either version 3 of the License, or
  (at your option) any later version.
  
  darktable is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License
  along with darktable.  If not, see <http://www.gnu.org/licenses/>.
*/
 
#include "common/image.h"
#include "common/paths.h"   // DT_PATH_MAX
#include "common/image_notify.h"
#include "common/thumbnail_notify.h"
#include "common/act_on.h"
#include "common/history_actions.h"
#include "develop/imageop.h"
#include "common/collection.h"
#include "system/dtpthread.h"
#include "common/xmp_sidecar.h"
#include "metadata/exif.h"
#include "common/file_location.h"
#include "common/grouping.h"
#include "common/film.h"
#include "history/history.h"
#include "database/database.h"
#include "database/history_repository.h"
#include "database/film_repository.h"
#include "database/image_repository.h"
#include "database/removed_image_repository.h"
#include "develop/history_merge.h"
#include "history/history_snapshot.h"
#include "caches/image_cache.h"
#include "common/image_extensions.h"
#include "imageio/imageio_core.h"
#include "caches/mipmap_cache.h"
#include "metadata/notify.h"
#include "metadata/ratings.h"
#include "metadata/tags.h"
#include "common/undo.h"
#include "common/selection.h"
#include "common/datetime.h"
#include "common/conf.h"
#include "control/control.h"
#include "system/atomic.h"
#include "develop/lightroom.h"
#include "develop/develop.h"
#include "views/view.h"
#include "win/filepath.h"
#include <assert.h>
#include <ctype.h>
#include <inttypes.h>
#include <math.h>
#include <sqlite3.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#ifndef _WIN32
#include <glob.h>
#endif
#include <glib/gstdio.h>
#include "common/utility.h"
 
typedef struct dt_undo_monochrome_t
{
  int32_t imgid;
  gboolean before;
  gboolean after;
} dt_undo_monochrome_t;
 
typedef struct dt_undo_datetime_t
{
  int32_t imgid;
  char before[DT_DATETIME_LENGTH];
  char after[DT_DATETIME_LENGTH];
} dt_undo_datetime_t;
 
typedef struct dt_undo_geotag_t
{
  int32_t imgid;
  dt_image_geoloc_t before;
  dt_image_geoloc_t after;
} dt_undo_geotag_t;
 
typedef struct dt_undo_duplicate_t
{
  int32_t orig_imgid;
  int32_t version;
  int32_t new_imgid;
} dt_undo_duplicate_t;
 
/* An image removed from the library holds nothing in memory worth keeping: the whole state
 * is the rows database/removed_image_repository.c staged, and this is the ticket to them. */
typedef struct dt_undo_remove_t
{
  int32_t imgid;
  int snap_id;
} dt_undo_remove_t;
 
static void _pop_undo_execute(const int32_t imgid, const gboolean before, const gboolean after);
static int32_t _image_duplicate_with_version(const int32_t imgid, const int32_t newversion, const gboolean undo,
                                             const gboolean reload);
static void _image_remove(const int32_t imgid, const gboolean undo);
static void _pop_undo(gpointer user_data, const dt_undo_type_t type, dt_undo_data_t data, const dt_undo_action_t action, GList **imgs);
 
static void _copy_text_sidecar_if_present(const char *src_image_path, const char *dest_image_path);
static void _move_text_sidecar_if_present(const char *src_image_path, const char *dest_image_path, const gboolean overwrite);
 
// NULL terminated list of supported non-RAW extensions
//  const char *dt_non_raw_extensions[]
//    = { ".jpeg", ".jpg",  ".pfm", ".hdr", ".exr", ".pxn", ".tif", ".tiff", ".png",
//        ".j2c",  ".j2k",  ".jp2", ".jpc", ".gif", ".jpc", ".jp2", ".bmp",  ".dcm",
//        ".jng",  ".miff", ".mng", ".pbm", ".pnm", ".ppm", ".pgm", NULL };
gboolean dt_image_is_raw(const dt_image_t *img)
{
  return (img->flags & DT_IMAGE_RAW) != 0;
}
 
// LDR / HDR are flag-only predicates now: no filename-extension sniffing. The flags are set by the
// decoding codec from the real file content (the buffer datatype is not a dynamic-range signal:
// every raster decodes into a TYPE_FLOAT working buffer regardless of LDR/HDR), so these report what
// was really loaded, not what the file name suggested. They are kept as a public API so callers
// don't open-code the bitmask test.
gboolean dt_image_is_ldr(const dt_image_t *img)
{
  return (img->flags & DT_IMAGE_LDR) != 0;
}
 
gboolean dt_image_is_hdr(const dt_image_t *img)
{
  return (img->flags & DT_IMAGE_HDR) != 0;
}
 
gboolean dt_image_is_monochrome(const dt_image_t *img)
{
  return (img->flags & (DT_IMAGE_MONOCHROME | DT_IMAGE_MONOCHROME_BAYER)) ? TRUE : FALSE;
}
 
static void _image_set_monochrome_flag(const int32_t imgid, gboolean monochrome, gboolean undo_on)
{
  dt_image_t *img = NULL;
  gboolean changed = FALSE;
 
  img = dt_image_cache_get(imgid, 'r');
  if(img)
  {
    const int mask_bw = dt_image_monochrome_flags(img);
    dt_image_cache_read_release(img);
 
    if((!monochrome) && (mask_bw & DT_IMAGE_MONOCHROME_PREVIEW))
    {
      // wanting it to be color found preview
      img = dt_image_cache_get(imgid, 'w');
      img->flags &= ~(DT_IMAGE_MONOCHROME_PREVIEW | DT_IMAGE_MONOCHROME_WORKFLOW);
      changed = TRUE;
    }
    if(monochrome && ((mask_bw == 0) || (mask_bw == DT_IMAGE_MONOCHROME_PREVIEW)))
    {
      // wanting monochrome and found color or just preview without workflow activation
      img = dt_image_cache_get(imgid, 'w');
      img->flags |= (DT_IMAGE_MONOCHROME_PREVIEW | DT_IMAGE_MONOCHROME_WORKFLOW);
      changed = TRUE;
    }
    if(changed)
    {
      const int mask = dt_image_monochrome_flags(img);
      dt_image_cache_write_release(img, DT_IMAGE_CACHE_RELAXED);
 
      if(undo_on)
      {
        dt_undo_monochrome_t *undomono = (dt_undo_monochrome_t *)malloc(sizeof(dt_undo_monochrome_t));
        undomono->imgid = imgid;
        undomono->before = mask_bw;
        undomono->after = mask;
        dt_undo_record(dt_undo_get_global(), NULL, DT_UNDO_FLAGS, undomono, _pop_undo, g_free);
      }
    }
  }
  else
    fprintf(stderr,"[image] could not dt_image_cache_get imgid %i\n", imgid);
}
 
void dt_image_set_monochrome_flag(const int32_t imgid, gboolean monochrome)
{
  _image_set_monochrome_flag(imgid, monochrome, TRUE);
}
 
static void _pop_undo_execute(const int32_t imgid, const gboolean before, const gboolean after)
{
  _image_set_monochrome_flag(imgid, after, FALSE);
}
 
static gboolean _image_matrix_has_data(const float *matrix, const int count)
{
  float sum = 0.0f;
  for(int i = 0; i < count; i++)
  {
    if(!isfinite(matrix[i])) return FALSE;
    sum += fabsf(matrix[i]);
  }
  return sum > 0.0f;
}
 
gboolean dt_image_is_matrix_correction_supported(const dt_image_t *img)
{
  // Whether a camera input matrix (and the white balance / color calibration it feeds) applies
  // depends only on the colorimetry axis, never on the mosaic axis: an already-demosaiced raw
  // (sRAW / linear DNG, e.g. DxO PureRAW) still carries raw colorimetry and an embedded matrix
  // and must get matrix correction. A previous `S_RAW && dsc.filters == 0 -> FALSE` short-circuit
  // wrongly excluded exactly those files, leaving white balance and color calibration disabled
  // (green renders, issue #729).
  if(!(img->flags & (DT_IMAGE_RAW | DT_IMAGE_S_RAW))) return FALSE;
  if(img->flags & DT_IMAGE_MONOCHROME) return FALSE;
 
  const gboolean has_d65 = _image_matrix_has_data(img->d65_color_matrix, 9);
  const gboolean has_adobe = _image_matrix_has_data(&img->adobe_XYZ_to_CAM[0][0], 9);
 
  return (has_d65 || has_adobe) ? TRUE : FALSE;
}
 
gboolean dt_image_use_monochrome_workflow(const dt_image_t *img)
{
  return ((img->flags & (DT_IMAGE_MONOCHROME | DT_IMAGE_MONOCHROME_BAYER)) ||
          ((img->flags & DT_IMAGE_MONOCHROME_PREVIEW) && (img->flags & DT_IMAGE_MONOCHROME_WORKFLOW)));
}
 
int dt_image_monochrome_flags(const dt_image_t *img)
{
  return (img->flags & (DT_IMAGE_MONOCHROME | DT_IMAGE_MONOCHROME_PREVIEW | DT_IMAGE_MONOCHROME_BAYER));
}
 
/* ------------------------------------------------------------------------------------------
 * Canonical image-type API. See image.h and src/doc/image-type-detection.md.
 * Each predicate tests exactly one independent flag fact; no filename sniffing here.
 * ------------------------------------------------------------------------------------------ */
 
gboolean dt_image_pipe_class_is_provisional(const dt_image_t *img)
{
  return (img->flags & DT_IMAGE_BUFFER_RESOLVED) == 0;
}
 
gboolean dt_image_is_sraw(const dt_image_t *img)
{
  return (img->flags & DT_IMAGE_S_RAW) != 0;
}
 
gboolean dt_image_is_mosaiced(const dt_image_t *img)
{
  return (img->flags & DT_IMAGE_MOSAIC) != 0;
}
 
gboolean dt_image_needs_rawprepare(const dt_image_t *img)
{
  // both mosaiced raw and already-demosaiced sRAW/linear-DNG carry raw colorimetry
  return (img->flags & (DT_IMAGE_RAW | DT_IMAGE_S_RAW)) != 0;
}
 
dt_image_pipe_class_t dt_image_pipe_class(const dt_image_t *img)
{
  const gboolean resolved = (img->flags & DT_IMAGE_BUFFER_RESOLVED) != 0;
  const gboolean raw_colorimetry = (img->flags & (DT_IMAGE_RAW | DT_IMAGE_S_RAW)) != 0;
 
  // Raw colorimetry takes precedence over the LDR/HDR bit-depth flags: a float mosaiced raw
  // is flagged both RAW and HDR but must still be treated as a mosaiced raw, never as RGB HDR.
  if(raw_colorimetry)
  {
    // Once decoded, the mosaic bit is authoritative and tells a mosaiced raw apart from an
    // already-demosaiced raw (sRAW / linear DNG).
    if(resolved)
      return (img->flags & DT_IMAGE_MOSAIC) ? DT_IMAGE_PIPE_MOSAIC_RAW : DT_IMAGE_PIPE_LINEAR_RAW;
 
    // Provisional (not decoded yet, or a database predating DT_IMAGE_BUFFER_RESOLVED):
    // DT_IMAGE_S_RAW is only ever set by a codec on a real decode (the extension detector
    // never sets it), so it reliably means an already-demosaiced raw even without the resolved
    // bit. Otherwise assume the common case of a mosaiced sensor raw; dt_image_buffer_resolve_flags()
    // corrects the guess on first decode.
    if((img->flags & DT_IMAGE_S_RAW) && !(img->flags & DT_IMAGE_RAW))
      return DT_IMAGE_PIPE_LINEAR_RAW;
    return DT_IMAGE_PIPE_MOSAIC_RAW;
  }
 
  if(dt_image_is_ldr(img)) return DT_IMAGE_PIPE_RGB_LDR;
  if(dt_image_is_hdr(img)) return DT_IMAGE_PIPE_RGB_HDR;
 
  return DT_IMAGE_PIPE_UNKNOWN;
}
 
gboolean dt_image_needs_demosaic(const dt_image_t *img)
{
  return dt_image_pipe_class(img) == DT_IMAGE_PIPE_MOSAIC_RAW;
}
 
const char *dt_image_pipe_class_name(const dt_image_pipe_class_t klass)
{
  switch(klass)
  {
    case DT_IMAGE_PIPE_MOSAIC_RAW: return "mosaic-raw";
    case DT_IMAGE_PIPE_LINEAR_RAW: return "linear-raw";
    case DT_IMAGE_PIPE_RGB_LDR:    return "rgb-ldr";
    case DT_IMAGE_PIPE_RGB_HDR:    return "rgb-hdr";
    case DT_IMAGE_PIPE_UNKNOWN:
    default:                       return "unknown";
  }
}
 
void dt_image_buffer_resolve_flags(dt_image_t *img)
{
  if(IS_NULL_PTR(img)) return;
 
  // The decoded buffer descriptor is the authoritative source for the mosaic axis.
  if(img->dsc.filters != 0u)
    img->flags |= DT_IMAGE_MOSAIC;
  else
    img->flags &= ~DT_IMAGE_MOSAIC;
 
  // The LDR/HDR axis is NOT derivable from the buffer datatype: every raster codec decodes into a
  // TYPE_FLOAT working buffer in RAM regardless of the file's real dynamic range (sRGB JPEG/PNG/WebP
  // all land as float just like Radiance/PFM/EXR). The decoding codec is the authoritative source
  // and has already set DT_IMAGE_LDR / DT_IMAGE_HDR from the actual file content, so we must not
  // override it here from the datatype — doing so re-flagged every display-referred raster as HDR.
  //
  // The only normalization we still own is the raw-sensor case: mosaiced / sRAW integer data is
  // scene-linear sensor data, neither display LDR nor HDR. A float raw (e.g. HDRMerge, legacy float
  // DNG) is left flagged HDR exactly as the raw codec set it.
  if(dt_image_needs_rawprepare(img) && img->dsc.datatype != TYPE_FLOAT)
    img->flags &= ~(DT_IMAGE_LDR | DT_IMAGE_HDR);
 
  img->flags |= DT_IMAGE_BUFFER_RESOLVED;
}
 
void dt_image_set_provisional_dsc(dt_image_t *img)
{
  if(IS_NULL_PTR(img)) return;
  // Never clobber a descriptor that a codec has already produced.
  if(img->flags & DT_IMAGE_BUFFER_RESOLVED) return;
 
  switch(dt_image_pipe_class(img))
  {
    case DT_IMAGE_PIPE_RGB_LDR:
      img->dsc.channels = 4; img->dsc.datatype = TYPE_UINT8; img->dsc.filters = 0u; img->dsc.cst = IOP_CS_RGB;
      break;
    case DT_IMAGE_PIPE_RGB_HDR:
      img->dsc.channels = 4; img->dsc.datatype = TYPE_FLOAT; img->dsc.filters = 0u; img->dsc.cst = IOP_CS_RGB;
      break;
    case DT_IMAGE_PIPE_MOSAIC_RAW:
      // filters is unknown until decode; leave 0 as a placeholder (the class is carried by
      // flags, not by this provisional filters value).
      img->dsc.channels = 1; img->dsc.datatype = TYPE_UINT16; img->dsc.filters = 0u; img->dsc.cst = IOP_CS_RAW;
      break;
    case DT_IMAGE_PIPE_LINEAR_RAW:
      img->dsc.channels = 4; img->dsc.datatype = TYPE_FLOAT; img->dsc.filters = 0u; img->dsc.cst = IOP_CS_RAW;
      break;
    case DT_IMAGE_PIPE_UNKNOWN:
    default:
      return; // nothing reliable to seed
  }
  dt_iop_buffer_dsc_update_bpp(&img->dsc);
}
 
static const char *_image_buf_type_to_string(const dt_iop_buffer_type_t type)
{
  switch(type)
  {
    case TYPE_FLOAT:
      return "float";
    case TYPE_UINT16:
      return "uint16";
    case TYPE_UINT8:
      return "uint8";
    case TYPE_UNKNOWN:
    default:
      return "unknown";
  }
}
 
static const char *_image_colorspace_to_string(const dt_image_colorspace_t colorspace)
{
  switch(colorspace)
  {
    case DT_IMAGE_COLORSPACE_SRGB:
      return "sRGB";
    case DT_IMAGE_COLORSPACE_ADOBE_RGB:
      return "AdobeRGB";
    case DT_IMAGE_COLORSPACE_NONE:
    default:
      return "none";
  }
}
 
void dt_image_print_debug_info(const dt_image_t *img, const char *context)
{
  if(IS_NULL_PTR(img)) return;
 
  const char *ctx = context ? context : "image";
  const dt_iop_buffer_dsc_t *dsc = &img->dsc;
  const gboolean is_raw = dt_image_is_raw(img);
  const gboolean is_ldr = dt_image_is_ldr(img);
  const gboolean is_hdr = dt_image_is_hdr(img);
  const gboolean is_mono = dt_image_is_monochrome(img);
  const gboolean mono_workflow = dt_image_use_monochrome_workflow(img);
  const int mono_flags = dt_image_monochrome_flags(img);
  const gboolean mosaic = dsc->filters != 0u;
  const gboolean xtrans = dsc->filters == 9u;
  const gboolean bayer = mosaic && !xtrans;
  const size_t bpp = dsc->bpp;
  int bit_depth = 0;
 
  switch(dsc->datatype)
  {
    case TYPE_FLOAT:
      bit_depth = 32;
      break;
    case TYPE_UINT16:
      bit_depth = 16;
      break;
    case TYPE_UINT8:
      bit_depth = 8;
      break;
    case TYPE_UNKNOWN:
    default:
      if(dsc->channels != 0)
        bit_depth = (int)(bpp * 8 / dsc->channels);
      break;
  }
 
  const unsigned int flags = (unsigned int)img->flags;
 
  dt_print(DT_DEBUG_IMAGEIO,
           "[image debug] %s id=%d filename='%s' fullpath='%s'\n",
           ctx, img->id, img->filename, img->fullpath);
  dt_print(DT_DEBUG_IMAGEIO,
           "[image debug] %s size=%dx%d crop=%dx%d+%d+%d orientation=%d psize=%dx%d pixel_aspect=%.6f\n",
           ctx, img->width, img->height, img->crop_width, img->crop_height, img->crop_x, img->crop_y,
           img->orientation, img->p_width, img->p_height, img->pixel_aspect_ratio);
  dt_print(DT_DEBUG_IMAGEIO,
           "[image debug] %s flags=0x%08x raw=%d non_raw=%d ldr=%d hdr=%d sraw=%d 4bayer=%d mosaic=%d xtrans=%d "
           "bayer=%d mono=%d mono_flags=0x%x mono_workflow=%d mono_preview=%d mono_bayer=%d bw=%d bw_flow=%d "
           "local_copy=%d has_txt=%d has_wav=%d addl_dng=%d auto_presets=%d no_legacy_presets=%d rejected=%d remove=%d "
           "has_localcopy=%d has_audio=%d is_hdr_field=%d\n",
           ctx, flags, is_raw, !is_raw, is_ldr, is_hdr,
           (flags & DT_IMAGE_S_RAW) != 0, (flags & DT_IMAGE_4BAYER) != 0, mosaic, xtrans, bayer, is_mono,
           mono_flags, mono_workflow, (flags & DT_IMAGE_MONOCHROME_PREVIEW) != 0,
           (flags & DT_IMAGE_MONOCHROME_BAYER) != 0, img->is_bw, img->is_bw_flow,
           (flags & DT_IMAGE_LOCAL_COPY) != 0, (flags & DT_IMAGE_HAS_TXT) != 0, (flags & DT_IMAGE_HAS_WAV) != 0,
           (flags & DT_IMAGE_HAS_ADDITIONAL_DNG_TAGS) != 0, (flags & DT_IMAGE_AUTO_PRESETS_APPLIED) != 0,
           (flags & DT_IMAGE_NO_LEGACY_PRESETS) != 0, (flags & DT_IMAGE_REJECTED) != 0, (flags & DT_IMAGE_REMOVE) != 0,
           img->has_localcopy, img->has_audio, img->is_hdr);
  dt_print(DT_DEBUG_IMAGEIO,
           "[image debug] %s buf: channels=%u datatype=%s bit_depth=%d bpp=%" G_GSIZE_FORMAT " filters=%u cst=%d colorspace=%s "
           "processed_max=[%.4f %.4f %.4f %.4f]\n",
           ctx, dsc->channels, _image_buf_type_to_string(dsc->datatype), bit_depth, bpp, dsc->filters, dsc->cst,
           _image_colorspace_to_string(img->colorspace), dsc->processed_maximum[0], dsc->processed_maximum[1],
           dsc->processed_maximum[2], dsc->processed_maximum[3]);
  dt_print(DT_DEBUG_IMAGEIO,
           "[image debug] %s colorspace=enum:%s d65_color_matrix=[%.6f %.6f %.6f %.6f %.6f %.6f %.6f %.6f %.6f]\n",
           ctx, _image_colorspace_to_string(img->colorspace), img->d65_color_matrix[0], img->d65_color_matrix[1],
           img->d65_color_matrix[2], img->d65_color_matrix[3], img->d65_color_matrix[4], img->d65_color_matrix[5],
           img->d65_color_matrix[6], img->d65_color_matrix[7], img->d65_color_matrix[8]);
  dt_print(DT_DEBUG_IMAGEIO,
           "[image debug] %s raw: black=%u separate=[%u %u %u %u] white=%u rawprepare=[%u %u]\n",
           ctx, img->raw_black_level, img->raw_black_level_separate[0], img->raw_black_level_separate[1],
           img->raw_black_level_separate[2], img->raw_black_level_separate[3], img->raw_white_point,
           dsc->rawprepare.raw_black_level, dsc->rawprepare.raw_white_point);
  dt_print(DT_DEBUG_IMAGEIO,
           "[image debug] %s class=%s state=%s needs_rawprepare=%d needs_demosaic=%d is_mosaiced=%d is_sraw=%d has_matrix=%d\n",
           ctx, dt_image_pipe_class_name(dt_image_pipe_class(img)),
           dt_image_pipe_class_is_provisional(img) ? "provisional" : "resolved",
           dt_image_needs_rawprepare(img), dt_image_needs_demosaic(img), dt_image_is_mosaiced(img),
           dt_image_is_sraw(img), dt_image_is_matrix_correction_supported(img));
}
 
const char *dt_image_film_roll_name(const char *path)
{
  const char *folder = path + strlen(path);
  const int numparts = CLAMPS(dt_conf_get_int("show_folder_levels"), 1, 5);
  int count = 0;
  while(folder > path)
  {
 
#ifdef _WIN32
    // in Windows, both \ and / can be folder separator
    if(*folder == G_DIR_SEPARATOR || *folder == '/')
#else
    if(*folder == G_DIR_SEPARATOR)
#endif
 
      if(++count >= numparts)
      {
        ++folder;
        break;
      }
    --folder;
  }
  return folder;
}
 
void dt_image_film_roll_directory(const dt_image_t *img, char *pathname, size_t pathname_len)
{
  char *folder = dt_film_repository_get_folder(img->film_id);
  if(folder)
  {
    g_strlcpy(pathname, folder, pathname_len);
    dt_free(folder);
  }
  pathname[pathname_len - 1] = '\0';
}
 
void dt_image_film_roll(const dt_image_t *img, char *pathname, size_t pathname_len)
{
  if(img->film_id < 0)
  {
    g_strlcpy(pathname, _("orphaned image"), pathname_len);
    return;
  }
 
  char *folder = dt_film_repository_get_folder(img->film_id);
  if(folder)
  {
    g_strlcpy(pathname, dt_image_film_roll_name(folder), pathname_len);
    dt_free(folder);
  }
  else
  {
    g_strlcpy(pathname, _("orphaned image"), pathname_len);
  }
  pathname[pathname_len - 1] = '\0';
}
 
// dt_image_get_xmp_mode() is on the hot path: called from many threads (image cache
// write-release, export/save-xmp jobs...) for every single image. Re-parsing the conf string on
// every call meant every reader raced every writer on the same key -- a torn read could
// masquerade as an unrecognized/disabled value and silently skip writing a sidecar. The setting
// itself changes only when the user acts (preferences dialog) or once at startup, so it is read
// from conf exactly those two times and cached here the rest of the time -- same lifecycle as
// dt_mipmap_cache_settings_t in darktable.c: "read at startup, re-read when the user changes it,
// never anywhere else". dt_image_xmp_mode_refresh_from_conf() does the actual conf read+sanitize
// and is wired to DT_SIGNAL_PREFERENCES_CHANGE in dt_init().
static dt_atomic_int _xmp_mode_cache = 1; // matches the confgen default (TRUE) until first refresh
 
gboolean dt_image_get_xmp_mode()
{
  return dt_atomic_get_int(&_xmp_mode_cache) != 0;
}
 
void dt_image_xmp_mode_refresh_from_conf(void)
{
  // Write sidecars when the setting is absent, consistently with the default exposed in the preferences.
  gboolean res = TRUE;
  char *config = dt_conf_get_string("write_sidecar_files");
  if(!IS_NULL_PTR(config))
  {
    res = FALSE;
 
    // Darktable > 3.6
    if(!strcmp(config, "after edit") || !strcmp(config, "on import") || !strcmp(config, "always")) res = TRUE;
 
    // Darktable < 3.8 and Ansel
    else if(!strcmp(config, "TRUE"))
      res = TRUE;
  }
 
  // sanitize keys, but only if they actually need it -- this now only ever runs at startup and
  // on user-initiated preference changes, so there is no concurrent reader/writer race left to
  // worry about; the guard is just to avoid rewriting an already-canonical value.
  const char *sanitized = res ? "TRUE" : "FALSE";
  if(IS_NULL_PTR(config) || strcmp(config, sanitized)) dt_conf_set_string("write_sidecar_files", sanitized);
 
  dt_free(config);
 
  dt_atomic_set_int(&_xmp_mode_cache, res ? 1 : 0);
}
 
gboolean dt_image_safe_remove(const int32_t imgid)
{
  // always safe to remove if we do not have .xmp
  if(!dt_image_get_xmp_mode()) return TRUE;
 
  // check whether the original file is accessible
  char pathname[DT_PATH_MAX] = { 0 };
  gboolean from_cache = TRUE;
 
  dt_image_full_path(imgid,  pathname,  sizeof(pathname),  &from_cache, __FUNCTION__);
 
  if(!from_cache)
    return TRUE;
 
  else
  {
    // finally check if we have a .xmp for the local copy. If no modification done on the local copy it is safe
    // to remove.
    g_strlcat(pathname, ".xmp", sizeof(pathname));
    return !g_file_test(pathname, G_FILE_TEST_EXISTS);
  }
}
 
dt_image_path_source_t dt_image_choose_input_path(const dt_image_t *img, char *pathname,
                                                  size_t pathname_len, gboolean force_cache)
{
  if(IS_NULL_PTR(img) || IS_NULL_PTR(pathname)) return DT_IMAGE_PATH_NONE;
  pathname[0] = '\0';
 
  // Start with local copies as file I/O will be better if we have a choice
  if(img->flags & DT_IMAGE_LOCAL_COPY || force_cache)
  {
    if(img->local_copy_path[0] && g_file_test(img->local_copy_path, G_FILE_TEST_EXISTS))
    {
      g_strlcpy(pathname, img->local_copy_path, pathname_len);
      return DT_IMAGE_PATH_LOCAL_COPY;
    }
 
    if(img->local_copy_legacy_path[0] && g_file_test(img->local_copy_legacy_path, G_FILE_TEST_EXISTS))
    {
      g_strlcpy(pathname, img->local_copy_legacy_path, pathname_len);
      return DT_IMAGE_PATH_LOCAL_COPY_LEGACY;
    }
 
    // Local copy flag might be stale (cache cleared, moved file, etc.). If local copy is missing and
    // we are not explicitly forcing the cache, fall back to the original image.
    if(!force_cache && img->fullpath[0] && g_file_test(img->fullpath, G_FILE_TEST_EXISTS))
    {
      g_strlcpy(pathname, img->fullpath, pathname_len);
      return DT_IMAGE_PATH_ORIGINAL;
    }
  }
  // Forcing the cache should not consider the original image
  else
  {
    if(img->fullpath[0] && g_file_test(img->fullpath, G_FILE_TEST_EXISTS))
    { 
      g_strlcpy(pathname, img->fullpath, pathname_len);
      return DT_IMAGE_PATH_ORIGINAL;
    }
  }
 
  // In case the local copy flag was not set properly, but the original file failed
  // last-chance attempt at getting a possibly forgotten local copy to have some input
  if(img->local_copy_path[0] && g_file_test(img->local_copy_path, G_FILE_TEST_EXISTS))
  {
    g_strlcpy(pathname, img->local_copy_path, pathname_len);
    return DT_IMAGE_PATH_LOCAL_COPY;
  }
 
  return DT_IMAGE_PATH_NONE;
}
 
void dt_image_full_path(const int32_t imgid, char *pathname, size_t pathname_len, gboolean *from_cache, const char *calling_func)
{
  if(imgid < 0) return;
  dt_image_path_source_t source = DT_IMAGE_PATH_NONE;
  const gboolean prefer_cache = (from_cache && *from_cache);
 
  const dt_image_t *img = dt_image_cache_get(imgid, 'r');
  if(img)
  {
    // Preserve legacy semantics: `from_cache = TRUE` means "try local copy first, then fall back to original",
    // not "only local copy".
    if(prefer_cache)
    {
      source = dt_image_choose_input_path(img, pathname, pathname_len, TRUE);
      if(source == DT_IMAGE_PATH_NONE)
        source = dt_image_choose_input_path(img, pathname, pathname_len, FALSE);
    }
    else
    {
      source = dt_image_choose_input_path(img, pathname, pathname_len, FALSE);
    }
    dt_image_cache_read_release(img);
  }
 
  if(from_cache) 
    *from_cache = (source == DT_IMAGE_PATH_LOCAL_COPY 
                   || source == DT_IMAGE_PATH_LOCAL_COPY_LEGACY);
 
}
 
void dt_image_local_copy_paths_from_fullpath(const char *fullpath, int32_t imgid, char *local_copy_path,
                                             size_t local_copy_len, char *local_copy_legacy_path,
                                             size_t local_copy_legacy_len)
{
  if(local_copy_path) local_copy_path[0] = '\0';
  if(local_copy_legacy_path) local_copy_legacy_path[0] = '\0';
  if(IS_NULL_PTR(fullpath) || !*fullpath || imgid <= 0 || IS_NULL_PTR(local_copy_path) || IS_NULL_PTR(local_copy_legacy_path)) return;
 
  char *md5_filename = g_compute_checksum_for_string(G_CHECKSUM_MD5, fullpath, strlen(fullpath));
  if(IS_NULL_PTR(md5_filename)) return;
 
  char cachedir[DT_PATH_MAX] = { 0 };
  dt_loc_get_user_cache_dir(cachedir, sizeof(cachedir));
 
  const char *c = fullpath + strlen(fullpath);
  while(*c != '.' && c > fullpath) c--;
 
  // cache filename old format: <cachedir>/img-<id>-<MD5>.<ext>
  // for upward compatibility we check for the old name, if found we return it
  snprintf(local_copy_legacy_path, local_copy_legacy_len, "%s/img-%d-%s%s", cachedir, imgid, md5_filename, c);
 
  // cache filename format: <cachedir>/img-<MD5>.<ext>
  snprintf(local_copy_path, local_copy_len, "%s/img-%s%s", cachedir, md5_filename, c);
 
  dt_free(md5_filename);
}
 
void dt_image_path_append_version_no_db(int version, char *pathname, size_t pathname_len)
{
  // the "first" instance (version zero) does not get a version suffix
  if(version > 0)
  {
    // add version information:
    char *filename = g_strdup(pathname);
 
    char *c = pathname + strlen(pathname);
    while(*c != '.' && c > pathname) c--;
    snprintf(c, pathname + pathname_len - c, "_%02d", version);
    c = pathname + strlen(pathname);
    char *c2 = filename + strlen(filename);
    while(*c2 != '.' && c2 > filename) c2--;
    g_strlcpy(c, c2, pathname + pathname_len - c);
    dt_free(filename);
  }
}
 
void dt_image_path_append_version(const int32_t imgid, char *pathname, size_t pathname_len)
{
  // get duplicate suffix
  const int version = dt_image_repository_get_version(imgid);
 
  dt_image_path_append_version_no_db(version, pathname, pathname_len);
}
 
void dt_image_print_exif(const dt_image_t *img, char *line, size_t line_len)
{
  char *exposure_str = dt_util_format_exposure(img->exif_exposure);
 
  snprintf(line, line_len, "%s f/%.1f %dmm ISO %d", exposure_str, img->exif_aperture, (int)img->exif_focal_length,
           (int)img->exif_iso);
 
  dt_free(exposure_str);
}
 
int dt_image_get_xmp_rating_from_flags(const int flags)
{
  return (flags & DT_IMAGE_REJECTED)
    ? -1                              // rejected image = -1
    : (flags & DT_VIEW_RATINGS_MASK); // others = 0 .. 5
}
 
int dt_image_get_xmp_rating(const dt_image_t *img)
{
  return dt_image_get_xmp_rating_from_flags(img->flags);
}
 
void dt_image_set_xmp_rating(dt_image_t *img, const int rating)
{
  // clean flags stars and rejected
  img->flags &= ~(DT_IMAGE_REJECTED | DT_VIEW_RATINGS_MASK);
 
  if(rating == -2) // assuming that value -2 cannot be found
  {
    img->flags |= (DT_VIEW_RATINGS_MASK & 0);
  }
  else if(rating == -1)
  {
    img->flags |= DT_IMAGE_REJECTED;
  }
  else
  {
    img->flags |= (DT_VIEW_RATINGS_MASK & rating);
  }
}
 
void dt_image_get_location(const int32_t imgid, dt_image_geoloc_t *geoloc)
{
  const dt_image_t *img = dt_image_cache_get(imgid, 'r');
  geoloc->longitude = img->geoloc.longitude;
  geoloc->latitude = img->geoloc.latitude;
  geoloc->elevation = img->geoloc.elevation;
  dt_image_cache_read_release(img);
}
 
static void _set_location(const int32_t imgid, const dt_image_geoloc_t *geoloc)
{
  /* fetch image from cache */
  dt_image_t *image = dt_image_cache_get(imgid, 'w');
 
  memcpy(&image->geoloc, geoloc, sizeof(dt_image_geoloc_t));
 
  dt_image_cache_write_release(image, DT_IMAGE_CACHE_SAFE);
}
 
static void _set_datetime(const int32_t imgid, const char *datetime)
{
  /* fetch image from cache */
  dt_image_t *image = dt_image_cache_get(imgid, 'w');
 
  dt_datetime_exif_to_img(image, datetime);
 
  dt_image_cache_write_release(image, DT_IMAGE_CACHE_SAFE);
}
 
static void _pop_undo(gpointer user_data, const dt_undo_type_t type, dt_undo_data_t data, const dt_undo_action_t action, GList **imgs)
{
  if(type == DT_UNDO_GEOTAG)
  {
    int i = 0;
 
    for(GList *list = (GList *)data; list; list = g_list_next(list))
    {
      dt_undo_geotag_t *undogeotag = (dt_undo_geotag_t *)list->data;
      const dt_image_geoloc_t *geoloc = (action == DT_ACTION_UNDO) ? &undogeotag->before : &undogeotag->after;
 
      _set_location(undogeotag->imgid, geoloc);
 
      *imgs = g_list_prepend(*imgs, GINT_TO_POINTER(undogeotag->imgid));
      i++;
    }
    if(i > 1) dt_control_log((action == DT_ACTION_UNDO)
                              ? _("geo-location undone for %d images")
                              : _("geo-location re-applied to %d images"), i);
    // the handler copies the list; ours stays ours (the raise it replaces took ownership,
    // which is why this site used to g_list_copy)
    dt_metadata_geotags_changed(*imgs);
  }
  else if(type == DT_UNDO_DATETIME)
  {
    int i = 0;
 
    for(GList *list = (GList *)data; list; list = g_list_next(list))
    {
      dt_undo_datetime_t *undodatetime = (dt_undo_datetime_t *)list->data;
 
      _set_datetime(undodatetime->imgid, (action == DT_ACTION_UNDO)
                                         ? undodatetime->before : undodatetime->after);
 
      *imgs = g_list_prepend(*imgs, GINT_TO_POINTER(undodatetime->imgid));
      i++;
    }
    if(i > 1) dt_control_log((action == DT_ACTION_UNDO)
                              ? _("date/time undone for %d images")
                              : _("date/time re-applied to %d images"), i);
  }
  else if(type == DT_UNDO_DUPLICATE)
  {
    dt_undo_duplicate_t *undo = (dt_undo_duplicate_t *)data;
 
    if(action == DT_ACTION_UNDO)
    {
      // remove image
      dt_image_remove(undo->new_imgid);
    }
    else
    {
      // restore image, note that we record the new imgid created while
      // restoring the duplicate.
      undo->new_imgid = _image_duplicate_with_version(undo->orig_imgid, undo->version, FALSE, TRUE);
      *imgs = g_list_prepend(*imgs, GINT_TO_POINTER(undo->new_imgid));
    }
  }
  else if(type == DT_UNDO_REMOVE)
  {
    const dt_undo_remove_t *undo = (const dt_undo_remove_t *)data;
 
    /* Undo puts the rows back, redo takes them out again -- and in both directions the
     * snapshot stays: it is the undo record's, and the record can be popped either way any
     * number of times. It is dropped only when the record itself is, by
     * _remove_undo_data_free() below. */
    GList *one = g_list_prepend(NULL, GINT_TO_POINTER(undo->imgid));
 
    if(action == DT_ACTION_UNDO)
    {
      if(dt_removed_image_repository_restore(undo->snap_id, undo->imgid))
      {
        /* DT_IMAGE_REMOVE is written to the row BEFORE the removal runs, so it is part of
         * what was staged -- and every collection query filters that flag out, which would
         * bring the image back into the database and into no view. The image is not marked
         * for deletion any more, so the flag goes. */
        dt_image_repository_clear_flag_among(one, DT_IMAGE_REMOVE);
 
        /* The removal emptied the image cache and the mipmap cache of this image, and neither
         * refills itself: the mipmap cache regenerates only on explicit removal, and the image
         * cache's history_items -- the "altered" flag deciding raw processing over the unedited
         * embedded JPEG -- would be whatever the entry held before. Without this the image comes
         * back looking undeveloped, or as a thumbnail that never renders at all. Reloading also
         * re-reads the flags cleared just above, so no stale cache entry can write DT_IMAGE_REMOVE
         * back over the restored row. TRUE: this is a lighttable operation, the filmstrip may
         * refresh with the grid. */
        dt_image_history_changed(undo->imgid, TRUE);
 
        /* The collection is the last one: nothing about a restored row reaches the grid until
         * the query is re-run. */
        dt_collection_update_query(dt_collection_get_global(), DT_COLLECTION_CHANGE_RELOAD,
                                   DT_COLLECTION_PROP_UNDEF, g_list_copy(one));
        /* Restoring the last image of a film roll brings the roll back with it, so the
         * folder list has to be told; the notifier keeps that a layer-1 statement. */
        dt_film_notify_rolls_changed();
        *imgs = g_list_prepend(*imgs, GINT_TO_POINTER(undo->imgid));
      }
      else
        fprintf(stderr, "[dt_image_remove] fails to restore image %" PRId32 "\n", undo->imgid);
    }
    else
    {
      // not added to *imgs: dt_undo_do_undo() ends by writing the XMP of every image it
      // names, and this one no longer has a row to write from.
      _image_remove(undo->imgid, FALSE);
      dt_collection_update_query(dt_collection_get_global(), DT_COLLECTION_CHANGE_RELOAD,
                                 DT_COLLECTION_PROP_UNDEF, g_list_copy(one));
      dt_film_notify_rolls_changed();
    }
 
    g_list_free(one);
  }
  else if(type == DT_UNDO_FLAGS)
  {
    dt_undo_monochrome_t *undomono = (dt_undo_monochrome_t *)data;
 
    const gboolean before = (action == DT_ACTION_UNDO) ? undomono->after : undomono->before;
    const gboolean after  = (action == DT_ACTION_UNDO) ? undomono->before : undomono->after;
    _pop_undo_execute(undomono->imgid, before, after);
    *imgs = g_list_prepend(*imgs, GINT_TO_POINTER(undomono->imgid));
  }
}
 
static void _geotag_undo_data_free(gpointer data)
{
  GList *l = (GList *)data;
  g_list_free_full(l, dt_free_gpointer);
  l = NULL;
}
 
static void _image_set_location(GList *imgs, const dt_image_geoloc_t *geoloc, GList **undo, const gboolean undo_on)
{
  for(GList *images = imgs; images; images = g_list_next(images))
  {
    const int32_t imgid = GPOINTER_TO_INT(images->data);
 
    if(undo_on)
    {
      dt_undo_geotag_t *undogeotag = (dt_undo_geotag_t *)malloc(sizeof(dt_undo_geotag_t));
      undogeotag->imgid = imgid;
      dt_image_get_location(imgid, &undogeotag->before);
 
      memcpy(&undogeotag->after, geoloc, sizeof(dt_image_geoloc_t));
 
      *undo = g_list_append(*undo, undogeotag);
    }
 
    _set_location(imgid, geoloc);
  }
}
 
void dt_image_set_locations(const GList *imgs, const dt_image_geoloc_t *geoloc, const gboolean undo_on)
{
  if(imgs)
  {
    GList *undo = NULL;
    if(undo_on) dt_undo_start_group(dt_undo_get_global(), DT_UNDO_GEOTAG);
 
    _image_set_location((GList *)imgs, geoloc, &undo, undo_on);
 
    if(undo_on)
    {
      dt_undo_record(dt_undo_get_global(), NULL, DT_UNDO_GEOTAG, undo, _pop_undo, _geotag_undo_data_free);
      dt_undo_end_group(dt_undo_get_global());
    }
  }
}
 
void dt_image_set_location(const int32_t imgid, const dt_image_geoloc_t *geoloc, const gboolean undo_on, const gboolean group_on)
{
  GList *imgs = NULL;
  if(imgid == UNKNOWN_IMAGE)
    imgs = dt_act_on_get_images();
  else
    imgs = g_list_prepend(imgs, GINT_TO_POINTER(imgid));
  if(group_on) dt_grouping_add_grouped_images(&imgs);
  dt_image_set_locations(imgs, geoloc, undo_on);
  g_list_free(imgs);
  imgs = NULL;
}
 
static void _image_set_images_locations(const GList *img, const GArray *gloc,
                                        GList **undo, const gboolean undo_on)
{
  int i = 0;
  for(GList *imgs = (GList *)img; imgs; imgs = g_list_next(imgs))
  {
    const int32_t imgid = GPOINTER_TO_INT(imgs->data);
    const dt_image_geoloc_t *geoloc = &g_array_index(gloc, dt_image_geoloc_t, i);
    if(undo_on)
    {
      dt_undo_geotag_t *undogeotag = (dt_undo_geotag_t *)malloc(sizeof(dt_undo_geotag_t));
      undogeotag->imgid = imgid;
      dt_image_get_location(imgid, &undogeotag->before);
 
      memcpy(&undogeotag->after, geoloc, sizeof(dt_image_geoloc_t));
 
      *undo = g_list_prepend(*undo, undogeotag);
    }
 
    _set_location(imgid, geoloc);
    i++;
  }
}
 
void dt_image_set_images_locations(const GList *imgs, const GArray *gloc, const gboolean undo_on)
{
  if(IS_NULL_PTR(imgs) || IS_NULL_PTR(gloc) || (g_list_length((GList *)imgs) != gloc->len))
    return;
  GList *undo = NULL;
  if(undo_on) dt_undo_start_group(dt_undo_get_global(), DT_UNDO_GEOTAG);
 
  _image_set_images_locations(imgs, gloc, &undo, undo_on);
 
  if(undo_on)
  {
    dt_undo_record(dt_undo_get_global(), NULL, DT_UNDO_GEOTAG, undo, _pop_undo, _geotag_undo_data_free);
    dt_undo_end_group(dt_undo_get_global());
  }
}
 
void dt_image_history_changed(const int32_t imgid, const gboolean refresh_filmstrip)
{
  if(imgid <= 0) return;
 
  // Reload the cached image metadata from the DB. The caller has already persisted the new
  // history there; this refreshes history_items (the count of history entries) in the shared
  // image cache. history_items is the "altered" flag that the thumbnail regeneration uses to
  // pick raw processing over the (unedited) embedded JPEG, so a stale count makes edits and
  // rotations appear to have no effect on the thumbnail (issues #647, #861).
  dt_image_t *image = dt_image_cache_get_reload(imgid, 'w');
  if(image)
    dt_image_cache_write_release(image, DT_IMAGE_CACHE_RELAXED);
 
  // Drop the stale rendered thumbnail. The mipmap cache regenerates purely on explicit removal,
  // never by comparing history hashes, so this is mandatory after any development change.
  dt_mipmap_cache_remove(imgid, TRUE);
 
  // The filmstrip is best-effort: refreshing it spawns an export thread that competes with the
  // realtime darkroom main preview. Darkroom write paths pass FALSE; lighttable ops pass TRUE.
  // No "is there a GUI?" guard needed: with no handler registered this is a no-op.
  dt_thumbnail_notify_image_changed(imgid, refresh_filmstrip);
}
 
void dt_image_set_flip(const int32_t imgid, const dt_image_orientation_t orientation)
{
  // push new orientation to sql via additional history entry:
  const int iop_flip_MODVER = 2;
  const int num = dt_history_repository_get_next_num(imgid);
  dt_history_repository_write_item(imgid, num, "flip", &orientation, sizeof(int32_t), iop_flip_MODVER, 1,
                                   NULL, 0, 0, 0, "");
  dt_history_repository_set_end(imgid, num + 1);
  dt_control_save_xmp(imgid);
 
  // Refresh the cached metadata and thumbnail. Without this the stale history_items keeps the
  // image flagged unedited, so the rotated raw keeps showing its (unrotated) embedded JPEG and
  // the rotation appears to do nothing unless "never use embedded JPEG" is forced (issue #647).
  dt_image_history_changed(imgid, TRUE);
}
 
void dt_image_orient_boundingbox(const dt_boundingbox_t in, const dt_image_orientation_t orientation,
                                 dt_boundingbox_t out)
{
  float top = in[0];
  float left = in[1];
  float bottom = in[2];
  float right = in[3];
 
  /* Mirror, then swap, in the exact order iop/flip.c's distort_transform() applies to points:
   * x -> width - x, y -> height - y, then (x, y) -> (y, x). On normalized edge coordinates a
   * mirror exchanges the two edges of that axis, and the swap exchanges the two axes. */
  if(orientation & ORIENTATION_FLIP_X)
  {
    const float new_left = 1.0f - right;
    right = 1.0f - left;
    left = new_left;
  }
 
  if(orientation & ORIENTATION_FLIP_Y)
  {
    const float new_top = 1.0f - bottom;
    bottom = 1.0f - top;
    top = new_top;
  }
 
  if(orientation & ORIENTATION_SWAP_XY)
  {
    const float swap_top = left;
    const float swap_bottom = right;
    left = top;
    right = bottom;
    top = swap_top;
    bottom = swap_bottom;
  }
 
  out[0] = top;
  out[1] = left;
  out[2] = bottom;
  out[3] = right;
}
 
gboolean dt_image_get_usercrop(const dt_image_t *img, dt_boundingbox_t box)
{
  box[0] = box[1] = 0.0f;
  box[2] = box[3] = 1.0f;
 
  if(IS_NULL_PTR(img) || img->usercrop_status != DT_IMAGE_USERCROP_VALID) return FALSE;
 
  for(int i = 0; i < 4; i++) box[i] = img->usercrop[i];
  return TRUE;
}
 
gboolean dt_image_resolve_usercrop(const int32_t imgid, dt_boundingbox_t box)
{
  box[0] = box[1] = 0.0f;
  box[2] = box[3] = 1.0f;
 
  if(imgid < 1 || !dt_image_cache_is_ready()) return FALSE;
 
  char filename[DT_PATH_MAX] = { 0 };
  const dt_image_t *img = dt_image_cache_get(imgid, 'r');
  if(IS_NULL_PTR(img)) return FALSE;
 
  if(img->usercrop_status != DT_IMAGE_USERCROP_UNKNOWN)
  {
    // Already answered for this image, in this session.
    const gboolean valid = dt_image_get_usercrop(img, box);
    dt_image_cache_read_release(img);
    return valid;
  }
 
  /* DT_IMAGE_HAS_ADDITIONAL_DNG_TAGS is persisted and is set at import for every image that has
   * a usable DefaultUserCrop (it also covers DNG opcodes, so it is a superset -- which is what we
   * want here: no valid framing can hide behind a cleared bit). Images without it can be settled
   * without touching the file at all, which keeps this off the hot path for whole libraries. */
  if(!(img->flags & DT_IMAGE_HAS_ADDITIONAL_DNG_TAGS))
  {
    // Cheap enough to redo on every call -- deliberately not memoized, so that populating a whole
    // library's thumbnails does not take one image-cache write lock per image to record a "no".
    dt_image_cache_read_release(img);
    return FALSE;
  }
 
  const gboolean has_file
      = (dt_image_choose_input_path(img, filename, sizeof(filename), FALSE) != DT_IMAGE_PATH_NONE);
  dt_image_cache_read_release(img);
  if(!has_file) return FALSE;
 
  /* Parse into a throwaway image, holding no lock: the crop lives in runtime-only state, so a
   * thumbnail worker starting from a database-loaded cache entry has to read the file itself.
   * Two threads racing here compute the same answer from the same file, so the duplicated work
   * is wasteful at worst -- which is cheaper than serializing file I/O behind a write lock. */
  dt_image_t probe;
  dt_image_init(&probe);
  probe.id = imgid;
  dt_exif_read_usercrop(&probe, filename);
 
  dt_image_t *write = dt_image_cache_get(imgid, 'w');
  if(!IS_NULL_PTR(write))
  {
    write->usercrop_status = probe.usercrop_status;
    for(int i = 0; i < 4; i++) write->usercrop[i] = probe.usercrop[i];
    dt_image_cache_write_release(write, DT_IMAGE_CACHE_RELAXED);
  }
 
  return dt_image_get_usercrop(&probe, box);
}
 
gboolean dt_image_get_usercrop_oriented(const dt_image_t *img, const dt_image_orientation_t orientation,
                                        dt_boundingbox_t box)
{
  if(!dt_image_get_usercrop(img, box)) return FALSE;
 
  dt_image_orient_boundingbox(box, orientation, box);
  return TRUE;
}
 
static dt_image_orientation_t _image_get_history_orientation(const int32_t imgid)
{
  // find the flip module -- the pointer stays valid until darktable shuts down
  static dt_iop_module_so_t *flip = NULL;
  if(IS_NULL_PTR(flip))
  {
    for(const GList *modules = dt_iop_get_modules_so(); modules; modules = g_list_next(modules))
    {
      dt_iop_module_so_t *module = (dt_iop_module_so_t *)(modules->data);
      if(!strcmp(module->op, "flip"))
      {
        flip = module;
        break;
      }
    }
  }
 
  dt_image_orientation_t orientation = ORIENTATION_NULL;
 
  // db lookup flip params
  if(flip && flip->have_introspection && flip->get_p)
  {
    void *params = NULL;
    int32_t params_len = 0;
    if(dt_history_repository_get_last_enabled_params(imgid, "flip", &params, &params_len))
    {
      // use introspection to get the orientation from the binary params blob
      orientation = *((dt_image_orientation_t *)flip->get_p(params, "orientation"));
      dt_free(params);
    }
  }
 
  return orientation;
}
 
dt_image_orientation_t dt_image_get_effective_orientation(const dt_image_t *img)
{
  if(IS_NULL_PTR(img)) return ORIENTATION_NONE;
 
  const dt_image_orientation_t orientation = _image_get_history_orientation(img->id);
  return (orientation != ORIENTATION_NULL) ? orientation : dt_image_orientation(img);
}
 
dt_image_orientation_t dt_image_get_orientation(const int32_t imgid)
{
  dt_image_orientation_t orientation = _image_get_history_orientation(imgid);
 
  if(orientation == ORIENTATION_NULL)
  {
    const dt_image_t *img = dt_image_cache_get(imgid, 'r');
    if(IS_NULL_PTR(img)) return ORIENTATION_NONE;
    orientation = dt_image_orientation(img);
    dt_image_cache_read_release(img);
  }
 
  return orientation;
}
 
static dt_image_orientation_t _orientation_mirror_horizontally(dt_image_orientation_t orientation)
{
  if(orientation & ORIENTATION_SWAP_XY)
    return orientation ^ ORIENTATION_FLIP_Y;
  else
    return orientation ^ ORIENTATION_FLIP_X;
}
 
static dt_image_orientation_t _orientation_mirror_vertically(dt_image_orientation_t orientation)
{
  if(orientation & ORIENTATION_SWAP_XY)
    return orientation ^ ORIENTATION_FLIP_X;
  else
    return orientation ^ ORIENTATION_FLIP_Y;
}
 
void dt_image_flip(const int32_t imgid, const dt_image_transform_t transform)
{
  // this is light table only:
  const dt_view_t *cv = dt_view_manager_get_current_view(dt_view_manager_get_global());
  if(dt_dev_get_global()->image_storage.id == imgid && cv->view((dt_view_t *)cv) == DT_VIEW_DARKROOM) return;
 
  dt_undo_lt_history_t *hist = dt_history_snapshot_item_init();
  hist->imgid = imgid;
  dt_history_snapshot_undo_create(hist->imgid, &hist->before, &hist->before_history_end);
 
  dt_image_orientation_t orientation = dt_image_get_orientation(imgid);
 
  switch(transform)
  {
    case DT_IMAGE_TRANSFORM_ROTATE_CCW:
      // mirror horizontally, then transpose
      orientation = _orientation_mirror_horizontally(orientation) ^ ORIENTATION_SWAP_XY;
      break;
 
    case DT_IMAGE_TRANSFORM_ROTATE_CW:
      // mirror vertically, then transpose
      orientation = _orientation_mirror_vertically(orientation) ^ ORIENTATION_SWAP_XY;
      break;
 
    case DT_IMAGE_TRANSFORM_FLIP_HORIZONTALLY:
      orientation = _orientation_mirror_horizontally(orientation);
      break;
 
    case DT_IMAGE_TRANSFORM_FLIP_VERTICALLY:
      orientation = _orientation_mirror_vertically(orientation);
      break;
 
    case DT_IMAGE_TRANSFORM_RESET:
      // NULL, not NONE: hand the image back to whatever its EXIF says, rather than declaring it
      // upright. The previous code computed a quarter turn first and then threw it away.
      orientation = ORIENTATION_NULL;
      break;
  }
 
  // dt_image_set_flip() writes the new orientation history entry and notifies the caches/GUI
  // (mipmap invalidation + thumbnail refresh) via dt_image_history_changed().
  dt_image_set_flip(imgid, orientation);
 
  dt_history_snapshot_undo_create(hist->imgid, &hist->after, &hist->after_history_end);
  dt_undo_record(dt_undo_get_global(), NULL, DT_UNDO_LT_HISTORY, (dt_undo_data_t)hist,
                 dt_history_snapshot_undo_pop, dt_history_snapshot_undo_lt_history_data_free);
}
 
int32_t dt_image_duplicate(const int32_t imgid)
{
  return dt_image_duplicate_with_version(imgid, -1);
}
 
 
 
static int32_t _image_duplicate_with_version(const int32_t imgid, const int32_t newversion, const gboolean undo,
                                             const gboolean reload)
{
  const int32_t newid = dt_image_repository_duplicate(imgid, newversion);
 
  if(newid != -1)
  {
    if(undo)
    {
      dt_undo_duplicate_t *dupundo = (dt_undo_duplicate_t *)malloc(sizeof(dt_undo_duplicate_t));
      dupundo->orig_imgid = imgid;
      dupundo->version = newversion;
      dupundo->new_imgid = newid;
      dt_undo_record(dt_undo_get_global(), NULL, DT_UNDO_DUPLICATE, dupundo, _pop_undo, NULL);
    }
 
    // make sure that the duplicate doesn't have some magic darktable| tags
    if(dt_tag_detach_by_string("darktable|changed", newid, FALSE, FALSE)
       || dt_tag_detach_by_string("darktable|exported", newid, FALSE, FALSE))
      dt_metadata_tags_changed();
 
    const dt_image_t *img = dt_image_cache_get(imgid, 'r');
    const int grpid = img->group_id;
    dt_image_cache_read_release(img);
    dt_grouping_add_to_group(grpid, newid);
 
    // Reloading here exposes newid to the lighttable grid: it becomes eligible for thumbnail
    // generation immediately, against whatever history it has *right now*. Callers still about
    // to write history onto it (duplicate-with-history, style-on-duplicate) must skip this and
    // reload themselves once that is done -- see dt_image_duplicate_no_reload().
    if(reload)
      dt_collection_update_query(dt_collection_get_global(), DT_COLLECTION_CHANGE_RELOAD, DT_COLLECTION_PROP_UNDEF, NULL);
  }
  return newid;
}
 
int32_t dt_image_duplicate_with_version(const int32_t imgid, const int32_t newversion)
{
  return _image_duplicate_with_version(imgid, newversion, TRUE, TRUE);
}
 
int32_t dt_image_duplicate_no_reload(const int32_t imgid)
{
  return _image_duplicate_with_version(imgid, -1, TRUE, FALSE);
}
 
static void _remove_undo_data_free(gpointer data)
{
  dt_undo_remove_t *undo = (dt_undo_remove_t *)data;
 
  /* The record is gone, so the removal it could have undone is now permanent. Nothing to
   * drop once the connection is closed, though: the staging tables are `memory.` ones and
   * went with it.
   *
   * No reachable path takes that branch today, and the check stays anyway. dt_undo_cleanup()
   * does run after dt_database_close() at shutdown, but it finds an empty list: the GUI's
   * teardown calls dt_ctl_switch_mode_to(""), and switching to no view begins by clearing
   * DT_UNDO_ALL -- while the database is still open. Without a GUI the order reverses, but
   * then nothing can have recorded a removal either, both callers of dt_control_remove_images()
   * being GUI ones. So this guards an ordering that nothing enforces, for the cost of one
   * call: dropping it would cost an abort on quit in a debug build (the assert inside
   * DT_DEBUG_SQLITE3_PREPARE_V2) the day either half of that changes. */
  if(dt_database_is_open())
    dt_removed_image_repository_clear(undo->snap_id, undo->imgid);
 
  dt_free(undo);
}
 
static void _image_remove(const int32_t imgid, const gboolean undo)
{
  // if a local copy exists, remove it
 
  if(dt_image_local_copy_reset(imgid)) return;
 
  /* Snapshot before anything below runs, dt_grouping_remove_from_group() included: that call
   * rewrites the group_id of the images STAYING behind, and putting that back is part of
   * undoing this removal. A snapshot that cannot be taken is not recorded -- removing the
   * image anyway is better than an undo entry that restores nothing. */
  if(undo)
  {
    dt_undo_remove_t *rm = (dt_undo_remove_t *)g_malloc0(sizeof(dt_undo_remove_t));
    rm->imgid = imgid;
    rm->snap_id = dt_removed_image_repository_create(imgid);
 
    if(rm->snap_id >= 0)
      dt_undo_record(dt_undo_get_global(), NULL, DT_UNDO_REMOVE, rm, _pop_undo, _remove_undo_data_free);
    else
    {
      fprintf(stderr, "[dt_image_remove] fails to snapshot image %" PRId32 ", removing it without undo\n", imgid);
      dt_free(rm);
    }
  }
 
  const dt_image_t *img = dt_image_cache_get(imgid, 'r');
  dt_image_cache_read_release(img);
 
  // make sure we remove from the cache first, or else the cache will look for imgid in sql
  dt_image_cache_remove(imgid);
 
  dt_grouping_remove_from_group(imgid);
  dt_image_repository_delete(imgid);
 
  /* Every buffer, not just the thumbnails: the decoded raw input survives dt_mipmap_cache_remove()
   * by design, and an image that has left the library has no business keeping one. It is what the
   * pipeline slices through basebuffer when the image comes back on Ctrl+Z, and a stale entry
   * there renders as a husk that no later pass replaces. */
  dt_mipmap_cache_remove_all_sizes(imgid, TRUE);
}
 
void dt_image_remove(const int32_t imgid)
{
  _image_remove(imgid, FALSE);
}
 
void dt_image_remove_undoable(const int32_t imgid)
{
  _image_remove(imgid, TRUE);
}
 
uint32_t dt_image_altered(const int32_t imgid)
{
  return dt_history_repository_count_items(imgid);
}
 
#ifndef _WIN32
static int _valid_glob_match(const char *const name, size_t offset)
{
  // verify that the name matched by glob() is a valid sidecar name by checking whether we have an underscore
  // followed by a sequence of digits followed by a period at the given offset in the name
  if(strlen(name) < offset || name[offset] != '_')
    return FALSE;
  size_t i;
  for(i = offset+1; name[i] && name[i] != '.'; i++)
  {
    if(!isdigit(name[i]))
      return FALSE;
  }
  return name[i] == '.';
}
#endif /* !_WIN32 */
 
GList* dt_image_find_xmps(const char* filename)
{
  // find all duplicates of an image by looking for all possible sidecars for the file: file.ext.xmp, file_NN.ext.xmp,
  //   file_NNN.ext.xmp, and file_NNNN.ext.xmp
  // because a glob() needs to scan the entire directory, we minimize work for large directories by doing a single
  //   glob which might generate false matches (if the image name contains an underscore followed by a digit) and
  //   filter out the false positives afterward
#ifndef _WIN32
  // start by locating the extension, which we'll be referencing multiple times
  const size_t fn_len = strlen(filename);
  const char* ext = strrchr(filename,'.');  // find last dot
  if(IS_NULL_PTR(ext)) ext = filename;
  const size_t ext_offset = ext - filename;
 
  gchar pattern[DT_PATH_MAX] = { 0 };
  GList* files = NULL;
 
  // check for file.ext.xmp
  static const char xmp[] = ".xmp";
  const size_t xmp_len = strlen(xmp);
  // concatenate filename and sidecar extension
  g_strlcpy(pattern,  filename, sizeof(pattern));
  g_strlcpy(pattern + fn_len, xmp, sizeof(pattern) - fn_len);
  if(dt_util_test_image_file(pattern))
  {
    // the default sidecar exists, is readable and is a regular file with lenght > 0, so add it to the list
    files = g_list_prepend(files, g_strdup(pattern));
  }
 
  // now collect all file_N*N.ext.xmp matches
  static const char glob_pattern[] = "_[0-9]*[0-9]";
  const size_t gp_len = strlen(glob_pattern);
  if(fn_len + gp_len + xmp_len < sizeof(pattern)) // enough space to build pattern?
  {
    // add GLOB.ext.xmp to the root of the basename
    g_strlcpy(pattern + ext_offset, glob_pattern, sizeof(pattern) - fn_len);
    g_strlcpy(pattern + ext_offset + gp_len, ext, sizeof(pattern) - ext_offset - gp_len);
    g_strlcpy(pattern + fn_len + gp_len, xmp, sizeof(pattern) - fn_len - gp_len);
    glob_t globbuf;
    if(!glob(pattern, 0, NULL, &globbuf))
    {
      // for each match of the pattern
      for(size_t i = 0; i < globbuf.gl_pathc; i++)
      {
        if(_valid_glob_match(globbuf.gl_pathv[i], ext_offset))
        {
          // it's not a false positive, so add it to the list of sidecars
          files = g_list_prepend(files, g_strdup(globbuf.gl_pathv[i]));
        }
      }
      globfree(&globbuf);
    }
  }
  // we built the list in reverse order for speed, so un-reverse it
  return g_list_reverse(files);
 
#else
  return win_image_find_duplicates(filename);
#endif
}
 
// Search for duplicate's sidecar files and import them if found and not in DB yet
int dt_image_read_duplicates(const uint32_t id, const char *filename, const gboolean clear_selection)
{
  int count_xmps_processed = 0;
  gchar pattern[DT_PATH_MAX] = { 0 };
 
  GList *files = dt_image_find_xmps(filename);
 
  // we store the xmp filename without version part in pattern to speed up string comparison later
  g_snprintf(pattern, sizeof(pattern), "%s.xmp", filename);
 
  for(GList *file_iter = files; file_iter; file_iter = g_list_next(file_iter))
  {
    gchar *xmpfilename = file_iter->data;
    int version = -1;
 
    // we need to get the version number of the sidecar filename
    if(!strncmp(xmpfilename, pattern, sizeof(pattern)))
    {
      // this is an xmp file without version number which corresponds to version 0
      version = 0;
    }
    else
    {
      // we need to derive the version number from the filename
 
      gchar *c3 = xmpfilename + strlen(xmpfilename)
        - 5; // skip over .xmp extension; position c3 at character before the '.'
      while(*c3 != '.' && c3 > xmpfilename)
        c3--; // skip over filename extension; position c3 is at character '.'
      gchar *c4 = c3;
      while(*c4 != '_' && c4 > xmpfilename) c4--; // move to beginning of version number
      c4++;
 
      gchar *idfield = g_strndup(c4, c3 - c4);
 
      version = atoi(idfield);
      dt_free(idfield);
    }
 
    int newid = id;
    int grpid = -1;
 
    if(count_xmps_processed == 0)
    {
      // this is the first xmp processed, just update the passed-in id
      dt_image_repository_set_version(id, version);
    }
    else
    {
      // create a new duplicate based on the passed-in id. Note that we do not call
      // dt_image_duplicate_with_version() as this version also set the group which
      // is using DT_IMAGE_CACHE_SAFE and so will write the .XMP. But we must avoid
      // this has the xmp for the duplicate is read just below.
      newid = dt_image_repository_duplicate(id, version);
      const dt_image_t *img = dt_image_cache_get(id, 'r');
      if(!IS_NULL_PTR(img))
      {
        grpid = img->group_id;
        dt_image_cache_read_release(img);
      }
    }
    // make sure newid is not selected
    if(clear_selection) dt_selection_clear(dt_selection_get_global());
 
    // dt_image_repository_duplicate() above returns -1 when it cannot create the duplicate,
    // and dt_image_cache_get() returns NULL for an id it cannot load. Both were dereferenced
    // unchecked, which is Sentry 140350855: SIGSEGV on `img->version = version` while
    // importing a film roll whose sidecars name versions the duplicate could not be created
    // for. Skipping this sidecar leaves the rest of the import to proceed.
    dt_image_t *img = (newid > 0) ? dt_image_cache_get(newid, 'w') : NULL;
    if(IS_NULL_PTR(img))
    {
      fprintf(stderr, "[dt_image_read_duplicates] could not load image %i for sidecar '%s', skipping it\n",
              newid, xmpfilename);
      continue;
    }
    (void)dt_exif_xmp_read(img, xmpfilename, 0);
    img->version = version;
    dt_image_cache_write_release(img, DT_IMAGE_CACHE_RELAXED);
 
    if(grpid != -1)
    {
      // now it is safe to set the duplicate group-id
      dt_grouping_add_to_group(grpid, newid);
      dt_collection_update_query(dt_collection_get_global(), DT_COLLECTION_CHANGE_RELOAD, DT_COLLECTION_PROP_UNDEF, NULL);
    }
 
    count_xmps_processed++;
  }
 
  g_list_free_full(files, dt_free_gpointer);
  files = NULL;
  return count_xmps_processed;
}
 
static int32_t _image_import_internal(const int32_t film_id, const char *filename,
                                       gboolean lua_locking, gboolean raise_signals)
{
  char *normalized_filename = dt_util_normalize_path(filename);
  if(!normalized_filename || !dt_util_test_image_file(normalized_filename))
  {
    dt_free(normalized_filename);
    return 0;
  }
  const char *cc = normalized_filename + strlen(normalized_filename);
  for(; *cc != '.' && cc > normalized_filename; cc--)
    ;
  if(!strcasecmp(cc, ".dt") || !strcasecmp(cc, ".dttags") || !strcasecmp(cc, ".xmp"))
  {
    dt_free(normalized_filename);
    return 0;
  }
  char *ext = g_ascii_strdown(cc + 1, -1);
  if(!dt_image_ext_is_supported(ext))
  {
    dt_free(normalized_filename);
    dt_free(ext);
    return 0;
  }
  // select from images; if found => return
  gchar *imgfname = g_path_get_basename(normalized_filename);
  int32_t id = dt_image_get_id(film_id, imgfname);
  if(id > UNKNOWN_IMAGE)
  {
    dt_control_log(_("Image %s is already in library and will not be re-imported.\n"), imgfname);
    dt_free(imgfname);
    dt_free(ext);
    dt_free(normalized_filename);
    return id;
  }
 
  // also need to set the no-legacy bit, to make sure we get the right presets (new ones)
  uint32_t flags = 0;
  flags |= DT_IMAGE_NO_LEGACY_PRESETS;
  // and we set the type of image flag (from extension for now)
  gchar *extension = g_strrstr(imgfname, ".");
  flags |= dt_image_flags_from_extension(extension);
  // set the bits in flags that indicate if any of the extra files (.txt, .wav) are present
  char *extra_file = dt_image_get_audio_path_from_path(normalized_filename);
  if(extra_file)
  {
    flags |= DT_IMAGE_HAS_WAV;
    dt_free(extra_file);
  }
  extra_file = dt_image_get_text_path_from_path(normalized_filename);
  if(extra_file)
  {
    flags |= DT_IMAGE_HAS_TXT;
    dt_free(extra_file);
  }
 
  if(!dt_image_repository_insert_import(film_id, imgfname, flags, dt_datetime_now_to_gtimespan()))
    return 0;
 
  id = dt_image_get_id(film_id, imgfname);
 
  // Try to find out if this should be grouped already.
  gchar *basename = g_strdup(imgfname);
  gchar *cc2 = basename + strlen(basename);
  for(; *cc2 != '.' && cc2 > basename; cc2--)
    ;
  *cc2 = '\0';
  gchar *sql_pattern = g_strconcat(basename, ".%", NULL);
  int group_id;
  // in case we are not a jpg check if we need to change group representative
  if(strcmp(ext, "jpg") != 0 && strcmp(ext, "jpeg") != 0)
  {
    const int32_t other_group = dt_image_repository_find_group_for_pattern(film_id, sql_pattern, TRUE, id);
    // if we have a group already
    if(other_group != -1)
    {
      int other_id = other_group;
      dt_image_t *other_img = dt_image_cache_get(other_id, 'w');
      gchar *other_basename = g_strdup(other_img->filename);
      gchar *cc3 = other_basename + strlen(other_img->filename);
      for(; *cc3 != '.' && cc3 > other_basename; cc3--)
        ;
      ++cc3;
      gchar *ext_lowercase = g_ascii_strdown(cc3, -1);
      // if the group representative is a jpg, change group representative to this new imported image
      if(!strcmp(ext_lowercase, "jpg") || !strcmp(ext_lowercase, "jpeg"))
      {
        other_img->group_id = id;
        dt_image_cache_write_release(other_img, DT_IMAGE_CACHE_SAFE);
        GList *members = dt_image_repository_get_group_members(other_id, other_id);
        for(GList *l = members; l; l = g_list_next(l))
        {
          other_id = GPOINTER_TO_INT(l->data);
          dt_image_t *group_img = dt_image_cache_get(other_id, 'w');
          group_img->group_id = id;
          dt_image_cache_write_release(group_img, DT_IMAGE_CACHE_SAFE);
        }
        g_list_free(members);
        group_id = id;
      }
      else
      {
        dt_image_cache_write_release(other_img, DT_IMAGE_CACHE_RELAXED);
        group_id = other_id;
      }
      dt_free(ext_lowercase);
      dt_free(other_basename);
    }
    else
    {
      group_id = id;
    }
  }
  else
  {
    const int32_t other_group = dt_image_repository_find_group_for_pattern(film_id, sql_pattern, FALSE, id);
    group_id = (other_group != -1) ? other_group : id;
  }
  dt_image_repository_set_group(id, group_id);
 
  // printf("[image_import] importing `%s' to img id %d\n", imgfname, id);
 
  // lock as shortly as possible:
  dt_image_t *img = dt_image_cache_get(id, 'w');
  img->group_id = group_id;
 
  // read dttags and exif for database queries!
  (void)dt_exif_read(img, normalized_filename);
  char dtfilename[DT_PATH_MAX] = { 0 };
  g_strlcpy(dtfilename, normalized_filename, sizeof(dtfilename));
  // dt_image_path_append_version(id, dtfilename, sizeof(dtfilename));
  g_strlcat(dtfilename, ".xmp", sizeof(dtfilename));
 
  const int res = dt_exif_xmp_read(img, dtfilename, 0);
 
  // write through to db, but not to xmp.
  dt_image_cache_write_release(img, DT_IMAGE_CACHE_RELAXED);
 
  // read all sidecar files
  const int nb_xmp = dt_image_read_duplicates(id, normalized_filename, raise_signals);
 
  if((res != 0) && (nb_xmp == 0))
  {
    const gboolean lr_xmp = dt_lightroom_import(id, NULL, TRUE);
    if(lr_xmp) dt_control_save_xmp(id);
  }
 
  // add a tag with the file extension
  guint tagid = 0;
  char tagname[512];
  snprintf(tagname, sizeof(tagname), "darktable|format|%s", ext);
  dt_free(ext);
  dt_tag_new(tagname, &tagid);
  dt_tag_attach(tagid, id, FALSE, FALSE);
 
  // make sure that there are no stale thumbnails left
  dt_mipmap_cache_remove(id, TRUE);
 
  //synch database entries to xmp
  if(dt_image_get_xmp_mode()) dt_image_synch_all_xmp(normalized_filename);
 
  dt_free(imgfname);
  dt_free(basename);
  dt_free(sql_pattern);
  dt_free(normalized_filename);
 
  if(raise_signals)
  {
    dt_image_notify_imported(id);
    // the old raise handed the freshly built list to the signal; the handler copies now,
    // so this site frees its own
    GList *imgs = g_list_prepend(NULL, GINT_TO_POINTER(id));
    dt_metadata_geotags_changed(imgs);
    g_list_free(imgs);
  }
 
  // the following line would look logical with new_tags_set being the return value
  // from dt_tag_new above, but this could lead to too rapid signals, being able to lock up the
  // keywords side pane when trying to use it, which can lock up the whole dt GUI ..
  // if(new_tags_set) DT_DEBUG_CONTROL_SIGNAL_RAISE(dt_control_signal_get_global(),DT_SIGNAL_TAG_CHANGED);
  return id;
}
 
int32_t dt_image_get_id_full_path(const gchar *filename)
{
  gchar *dir = g_path_get_dirname(filename);
  gchar *file = g_path_get_basename(filename);
  const int32_t id = dt_image_repository_find_by_folder_and_filename(dir, file);
  dt_free(dir);
  dt_free(file);
 
  return id;
}
 
int32_t dt_image_get_id(int32_t film_id, const gchar *filename)
{
  return dt_image_repository_find_by_film_and_filename(film_id, filename);
}
 
/* Guess what kind of image a file holds from its extension alone.
 *
 * Lived in imageio/imageio_core.c, which put a six-line function over three predicates
 * from common/image_extensions.h and one enum from this file at layer 6 -- above every
 * one of its callers (common/image.c, common/exif.cc, caches/image_cache.c,
 * database/database.c, gui/import.c). Nothing in imageio/ or above ever called it. */
dt_image_flags_t dt_image_flags_from_extension(const char *extension)
{
  const char *ext = g_str_has_prefix(extension, ".") ? extension + 1 : extension;
  if(dt_image_ext_is_ambiguous(ext)) return 0;
  if(dt_image_ext_is_raw(ext)) return DT_IMAGE_RAW;
  if(dt_image_ext_is_hdr(ext)) return DT_IMAGE_HDR;
  if(dt_image_ext_is_ldr(ext)) return DT_IMAGE_LDR;
  return 0;
}
 
int32_t dt_image_import(const int32_t film_id, const char *filename, gboolean raise_signals)
{
  return _image_import_internal(film_id, filename, TRUE, raise_signals);
}
 
int32_t dt_image_import_lua(const int32_t film_id, const char *filename)
{
  return _image_import_internal(film_id, filename, FALSE, TRUE);
}
 
void dt_image_init(dt_image_t *img)
{
  memset(img, 0, sizeof(*img));
  img->width = img->height = 0;
  img->p_width = img->p_height = 0;
  img->crop_x = img->crop_y = img->crop_width = img->crop_height = 0;
  img->orientation = ORIENTATION_NULL;
 
  img->import_timestamp = img->change_timestamp = img->export_timestamp = img->print_timestamp = 0;
 
  img->legacy_flip.legacy = 0;
  img->legacy_flip.user_flip = 0;
 
  img->dsc = (dt_iop_buffer_dsc_t){ .channels = 0, .datatype = TYPE_UNKNOWN, .bpp = 0, .filters = 0u };
  img->film_id = UNKNOWN_IMAGE;
  img->group_id = UNKNOWN_IMAGE;
  img->group_members = 0;
  img->history_items = 0;
  img->history_hash = UINT64_MAX;
  img->mipmap_hash = 0; // don't default it to img->history_hash to not make it look like they are in sync
  img->self_hash = 0;
  img->flags = 0;
  img->id = UNKNOWN_IMAGE;
  img->version = -1;
  img->loader = LOADER_UNKNOWN;
  img->exif_inited = 0;
  img->camera_missing_sample = FALSE;
  dt_datetime_exif_to_img(img, "");
  memset(img->exif_maker, 0, sizeof(img->exif_maker));
  memset(img->exif_model, 0, sizeof(img->exif_model));
  memset(img->exif_lens, 0, sizeof(img->exif_lens));
  memset(img->camera_maker, 0, sizeof(img->camera_maker));
  memset(img->camera_model, 0, sizeof(img->camera_model));
  memset(img->camera_alias, 0, sizeof(img->camera_alias));
  memset(img->camera_makermodel, 0, sizeof(img->camera_makermodel));
  memset(img->camera_legacy_makermodel, 0, sizeof(img->camera_legacy_makermodel));
  memset(img->filename, 0, sizeof(img->filename));
  memset(img->fullpath, 0, sizeof(img->fullpath));
  memset(img->local_copy_path, 0, sizeof(img->local_copy_path));
  memset(img->local_copy_legacy_path, 0, sizeof(img->local_copy_legacy_path));
  memset(img->folder, 0, sizeof(img->folder));
  memset(img->filmroll, 0, sizeof(img->filmroll));
  memset(img->datetime, 0, sizeof(img->datetime));
  g_strlcpy(img->filename, "(unknown)", sizeof(img->filename));
  img->exif_crop = 1.0;
  img->exif_exposure = 0;
  img->exif_exposure_bias = NAN;
  img->exif_aperture = 0;
  img->exif_iso = 0;
  img->exif_focal_length = 0;
  img->exif_focus_distance = 0;
  img->geoloc.latitude = NAN;
  img->geoloc.longitude = NAN;
  img->geoloc.elevation = NAN;
  img->raw_black_level = 0;
  for(uint8_t i = 0; i < 4; i++) img->raw_black_level_separate[i] = 0;
  img->raw_white_point = 16384; // 2^14
  img->d65_color_matrix[0] = NAN;
  img->profile = NULL;
  img->profile_size = 0;
  img->embedded_profile = NULL;
  img->colorspace = DT_IMAGE_COLORSPACE_NONE;
  img->fuji_rotation_pos = 0;
  img->pixel_aspect_ratio = 1.0f;
  img->wb_coeffs[0] = NAN;
  img->wb_coeffs[1] = NAN;
  img->wb_coeffs[2] = NAN;
  img->wb_coeffs[3] = NAN;
  img->usercrop[0] = img->usercrop[1] = 0;
  img->usercrop[2] = img->usercrop[3] = 1;
  img->usercrop_status = DT_IMAGE_USERCROP_UNKNOWN;
  img->dng_gain_maps = NULL;
  img->cache_entry = 0;
  img->color_labels = 0;
  img->rating = 0;
  img->has_localcopy = FALSE;
  img->has_audio = FALSE;
  img->is_bw = FALSE;
  img->is_bw_flow = FALSE;
  img->is_hdr = FALSE;
 
  for(int k=0; k<4; k++)
    for(int i=0; i<3; i++)
      img->adobe_XYZ_to_CAM[k][i] = NAN;
}
 
void dt_image_refresh_makermodel(dt_image_t *img)
{
  if(!img->camera_maker[0] || !img->camera_model[0] || !img->camera_alias[0])
  {
    // We need to use the exif values, so let's get rawspeed to munge them
    dt_imageio_lookup_makermodel(img->exif_maker, img->exif_model,
                                 img->camera_maker, sizeof(img->camera_maker),
                                 img->camera_model, sizeof(img->camera_model),
                                 img->camera_alias, sizeof(img->camera_alias));
  }
 
  // Now we just create a makermodel by concatenation
  g_strlcpy(img->camera_makermodel, img->camera_maker, sizeof(img->camera_makermodel));
  const int len = strlen(img->camera_maker);
  img->camera_makermodel[len] = ' ';
  g_strlcpy(img->camera_makermodel+len+1, img->camera_model, sizeof(img->camera_makermodel)-len-1);
}
 
int32_t dt_image_rename(const int32_t imgid, const int32_t filmid, const gchar *newname)
{
  // TODO: several places where string truncation could occur unnoticed
  int32_t result = -1;
  gchar oldimg[DT_PATH_MAX] = { 0 };
  gchar newimg[DT_PATH_MAX] = { 0 };
  gboolean from_cache = FALSE;
  dt_image_full_path(imgid,  oldimg,  sizeof(oldimg),  &from_cache, __FUNCTION__);
  gchar *newdir = NULL;
 
  newdir = dt_film_repository_get_folder(filmid);
 
  gchar copysrcpath[DT_PATH_MAX] = { 0 };
  gchar copydestpath[DT_PATH_MAX] = { 0 };
  GFile *old = NULL, *new = NULL;
  if(newdir)
  {
    old = g_file_new_for_path(oldimg);
 
    if(newname)
    {
      g_snprintf(newimg, sizeof(newimg), "%s%c%s", newdir, G_DIR_SEPARATOR, newname);
      new = g_file_new_for_path(newimg);
      // 'newname' represents the file's new *basename* -- it must not
      // refer to a file outside of 'newdir'.
      gchar *newBasename = g_file_get_basename(new);
      if(g_strcmp0(newname, newBasename) != 0)
      {
        g_object_unref(old);
        old = NULL;
        g_object_unref(new);
        new = NULL;
      }
      dt_free(newBasename);
    }
    else
    {
      gchar *imgbname = g_path_get_basename(oldimg);
      g_snprintf(newimg, sizeof(newimg), "%s%c%s", newdir, G_DIR_SEPARATOR, imgbname);
      new = g_file_new_for_path(newimg);
      dt_free(imgbname);
    }
    dt_free(newdir);
  }
 
  if(new)
  {
    // get current local copy if any
    dt_image_t *img = dt_image_cache_get(imgid, 'r');
    if(img)
    {
      dt_image_choose_input_path(img, copysrcpath, sizeof(copysrcpath), TRUE);
      dt_image_cache_read_release(img);
    }
 
    // move image
    GError *moveError = NULL;
    gboolean moveStatus = g_file_move(old, new, 0, NULL, NULL, NULL, &moveError);
 
    if(moveStatus)
    {
      _move_text_sidecar_if_present(oldimg, newimg, FALSE);
 
      // first move xmp files of image and duplicates
      GList *dup_list = dt_image_repository_get_duplicate_ids(imgid);
      for(GList *l = dup_list; l; l = g_list_next(l))
      {
        const int32_t id = GPOINTER_TO_INT(l->data);
        gchar oldxmp[DT_PATH_MAX] = { 0 }, newxmp[DT_PATH_MAX] = { 0 };
        g_strlcpy(oldxmp, oldimg, sizeof(oldxmp));
        g_strlcpy(newxmp, newimg, sizeof(newxmp));
        dt_image_path_append_version(id, oldxmp, sizeof(oldxmp));
        dt_image_path_append_version(id, newxmp, sizeof(newxmp));
        g_strlcat(oldxmp, ".xmp", sizeof(oldxmp));
        g_strlcat(newxmp, ".xmp", sizeof(newxmp));
 
        GFile *goldxmp = g_file_new_for_path(oldxmp);
        GFile *gnewxmp = g_file_new_for_path(newxmp);
 
        g_file_move(goldxmp, gnewxmp, 0, NULL, NULL, NULL, NULL);
 
        g_object_unref(goldxmp);
        g_object_unref(gnewxmp);
      }
 
      // then update database and cache
      // if update was performed in above loop, dt_image_path_append_version()
      // would return wrong version!
      while(dup_list)
      {
        const int id = GPOINTER_TO_INT(dup_list->data);
        img = dt_image_cache_get(id, 'w');
        img->film_id = filmid;
        if(newname) g_strlcpy(img->filename, newname, DT_MAX_FILENAME_LEN);
        // write through to db and queue xmp write
        dt_image_cache_write_release(img, DT_IMAGE_CACHE_SAFE);
        dup_list = g_list_delete_link(dup_list, dup_list);
      }
      g_list_free(dup_list);
      dup_list = NULL;
 
      // finally, rename local copy if any
      if(g_file_test(copysrcpath, G_FILE_TEST_EXISTS))
      {
        // get new name
        img = dt_image_cache_get(imgid, 'r');
        if(img)
        {
          dt_image_choose_input_path(img, copydestpath, sizeof(copydestpath), TRUE);
          dt_image_cache_read_release(img);
        }
 
        GFile *cold = g_file_new_for_path(copysrcpath);
        GFile *cnew = g_file_new_for_path(copydestpath);
 
        g_clear_error(&moveError);
        moveStatus = g_file_move(cold, cnew, 0, NULL, NULL, NULL, &moveError);
        if(!moveStatus)
        {
          fprintf(stderr, "[dt_image_rename] error moving local copy `%s' -> `%s'\n", copysrcpath, copydestpath);
 
          if(g_error_matches(moveError, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
          {
            gchar *oldBasename = g_path_get_basename(copysrcpath);
            dt_control_log(_("cannot access local copy `%s'"), oldBasename);
            dt_free(oldBasename);
          }
          else if(g_error_matches(moveError, G_IO_ERROR, G_IO_ERROR_EXISTS)
                  || g_error_matches(moveError, G_IO_ERROR, G_IO_ERROR_IS_DIRECTORY))
          {
            gchar *newBasename = g_path_get_basename(copydestpath);
            dt_control_log(_("cannot write local copy `%s'"), newBasename);
            dt_free(newBasename);
          }
          else
          {
            gchar *oldBasename = g_path_get_basename(copysrcpath);
            gchar *newBasename = g_path_get_basename(copydestpath);
            dt_control_log(_("error moving local copy `%s' -> `%s'"), oldBasename, newBasename);
            dt_free(oldBasename);
            dt_free(newBasename);
          }
        }
 
        g_object_unref(cold);
        g_object_unref(cnew);
      }
 
      result = 0;
    }
    else
    {
      if(g_error_matches(moveError, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
      {
        dt_control_log(_("error moving `%s': file not found"), oldimg);
      }
      // only display error message if newname is set (renaming and
      // not moving) as when moving it can be the case where a
      // duplicate is being moved, so only the .xmp are present but
      // the original file may already have been moved.
      else if(newname
              && (g_error_matches(moveError, G_IO_ERROR, G_IO_ERROR_EXISTS)
                  || g_error_matches(moveError, G_IO_ERROR, G_IO_ERROR_IS_DIRECTORY)))
      {
        dt_control_log(_("error moving `%s' -> `%s': file exists"), oldimg, newimg);
      }
      else if(newname)
      {
        dt_control_log(_("error moving `%s' -> `%s'"), oldimg, newimg);
      }
    }
 
    g_clear_error(&moveError);
    g_object_unref(old);
    g_object_unref(new);
  }
 
  return result;
}
 
int32_t dt_image_move(const int32_t imgid, const int32_t filmid)
{
  return dt_image_rename(imgid, filmid, NULL);
}
 
int32_t dt_image_copy_rename(const int32_t imgid, const int32_t filmid, const gchar *newname)
{
  int32_t newid = -1;
  gchar srcpath[DT_PATH_MAX] = { 0 };
  gchar *newdir = NULL;
  gboolean from_cache = FALSE;
  gchar *oldFilename = NULL;
  gchar *newFilename = NULL;
 
  newdir = dt_film_repository_get_folder(filmid);
 
  GFile *src = NULL, *dest = NULL;
  if(newdir)
  {
    dt_image_full_path(imgid,  srcpath,  sizeof(srcpath),  &from_cache, __FUNCTION__);
    oldFilename = g_path_get_basename(srcpath);
    gchar *destpath;
    if(newname)
    {
      newFilename = g_strdup(newname);
      destpath = g_build_filename(newdir, newname, NULL);
      dest = g_file_new_for_path(destpath);
      // 'newname' represents the file's new *basename* -- it must not
      // refer to a file outside of 'newdir'.
      gchar *destBasename = g_file_get_basename(dest);
      if(g_strcmp0(newname, destBasename) != 0)
      {
        g_object_unref(dest);
        dest = NULL;
      }
      dt_free(destBasename);
    }
    else
    {
      newFilename = g_path_get_basename(srcpath);
      destpath = g_build_filename(newdir, newFilename, NULL);
      dest = g_file_new_for_path(destpath);
    }
    if(dest)
    {
      src = g_file_new_for_path(srcpath);
    }
    dt_free(newdir);
    dt_free(destpath);
  }
 
  if(dest)
  {
    // copy image to new folder
    // if image file already exists, continue
    GError *gerror = NULL;
    gboolean copyStatus = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
 
    if(copyStatus || g_error_matches(gerror, G_IO_ERROR, G_IO_ERROR_EXISTS))
    {
      gchar *dest_image_path = g_file_get_path(dest);
      if(dest_image_path)
      {
        _copy_text_sidecar_if_present(srcpath, dest_image_path);
        dt_free(dest_image_path);
      }
 
      newid = dt_image_repository_copy_to_film(imgid, filmid, newFilename, oldFilename);
 
      if(newid != -1)
      {
        // also copy over on-disk thumbnails, if any
        dt_mipmap_cache_copy_thumbnails(newid, imgid);
 
        dt_history_copy_and_paste_on_image(imgid, newid, NULL, TRUE, DT_HISTORY_MERGE_REPLACE, FALSE, NULL);
 
        dt_collection_update_query(dt_collection_get_global(), DT_COLLECTION_CHANGE_RELOAD, DT_COLLECTION_PROP_UNDEF,
                                   NULL);
      }
    }
    else
    {
      fprintf(stderr, "Failed to copy image %s: %s\n", srcpath, gerror->message);
    }
    g_object_unref(dest);
    g_object_unref(src);
    g_clear_error(&gerror);
  }
  dt_free(oldFilename);
  dt_free(newFilename);
 
  return newid;
}
 
int32_t dt_image_copy(const int32_t imgid, const int32_t filmid)
{
  return dt_image_copy_rename(imgid, filmid, NULL);
}
 
int dt_image_local_copy_set(const int32_t imgid)
{
  gchar srcpath[DT_PATH_MAX] = { 0 };
  gchar destpath[DT_PATH_MAX] = { 0 };
  dt_image_path_source_t source = DT_IMAGE_PATH_NONE;
  char local_copy_path[DT_PATH_MAX] = { 0 };
  char local_copy_legacy_path[DT_PATH_MAX] = { 0 };
  dt_image_t *img = dt_image_cache_get(imgid, 'r');
  if(img)
  {
    g_strlcpy(srcpath, img->fullpath, DT_PATH_MAX);
    g_strlcpy(local_copy_path, img->local_copy_path, sizeof(local_copy_path));
    g_strlcpy(local_copy_legacy_path, img->local_copy_legacy_path, sizeof(local_copy_legacy_path));
    char existing[DT_PATH_MAX] = { 0 };
    source = dt_image_choose_input_path(img, existing, sizeof(existing), TRUE);
    if(source == DT_IMAGE_PATH_LOCAL_COPY || source == DT_IMAGE_PATH_LOCAL_COPY_LEGACY)
      g_strlcpy(destpath, existing, sizeof(destpath));
    else
      g_strlcpy(destpath, local_copy_path, sizeof(destpath));
    dt_image_cache_read_release(img);
  }
  else
  {
    return 1;
  }
 
  // check that the src file is readable
  if(!g_file_test(srcpath, G_FILE_TEST_IS_REGULAR))
  {
    dt_control_log(_("cannot create local copy when the original file is not accessible."));
    return 1;
  }
 
  if(source != DT_IMAGE_PATH_LOCAL_COPY && source != DT_IMAGE_PATH_LOCAL_COPY_LEGACY)
  {
    GFile *src = g_file_new_for_path(srcpath);
    GFile *dest = g_file_new_for_path(destpath);
 
    // copy image to cache directory
    GError *gerror = NULL;
 
    if(!g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror))
    {
      dt_control_log(_("cannot create local copy."));
      g_object_unref(dest);
      g_object_unref(src);
      return 1;
    }
 
    g_object_unref(dest);
    g_object_unref(src);
  }
 
  _copy_text_sidecar_if_present(srcpath, destpath);
 
  // update cache local copy flags, do this even if the local copy already exists as we need to set the flags
  // for duplicate
  img = dt_image_cache_get(imgid, 'w');
  img->flags |= DT_IMAGE_LOCAL_COPY;
  dt_image_cache_write_release(img, DT_IMAGE_CACHE_RELAXED);
 
  dt_control_queue_redraw_center();
  return 0;
}
 
static int _nb_other_local_copy_for(const int32_t imgid)
{
  return dt_image_repository_count_others_with_flag(imgid, DT_IMAGE_LOCAL_COPY);
}
 
int dt_image_local_copy_reset(const int32_t imgid)
{
  gchar destpath[DT_PATH_MAX] = { 0 };
  gchar locppath[DT_PATH_MAX] = { 0 };
  gchar cachedir[DT_PATH_MAX] = { 0 };
 
  // check that a local copy exists, otherwise there is nothing to do
  dt_image_t *imgr = dt_image_cache_get(imgid, 'r');
  if(!imgr) return 0;
  const gboolean local_copy_exists = (imgr->flags & DT_IMAGE_LOCAL_COPY) == DT_IMAGE_LOCAL_COPY ? TRUE : FALSE;
  dt_image_cache_read_release(imgr);
 
  if(!local_copy_exists)
    return 0;
 
  // check that the original file is accessible
 
  gboolean from_cache = FALSE;
  dt_image_full_path(imgid,  destpath,  sizeof(destpath),  &from_cache, __FUNCTION__);
 
  from_cache = TRUE;
  dt_image_full_path(imgid,  locppath,  sizeof(locppath),  &from_cache, __FUNCTION__);
  dt_image_path_append_version(imgid, locppath, sizeof(locppath));
  g_strlcat(locppath, ".xmp", sizeof(locppath));
 
  // a local copy exists, but the original is not accessible
 
  if(g_file_test(locppath, G_FILE_TEST_EXISTS) && !g_file_test(destpath, G_FILE_TEST_EXISTS))
  {
    dt_control_log(_("cannot remove local copy when the original file is not accessible."));
    return 1;
  }
 
  // get name of local copy
 
  locppath[0] = '\0';
  dt_image_t *img = dt_image_cache_get(imgid, 'r');
  if(img)
  {
    dt_image_choose_input_path(img, locppath, sizeof(locppath), TRUE);
    dt_image_cache_read_release(img);
  }
 
  // remove cached file, but double check that this is really into the cache. We really want to avoid deleting
  // a user's original file.
 
  dt_loc_get_user_cache_dir(cachedir, sizeof(cachedir));
 
  if(g_file_test(locppath, G_FILE_TEST_EXISTS) && strstr(locppath, cachedir))
  {
    GFile *dest = g_file_new_for_path(locppath);
 
    // first sync the xmp with the original picture
 
    dt_control_save_xmp(imgid);
 
    // delete image from cache directory only if there is no other local cache image referencing it
    // for example duplicates are all referencing the same base picture.
 
    if(_nb_other_local_copy_for(imgid) == 0)
    {
      _move_text_sidecar_if_present(locppath, destpath, TRUE);
      g_file_delete(dest, NULL, NULL);
    }
 
    g_object_unref(dest);
 
    // delete xmp if any
    dt_image_path_append_version(imgid, locppath, sizeof(locppath));
    g_strlcat(locppath, ".xmp", sizeof(locppath));
    dest = g_file_new_for_path(locppath);
 
    if(g_file_test(locppath, G_FILE_TEST_EXISTS)) g_file_delete(dest, NULL, NULL);
    g_object_unref(dest);
  }
 
  // update cache, remove local copy flags, this is done in all cases here as when we
  // reach this point the local-copy flag is present and the file has been either removed
  // or is not present.
 
  img = dt_image_cache_get(imgid, 'w');
  img->flags &= ~DT_IMAGE_LOCAL_COPY;
  dt_image_cache_write_release(img, DT_IMAGE_CACHE_RELAXED);
 
  dt_control_queue_redraw_center();
 
  return 0;
}
 
// *******************************************************
// xmp stuff
// *******************************************************
 
static gboolean _sidecar_is_up_to_date(const dt_image_t *img)
{
  if(IS_NULL_PTR(img) || img->id <= 0) return FALSE;
  if(img->change_timestamp <= 0) return FALSE;
 
  const int64_t write_timestamp = dt_image_repository_get_write_timestamp(img->id);
  if(write_timestamp <= 0) return FALSE;
 
  GDateTime *gdt = dt_datetime_gtimespan_to_gdatetime(img->change_timestamp);
  if(IS_NULL_PTR(gdt)) return FALSE;
  const int64_t change_timestamp_unix = g_date_time_to_unix(gdt);
  g_date_time_unref(gdt);
  dt_print(DT_DEBUG_CONTROL,
           "[xmp] imgid %d change_ts=%lld write_ts=%lld\n",
           img->id, (long long)change_timestamp_unix, (long long)write_timestamp);
  return change_timestamp_unix <= write_timestamp;
}
 
static dt_image_write_sidecar_result_t _write_sidecar_file_from_image_locked(const dt_image_t *img,
                                                                               const gboolean force_write)
{
  if(IS_NULL_PTR(img) || img->id <= 0) return DT_IMAGE_WRITE_SIDECAR_DISABLED;
 
  char imgpath[DT_PATH_MAX] = { 0 };
  if(dt_image_choose_input_path(img, imgpath, sizeof(imgpath), FALSE) == DT_IMAGE_PATH_NONE)
  {
    dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d no source path available\n", img->id);
    return DT_IMAGE_WRITE_SIDECAR_NO_SOURCE_PATH;
  }
 
  char filename[DT_PATH_MAX] = { 0 };
  g_strlcpy(filename, imgpath, sizeof(filename));
  dt_image_path_append_version(img->id, filename, sizeof(filename));
  g_strlcat(filename, ".xmp", sizeof(filename));
 
  if(!force_write && g_file_test(filename, G_FILE_TEST_EXISTS))
  {
    if(_sidecar_is_up_to_date(img))
    {
      dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d sidecar up-to-date, skip\n", img->id);
      return DT_IMAGE_WRITE_SIDECAR_OK;
    }
  }
 
  dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d writing sidecar %s\n", img->id, filename);
  if(dt_exif_xmp_write_with_imgpath(img, filename, imgpath) != 0)
    return DT_IMAGE_WRITE_SIDECAR_IO_ERROR;
 
  dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d updating write_timestamp\n", img->id);
  dt_image_repository_touch_write_timestamp(img->id);
  return DT_IMAGE_WRITE_SIDECAR_OK;
}
 
dt_image_write_sidecar_result_t dt_image_write_sidecar_file(const int32_t imgid)
{
  if(imgid <= 0 || !dt_image_get_xmp_mode()) return DT_IMAGE_WRITE_SIDECAR_DISABLED;
 
  dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d write start\n", imgid);
  dt_image_t *img = dt_image_cache_get(imgid, 'w');
  if(IS_NULL_PTR(img))
  {
    dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d cache lock failed\n", imgid);
    return DT_IMAGE_WRITE_SIDECAR_CACHE_BUSY;
  }
  dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d cache lock acquired (write)\n", imgid);
 
  const dt_image_write_sidecar_result_t res = _write_sidecar_file_from_image_locked(img, FALSE);
  dt_image_cache_write_release(img, DT_IMAGE_CACHE_MINIMAL);
  dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d cache lock released (write minimal)\n", imgid);
  return res;
}
 
dt_image_write_sidecar_result_t dt_image_write_sidecar_file_forced(const int32_t imgid)
{
  if(imgid <= 0) return DT_IMAGE_WRITE_SIDECAR_DISABLED;
 
  dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d forced write start\n", imgid);
  dt_image_t *img = dt_image_cache_get(imgid, 'w');
  if(IS_NULL_PTR(img))
  {
    dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d cache lock failed\n", imgid);
    return DT_IMAGE_WRITE_SIDECAR_CACHE_BUSY;
  }
  dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d cache lock acquired (write)\n", imgid);
 
  const dt_image_write_sidecar_result_t res = _write_sidecar_file_from_image_locked(img, TRUE);
  dt_image_cache_write_release(img, DT_IMAGE_CACHE_MINIMAL);
  dt_print(DT_DEBUG_CONTROL, "[xmp] imgid %d cache lock released (write minimal)\n", imgid);
  return res;
}
 
void dt_image_synch_xmps(const GList *img)
{
  if(IS_NULL_PTR(img)) return;
  if(dt_image_get_xmp_mode())
  {
    dt_control_save_xmps(img, FALSE);
  }
}
 
void dt_image_synch_xmp(const int selected)
{
  if(selected > 0)
  {
    GList *imgs = g_list_append(NULL, GINT_TO_POINTER(selected));
    dt_control_save_xmps(imgs, FALSE);
    g_list_free(imgs);
    imgs = NULL;
  }
  else
  {
    GList *imgs = dt_act_on_get_images();
    dt_image_synch_xmps(imgs);
    g_list_free(imgs);
    imgs = NULL;
  }
}
 
void dt_image_synch_all_xmp(const gchar *pathname)
{
  if(dt_image_get_xmp_mode())
  {
    const int32_t imgid = dt_image_get_id_full_path(pathname);
    if(imgid != UNKNOWN_IMAGE)
    {
      GList *imgs = g_list_append(NULL, GINT_TO_POINTER(imgid));
      dt_control_save_xmps(imgs, FALSE);
      g_list_free(imgs);
      imgs = NULL;
    }
  }
}
 
void dt_image_local_copy_synch()
{
  GList *imgs = NULL;
 
  GList *candidates = dt_image_repository_get_ids_with_flag(DT_IMAGE_LOCAL_COPY);
  for(GList *l = candidates; l; l = g_list_next(l))
  {
    const int32_t imgid = GPOINTER_TO_INT(l->data);
    gboolean from_cache = FALSE;
    char filename[DT_PATH_MAX] = { 0 };
    dt_image_full_path(imgid,  filename,  sizeof(filename),  &from_cache, __FUNCTION__);
 
    if(g_file_test(filename, G_FILE_TEST_EXISTS))
    {
      imgs = g_list_prepend(imgs, GINT_TO_POINTER(imgid));
    }
  }
  g_list_free(candidates);
 
  const int count = g_list_length(imgs);
  if(count > 0)
  {
    dt_control_save_xmps_forced(imgs);
    dt_control_log(ngettext("%d local copy has been synchronized",
                            "%d local copies have been synchronized", count),
                   count);
  }
  g_list_free(imgs);
  imgs = NULL;
}
 
void dt_image_get_datetime(const int32_t imgid, char *datetime)
{
  if(IS_NULL_PTR(datetime)) return;
  datetime[0] = '\0';
  const dt_image_t *cimg = dt_image_cache_get(imgid, 'r');
  if(IS_NULL_PTR(cimg)) return;
  dt_datetime_img_to_exif(datetime, DT_DATETIME_LENGTH, cimg);
  dt_image_cache_read_release(cimg);
}
 
static void _datetime_undo_data_free(gpointer data)
{
  GList *l = (GList *)data;
  g_list_free_full(l, dt_free_gpointer);
  l = NULL;
}
 
typedef struct _datetime_t
{
  char dt[DT_DATETIME_LENGTH];
} _datetime_t;
 
static void _image_set_datetimes(const GList *img, const GArray *dtime,
                                 GList **undo, const gboolean undo_on)
{
  int i = 0;
  for(GList *imgs = (GList *)img; imgs; imgs = g_list_next(imgs))
  {
    const int32_t imgid = GPOINTER_TO_INT(imgs->data);
    // if char *datetime, the returned pointer is not correct => use of _datetime_t
    const _datetime_t *datetime = &g_array_index(dtime, _datetime_t, i);
    if(undo_on)
    {
      dt_undo_datetime_t *undodatetime = (dt_undo_datetime_t *)malloc(sizeof(dt_undo_datetime_t));
      undodatetime->imgid = imgid;
      dt_image_get_datetime(imgid, undodatetime->before);
 
      memcpy(&undodatetime->after, datetime->dt, DT_DATETIME_LENGTH);
 
      *undo = g_list_prepend(*undo, undodatetime);
    }
 
    _set_datetime(imgid, datetime->dt);
    i++;
  }
}
 
void dt_image_set_datetimes(const GList *imgs, const GArray *dtime, const gboolean undo_on)
{
  if(IS_NULL_PTR(imgs) || IS_NULL_PTR(dtime) || (g_list_length((GList *)imgs) != dtime->len))
    return;
  GList *undo = NULL;
  if(undo_on) dt_undo_start_group(dt_undo_get_global(), DT_UNDO_DATETIME);
 
  _image_set_datetimes(imgs, dtime, &undo, undo_on);
 
  if(undo_on)
  {
    dt_undo_record(dt_undo_get_global(), NULL, DT_UNDO_DATETIME, undo, _pop_undo, _datetime_undo_data_free);
    dt_undo_end_group(dt_undo_get_global());
  }
}
 
static void _image_set_datetime(const GList *img, const char *datetime,
                                GList **undo, const gboolean undo_on)
{
  for(GList *imgs = (GList *)img; imgs;  imgs = g_list_next(imgs))
  {
    const int32_t imgid = GPOINTER_TO_INT(imgs->data);
    if(undo_on)
    {
      dt_undo_datetime_t *undodatetime = (dt_undo_datetime_t *)malloc(sizeof(dt_undo_datetime_t));
      undodatetime->imgid = imgid;
      dt_image_get_datetime(imgid, undodatetime->before);
 
      memcpy(&undodatetime->after, datetime, DT_DATETIME_LENGTH);
 
      *undo = g_list_prepend(*undo, undodatetime);
    }
 
    _set_datetime(imgid, datetime);
  }
}
 
void dt_image_set_datetime(const GList *imgs, const char *datetime, const gboolean undo_on)
{
  if(IS_NULL_PTR(imgs))
    return;
  GList *undo = NULL;
  if(undo_on) dt_undo_start_group(dt_undo_get_global(), DT_UNDO_DATETIME);
 
  _image_set_datetime(imgs, datetime, &undo, undo_on);
 
  if(undo_on)
  {
    dt_undo_record(dt_undo_get_global(), NULL, DT_UNDO_DATETIME, undo, _pop_undo, _datetime_undo_data_free);
    dt_undo_end_group(dt_undo_get_global());
  }
}
 
char *dt_image_get_audio_path_from_path(const char *image_path)
{
  size_t len = strlen(image_path);
  const char *c = image_path + len;
  while((c > image_path) && (*c != '.')) c--;
  len = c - image_path + 1;
 
  char *result = g_strndup(image_path, len + 3);
 
  result[len] = 'w';
  result[len + 1] = 'a';
  result[len + 2] = 'v';
  if(g_file_test(result, G_FILE_TEST_EXISTS)) return result;
 
  result[len] = 'W';
  result[len + 1] = 'A';
  result[len + 2] = 'V';
  if(g_file_test(result, G_FILE_TEST_EXISTS)) return result;
 
  dt_free(result);
  return NULL;
}
 
char *dt_image_get_audio_path(const int32_t imgid)
{
  gboolean from_cache = FALSE;
  char image_path[DT_PATH_MAX] = { 0 };
  dt_image_full_path(imgid,  image_path,  sizeof(image_path),  &from_cache, __FUNCTION__);
 
  return dt_image_get_audio_path_from_path(image_path);
}
 
static char *_text_path_legacy_if_exists(const char *image_path)
{
  size_t len = strlen(image_path);
  const char *c = image_path + len;
  while((c > image_path) && (*c != '.')) c--;
  len = c - image_path + 1;
 
  char *result = g_strndup(image_path, len + 3);
 
  result[len] = 't';
  result[len + 1] = 'x';
  result[len + 2] = 't';
  if(g_file_test(result, G_FILE_TEST_EXISTS)) return result;
 
  result[len] = 'T';
  result[len + 1] = 'X';
  result[len + 2] = 'T';
  if(g_file_test(result, G_FILE_TEST_EXISTS)) return result;
 
  dt_free(result);
  return NULL;
}
 
static char *_text_path_legacy_build(const char *image_path)
{
  size_t len = strlen(image_path);
  const char *c = image_path + len;
  while((c > image_path) && (*c != '.')) c--;
  len = c - image_path + 1;
 
  char *result = g_strndup(image_path, len + 3);
  result[len] = 't';
  result[len + 1] = 'x';
  result[len + 2] = 't';
  return result;
}
 
char *dt_image_get_text_path_from_path(const char *image_path)
{
  if(IS_NULL_PTR(image_path)) return NULL;
 
  return _text_path_legacy_if_exists(image_path);
}
 
char *dt_image_build_text_path_from_path(const char *image_path)
{
  if(IS_NULL_PTR(image_path)) return NULL;
 
  return _text_path_legacy_build(image_path);
}
 
static void _copy_text_sidecar_if_present(const char *src_image_path, const char *dest_image_path)
{
  if(IS_NULL_PTR(src_image_path) || IS_NULL_PTR(dest_image_path)) return;
 
  char *src_txt = dt_image_get_text_path_from_path(src_image_path);
  if(IS_NULL_PTR(src_txt)) return;
 
  char *dest_txt = _text_path_legacy_build(dest_image_path);
  if(dest_txt)
  {
    GFile *src = g_file_new_for_path(src_txt);
    GFile *dest = g_file_new_for_path(dest_txt);
    GError *gerror = NULL;
    gboolean copyStatus = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
 
    if(!copyStatus && g_error_matches(gerror, G_IO_ERROR, G_IO_ERROR_EXISTS))
      copyStatus = TRUE;
 
    if(!copyStatus && gerror)
      fprintf(stderr, "[dt_image] failed to copy text sidecar `%s' -> `%s': %s\n",
              src_txt, dest_txt, gerror->message);
 
    g_clear_error(&gerror);
    g_object_unref(dest);
    g_object_unref(src);
  }
 
  dt_free(dest_txt);
  dt_free(src_txt);
}
 
static void _move_text_sidecar_if_present(const char *src_image_path, const char *dest_image_path, const gboolean overwrite)
{
  if(IS_NULL_PTR(src_image_path) || IS_NULL_PTR(dest_image_path)) return;
 
  char *src_txt = dt_image_get_text_path_from_path(src_image_path);
  if(IS_NULL_PTR(src_txt)) return;
 
  char *dest_txt = _text_path_legacy_build(dest_image_path);
  if(dest_txt)
  {
    GFile *src = g_file_new_for_path(src_txt);
    GFile *dest = g_file_new_for_path(dest_txt);
    GError *gerror = NULL;
    const GFileCopyFlags flags = overwrite ? G_FILE_COPY_OVERWRITE : G_FILE_COPY_NONE;
    gboolean moveStatus = g_file_move(src, dest, flags, NULL, NULL, NULL, &gerror);
 
    if(!moveStatus && gerror)
      fprintf(stderr, "[dt_image] failed to move text sidecar `%s' -> `%s': %s\n",
              src_txt, dest_txt, gerror->message);
 
    g_clear_error(&gerror);
    g_object_unref(dest);
    g_object_unref(src);
  }
 
  dt_free(dest_txt);
  dt_free(src_txt);
}
 
char *dt_image_get_text_path(const int32_t imgid)
{
  char image_path[DT_PATH_MAX] = { 0 };
 
  gboolean from_cache = FALSE;
  dt_image_full_path(imgid,  image_path,  sizeof(image_path),  &from_cache, __FUNCTION__);
 
  if(image_path[0] != '\0' && g_file_test(image_path, G_FILE_TEST_EXISTS))
    return dt_image_get_text_path_from_path(image_path);
 
  from_cache = TRUE;
  dt_image_full_path(imgid,  image_path,  sizeof(image_path),  &from_cache, __FUNCTION__);
 
  if(from_cache && image_path[0] != '\0' && g_file_test(image_path, G_FILE_TEST_EXISTS))
    return dt_image_get_text_path_from_path(image_path);
 
  return NULL;
}
 
float dt_image_get_exposure_bias(const struct dt_image_t *image_storage)
{
  // just check that pointers exist and are initialized
  if((image_storage) && (image_storage->exif_exposure_bias))
  {
    // sanity checks because I don't trust exif tags too much
    if(isnan(image_storage->exif_exposure_bias)
       || CLAMP(image_storage->exif_exposure_bias, -5.0f, 5.0f) != image_storage->exif_exposure_bias)
      return 0.0f; // isnan
    else
      return CLAMP(image_storage->exif_exposure_bias, -5.0f, 5.0f);
  }
  else
    return 0.0f;
}
 
char *dt_image_camera_missing_sample_message(const struct dt_image_t *img, gboolean logmsg)
{
  const char *T1 = _("<b>WARNING</b>: camera is missing samples!");
  const char *T2 = _("You must provide samples in <a href='https://raw.pixls.us/'>https://raw.pixls.us/</a>");
  char *T3 = g_strdup_printf(_("for `%s' `%s'\n"
                               "in as many format/compression/bit depths as possible"),
                             img->camera_maker, img->camera_model);
  const char *T4 = _("or the <b>RAW won't be readable</b> in next version.");
 
  char *NL     = logmsg ? "\n\n" : "\n";
  char *PREFIX = logmsg ? "<big>" : "";
  char *SUFFIX = logmsg ? "</big>" : "";
 
  char *msg = g_strconcat(PREFIX, T1, NL, T2, NL, T3, NL, T4, SUFFIX, NULL);
 
  if(logmsg)
  {
    char *newmsg = dt_util_str_replace(msg, "<b>", "<span foreground='red'><b>");
    dt_free(msg);
    msg = dt_util_str_replace(newmsg, "</b>", "</b></span>");
    dt_free(newmsg);
  }
 
  dt_free(T3);
  return msg;
}
 
void dt_image_check_camera_missing_sample(const struct dt_image_t *img)
{
  if(img->camera_missing_sample)
  {
    char *msg = dt_image_camera_missing_sample_message(img, TRUE);
    dt_control_log(msg, (char *)NULL);
    dt_free(msg);
  }
}
 
void dt_get_dirname_from_imgid(gchar *dir, const int32_t imgid)
{
  gchar path[DT_PATH_MAX] = { 0 };
  gboolean from_cache = FALSE;
  dt_image_full_path(imgid, path, sizeof(path), &from_cache, __FUNCTION__);
  gchar *folder = g_path_get_dirname(path);
  g_strlcpy(dir, folder, DT_PATH_MAX); // callers all provide a DT_PATH_MAX-sized dir buffer
  dt_free(folder);
}
 
// clang-format off
// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
// vim: shiftwidth=2 expandtab tabstop=2 cindent
// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
// clang-format on