Ansel 0.0
A darktable fork - bloat + design vision
Loading...
Searching...
No Matches
database.c
Go to the documentation of this file.
1/*
2 This file is part of darktable,
3 Copyright (C) 2011-2012 Edouard Gomez.
4 Copyright (C) 2011 Henrik Andersson.
5 Copyright (C) 2011-2012 johannes hanika.
6 Copyright (C) 2011 Moritz Lipp.
7 Copyright (C) 2012, 2017 Christian Tellefsen.
8 Copyright (C) 2012 José Carlos García Sogo.
9 Copyright (C) 2012 Jérémy Rosen.
10 Copyright (C) 2012 Richard Wonka.
11 Copyright (C) 2012 Simon Spannagel.
12 Copyright (C) 2012-2020 Tobias Ellinghaus.
13 Copyright (C) 2013-2014, 2018-2022 Pascal Obry.
14 Copyright (C) 2014-2016 Roman Lebedev.
15 Copyright (C) 2015 Pedro Côrte-Real.
16 Copyright (C) 2016 piterdias.
17 Copyright (C) 2017, 2019, 2021 luzpaz.
18 Copyright (C) 2017-2018 Rikard Öxler.
19 Copyright (C) 2018-2019 Edgardo Hoszowski.
20 Copyright (C) 2018 Mario Lueder.
21 Copyright (C) 2018 Peter Budai.
22 Copyright (C) 2019 jakubfi.
23 Copyright (C) 2019-2022 Philippe Weyland.
24 Copyright (C) 2020 Heiko Bauke.
25 Copyright (C) 2020-2021 Hubert Kowalski.
26 Copyright (C) 2020 JP Verrue.
27 Copyright (C) 2020 jpverrue.
28 Copyright (C) 2021 Diederik Ter Rahe.
29 Copyright (C) 2021 Hanno Schwalm.
30 Copyright (C) 2021 HansBull.
31 Copyright (C) 2021 Harald.
32 Copyright (C) 2021 Ralf Brown.
33 Copyright (C) 2022 Aldric Renaudin.
34 Copyright (C) 2022-2023, 2025-2026 Aurélien PIERRE.
35 Copyright (C) 2022 Martin Bařinka.
36 Copyright (C) 2022 Miloš Komarčević.
37 Copyright (C) 2023 Maurizio Paglia.
38 Copyright (C) 2025 Alynx Zhou.
39 Copyright (C) 2025-2026 Guillaume Stutin.
40
41 darktable is free software: you can redistribute it and/or modify
42 it under the terms of the GNU General Public License as published by
43 the Free Software Foundation, either version 3 of the License, or
44 (at your option) any later version.
45
46 darktable is distributed in the hope that it will be useful,
47 but WITHOUT ANY WARRANTY; without even the implied warranty of
48 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
49 GNU General Public License for more details.
50
51 You should have received a copy of the GNU General Public License
52 along with darktable. If not, see <http://www.gnu.org/licenses/>.
53*/
54
55#ifdef HAVE_CONFIG_H
56#include "config.h"
57#endif
58
59#include "system/atomic.h"
60#include "common/paths.h" // DT_PATH_MAX
61#include "system/dtpthread.h"
62#include "common/image.h"
63#include "database/database.h"
64#include "database/sql_debug.h"
65#include "common/datetime.h"
66#include "common/file_location.h"
67#include "develop/iop_order.h"
68#ifdef HAVE_ICU
69/* sqlite3IcuInit(), called by _load_icu_collation() below. This block was empty: the
70 * declaration had been lost, and since HAVE_ICU is off in every local and CI build the
71 * only configuration that would have caught it is one nobody runs. */
72#include "database/sqliteicu.h" // conditional-ok: sqlite3IcuInit() is only called inside this same #ifdef, and sqliteicu.c is only compiled with HAVE_ICU
73#endif
74#include "database/legacy_presets.h"
75
76#include <gio/gio.h>
77#include <glib.h>
78#include <glib/gstdio.h>
79
80#include <errno.h>
81#include <fcntl.h>
82#include <signal.h>
83#include <sys/stat.h>
84#include <sys/types.h>
85#include "common/utility.h"
86
87// whenever _create_*_schema() gets changed you HAVE to bump this version and add an update path to
88// _upgrade_*_schema_step()!
89#define CURRENT_DATABASE_VERSION_LIBRARY 36
90#define CURRENT_DATABASE_VERSION_DATA 9
91
92// #define USE_NESTED_TRANSACTIONS
93#define MAX_NESTED_TRANSACTIONS 0
94/* transaction id */
95static dt_atomic_int _trxid;
96static dt_atomic_int _trx_batch_level;
97static gpointer _trx_owner = NULL;
98static gpointer _trx_batch_owner = NULL;
99
100typedef struct dt_database_t
101{
102 gboolean lock_acquired;
103
104 /* data database filename */
105 gchar *dbfilename_data, *lockfile_data;
106
107 /* library database filename */
108 gchar *dbfilename_library, *lockfile_library;
109
110 /* ondisk DB */
111 sqlite3 *handle;
112
113 gchar *error_message, *error_dbfilename;
114 int error_other_pid;
115} dt_database_t;
116
117/* ---------------------------------------------------------------------------------------
118 * Module state. All of it private: the connection, the policy, and the session constants
119 * the orchestrator told us at open time.
120 * ------------------------------------------------------------------------------------- */
121
123static dt_database_t *_db = NULL;
124
141static dt_pthread_rwlock_t _db_lock;
142static gboolean _db_lock_inited = FALSE;
143
147static dt_database_settings_t _settings = { NULL, 0, NULL, 0 };
148static dt_pthread_mutex_t _settings_lock;
149static gboolean _settings_lock_inited = FALSE;
150
152static gboolean _has_gui = FALSE;
153static gboolean _verbose = FALSE;
154
156static dt_database_renamed_handler_t _renamed_handler = NULL;
157
161#define _db_print(...) \
162 do { \
163 if(_verbose) dt_print(DT_DEBUG_SQL, __VA_ARGS__); \
164 } while(0)
165
166
167/* migrates database from old place to new */
168static gchar *_database_migrate_to_xdg_structure(const char *configured_db);
169
170/* delete old mipmaps files */
171static void _database_delete_mipmaps_files();
172
173/* tear down one connection; used both by dt_database_close() and by the failure paths of
174 * _database_init(), which have a database to free but nothing published yet */
175static void _database_free(dt_database_t *db);
176
177#define _SQLITE3_EXEC(a, b, c, d, e) \
178 if(sqlite3_exec(a, b, c, d, e) != SQLITE_OK) \
179 { \
180 all_ok = FALSE; \
181 failing_query = b; \
182 goto end; \
183 }
184
185/* migrate from the legacy db format (with the 'settings' blob) to the first version this system knows */
186static gboolean _migrate_schema(dt_database_t *db, int version)
187{
188 gboolean all_ok = TRUE;
189 const char *failing_query = NULL;
190 sqlite3_stmt *stmt;
191 sqlite3_stmt *innerstmt;
192
193 if(version != 36) // if anyone shows up with an older db we can probably add extra code
194 return FALSE;
195
196 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
197 // clang-format off
198 // remove stuff that is either no longer needed or that got renamed
199 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.lock", NULL, NULL, NULL);
200 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.settings", NULL, NULL, NULL); // yes, we do this in many
201 // places. because it's really
202 // important to not miss it in
203 // any code path.
204 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.group_id_index", NULL, NULL, NULL);
205 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.imgid_index", NULL, NULL, NULL);
206 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.mipmaps", NULL, NULL, NULL);
207 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.mipmap_timestamps", NULL, NULL, NULL);
208 _SQLITE3_EXEC(db->handle, "DROP TABLE IF EXISTS main.dt_migration_table", NULL, NULL, NULL);
209
210 // using _create_library_schema() and filling that with the old data doesn't work since we always want to generate
211 // version 1 tables
213 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)",
214 NULL, NULL, NULL);
215 _SQLITE3_EXEC(db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', 1)",
216 NULL, NULL, NULL);
218 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.film_rolls_folder_index ON film_rolls (folder)",
219 NULL, NULL, NULL);
221 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN orientation INTEGER", NULL, NULL, NULL);
222 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN focus_distance REAL", NULL, NULL, NULL);
223 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN group_id INTEGER", NULL, NULL, NULL);
224 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN histogram BLOB", NULL, NULL, NULL);
225 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN lightmap BLOB", NULL, NULL, NULL);
226 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN longitude REAL", NULL, NULL, NULL);
227 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN latitude REAL", NULL, NULL, NULL);
228 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN color_matrix BLOB", NULL, NULL, NULL);
229 // the colorspace as specified in some image types
230 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN colorspace INTEGER", NULL, NULL, NULL);
231 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN version INTEGER", NULL, NULL, NULL);
232 sqlite3_exec(db->handle, "ALTER TABLE main.images ADD COLUMN max_version INTEGER", NULL, NULL, NULL);
233 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET orientation = -1 WHERE orientation IS NULL", NULL, NULL, NULL);
234 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET focus_distance = -1 WHERE focus_distance IS NULL",
235 NULL, NULL, NULL);
236 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET group_id = id WHERE group_id IS NULL", NULL, NULL, NULL);
237 _SQLITE3_EXEC(db->handle, "UPDATE main.images SET max_version = (SELECT COUNT(*)-1 FROM main.images i WHERE "
238 "i.filename = main.images.filename AND "
239 "i.film_id = main.images.film_id) WHERE max_version IS NULL",
240 NULL, NULL, NULL);
241 _SQLITE3_EXEC(
242 db->handle,
243 "UPDATE main.images SET version = (SELECT COUNT(*) FROM main.images i "
244 "WHERE i.filename = main.images.filename AND "
245 "i.film_id = main.images.film_id AND i.id < main.images.id) WHERE version IS NULL",
246 NULL, NULL, NULL);
247 // make sure we have AUTOINCREMENT on imgid --> move the whole thing away and recreate the table :(
248 _SQLITE3_EXEC(db->handle, "ALTER TABLE main.images RENAME TO dt_migration_table", NULL, NULL, NULL);
249 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.images_group_id_index", NULL, NULL, NULL);
250 _SQLITE3_EXEC(db->handle, "DROP INDEX IF EXISTS main.images_film_id_index", NULL, NULL, NULL);
251 _SQLITE3_EXEC(
252 db->handle,
253 "CREATE TABLE main.images (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
254 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
255 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
256 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
257 "output_width INTEGER, output_height INTEGER, crop REAL, "
258 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
259 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
260 "caption VARCHAR, description VARCHAR, license VARCHAR, sha1sum CHAR(40), "
261 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
262 "latitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, max_version INTEGER)",
263 NULL, NULL, NULL);
264 _SQLITE3_EXEC(db->handle, "CREATE INDEX main.images_group_id_index ON images (group_id)", NULL, NULL, NULL);
265 _SQLITE3_EXEC(db->handle, "CREATE INDEX main.images_film_id_index ON images (film_id)", NULL, NULL, NULL);
266 _SQLITE3_EXEC(
267 db->handle,
268 "INSERT INTO main.images (id, group_id, film_id, width, height, filename, maker, model, "
269 "lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags, "
270 "output_width, output_height, crop, raw_parameters, raw_denoise_threshold, "
271 "raw_auto_bright_threshold, raw_black, raw_maximum, caption, description, license, sha1sum, "
272 "orientation, histogram, lightmap, longitude, latitude, color_matrix, colorspace, version, "
273 "max_version) "
274 "SELECT id, group_id, film_id, width, height, filename, maker, model, lens, exposure, aperture, iso, "
275 "focal_length, focus_distance, datetime_taken, flags, output_width, output_height, crop, "
276 "raw_parameters, raw_denoise_threshold, raw_auto_bright_threshold, raw_black, raw_maximum, "
277 "caption, description, license, sha1sum, orientation, histogram, lightmap, longitude, "
278 "latitude, color_matrix, colorspace, version, max_version FROM dt_migration_table",
279 NULL, NULL, NULL);
280 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
282 // selected_images should have a primary key. add it if it's missing:
283 _SQLITE3_EXEC(db->handle, "CREATE TEMPORARY TABLE dt_migration_table (imgid INTEGER)", NULL, NULL, NULL);
284 _SQLITE3_EXEC(db->handle, "INSERT INTO dt_migration_table SELECT imgid FROM main.selected_images", NULL, NULL,
285 NULL);
286 _SQLITE3_EXEC(db->handle, "DROP TABLE main.selected_images", NULL, NULL, NULL);
287 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.selected_images (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
288 _SQLITE3_EXEC(db->handle, "INSERT OR IGNORE INTO main.selected_images SELECT imgid FROM dt_migration_table",
289 NULL, NULL, NULL);
290 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
292 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
293 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
294 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
295 sqlite3_exec(db->handle, "ALTER TABLE main.history ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
296 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.history_imgid_index ON history (imgid)", NULL, NULL,
297 NULL);
298 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
299 NULL, NULL);
300 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET multi_priority = 0 WHERE multi_priority IS NULL", NULL, NULL,
301 NULL);
302 _SQLITE3_EXEC(db->handle, "UPDATE main.history SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL, NULL);
304 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.mask (imgid INTEGER, formid INTEGER, form INTEGER, "
305 "name VARCHAR(256), version INTEGER, "
306 "points BLOB, points_count INTEGER, source BLOB)",
307 NULL, NULL, NULL);
308 sqlite3_exec(db->handle, "ALTER TABLE main.mask ADD COLUMN source BLOB", NULL, NULL,
309 NULL); // in case the table was there already but missed that column
311 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.tagged_images_tagid_index ON tagged_images (tagid)",
312 NULL, NULL, NULL);
314 _SQLITE3_EXEC(db->handle,
315 "CREATE TABLE IF NOT EXISTS main.styles (id INTEGER, name VARCHAR, description VARCHAR)", NULL,
316 NULL, NULL);
317 sqlite3_exec(db->handle, "ALTER TABLE main.styles ADD COLUMN id INTEGER", NULL, NULL, NULL);
318 _SQLITE3_EXEC(db->handle, "UPDATE main.styles SET id = rowid WHERE id IS NULL", NULL, NULL, NULL);
320 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.style_items (styleid INTEGER, num INTEGER, module "
321 "INTEGER, operation VARCHAR(256), op_params BLOB, "
322 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, multi_priority "
323 "INTEGER, multi_name VARCHAR(256))",
324 NULL, NULL, NULL);
325 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
326 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
327 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
328 sqlite3_exec(db->handle, "ALTER TABLE main.style_items ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
329 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
330 NULL, NULL);
331 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET multi_priority = 0 WHERE multi_priority IS NULL", NULL,
332 NULL, NULL);
333 _SQLITE3_EXEC(db->handle, "UPDATE main.style_items SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL,
334 NULL);
336 // color_labels could have a PRIMARY KEY that we don't want
337 _SQLITE3_EXEC(db->handle, "CREATE TEMPORARY TABLE dt_migration_table (imgid INTEGER, color INTEGER)", NULL,
338 NULL, NULL);
339 _SQLITE3_EXEC(db->handle, "INSERT INTO dt_migration_table SELECT imgid, color FROM main.color_labels", NULL,
340 NULL, NULL);
341 _SQLITE3_EXEC(db->handle, "DROP TABLE main.color_labels", NULL, NULL, NULL);
342 _SQLITE3_EXEC(db->handle, "CREATE TABLE main.color_labels (imgid INTEGER, color INTEGER)", NULL, NULL, NULL);
343 _SQLITE3_EXEC(db->handle, "CREATE UNIQUE INDEX main.color_labels_idx ON color_labels (imgid, color)", NULL, NULL,
344 NULL);
345 _SQLITE3_EXEC(db->handle, "INSERT OR IGNORE INTO main.color_labels SELECT imgid, color FROM dt_migration_table",
346 NULL, NULL, NULL);
347 _SQLITE3_EXEC(db->handle, "DROP TABLE dt_migration_table", NULL, NULL, NULL);
349 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.meta_data (id INTEGER, key INTEGER, value VARCHAR)",
350 NULL, NULL, NULL);
351 _SQLITE3_EXEC(db->handle, "CREATE INDEX IF NOT EXISTS main.metadata_index ON meta_data (id, key)", NULL, NULL,
352 NULL);
354 _SQLITE3_EXEC(db->handle, "CREATE TABLE IF NOT EXISTS main.presets (name VARCHAR, description VARCHAR, "
355 "operation VARCHAR, op_version INTEGER, op_params BLOB, "
356 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, multi_priority "
357 "INTEGER, multi_name VARCHAR(256), "
358 "model VARCHAR, maker VARCHAR, lens VARCHAR, iso_min REAL, iso_max REAL, "
359 "exposure_min REAL, exposure_max REAL, "
360 "aperture_min REAL, aperture_max REAL, focal_length_min REAL, focal_length_max "
361 "REAL, writeprotect INTEGER, "
362 "autoapply INTEGER, filter INTEGER, def INTEGER, isldr INTEGER)",
363 NULL, NULL, NULL);
364 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN op_version INTEGER", NULL, NULL, NULL);
365 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN blendop_params BLOB", NULL, NULL, NULL);
366 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN blendop_version INTEGER", NULL, NULL, NULL);
367 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN multi_priority INTEGER", NULL, NULL, NULL);
368 sqlite3_exec(db->handle, "ALTER TABLE main.presets ADD COLUMN multi_name VARCHAR(256)", NULL, NULL, NULL);
369 // the unique index only works if the db doesn't have any (name, operation, op_version) more than once.
370 // apparently there are dbs out there which do have that. :(
371 sqlite3_prepare_v2(db->handle,
372 "SELECT p.rowid, p.name, p.operation, p.op_version FROM main.presets p INNER JOIN "
373 "(SELECT * FROM (SELECT rowid, name, operation, op_version, COUNT(*) AS count "
374 "FROM main.presets GROUP BY name, operation, op_version) WHERE count > 1) s "
375 "ON p.name = s.name AND p.operation = s.operation AND p.op_version = s.op_version",
376 -1, &stmt, NULL);
377 // clang-format on
378 char *last_name = NULL, *last_operation = NULL;
379 int last_op_version = 0;
380 int i = 0;
381 while(sqlite3_step(stmt) == SQLITE_ROW)
382 {
383 const int rowid = sqlite3_column_int(stmt, 0);
384 const char *name = (const char *)sqlite3_column_text(stmt, 1);
385 const char *operation = (const char *)sqlite3_column_text(stmt, 2);
386 const int op_version = sqlite3_column_int(stmt, 3);
387
388 // is it still the same (name, operation, op_version) triple?
389 if(IS_NULL_PTR(last_name) || strcmp(last_name, name) || IS_NULL_PTR(last_operation) || strcmp(last_operation, operation)
390 || last_op_version != op_version)
391 {
392 dt_free(last_name);
393 dt_free(last_operation);
394 last_name = g_strdup(name);
395 last_operation = g_strdup(operation);
396 last_op_version = op_version;
397 i = 0;
398 }
399
400 // find the next free amended version of name
401 // clang-format off
402 sqlite3_prepare_v2(db->handle, "SELECT name FROM main.presets WHERE name = ?1 || ' (' || ?2 || ')' AND "
403 "operation = ?3 AND op_version = ?4",
404 -1, &innerstmt, NULL);
405 // clang-format on
406 while(1)
407 {
408 sqlite3_bind_text(innerstmt, 1, name, -1, SQLITE_TRANSIENT);
409 sqlite3_bind_int(innerstmt, 2, i);
410 sqlite3_bind_text(innerstmt, 3, operation, -1, SQLITE_TRANSIENT);
411 sqlite3_bind_int(innerstmt, 4, op_version);
412 if(sqlite3_step(innerstmt) != SQLITE_ROW) break;
413 sqlite3_reset(innerstmt);
414 sqlite3_clear_bindings(innerstmt);
415 i++;
416 }
417 sqlite3_finalize(innerstmt);
418
419 // rename preset
420 // clang-format off
421 const char *query = "UPDATE main.presets SET name = name || ' (' || ?1 || ')' WHERE rowid = ?2";
422 // clang-format on
423 sqlite3_prepare_v2(db->handle, query, -1, &innerstmt, NULL);
424 sqlite3_bind_int(innerstmt, 1, i);
425 sqlite3_bind_int(innerstmt, 2, rowid);
426 if(sqlite3_step(innerstmt) != SQLITE_DONE)
427 {
428 all_ok = FALSE;
429 failing_query = query;
430 goto end;
431 }
432 sqlite3_finalize(innerstmt);
433 }
434 sqlite3_finalize(stmt);
435 dt_free(last_name);
436 dt_free(last_operation);
437 // now we should be able to create the index
438 // clang-format off
439 _SQLITE3_EXEC(db->handle,
440 "CREATE UNIQUE INDEX IF NOT EXISTS main.presets_idx ON presets (name, operation, op_version)",
441 NULL, NULL, NULL);
442 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET blendop_version = 1 WHERE blendop_version IS NULL", NULL,
443 NULL, NULL);
444 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET multi_priority = 0 WHERE multi_priority IS NULL", NULL, NULL,
445 NULL);
446 _SQLITE3_EXEC(db->handle, "UPDATE main.presets SET multi_name = ' ' WHERE multi_name IS NULL", NULL, NULL, NULL);
447 // clang-format on
448
449
450 // There are systems where absolute paths don't start with '/' (like Windows).
451 // Since the bug which introduced absolute paths to the db was fixed before a
452 // Windows build was available this shouldn't matter though.
453 // clang-format off
454 sqlite3_prepare_v2(db->handle, "SELECT id, filename FROM main.images WHERE filename LIKE '/%'", -1, &stmt, NULL);
455 sqlite3_prepare_v2(db->handle, "UPDATE main.images SET filename = ?1 WHERE id = ?2", -1, &innerstmt, NULL);
456 // clang-format on
457 while(sqlite3_step(stmt) == SQLITE_ROW)
458 {
459 const int id = sqlite3_column_int(stmt, 0);
460 const char *path = (const char *)sqlite3_column_text(stmt, 1);
461 gchar *filename = g_path_get_basename(path);
462 sqlite3_bind_text(innerstmt, 1, filename, -1, SQLITE_TRANSIENT);
463 sqlite3_bind_int(innerstmt, 2, id);
464 sqlite3_step(innerstmt);
465 sqlite3_reset(innerstmt);
466 sqlite3_clear_bindings(innerstmt);
467 dt_free(filename);
468 }
469 sqlite3_finalize(stmt);
470 sqlite3_finalize(innerstmt);
471
472 // We used to insert datetime_taken entries with '-' as date separators. Since that doesn't work well with
473 // the regular ':' when parsing
474 // or sorting we changed it to ':'. This takes care to change what we have as leftovers
475 // clang-format off
476 _SQLITE3_EXEC(
477 db->handle,
478 "UPDATE main.images SET datetime_taken = REPLACE(datetime_taken, '-', ':') WHERE datetime_taken LIKE '%-%'",
479 NULL, NULL, NULL);
480 // clang-format on
481
482end:
483 if(all_ok)
484 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
485 else
486 {
487 fprintf(stderr, "[init] failing query: `%s'\n", failing_query);
488 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle));
489 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL);
490 }
491
492 return all_ok;
493}
494
495#undef _SQLITE3_EXEC
496
497#define TRY_EXEC(_query, _message) \
498 do \
499 { \
500 if(sqlite3_exec(db->handle, _query, NULL, NULL, NULL) != SQLITE_OK) \
501 { \
502 fprintf(stderr, _message); \
503 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
504 FINALIZE; \
505 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
506 return version; \
507 } \
508 } while(0)
509
510#define TRY_STEP(_stmt, _expected, _message) \
511 do \
512 { \
513 if(sqlite3_step(_stmt) != _expected) \
514 { \
515 fprintf(stderr, _message); \
516 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
517 FINALIZE; \
518 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
519 return version; \
520 } \
521 } while(0)
522
523#define TRY_PREPARE(_stmt, _query, _message) \
524 do \
525 { \
526 if(sqlite3_prepare_v2(db->handle, _query, -1, &_stmt, NULL) != SQLITE_OK) \
527 { \
528 fprintf(stderr, _message); \
529 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
530 FINALIZE; \
531 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
532 return version; \
533 } \
534 } while(0)
535
536// redefine this where needed
537#define FINALIZE
538
539/* do the real migration steps, returns the version the db was converted to */
540static int _upgrade_library_schema_step(dt_database_t *db, int version)
541{
542 sqlite3_stmt *stmt;
543 int new_version = version;
544 if(version == CURRENT_DATABASE_VERSION_LIBRARY)
545 return version;
546 else if(version == 0)
547 {
548 // this can't happen, we started with 1, but it's a good example how this function works
549 // <do some magic to the db>
550 new_version = 1; // the version we transformed the db to. this way it might be possible to roll back or
551 // add fast paths
552 }
553 else if(version == 1)
554 {
555 // 1 -> 2 added write_timestamp
556 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
557 // clang-format off
558 TRY_EXEC("ALTER TABLE main.images ADD COLUMN write_timestamp INTEGER",
559 "[init] can't add `write_timestamp' column to database\n");
560 TRY_EXEC("UPDATE main.images SET write_timestamp = STRFTIME('%s', 'now') WHERE write_timestamp IS NULL",
561 "[init] can't initialize `write_timestamp' with current point in time\n");
562 // clang-format on
563 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
564 new_version = 2;
565 }
566 else if(version == 2)
567 {
568 // 2 -> 3 reset raw_black and raw_maximum. in theory we should change the columns from REAL to INTEGER,
569 // but sqlite doesn't care about types so whatever
570 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
571 // clang-format off
572 TRY_EXEC("UPDATE main.images SET raw_black = 0, raw_maximum = 16384",
573 "[init] can't reset raw_black and raw_maximum\n");
574 // clang-format on
575 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
576 new_version = 3;
577 }
578 else if(version == 3)
579 {
580 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
581 // clang-format off
582 TRY_EXEC("CREATE TRIGGER insert_tag AFTER INSERT ON main.tags"
583 " BEGIN"
584 " INSERT INTO tagxtag SELECT id, new.id, 0 FROM TAGS;"
585 " UPDATE tagxtag SET count = 1000000 WHERE id1=new.id AND id2=new.id;"
586 " END",
587 "[init] can't create insert_tag trigger\n");
588 TRY_EXEC("CREATE TRIGGER delete_tag BEFORE DELETE ON main.tags"
589 " BEGIN"
590 " DELETE FROM tagxtag WHERE id1=old.id OR id2=old.id;"
591 " DELETE FROM tagged_images WHERE tagid=old.id;"
592 " END",
593 "[init] can't create delete_tag trigger\n");
594 TRY_EXEC("CREATE TRIGGER attach_tag AFTER INSERT ON main.tagged_images"
595 " BEGIN"
596 " UPDATE tagxtag"
597 " SET count = count + 1"
598 " WHERE (id1=new.tagid AND id2 IN (SELECT tagid FROM tagged_images WHERE imgid=new.imgid))"
599 " OR (id2=new.tagid AND id1 IN (SELECT tagid FROM tagged_images WHERE imgid=new.imgid));"
600 " END",
601 "[init] can't create attach_tag trigger\n");
602 TRY_EXEC("CREATE TRIGGER detach_tag BEFORE DELETE ON main.tagged_images"
603 " BEGIN"
604 " UPDATE tagxtag"
605 " SET count = count - 1"
606 " WHERE (id1=old.tagid AND id2 IN (SELECT tagid FROM tagged_images WHERE imgid=old.imgid))"
607 " OR (id2=old.tagid AND id1 IN (SELECT tagid FROM tagged_images WHERE imgid=old.imgid));"
608 " END",
609 "[init] can't create detach_tag trigger\n");
610 // clang-format on
611 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
612 new_version = 4;
613 }
614 else if(version == 4)
615 {
616 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
617 // clang-format off
618 TRY_EXEC("ALTER TABLE main.presets RENAME TO tmp_presets", "[init] can't rename table presets\n");
619
620 TRY_EXEC("CREATE TABLE main.presets (name VARCHAR, description VARCHAR, operation VARCHAR, op_params BLOB,"
621 "enabled INTEGER, blendop_params BLOB, model VARCHAR, maker VARCHAR, lens VARCHAR,"
622 "iso_min REAL, iso_max REAL, exposure_min REAL, exposure_max REAL, aperture_min REAL,"
623 "aperture_max REAL, focal_length_min REAL, focal_length_max REAL, writeprotect INTEGER,"
624 "autoapply INTEGER, filter INTEGER, def INTEGER, format INTEGER, op_version INTEGER,"
625 "blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
626 "[init] can't create new presets table\n");
627
628 TRY_EXEC("INSERT INTO main.presets (name, description, operation, op_params, enabled, blendop_params, model, "
629 "maker, lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, aperture_max,"
630 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, def, format, op_version, "
631 "blendop_version, multi_priority, multi_name) SELECT name, description, operation, op_params, "
632 "enabled, blendop_params, model, maker, lens, iso_min, iso_max, exposure_min, exposure_max, "
633 "aperture_min, aperture_max, focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
634 "def, isldr, op_version, blendop_version, multi_priority, multi_name FROM tmp_presets",
635 "[init] can't populate presets table from tmp_presets\n");
636
637 TRY_EXEC("DROP TABLE tmp_presets", "[init] can't delete table tmp_presets\n");
638 // clang-format on
639 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
640 new_version = 5;
641 }
642 else if(version == 5)
643 {
644 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
645 // clang-format off
646 TRY_EXEC("CREATE INDEX main.images_filename_index ON images (filename)",
647 "[init] can't create index on image filename\n");
648 // clang-format on
649 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
650 new_version = 6;
651 }
652 else if(version == 6)
653 {
654 // some ancient tables can have the styleid column of style_items be called style_id. fix that.
655 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
656
657 if(sqlite3_exec(db->handle, "SELECT style_id FROM main.style_items", NULL, NULL, NULL) == SQLITE_OK)
658 {
659 // clang-format off
660 TRY_EXEC("ALTER TABLE main.style_items RENAME TO tmp_style_items",
661 "[init] can't rename table style_items\n");
662
663 TRY_EXEC("CREATE TABLE main.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
664 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
665 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
666 "[init] can't create new style_items table\n");
667
668 TRY_EXEC("INSERT INTO main.style_items (styleid, num, module, operation, op_params, enabled,"
669 " blendop_params, blendop_version, multi_priority, multi_name)"
670 " SELECT style_id, num, module, operation, op_params, enabled,"
671 " blendop_params, blendop_version, multi_priority, multi_name"
672 " FROM tmp_style_items",
673 "[init] can't populate style_items table from tmp_style_items\n");
674
675 TRY_EXEC("DROP TABLE tmp_style_items", "[init] can't delete table tmp_style_items\n");
676 // clang-format on
677 }
678
679 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
680 new_version = 7;
681 }
682 else if(version == 7)
683 {
684 // make sure that we have no film rolls with a NULL folder
685 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
686 // clang-format off
687 TRY_EXEC("ALTER TABLE main.film_rolls RENAME TO tmp_film_rolls", "[init] can't rename table film_rolls\n");
688
689 TRY_EXEC("CREATE TABLE main.film_rolls "
690 "(id INTEGER PRIMARY KEY, datetime_accessed CHAR(20), "
691 "folder VARCHAR(1024) NOT NULL)",
692 "[init] can't create new film_rolls table\n");
693
694 TRY_EXEC("INSERT INTO main.film_rolls (id, datetime_accessed, folder) "
695 "SELECT id, datetime_accessed, folder "
696 "FROM tmp_film_rolls "
697 "WHERE folder IS NOT NULL",
698 "[init] can't populate film_rolls table from tmp_film_rolls\n");
699
700 TRY_EXEC("DROP TABLE tmp_film_rolls", "[init] can't delete table tmp_film_rolls\n");
701 // clang-format on
702 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
703 new_version = 8;
704 }
705 else if(version == 8)
706 {
707 // 8 -> 9 added history_end column to images
708 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
709 // clang-format off
710 TRY_EXEC("ALTER TABLE main.images ADD COLUMN history_end INTEGER",
711 "[init] can't add `history_end' column to database\n");
712
713 TRY_EXEC("UPDATE main.images SET history_end = (SELECT IFNULL(MAX(num) + 1, 0) FROM main.history "
714 "WHERE imgid = id)", "[init] can't initialize `history_end' with last history entry\n");
715 // clang-format on
716 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
717 new_version = 9;
718 }
719 else if(version == 9)
720 {
721 // 9 -> 10 cleanup of last update :(
722 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
723 // clang-format off
724 TRY_EXEC("UPDATE main.images SET history_end = (SELECT IFNULL(MAX(num) + 1, 0) FROM main.history "
725 "WHERE imgid = id)", "[init] can't set `history_end' to 0 where it was NULL\n");
726 // clang-format on
727 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
728 new_version = 10;
729 }
730 else if(version == 10)
731 {
732 // 10 -> 11 added altitude column to images
733 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
734 // clang-format off
735 TRY_EXEC("ALTER TABLE main.images ADD COLUMN altitude REAL",
736 "[init] can't add `altitude' column to database\n");
737
738 TRY_EXEC("UPDATE main.images SET altitude = NULL", "[init] can't initialize `altitude' with NULL\n");
739 // clang-format on
740 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
741 new_version = 11;
742 }
743 else if(version == 11)
744 {
745 // 11 -> 12 tagxtag was removed in order to reduce database size
746 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
747 // clang-format off
748 TRY_EXEC("DROP TRIGGER main.detach_tag", "[init] can't drop trigger `detach_tag' from database\n");
749
750 TRY_EXEC("DROP TRIGGER main.attach_tag", "[init] can't drop trigger `attach_tag' from database\n");
751
752 TRY_EXEC("DROP TRIGGER main.delete_tag", "[init] can't drop trigger `delete_tag' from database\n");
753
754 TRY_EXEC("DROP TRIGGER main.insert_tag", "[init] can't drop trigger `insert_tag' from database\n");
755
756 TRY_EXEC("DROP TABLE main.tagxtag", "[init] can't drop table `tagxtag' from database\n");
757 // clang-format on
758 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
759 new_version = 12;
760 }
761 else if(version == 12)
762 {
763 // 11 -> 12 move presets, styles and tags over to the data database
764 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
765
767#undef FINALIZE
768#define FINALIZE \
769 do \
770 { \
771 sqlite3_finalize(stmt); \
772 sqlite3_finalize(select_stmt); \
773 sqlite3_finalize(count_clashes_stmt); \
774 sqlite3_finalize(update_name_stmt); \
775 sqlite3_finalize(insert_stmt); \
776 sqlite3_finalize(delete_stmt); \
777 } while(0)
778
779 stmt = NULL;
780 sqlite3_stmt *insert_stmt = NULL, *delete_stmt = NULL, *select_stmt = NULL, *count_clashes_stmt = NULL,
781 *update_name_stmt = NULL;
782 // clang-format off
783 // remove presets that are already in data.
784 // we can't use a NATURAL JOIN here as that fails when columns have NULL values. :-(
785 TRY_EXEC("DELETE FROM main.presets WHERE rowid IN (SELECT p1.rowid FROM main.presets p1 "
786 "JOIN data.presets p2 ON "
787 "p1.name IS p2.name AND "
788 "p1.description IS p2.description AND "
789 "p1.operation IS p2.operation AND "
790 "p1.op_version IS p2.op_version AND "
791 "p1.op_params IS p2.op_params AND "
792 "p1.enabled IS p2.enabled AND "
793 "p1.blendop_params IS p2.blendop_params AND "
794 "p1.blendop_version IS p2.blendop_version AND "
795 "p1.multi_priority IS p2.multi_priority AND "
796 "p1.multi_name IS p2.multi_name AND "
797 "p1.model IS p2.model AND "
798 "p1.maker IS p2.maker AND "
799 "p1.lens IS p2.lens AND "
800 "p1.iso_min IS p2.iso_min AND "
801 "p1.iso_max IS p2.iso_max AND "
802 "p1.exposure_min IS p2.exposure_min AND "
803 "p1.exposure_max IS p2.exposure_max AND "
804 "p1.aperture_min IS p2.aperture_min AND "
805 "p1.aperture_max IS p2.aperture_max AND "
806 "p1.focal_length_min IS p2.focal_length_min AND "
807 "p1.focal_length_max IS p2.focal_length_max AND "
808 "p1.writeprotect IS p2.writeprotect AND "
809 "p1.autoapply IS p2.autoapply AND "
810 "p1.filter IS p2.filter AND "
811 "p1.def IS p2.def AND "
812 "p1.format IS p2.format "
813 "WHERE p1.writeprotect = 0)",
814 "[init] can't delete already migrated presets from database\n");
815
816 // find all presets that are clashing with something else in presets. that can happen as we introduced an
817 // index on presets in data which wasn't in place in library.
818 TRY_PREPARE(select_stmt, "SELECT p.rowid, r FROM main.presets AS p, (SELECT rowid AS r, name, operation, "
819 "op_version FROM main.presets GROUP BY name, operation, op_version HAVING "
820 "COUNT(*) > 1) USING (name, operation, op_version) WHERE p.rowid != r",
821 "[init] can't prepare selecting presets with same name, operation, op_version from database\n");
822
823 // see if an updated preset name still causes problems
824 TRY_PREPARE(count_clashes_stmt, "SELECT COUNT(*) FROM main.presets AS p, (SELECT name, operation, op_version "
825 "FROM main.presets WHERE rowid = ?1) AS i ON p.name = i.name || \" #\" || ?2 "
826 "AND p.operation = i.operation AND p.op_version = i.op_version",
827 "[init] can't prepare selection of preset count by name from database\n");
828
829 // update the preset name for good
830 TRY_PREPARE(update_name_stmt, "UPDATE main.presets SET name = name || \" #\" || ?1 WHERE rowid = ?2",
831 "[init] can't prepare updating of preset name in database\n");
832
833 // find all presets that would be clashing with something in data
834 TRY_PREPARE(stmt, "SELECT p1.rowid FROM main.presets p1 INNER JOIN data.presets p2 "
835 "USING (name, operation, op_version) WHERE p1.writeprotect = 0",
836 "[init] can't access table `presets' in database\n");
837
838 // ... and move them over with a new name
839 TRY_PREPARE(insert_stmt, "INSERT OR FAIL INTO data.presets (name, description, operation, op_version, "
840 "op_params, enabled, blendop_params, blendop_version, multi_priority, multi_name, "
841 "model, maker, lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, "
842 "aperture_max, focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
843 "def, format) "
844 "SELECT name || \" #\" || ?1, description, operation, op_version, op_params, "
845 "enabled, blendop_params, blendop_version, multi_priority, multi_name, model, maker, "
846 "lens, iso_min, iso_max, exposure_min, exposure_max, aperture_min, aperture_max, "
847 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, def, format "
848 "FROM main.presets p1 WHERE p1.rowid = ?2",
849 "[init] can't prepare insertion statement\n");
850
851 TRY_PREPARE(delete_stmt, "DELETE FROM main.presets WHERE rowid = ?1", "[init] can't prepare deletion statement\n");
852 // clang-format on
853 // first rename presets with (name, operation, op_version) not being unique
854 while(sqlite3_step(select_stmt) == SQLITE_ROW)
855 {
856 const int own_rowid = sqlite3_column_int(select_stmt, 0);
857 const int other_rowid = sqlite3_column_int(select_stmt, 1);
858 int preset_version = 0;
859
860 do
861 {
862 preset_version++;
863 sqlite3_reset(count_clashes_stmt);
864 sqlite3_clear_bindings(count_clashes_stmt);
865 sqlite3_bind_int(count_clashes_stmt, 1, other_rowid);
866 sqlite3_bind_int(count_clashes_stmt, 2, preset_version);
867 }
868 while(sqlite3_step(count_clashes_stmt) == SQLITE_ROW && sqlite3_column_int(count_clashes_stmt, 0) > 0);
869
870 sqlite3_bind_int(update_name_stmt, 1, preset_version);
871 sqlite3_bind_int(update_name_stmt, 2, own_rowid);
872 TRY_STEP(update_name_stmt, SQLITE_DONE, "[init] can't rename preset in database\n");
873 sqlite3_reset(update_name_stmt);
874 sqlite3_reset(update_name_stmt);
875 }
876
877 // now rename to avoid clashes with data.presets
878 while(sqlite3_step(stmt) == SQLITE_ROW)
879 {
880 int preset_version = 0;
881 const int rowid = sqlite3_column_int(stmt, 0);
882
883 do
884 {
885 preset_version++;
886 sqlite3_reset(insert_stmt);
887 sqlite3_clear_bindings(insert_stmt);
888 sqlite3_bind_int(insert_stmt, 1, preset_version);
889 sqlite3_bind_int(insert_stmt, 2, rowid);
890 } while(sqlite3_step(insert_stmt) != SQLITE_DONE);
891
892 sqlite3_reset(delete_stmt);
893 sqlite3_clear_bindings(delete_stmt);
894 sqlite3_bind_int(delete_stmt, 1, rowid);
895 TRY_STEP(delete_stmt, SQLITE_DONE, "[init] can't delete preset from database\n");
896 }
897 // clang-format off
898 // all that is left in presets should be those that can be moved over without any further concerns
899 TRY_EXEC("INSERT OR FAIL INTO data.presets SELECT name, description, operation, "
900 "op_version, op_params, enabled, blendop_params, blendop_version, "
901 "multi_priority, multi_name, model, maker, lens, iso_min, iso_max, "
902 "exposure_min, exposure_max, aperture_min, aperture_max, "
903 "focal_length_min, focal_length_max, writeprotect, autoapply, filter, "
904 "def, format FROM main.presets WHERE writeprotect = 0",
905 "[init] can't copy presets to the data database\n");
906 // ... delete them on the old side
907 TRY_EXEC("DELETE FROM main.presets WHERE writeprotect = 0",
908 "[init] can't copy presets to the data database\n");
909 // clang-format on
910
911 FINALIZE;
912#undef FINALIZE
913
915#define FINALIZE \
916 do \
917 { \
918 sqlite3_finalize(stmt); \
919 sqlite3_finalize(insert_stmt); \
920 sqlite3_finalize(select_stmt); \
921 sqlite3_finalize(delete_stmt); \
922 sqlite3_finalize(update_name_stmt); \
923 sqlite3_finalize(select_new_stmt); \
924 sqlite3_finalize(copy_style_items_stmt); \
925 sqlite3_finalize(delete_style_items_stmt); \
926 } while(0)
927
928 stmt = NULL;
929 select_stmt = NULL;
930 update_name_stmt = NULL;
931 insert_stmt = NULL;
932 delete_stmt = NULL;
933 sqlite3_stmt *select_new_stmt = NULL, *copy_style_items_stmt = NULL, *delete_style_items_stmt = NULL;
934 // clang-format off
935 TRY_PREPARE(stmt, "SELECT id, name FROM main.styles", "[init] can't prepare style selection from database\n");
936 TRY_PREPARE(select_stmt, "SELECT rowid FROM data.styles WHERE name = ?1 LIMIT 1",
937 "[init] can't prepare style item selection from database\n");
938 TRY_PREPARE(update_name_stmt, "UPDATE main.styles SET name = ?1 WHERE id = ?2",
939 "[init] can't prepare style name update\n");
940 TRY_PREPARE(insert_stmt, "INSERT INTO data.styles (id, name, description) "
941 "SELECT (SELECT COALESCE(MAX(id),0)+1 FROM data.styles), name, description "
942 "FROM main.styles where id = ?1",
943 "[init] can't prepare style insertion for database\n");
944 TRY_PREPARE(delete_stmt, "DELETE FROM main.styles WHERE id = ?1",
945 "[init] can't prepare style deletion for database\n");
946 TRY_PREPARE(select_new_stmt, "SELECT id FROM data.styles WHERE rowid = ?1",
947 "[init] can't prepare style selection from data database\n");
948 TRY_PREPARE(copy_style_items_stmt, "INSERT INTO data.style_items "
949 "(styleid, num, module, operation, op_params, enabled, blendop_params, "
950 "blendop_version, multi_priority, multi_name) "
951 "SELECT ?1, num, module, operation, op_params, enabled, blendop_params, "
952 "blendop_version, multi_priority, multi_name FROM main.style_items "
953 "WHERE styleid = ?2",
954 "[init] can't prepare style item copy into data database\n");
955 TRY_PREPARE(delete_style_items_stmt, "DELETE FROM main.style_items WHERE styleid = ?1",
956 "[init] can't prepare style item deletion for database\n");
957 // clang-format on
958 while(sqlite3_step(stmt) == SQLITE_ROW)
959 {
960 const int id = sqlite3_column_int(stmt, 0);
961 const char *name = (const char *)sqlite3_column_text(stmt, 1);
962
963 // find a unique name of the style for data.styles
964 sqlite3_bind_text(select_stmt, 1, name, -1, SQLITE_TRANSIENT);
965 if(sqlite3_step(select_stmt) == SQLITE_ROW)
966 {
967 // we need to append a version
968 int style_version = 0;
969 char *new_name = NULL;
970 do
971 {
972 style_version++;
973 dt_free(new_name);
974 new_name = g_strdup_printf("%s #%d", name, style_version);
975 sqlite3_reset(select_stmt);
976 sqlite3_clear_bindings(select_stmt);
977 sqlite3_bind_text(select_stmt, 1, new_name, -1, SQLITE_TRANSIENT);
978 } while(sqlite3_step(select_stmt) == SQLITE_ROW);
979
980 // update the name in the old place
981 sqlite3_bind_text(update_name_stmt, 1, new_name, -1, SQLITE_TRANSIENT);
982 sqlite3_bind_int(update_name_stmt, 2, id);
983 TRY_STEP(update_name_stmt, SQLITE_DONE, "[init] can't update name of style in database\n");
984 sqlite3_reset(update_name_stmt);
985 sqlite3_clear_bindings(update_name_stmt);
986 dt_free(new_name);
987 }
988
989 // move the style to data.styles and get the rowid
990 sqlite3_bind_int(insert_stmt, 1, id);
991 TRY_STEP(insert_stmt, SQLITE_DONE, "[init] can't insert style into data database\n");
992 sqlite3_int64 last_rowid = sqlite3_last_insert_rowid(db->handle);
993
994 // delete style from styles
995 sqlite3_bind_int(delete_stmt, 1, id);
996 TRY_STEP(delete_stmt, SQLITE_DONE, "[init] can't delete style from database\n");
997
998 sqlite3_bind_int(select_new_stmt, 1, last_rowid);
999 TRY_STEP(select_new_stmt, SQLITE_ROW, "[init] can't select new style from data database\n");
1000 const int new_id = sqlite3_column_int(select_new_stmt, 0);
1001
1002 // now that we have the style over in data.styles and the new id we can just copy over all style items
1003 sqlite3_bind_int(copy_style_items_stmt, 1, new_id);
1004 sqlite3_bind_int(copy_style_items_stmt, 2, id);
1005 TRY_STEP(copy_style_items_stmt, SQLITE_DONE, "[init] can't copy style items into data database\n");
1006
1007 // delete the style items from the old table
1008 sqlite3_bind_int(delete_style_items_stmt, 1, id);
1009 TRY_STEP(delete_style_items_stmt, SQLITE_DONE, "[init] can't delete style items from database\n");
1010
1011 // cleanup for the next round
1012 sqlite3_reset(insert_stmt);
1013 sqlite3_clear_bindings(insert_stmt);
1014 sqlite3_reset(select_stmt);
1015 sqlite3_clear_bindings(select_stmt);
1016 sqlite3_reset(delete_stmt);
1017 sqlite3_clear_bindings(delete_stmt);
1018 sqlite3_reset(select_new_stmt);
1019 sqlite3_clear_bindings(select_new_stmt);
1020 sqlite3_reset(copy_style_items_stmt);
1021 sqlite3_clear_bindings(copy_style_items_stmt);
1022 sqlite3_reset(delete_style_items_stmt);
1023 sqlite3_clear_bindings(delete_style_items_stmt);
1024 }
1025 FINALIZE;
1026#undef FINALIZE
1027
1029#define FINALIZE
1030 // clang-format off
1031 // tags
1032 TRY_EXEC("INSERT OR IGNORE INTO data.tags (name, icon, description, flags) "
1033 "SELECT name, icon, description, flags FROM main.tags",
1034 "[init] can't prepare insertion of used tags into data database\n");
1035
1036 // tagged images
1037 // we need a temp table to update tagged_images due to its primary key
1038 TRY_EXEC("CREATE TEMPORARY TABLE tagged_images_tmp (imgid INTEGER, tagid INTEGER)",
1039 "[init] can't create temporary table for updating `tagged_images'\n");
1040
1041 TRY_EXEC("INSERT INTO tagged_images_tmp (imgid, tagid) "
1042 "SELECT imgid, (SELECT t2.id FROM main.tags t1, data.tags t2 USING (name) WHERE t1.id = tagid) "
1043 "FROM main.tagged_images", "[init] can't insert into `tagged_images_tmp'\n");
1044
1045 TRY_EXEC("DELETE FROM main.tagged_images", "[init] can't delete tagged images in database\n");
1046
1047 TRY_EXEC("INSERT OR IGNORE INTO main.tagged_images (imgid, tagid) SELECT imgid, tagid FROM tagged_images_tmp",
1048 "[init] can't copy updated values back to `tagged_images'\n");
1049
1050 TRY_EXEC("DROP TABLE tagged_images_tmp", "[init] can't drop table `tagged_images_tmp' from database\n");
1051
1053 TRY_EXEC("DROP INDEX IF EXISTS main.presets_idx", "[init] can't drop index `presets_idx' from database\n");
1054 TRY_EXEC("DROP TABLE main.presets", "[init] can't drop table `presets' from database\n");
1055 TRY_EXEC("DROP TABLE main.style_items", "[init] can't drop table `style_items' from database\n");
1056 TRY_EXEC("DROP TABLE main.styles", "[init] can't drop table `styles' from database\n");
1057 TRY_EXEC("DROP TABLE main.tags", "[init] can't drop table `tags' from database\n");
1058 // clang-format on
1059 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1060 new_version = 13;
1061 } else if(version == 13)
1062 {
1063 // 12 -> 13 bring back the used tag names to library.db so people can use it independently of data.db
1064 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1065 // clang-format off
1066 TRY_EXEC("CREATE TABLE main.used_tags (id INTEGER, name VARCHAR NOT NULL)",
1067 "[init] can't create `used_tags` table\n");
1068
1069 TRY_EXEC("CREATE INDEX main.used_tags_idx ON used_tags (id, name)",
1070 "[init] can't create index on table `used_tags' in database\n");
1071
1072 TRY_EXEC("INSERT INTO main.used_tags (id, name) SELECT t.id, t.name FROM data.tags AS t, main.tagged_images "
1073 "AS i ON t.id = i.tagid GROUP BY t.id",
1074 "[init] can't insert used tags into `used_tags` table in database\n");
1075 // clang-format on
1076 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1077 new_version = 14;
1078 }
1079 else if(version == 14)
1080 {
1081 // 13 -> fix the index on used_tags to be a UNIQUE index :-/
1082 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1083 // clang-format off
1084 TRY_EXEC("DELETE FROM main.used_tags WHERE rowid NOT IN (SELECT rowid FROM used_tags GROUP BY id)",
1085 "[init] can't delete duplicated entries from `used_tags' in database\n");
1086
1087 TRY_EXEC("DROP INDEX main.used_tags_idx", "[init] can't drop index `used_tags_idx' from database\n");
1088
1089 TRY_EXEC("CREATE UNIQUE INDEX main.used_tags_idx ON used_tags (id, name)",
1090 "[init] can't create index `used_tags_idx' in database\n");
1091
1092 TRY_EXEC("DELETE FROM main.tagged_images WHERE tagid IS NULL",
1093 "[init] can't delete NULL entries from `tagged_images' in database");
1094
1095 TRY_EXEC("DELETE FROM main.used_tags WHERE id NOT IN (SELECT DISTINCT tagid FROM main.tagged_images)",
1096 "[init] can't delete unused tags from `used_tags' in database\n");
1097 // clang-format on
1098 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1099 new_version = 15;
1100 }
1101 else if(version == 15)
1102 {
1103 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1105 // clang-format off
1106 TRY_EXEC("ALTER TABLE main.images ADD COLUMN position INTEGER",
1107 "[init] can't add `position' column to images table in database\n");
1108 TRY_EXEC("CREATE INDEX main.image_position_index ON images (position)",
1109 "[init] can't create index for custom image order table\n");
1110
1111 // Set the initial image sequence. The image id - the sequence images were imported -
1112 // defines the initial order of images.
1113 //
1114 // An int64 is used for the position index. The upper 31 bits define the initial order.
1115 // The lower 32bit provide space to reorder images.
1116 //
1117 // see: dt_collection_move_before()
1118 //
1119 TRY_EXEC("UPDATE main.images SET position = id << 32",
1120 "[init] can't update positions custom image order table\n");
1121 // clang-format on
1122 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1123 new_version = 16;
1124 }
1125 else if(version == 16)
1126 {
1127 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1129 // clang-format off
1130 TRY_EXEC("ALTER TABLE main.images ADD COLUMN aspect_ratio REAL",
1131 "[init] can't add `aspect_ratio' column to images table in database\n");
1132 TRY_EXEC("UPDATE main.images SET aspect_ratio = 0.0",
1133 "[init] can't update aspect_ratio in database\n");
1134 // clang-format on
1135 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1136 new_version = 17;
1137 }
1138 else if(version == 17)
1139 {
1140 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1141
1143 // clang-format off
1144 TRY_EXEC("CREATE TABLE main.masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, name VARCHAR(256), "
1145 "version INTEGER, points BLOB, points_count INTEGER, source BLOB)",
1146 "[init] can't create `masks_history` table\n");
1147
1148 TRY_EXEC("CREATE INDEX main.masks_history_imgid_index ON masks_history (imgid)",
1149 "[init] can't create index `masks_history_imgid_index' in database\n");
1150
1151 // to speed up the mask look-up, and makes the following UPDATE instantaneous whereas it could takes hours
1152 TRY_EXEC("CREATE INDEX main.mask_imgid_index ON mask (imgid);",
1153 "[init] can't create index `mask_imgid_index' in database\n");
1154
1155 // create a mask manager entry on history for all images containing all forms
1156 // make room for mask manager history entry
1157 TRY_EXEC("UPDATE main.history SET num=num+1 WHERE imgid IN (SELECT imgid FROM main.mask WHERE main.mask.imgid=main.history.imgid)",
1158 "[init] can't update `num' with num+1\n");
1159
1160 // update history end
1161 TRY_EXEC("UPDATE main.images SET history_end = history_end+1 WHERE id IN (SELECT imgid FROM main.mask WHERE main.mask.imgid=main.images.id)",
1162 "[init] can't update `history_end' with history_end+1\n");
1163
1164 // copy all masks into history
1165 TRY_EXEC("INSERT INTO main.masks_history (imgid, num, formid, form, name, version, points, points_count, source) SELECT "
1166 "imgid, 0, formid, form, name, version, points, points_count, source FROM main.mask",
1167 "[init] can't insert into masks_history\n");
1168
1169 // create a mask manager entry for each image that has masks
1170 TRY_EXEC("INSERT INTO main.history (imgid, num, operation, op_params, module, enabled, "
1171 "blendop_params, blendop_version, multi_priority, multi_name) "
1172 "SELECT DISTINCT imgid, 0, 'mask_manager', NULL, 1, 0, NULL, 0, 0, '' FROM main.mask "
1173 "GROUP BY imgid",
1174 "[init] can't insert mask manager into history\n");
1175
1176 TRY_EXEC("DROP TABLE main.mask", "[init] can't drop table `mask' from database\n");
1177
1179 GList *prior_v1 = dt_ioppr_get_iop_order_list_version(DT_IOP_ORDER_LEGACY);
1180
1181 TRY_EXEC("ALTER TABLE main.images ADD COLUMN iop_order_version INTEGER",
1182 "[init] can't add `iop_order_version' column to images table in database\n");
1183
1184 TRY_EXEC("UPDATE main.images SET iop_order_version = 0",
1185 "[init] can't update iop_order_version in database\n");
1186
1187 TRY_EXEC("UPDATE main.images SET iop_order_version = 1 WHERE "
1188 "EXISTS(SELECT * FROM main.history WHERE main.history.imgid = main.images.id)",
1189 "[init] can't update iop_order_version in database\n");
1190
1191 TRY_EXEC("ALTER TABLE main.history ADD COLUMN iop_order REAL",
1192 "[init] can't add `iop_order' column to history table in database\n");
1193
1194 // create a temp table with the previous priorities
1195 TRY_EXEC("CREATE TEMPORARY TABLE iop_order_tmp (iop_order REAL, operation VARCHAR(256))",
1196 "[init] can't create temporary table for updating `main.history'\n");
1197 // clang-format on
1198 // fill temp table with all operations up to this release
1199 // it will be used to create the pipe and update the iop_order on history
1200 for(GList *priorities = prior_v1; priorities; priorities = g_list_next(priorities))
1201 {
1202 dt_iop_order_entry_t *prior = (dt_iop_order_entry_t *)priorities->data;
1203
1204 sqlite3_prepare_v2(
1205 db->handle,
1206 "INSERT INTO iop_order_tmp (iop_order, operation) VALUES (?1, ?2)",
1207 -1, &stmt, NULL);
1208 sqlite3_bind_double(stmt, 1, prior->o.iop_order_f);
1209 sqlite3_bind_text(stmt, 2, prior->operation, -1, SQLITE_TRANSIENT);
1210 TRY_STEP(stmt, SQLITE_DONE, "[init] can't insert default value in iop_order_tmp\n");
1211 sqlite3_finalize(stmt);
1212 }
1213 g_list_free_full(prior_v1, dt_free_gpointer);
1214 prior_v1 = NULL;
1215
1216 // create the order of the pipe
1217 // iop_order is by default the module priority
1218 // if there's multi-instances we add the multi_priority
1219 // multi_priority is in reverse order in this version,
1220 // so we assume that is always less than 1000 and reverse it
1221 // it is possible that multi_priority = 0 don't appear in history
1222 // so just in case 1 / 1000 to every instance
1223 // clang-format off
1224 TRY_EXEC("UPDATE main.history SET iop_order = ((("
1225 "SELECT MAX(multi_priority) FROM main.history hist1 WHERE hist1.imgid = main.history.imgid AND hist1.operation = main.history.operation "
1226 ") + 1. - multi_priority) / 1000.) + "
1227 "IFNULL((SELECT iop_order FROM iop_order_tmp WHERE iop_order_tmp.operation = "
1228 "main.history.operation), -999999.) ",
1229 "[init] can't update iop_order in history table\n");
1230
1231 // check if there's any entry in history that was not updated
1232 sqlite3_stmt *sel_stmt;
1233 TRY_PREPARE(sel_stmt, "SELECT DISTINCT operation FROM main.history WHERE iop_order <= 0 OR iop_order IS NULL",
1234 "[init] can't prepare selecting history iop_order\n");
1235 // clang-format on
1236 while(sqlite3_step(sel_stmt) == SQLITE_ROW)
1237 {
1238 const char *op_name = (const char *)sqlite3_column_text(sel_stmt, 0);
1239 printf("operation %s with no iop_order while upgrading database\n", op_name);
1240 }
1241 sqlite3_finalize(sel_stmt);
1242 // clang-format off
1243 TRY_EXEC("DROP TABLE iop_order_tmp", "[init] can't drop table `iop_order_tmp' from database\n");
1244 // clang-format on
1245 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1246 new_version = 18;
1247 }
1248 // maybe in the future, see commented out code elsewhere
1249 // else if(version == XXX)
1250 // {
1251 // sqlite3_exec(db->handle, "ALTER TABLE film_rolls ADD COLUMN external_drive VARCHAR(1024)", NULL,
1252 // NULL, NULL);
1253 // }
1254 else if(version == 18)
1255 {
1256 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1257 // clang-format off
1258 TRY_EXEC("UPDATE images SET orientation=-2 WHERE orientation=1;",
1259 "[init] can't update images orientation 1 from database\n");
1260
1261 TRY_EXEC("UPDATE images SET orientation=1 WHERE orientation=2;",
1262 "[init] can't update images orientation 2 from database\n");
1263
1264 TRY_EXEC("UPDATE images SET orientation=-6 WHERE orientation=5;",
1265 "[init] can't update images orientation 5 from database\n");
1266
1267 TRY_EXEC("UPDATE images SET orientation=5 WHERE orientation=6;",
1268 "[init] can't update images orientation 6 from database\n");
1269
1270 TRY_EXEC("UPDATE images SET orientation=2 WHERE orientation=-2;",
1271 "[init] can't update images orientation -1 from database\n");
1272
1273 TRY_EXEC("UPDATE images SET orientation=6 WHERE orientation=-6;",
1274 "[init] can't update images orientation -6 from database\n");
1275 // clang-format on
1276 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1277 new_version = 19;
1278 }
1279 else if(version == 19)
1280 {
1281 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1282 // clang-format off
1283 // create a temp table to invert all multi_priority
1284 TRY_EXEC("CREATE TEMPORARY TABLE m_prio (id INTEGER, operation VARCHAR(256), prio INTEGER)",
1285 "[init] can't create temporary table for updating `history and style_items'\n");
1286
1287 TRY_EXEC("CREATE INDEX m_prio_id_index ON m_prio (id)",
1288 "[init] can't create temporary index for updating `history and style_items'\n");
1289 TRY_EXEC("CREATE INDEX m_prio_op_index ON m_prio (operation)",
1290 "[init] can't create temporary index for updating `history and style_items'\n");
1291
1292 TRY_EXEC("INSERT INTO m_prio SELECT imgid, operation, MAX(multi_priority)"
1293 " FROM main.history GROUP BY imgid, operation",
1294 "[init] can't populate m_prio\n");
1295
1296 TRY_EXEC("UPDATE main.history SET multi_priority = "
1297 "(SELECT prio FROM m_prio "
1298 " WHERE main.history.operation = operation AND main.history.imgid = id) - main.history.multi_priority",
1299 "[init] can't update multi_priority for history\n");
1300
1301 TRY_EXEC("DROP TABLE m_prio", "[init] can't drop table `m_prio' from database\n");
1302 // clang-format on
1303 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1304 new_version = 20;
1305 }
1306 else if(version == 20)
1307 {
1308 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1309 // clang-format off
1310 TRY_EXEC("DROP INDEX IF EXISTS main.used_tags_idx", "[init] can't drop index `used_tags_idx' from database\n");
1311 TRY_EXEC("DROP TABLE used_tags", "[init] can't delete table used_tags\n");
1312 // clang-format on
1313 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1314
1315 new_version = 21;
1316 }
1317 else if(version == 21)
1318 {
1319 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1320 // create a temp table to invert all multi_priority
1321 // clang-format off
1322 TRY_EXEC("CREATE TABLE module_order (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR)",
1323 "[init] can't create module_order table'\n");
1324
1325 // for all images:
1326 sqlite3_stmt *mig_stmt;
1327 TRY_PREPARE(mig_stmt, "SELECT imgid, operation, multi_priority, iop_order, mi.iop_order_version"
1328 " FROM main.history AS hi, main.images AS mi"
1329 " WHERE hi.imgid = mi.id"
1330 " GROUP BY imgid, operation, multi_priority"
1331 " ORDER BY imgid, iop_order",
1332 "[init] can't prepare selecting history for iop_order migration (v21)\n");
1333 // clang-format on
1334 GList *item_list = NULL;
1335 int current_imgid = UNKNOWN_IMAGE;
1336 int current_order_version = -1;
1337
1338 gboolean has_row = (sqlite3_step(mig_stmt) == SQLITE_ROW);
1339
1340 while(has_row)
1341 {
1342 const int32_t imgid = sqlite3_column_int(mig_stmt, 0);
1343 char operation[20] = { 0 };
1344 g_strlcpy(operation, (const char *)sqlite3_column_text(mig_stmt, 1), sizeof(operation));
1345 const int multi_priority = sqlite3_column_int(mig_stmt, 2);
1346 const double iop_order = sqlite3_column_double(mig_stmt, 3);
1347 const int iop_order_version = sqlite3_column_int(mig_stmt, 4);
1348
1349 has_row = (sqlite3_step(mig_stmt) == SQLITE_ROW);
1350
1351 // a new image, let's initialize the iop_order_version
1352 if(imgid != current_imgid || !has_row)
1353 {
1354 // new image, let's handle it
1355 if(!IS_NULL_PTR(item_list))
1356 {
1357 // we keep legacy, everything else is migrated to v3.0
1358 const dt_iop_order_t new_order_version = current_order_version == 2 ? DT_IOP_ORDER_LEGACY : DT_IOP_ORDER_ANSEL_RAW;
1359
1360 GList *iop_order_list = dt_ioppr_get_iop_order_list_version(new_order_version);
1361
1362 // merge entries into iop_order_list
1363
1364 // first remove all item_list iop from the iop_order_list
1365
1366 GList *e = item_list;
1367 GList *n = NULL;
1368 dt_iop_order_entry_t *n_entry = NULL;
1369
1370 while(e)
1371 {
1372 dt_iop_order_entry_t *e_entry = (dt_iop_order_entry_t *)e->data;
1373
1374 GList *s = iop_order_list;
1375 while(s && strcmp(((dt_iop_order_entry_t *)s->data)->operation, e_entry->operation))
1376 {
1377 s = g_list_next(s);
1378 }
1379 if(s)
1380 {
1381 iop_order_list = g_list_delete_link(iop_order_list, s);
1382 }
1383
1384 // skip all multipe instances
1385 n = e;
1386 do
1387 {
1388 n = g_list_next(n);
1389 if(IS_NULL_PTR(n)) break;
1390 n_entry = (dt_iop_order_entry_t *)n->data;
1391 } while(!strcmp(n_entry->operation, e_entry->operation));
1392 e = n;
1393 }
1394
1395 // then add all item_list into iop_order_list
1396
1397 for(e = item_list; e; e = g_list_next(e))
1398 {
1399 dt_iop_order_entry_t *e_entry = (dt_iop_order_entry_t *)e->data;
1400 iop_order_list = g_list_prepend(iop_order_list, e_entry);
1401 }
1402
1403 // and finally reorder the full list based on the iop-order
1404
1405 iop_order_list = g_list_sort(iop_order_list, dt_sort_iop_list_by_order_f);
1406
1407 const dt_iop_order_t kind = dt_ioppr_get_iop_order_list_kind(iop_order_list);
1408
1409 // check if we have some multi-instances
1410
1411 gboolean has_multiple_instances = FALSE;
1412 GList *l = iop_order_list;
1413
1414 while(l)
1415 {
1416 GList *next = g_list_next(l);
1417 if(next
1418 && (strcmp(((dt_iop_order_entry_t *)(l->data))->operation,
1419 ((dt_iop_order_entry_t *)(next->data))->operation) == 0))
1420 {
1421 has_multiple_instances = TRUE;
1422 break;
1423 }
1424 l = next;
1425 }
1426
1427 // write iop_order_list and/or version into module_order
1428
1429 sqlite3_stmt *ins_stmt = NULL;
1430 if(kind == DT_IOP_ORDER_CUSTOM || has_multiple_instances)
1431 {
1432 char *iop_list_txt = dt_ioppr_serialize_text_iop_order_list(iop_order_list);
1433
1434 sqlite3_prepare_v2(db->handle,
1435 "INSERT INTO module_order VALUES (?1, ?2, ?3)", -1,
1436 &ins_stmt, NULL);
1437 sqlite3_bind_int(ins_stmt, 1, current_imgid);
1438 sqlite3_bind_int(ins_stmt, 2, kind);
1439 sqlite3_bind_text(ins_stmt, 3, iop_list_txt, -1, SQLITE_TRANSIENT);
1440 TRY_STEP(ins_stmt, SQLITE_DONE, "[init] can't insert into module_order (custom order)\n");
1441 sqlite3_finalize(ins_stmt);
1442
1443 dt_free(iop_list_txt);
1444 }
1445 else
1446 {
1447 sqlite3_prepare_v2(db->handle,
1448 "INSERT INTO module_order VALUES (?1, ?2, NULL)", -1,
1449 &ins_stmt, NULL);
1450 sqlite3_bind_int(ins_stmt, 1, current_imgid);
1451 sqlite3_bind_int(ins_stmt, 2, kind);
1452 TRY_STEP(ins_stmt, SQLITE_DONE, "[init] can't insert into module_order (standard order)\n");
1453 sqlite3_finalize(ins_stmt);
1454 }
1455
1456 g_list_free(item_list);
1457 item_list = NULL;
1458 g_list_free_full(iop_order_list, dt_free_gpointer);
1459 iop_order_list = NULL;
1460
1461 item_list = NULL;
1462 }
1463
1464 current_imgid = imgid;
1465 current_order_version = iop_order_version;
1466 }
1467
1468 dt_iop_order_entry_t *item = (dt_iop_order_entry_t *)malloc(sizeof(dt_iop_order_entry_t));
1469 memcpy(item->operation, operation, sizeof(item->operation));
1470 item->instance = multi_priority;
1471 item->o.iop_order_f = iop_order; // used to order the enties only
1472 item_list = g_list_append(item_list, item);
1473 }
1474 sqlite3_finalize(mig_stmt);
1475
1476 // remove iop_order from history table
1477 // clang-format off
1478 TRY_EXEC("CREATE TABLE h (imgid INTEGER, num INTEGER, module INTEGER, "
1479 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
1480 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
1481 "[init] can't create module_order table\n");
1482 TRY_EXEC("CREATE INDEX h_imgid_index ON h (imgid)",
1483 "[init] can't create index h_imgid_index\n");
1484 TRY_EXEC("INSERT INTO h SELECT imgid, num, module, operation, op_params, enabled, "
1485 "blendop_params, blendop_version, multi_priority, multi_name FROM main.history",
1486 "[init] can't create module_order table\n");
1487 TRY_EXEC("DROP TABLE history",
1488 "[init] can't drop table history\n");
1489 TRY_EXEC("ALTER TABLE h RENAME TO history",
1490 "[init] can't rename h to history\n");
1491 TRY_EXEC("DROP INDEX h_imgid_index",
1492 "[init] can't drop index h_imgid_index\n");
1493 TRY_EXEC("CREATE INDEX main.history_imgid_index ON history (imgid)",
1494 "[init] can't create index images_imgid_index\n");
1495
1496 // remove iop_order_version from images
1497
1498 TRY_EXEC("CREATE TABLE i (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1499 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1500 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1501 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
1502 "output_width INTEGER, output_height INTEGER, crop REAL, "
1503 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1504 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1505 "caption VARCHAR, description VARCHAR, license VARCHAR, sha1sum CHAR(40), "
1506 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1507 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1508 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, aspect_ratio REAL)",
1509 "[init] can't create table i\n");
1510 TRY_EXEC("INSERT INTO i SELECT id, group_id, film_id, width, height, filename, maker, model,"
1511 " lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags,"
1512 " output_width, output_height, crop, raw_parameters, raw_denoise_threshold,"
1513 " raw_auto_bright_threshold, raw_black, raw_maximum, caption, description, license, sha1sum,"
1514 " orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version,"
1515 " max_version, write_timestamp, history_end, position, aspect_ratio "
1516 "FROM images",
1517 "[init] can't populate table h\n");
1518 TRY_EXEC("DROP TABLE images",
1519 "[init] can't drop table images\n");
1520 TRY_EXEC("ALTER TABLE i RENAME TO images",
1521 "[init] can't rename i to images\n");
1522 // clang-format on
1523 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1524
1525 new_version = 22;
1526 }
1527 else if(version == 22)
1528 {
1529 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1530 // clang-format off
1531 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_group_id_index ON images (group_id)",
1532 "[init] can't create group_id index on image\n");
1533 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_film_id_index ON images (film_id)",
1534 "[init] can't create film_id index on image\n");
1535 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_filename_index ON images (filename)",
1536 "[init] can't create filename index on image\n");
1537 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.image_position_index ON images (position)",
1538 "[init] can't create position index on image\n");
1539
1540 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.film_rolls_folder_index ON film_rolls (folder)",
1541 "[init] can't create folder index on film_rolls\n");
1542 // clang-format on
1543 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1544
1545 new_version = 23;
1546 }
1547 else if(version == 23)
1548 {
1549 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1550 // clang-format off
1551 TRY_EXEC("CREATE TABLE main.history_hash (imgid INTEGER PRIMARY KEY, "
1552 "basic_hash BLOB, auto_hash BLOB, current_hash BLOB)",
1553 "[init] can't create table history_hash\n");
1554 // clang-format on
1555
1556 // history_hash backfill is deferred: it will be filled through the image cache.
1557
1558 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1559
1560 new_version = 24;
1561 }
1562 else if(version == 24)
1563 {
1564 // clang-format off
1565 TRY_EXEC("ALTER TABLE main.history_hash ADD COLUMN mipmap_hash BLOB",
1566 "[init] can't add `mipmap_hash' column to history_hash table in database\n");
1567 // clang-format on
1568
1569 new_version = 25;
1570 }
1571 else if(version == 25)
1572 {
1573 // clang-format of
1574 TRY_EXEC("ALTER TABLE main.images ADD COLUMN exposure_bias REAL",
1575 "[init] can't add `exposure_bias' column to images table in database\n");
1576 // clang-format on
1577
1578 new_version = 26;
1579 }
1580 else if(version == 26)
1581 {
1582 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1583 // clang-format off
1584 TRY_EXEC("CREATE TABLE main.new_film_rolls "
1585 "(id INTEGER PRIMARY KEY, "
1586 "access_timestamp INTEGER, "
1587 "folder VARCHAR(1024) NOT NULL)",
1588 "[init] can't create new_film_rolls table\n");
1589
1590 TRY_EXEC("INSERT INTO main.new_film_rolls"
1591 "(id, access_timestamp, folder) "
1592 "SELECT id, "
1593 "strftime('%s', replace(substr(datetime_accessed, 1, 10), ':', '-') || substr(datetime_accessed, 11), 'utc'), "
1594 "folder "
1595 "FROM film_rolls "
1596 "WHERE folder IS NOT NULL",
1597 "[init] can't populate new_film_rolls table from film_rolls\n");
1598
1599 TRY_EXEC("DROP TABLE film_rolls",
1600 "[init] can't delete table film_rolls\n");
1601
1602 TRY_EXEC("ALTER TABLE main.new_film_rolls RENAME TO film_rolls",
1603 "[init] can't rename table new_film_rolls to film_rolls\n");
1604
1605 TRY_EXEC("CREATE INDEX main.film_rolls_folder_index ON film_rolls (folder)",
1606 "[init] can't create index `film_rolls_folder_index' on table `film_rolls'\n");
1607 // clang-format on
1608 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1609 new_version = 27;
1610 }
1611 else if(version == 27)
1612 {
1613 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1614 // clang-format off
1615 TRY_EXEC("ALTER TABLE main.images ADD COLUMN import_timestamp INTEGER DEFAULT -1",
1616 "[init] can't add `import_timestamp' column to images table in database\n");
1617 TRY_EXEC("ALTER TABLE main.images ADD COLUMN change_timestamp INTEGER DEFAULT -1",
1618 "[init] can't add `change_timestamp' column to images table in database\n");
1619 TRY_EXEC("ALTER TABLE main.images ADD COLUMN export_timestamp INTEGER DEFAULT -1",
1620 "[init] can't add `export_timestamp' column to images table in database\n");
1621 TRY_EXEC("ALTER TABLE main.images ADD COLUMN print_timestamp INTEGER DEFAULT -1",
1622 "[init] can't add `print_timestamp' column to images table in database\n");
1623
1624 TRY_EXEC("UPDATE main.images SET import_timestamp = (SELECT access_timestamp "
1625 "FROM main.film_rolls WHERE film_rolls.id = images.film_id)",
1626 "[init] can't populate import_timestamp column from film_rolls.access_timestamp.\n");
1627
1628 TRY_EXEC("UPDATE main.images SET change_timestamp = images.write_timestamp "
1629 "WHERE images.write_timestamp IS NOT NULL "
1630 "AND images.id = (SELECT imgid FROM tagged_images "
1631 "JOIN data.tags ON tags.id = tagged_images.tagid "
1632 "WHERE data.tags.name = 'darktable|changed')",
1633 "[init] can't populate change_timestamp column from images.write_timestamp.\n");
1634 // clang-format on
1635 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1636 new_version = 28;
1637 }
1638 else if(version == 28)
1639 {
1640 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1641 // clang-format off
1642 // clear flag DT_IMAGE_REJECTED (was not used)
1643 TRY_EXEC("UPDATE main.images SET flags = (flags & ~8)",
1644 "[init] can't clear rejected flags");
1645
1646 // add DT_IMAGE_REJECTED and clear rating for all images being rejected
1647 TRY_EXEC("UPDATE main.images SET flags = (flags | 8) & ~7 WHERE (flags & 7) = 6",
1648 "[init] can't set rejected flags");
1649 // clang-format on
1650 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1651 new_version = 29;
1652 }
1653 else if(version == 29)
1654 {
1655 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1656 // clang-format off
1657 // add position in tagged_images table
1658 TRY_EXEC("ALTER TABLE main.tagged_images ADD COLUMN position INTEGER",
1659 "[init] can't add `position' column to tagged_images table in database\n");
1660
1661 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.tagged_images_imgid_index ON tagged_images (imgid)",
1662 "[init] can't create image index on tagged_images\n");
1663 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.tagged_images_position_index ON tagged_images (position)",
1664 "[init] can't create position index on tagged_images\n");
1665 TRY_EXEC("UPDATE main.tagged_images SET position = (tagid + imgid) << 32",
1666 "[init] can't populate position on tagged_images\n");
1667
1668 // remove caption and description fields from images table
1669
1670 TRY_EXEC("CREATE TABLE main.i (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1671 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
1672 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
1673 "focus_distance REAL, datetime_taken CHAR(20), flags INTEGER, "
1674 "output_width INTEGER, output_height INTEGER, crop REAL, "
1675 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
1676 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
1677 "license VARCHAR, sha1sum CHAR(40), "
1678 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
1679 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
1680 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
1681 "aspect_ratio REAL, exposure_bias REAL, "
1682 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, "
1683 "export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1)",
1684 "[init] can't create table i\n");
1685
1686 TRY_EXEC("INSERT INTO main.i SELECT id, group_id, film_id, width, height, filename, maker, model,"
1687 " lens, exposure, aperture, iso, focal_length, focus_distance, datetime_taken, flags,"
1688 " output_width, output_height, crop, raw_parameters, raw_denoise_threshold,"
1689 " raw_auto_bright_threshold, raw_black, raw_maximum, license, sha1sum,"
1690 " orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version,"
1691 " max_version, write_timestamp, history_end, position, aspect_ratio, exposure_bias,"
1692 " import_timestamp, change_timestamp, export_timestamp, print_timestamp "
1693 "FROM main.images",
1694 "[init] can't populate table i\n");
1695 TRY_EXEC("DROP TABLE main.images",
1696 "[init] can't drop table images\n");
1697 TRY_EXEC("ALTER TABLE main.i RENAME TO images",
1698 "[init] can't rename i to images\n");
1699
1700 TRY_EXEC("CREATE INDEX main.images_group_id_index ON images (group_id)",
1701 "[init] can't create group_id index on images table\n");
1702 TRY_EXEC("CREATE INDEX main.images_film_id_index ON images (film_id)",
1703 "[init] can't create film_id index on images table\n");
1704 TRY_EXEC("CREATE INDEX main.images_filename_index ON images (filename)",
1705 "[init] can't create filename index on images table\n");
1706 TRY_EXEC("CREATE INDEX main.image_position_index ON images (position)",
1707 "[init] can't create position index on images table\n");
1708 // clang-format on
1709 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1710 new_version = 30;
1711 }
1712 else if(version == 30)
1713 {
1714 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1715 // clang-format off
1716 // add second columns to speed up sorting
1717 TRY_EXEC("DROP INDEX IF EXISTS `history_imgid_index`",
1718 "[init] can't drop history_imgid_index\n");
1719 TRY_EXEC("CREATE INDEX `history_imgid_index` ON `history` ( `imgid`, `operation` )",
1720 "[init] can't recreate history_imgid_index\n");
1721
1722 TRY_EXEC("DROP INDEX IF EXISTS `images_filename_index`",
1723 "[init] can't drop images_filename_index\n");
1724 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
1725 "[init] can't recreate images_filename_index\n");
1726
1727 TRY_EXEC("DROP INDEX IF EXISTS `images_film_id_index`",
1728 "[init] can't drop images_film_id_index\n");
1729 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
1730 "[init] can't recreate images_film_id_index\n");
1731
1732 TRY_EXEC("DROP INDEX IF EXISTS `images_group_id_index`",
1733 "[init] can't drop images_group_id_index\n");
1734 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
1735 "[init] can't recreate images_group_id_index\n");
1736
1737 TRY_EXEC("DROP INDEX IF EXISTS `masks_history_imgid_index`",
1738 "[init] can't drop masks_history_imgid_index\n");
1739 TRY_EXEC("CREATE INDEX `masks_history_imgid_index` ON `masks_history` ( `imgid`, `num` )",
1740 "[init] can't recreate masks_history_imgid_index\n");
1741
1742 // map refinement: avoid full table scan
1743 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( `latitude` DESC, `longitude` DESC )",
1744 "[init] can't create images_latlong_index\n");
1745 // clang-format on
1746 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1747 new_version = 31;
1748 }
1749 else if(version == 31)
1750 {
1751 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1752 // clang-format off
1753 // remove duplicates
1754 TRY_EXEC("DELETE FROM main.meta_data WHERE rowid NOT IN (SELECT MIN(rowid) "
1755 "FROM main.meta_data GROUP BY id, key)",
1756 "[init] can't remove duplicates from meta_data\n");
1757
1758 // recreate the index with UNIQUE option
1759 TRY_EXEC("DROP INDEX IF EXISTS metadata_index",
1760 "[init] can't drop metadata_index\n");
1761 TRY_EXEC("CREATE UNIQUE INDEX main.metadata_index ON meta_data (id, key)",
1762 "[init] can't create metadata_index\n");
1763 // clang-format on
1764 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
1765 new_version = 32;
1766 }
1767 else if(version == 32)
1768 {
1769 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
1770 // clang-format off
1771 // add foreign keys for database consistency. ON UPDATE CASCADE since you never know
1772 // if a future version will change image_id
1773 // Unfortunately sqlite does not support adding foreign keys to existing tables
1774 // so we have to rename the existing tables, recreate them and copy back the old values
1775 // images first
1776 // needs to delete orphaned entries
1777 TRY_EXEC("ALTER TABLE `images` RENAME TO `images_old`",
1778 "[init] can't rename images\n");
1779
1780 TRY_EXEC("CREATE TABLE `images` (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
1781 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, lens VARCHAR, "
1782 "exposure REAL, aperture REAL, iso REAL, focal_length REAL, focus_distance REAL, datetime_taken CHAR(20), "
1783 "flags INTEGER, output_width INTEGER, output_height INTEGER, crop REAL, "
1784 "raw_parameters INTEGER, raw_denoise_threshold REAL, raw_auto_bright_threshold REAL, "
1785 "raw_black INTEGER, raw_maximum INTEGER, license VARCHAR, sha1sum CHAR(40), "
1786 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, latitude REAL, altitude REAL, "
1787 "color_matrix BLOB, colorspace INTEGER, version INTEGER, max_version INTEGER, write_timestamp INTEGER, "
1788 "history_end INTEGER, position INTEGER, aspect_ratio REAL, exposure_bias REAL, "
1789 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1, "
1790 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
1791 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
1792 "[init] can't create new images table\n");
1793
1794 // corner case: database inconsistency with images having invalid film id
1795 TRY_EXEC("DELETE FROM `images_old` WHERE film_id NOT IN (SELECT id FROM `film_rolls`)",
1796 "[init] can't delete images with invalid film id\n");
1797
1798 TRY_EXEC("UPDATE `images_old` SET group_id=id WHERE group_id NOT IN (SELECT id from `images_old`)",
1799 "[init] can't fix invalid group ids\n");
1800
1801 TRY_EXEC("INSERT INTO `images` SELECT * FROM `images_old`",
1802 "[init] can't copy back from images_old\n");
1803
1804 // pita: need to recreate index
1805 TRY_EXEC("DROP INDEX IF EXISTS `image_position_index`",
1806 "[init] can't drop image_position_index\n");
1807 TRY_EXEC("CREATE INDEX `image_position_index` ON `images` (position)",
1808 "[init] can't add image_position_index\n");
1809
1810 // second columns
1811 TRY_EXEC("DROP INDEX IF EXISTS `images_filename_index`",
1812 "[init] can't drop images_filename_index\n");
1813 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
1814 "[init] can't recreate images_filename_index\n");
1815
1816 TRY_EXEC("DROP INDEX IF EXISTS `images_film_id_index`",
1817 "[init] can't drop images_film_id_index\n");
1818 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
1819 "[init] can't recreate images_film_id_index\n");
1820
1821 TRY_EXEC("DROP INDEX IF EXISTS `images_group_id_index`",
1822 "[init] can't drop images_group_id_index\n");
1823 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
1824 "[init] can't recreate images_group_id_index\n");
1825
1826 TRY_EXEC("DROP INDEX IF EXISTS `images_latlong_index`",
1827 "[init] can't drop images_latlong_index\n");
1828 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( latitude DESC, longitude DESC )",
1829 "[init] can't add images_latlong_index\n");
1830
1831 TRY_EXEC("DROP TABLE `images_old`",
1832 "[init] can't drop table images_old\n");
1833
1834 // history
1835 TRY_EXEC("ALTER TABLE `history` RENAME TO `history_old`",
1836 "[init] can't rename history\n");
1837
1838 TRY_EXEC("CREATE TABLE `history` (imgid INTEGER, num INTEGER, module INTEGER, "
1839 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, "
1840 "multi_priority INTEGER, multi_name VARCHAR(256), "
1841 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1842 "[init] can't create new history table\n");
1843
1844 TRY_EXEC("DELETE FROM `history_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1845 "[init] can't delete orphaned history elements\n");
1846
1847 TRY_EXEC("INSERT INTO history SELECT * FROM history_old",
1848 "[init] can't copy back from history_old\n");
1849
1850 TRY_EXEC("DROP INDEX IF EXISTS `history_imgid_index`",
1851 "[init] can't drop history_imgid_index\n");
1852 TRY_EXEC("CREATE INDEX `history_imgid_op_index` ON `history` ( `imgid`, `operation` )",
1853 "[init] can't recreate history_imgid_index\n");
1854 TRY_EXEC("CREATE INDEX `history_imgid_num_index` ON `history` ( `imgid`, `num` DESC )",
1855 "[init] can't recreate history_imgid_index\n");
1856
1857 TRY_EXEC("DROP TABLE `history_old`",
1858 "[init] can't drop table history_old\n");
1859
1860 // history hash
1861 TRY_EXEC("ALTER TABLE `history_hash` RENAME TO `history_hash_old`",
1862 "[init] can't rename history_hash\n");
1863
1864 TRY_EXEC("CREATE TABLE `history_hash` (imgid INTEGER PRIMARY KEY, basic_hash BLOB, auto_hash BLOB, current_hash BLOB, "
1865 "mipmap_hash BLOB, FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1866 "[init] can't create new history_hash table\n");
1867
1868 TRY_EXEC("DELETE FROM `history_hash_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1869 "[init] can't delete orphaned history_hash elements\n");
1870
1871 TRY_EXEC("INSERT INTO `history_hash` SELECT * FROM `history_hash_old`",
1872 "[init] can't copy back from history_hash_old\n");
1873
1874 TRY_EXEC("DROP TABLE `history_hash_old`",
1875 "[init] can't drop table history_hash_old\n");
1876
1877 // tagged images
1878 TRY_EXEC("ALTER TABLE `tagged_images` RENAME TO `tagged_images_old`",
1879 "[init] can't rename tagged_images\n");
1880
1881 TRY_EXEC("CREATE TABLE `tagged_images` (imgid integer, tagid integer, position INTEGER, "
1882 "primary key(imgid, tagid), FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1883 "[init] can't create new tagged_images table\n");
1884
1885 TRY_EXEC("DELETE FROM `tagged_images_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1886 "[init] can't delete orphaned tagged_images elements\n");
1887
1888 TRY_EXEC("INSERT INTO `tagged_images` SELECT * FROM `tagged_images_old`",
1889 "[init] can't copy back from tagged_images_old\n");
1890
1891 // old indices
1892 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_imgid_index",
1893 "[init] can't drop tagged_images_imgid_index\n");
1894 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_position_index",
1895 "[init] can't drop tagged_images_position_index\n");
1896 TRY_EXEC("CREATE INDEX tagged_images_position_index ON tagged_images (position)",
1897 "[init] can't add index tagged_images_position_index\n");
1898 TRY_EXEC("DROP INDEX IF EXISTS tagged_images_tagid_index",
1899 "[init] can't drop tagged_images_tagid_index\n");
1900 TRY_EXEC("CREATE INDEX tagged_images_tagid_index ON tagged_images (tagid)",
1901 "[init] can't add index tagged_images_tagid_index\n");
1902
1903 TRY_EXEC("DROP TABLE `tagged_images_old`",
1904 "[init] can't drop table tagged_images_old\n");
1905
1906 // masks history
1907 TRY_EXEC("ALTER TABLE `masks_history` RENAME TO `masks_history_old`",
1908 "[init] can't rename masks_history\n");
1909
1910 TRY_EXEC("CREATE TABLE masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, "
1911 "name VARCHAR(256), version INTEGER, points BLOB, points_count INTEGER, source BLOB, "
1912 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1913 "[init] can't create new masks_history table\n");
1914
1915 TRY_EXEC("DELETE FROM `masks_history_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1916 "[init] can't delete orphaned masks_history elements\n");
1917
1918 TRY_EXEC("INSERT INTO `masks_history` SELECT * FROM `masks_history_old`",
1919 "[init] can't copy back from masks_history\n");
1920
1921 TRY_EXEC("DROP INDEX IF EXISTS `masks_history_imgid_index`",
1922 "[init] can't drop masks_history_imgid_index\n");
1923 TRY_EXEC("CREATE INDEX `masks_history_imgid_index` ON `masks_history` ( imgid, num )",
1924 "[init] can't recreate masks_history_imgid_index\n");
1925
1926 TRY_EXEC("DROP TABLE masks_history_old",
1927 "[init] can't drop table masks_history_old\n");
1928
1929 // color labels
1930 TRY_EXEC("ALTER TABLE `color_labels` RENAME TO `color_labels_old`",
1931 "[init] can't rename color_labels\n");
1932 TRY_EXEC("CREATE TABLE `color_labels` (imgid INTEGER, color INTEGER, "
1933 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1934 "[init] can't create new color_labels table\n");
1935
1936 TRY_EXEC("DELETE FROM `color_labels_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1937 "[init] can't delete orphaned color_labels elements\n");
1938
1939 TRY_EXEC("INSERT INTO `color_labels` SELECT * FROM `color_labels_old`",
1940 "[init] can't copy back from color_labels\n");
1941
1942 TRY_EXEC("DROP TABLE color_labels_old",
1943 "[init] can't drop table color_labels_old\n");
1944
1945 TRY_EXEC("CREATE UNIQUE INDEX `color_labels_idx` ON `color_labels` (imgid, color)",
1946 "[init] can't recreate color_labels_idx\n");
1947
1948 // meta data
1949 TRY_EXEC("ALTER TABLE `meta_data` RENAME TO `meta_data_old`",
1950 "[init] can't rename meta_data\n");
1951 TRY_EXEC("CREATE TABLE `meta_data` (id integer, key integer, value varchar, "
1952 "FOREIGN KEY(id) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1953 "[init] can't create new meta_data table\n");
1954
1955 TRY_EXEC("DELETE FROM `meta_data_old` WHERE id NOT IN (SELECT id FROM `images`)",
1956 "[init] can't delete orphaned meta_data elements\n");
1957
1958 TRY_EXEC("INSERT INTO `meta_data` SELECT * FROM `meta_data_old`",
1959 "[init] can't copy back from meta_data\n");
1960
1961 TRY_EXEC("DROP TABLE meta_data_old",
1962 "[init] can't drop table meta_data_old\n");
1963
1964 TRY_EXEC("CREATE UNIQUE INDEX `metadata_index` ON `meta_data` (id, key, value)",
1965 "[init] can't recreate metadata_index\n");
1966
1967 // selected images
1968 TRY_EXEC("ALTER TABLE `selected_images` RENAME TO `selected_images_old`",
1969 "[init] can't rename selected_images\n");
1970 TRY_EXEC("CREATE TABLE `selected_images` (imgid INTEGER PRIMARY KEY, "
1971 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1972 "[init] can't create new selected_images table\n");
1973
1974 TRY_EXEC("DELETE FROM `selected_images_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1975 "[init] can't delete orphaned selected_images elements\n");
1976
1977 TRY_EXEC("INSERT INTO `selected_images` SELECT * FROM `selected_images_old`",
1978 "[init] can't copy back selected_images meta_data\n");
1979
1980 TRY_EXEC("DROP TABLE selected_images_old",
1981 "[init] can't drop table selected_images_old\n");
1982
1983 // module order
1984 TRY_EXEC("ALTER TABLE `module_order` RENAME TO `module_order_old`",
1985 "[init] can't rename module_order\n");
1986 TRY_EXEC("CREATE TABLE `module_order` (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR, "
1987 "FOREIGN KEY(imgid) REFERENCES images(id) ON DELETE CASCADE ON UPDATE CASCADE)",
1988 "[init] can't create new module_order table\n");
1989
1990 TRY_EXEC("DELETE FROM `module_order_old` WHERE imgid NOT IN (SELECT id FROM `images`)",
1991 "[init] can't delete orphaned module_order elements\n");
1992
1993 TRY_EXEC("INSERT INTO `module_order` SELECT * FROM `module_order_old`",
1994 "[init] can't copy back module_order meta_data\n");
1995
1996 TRY_EXEC("DROP TABLE module_order_old",
1997 "[init] can't drop table module_order_old\n");
1998 // clang-format on
1999 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2000 new_version = 33;
2001 }
2002 else if(version == 33)
2003 {
2004 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2005
2006 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.images_datetime_taken_nc ON images (datetime_taken COLLATE NOCASE)",
2007 "[init] can't create images_datetime_taken\n");
2008 TRY_EXEC("CREATE INDEX IF NOT EXISTS main.metadata_index_key ON meta_data (key)",
2009 "[init] can't create metadata_index_key\n");
2010
2011 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2012 new_version = 34;
2013 }
2014 else if(version == 34)
2015 {
2016 sqlite3_exec(db->handle, "PRAGMA foreign_keys = OFF", NULL, NULL, NULL);
2017 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2018
2019 TRY_EXEC("CREATE TABLE main.images_new (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
2020 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
2021 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
2022 "focus_distance REAL, datetime_taken INTEGER, flags INTEGER, "
2023 "output_width INTEGER, output_height INTEGER, crop REAL, "
2024 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
2025 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
2026 "license VARCHAR, sha1sum CHAR(40), "
2027 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
2028 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
2029 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
2030 "aspect_ratio REAL, exposure_bias REAL, "
2031 "import_timestamp INTEGER, change_timestamp INTEGER, "
2032 "export_timestamp INTEGER, print_timestamp INTEGER, "
2033 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
2034 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
2035 "[init] can't create new images table\n");
2036
2037 TRY_EXEC("INSERT INTO `images_new` SELECT "
2038 "id, group_id, film_id, width, height, filename, maker, model, "
2039 "lens, exposure, aperture, iso, focal_length, focus_distance, NULL AS datetime_taken, flags, "
2040 "output_width, output_height, crop, raw_parameters, raw_denoise_threshold, raw_auto_bright_threshold, raw_black, raw_maximum, "
2041 "license, sha1sum, orientation, histogram, lightmap, longitude, latitude, altitude, color_matrix, colorspace, version, "
2042 "max_version, write_timestamp, history_end, position, aspect_ratio, exposure_bias, "
2043 "NULL AS import_timestamp, NULL AS change_timestamp, NULL AS export_timestamp, NULL AS print_timestamp "
2044 "FROM `images`",
2045 "[init] can't copy back from images\n");
2046
2047 TRY_PREPARE(stmt, "SELECT id,"
2048 " CASE WHEN datetime_taken = '' THEN NULL ELSE datetime_taken END,"
2049 " CASE WHEN import_timestamp = -1 THEN NULL ELSE import_timestamp END,"
2050 " CASE WHEN change_timestamp = -1 THEN NULL ELSE change_timestamp END,"
2051 " CASE WHEN export_timestamp = -1 THEN NULL ELSE export_timestamp END,"
2052 " CASE WHEN print_timestamp = -1 THEN NULL ELSE print_timestamp END "
2053 "FROM `images`",
2054 "[init] can't get datetime from images\n");
2055 while(sqlite3_step(stmt) == SQLITE_ROW)
2056 {
2057 sqlite3_stmt *stmt2;
2058 sqlite3_prepare_v2(db->handle,
2059 "UPDATE `images_new` SET"
2060 " (datetime_taken, import_timestamp,"
2061 " change_timestamp, export_timestamp, print_timestamp) = "
2062 " (?2, ?3, ?4, ?5, ?6) WHERE id = ?1",
2063 -1, &stmt2, NULL);
2064 sqlite3_bind_int(stmt2, 1, sqlite3_column_int(stmt, 0));
2065 if(sqlite3_column_type(stmt, 1) != SQLITE_NULL)
2066 {
2067 GDateTime *gdt = dt_datetime_exif_to_gdatetime((const char *)sqlite3_column_text(stmt, 1), dt_datetime_utc_tz());
2068 if(gdt)
2069 {
2070 sqlite3_bind_int64(stmt2, 2, dt_datetime_gdatetime_to_gtimespan(gdt));
2071 g_date_time_unref(gdt);
2072 }
2073 }
2074 for(int i = 0; i < 4; i++)
2075 {
2076 if(sqlite3_column_type(stmt, i + 2) != SQLITE_NULL)
2077 {
2078 GDateTime *gdt = g_date_time_new_from_unix_utc(sqlite3_column_int(stmt, i + 2));
2079 if(gdt)
2080 {
2081 sqlite3_bind_int64(stmt2, i + 3, dt_datetime_gdatetime_to_gtimespan(gdt));
2082 g_date_time_unref(gdt);
2083 }
2084 }
2085 }
2086 TRY_STEP(stmt2, SQLITE_DONE, "[init] can't update datetimes into images_new table\n");
2087 sqlite3_finalize(stmt2);
2088 }
2089 sqlite3_finalize(stmt);
2090
2091 TRY_EXEC("DROP TABLE `images`", "[init] can't drop images table\n");
2092 // that's the way to keep the other tables foreign keys references valid
2093 TRY_EXEC("ALTER TABLE `images_new` RENAME TO `images`", "[init] can't rename images_new table to images");
2094
2095 // pita: need to recreate indexes
2096 TRY_EXEC("CREATE INDEX `image_position_index` ON `images` (position)",
2097 "[init] can't add image_position_index\n");
2098 TRY_EXEC("CREATE INDEX `images_filename_index` ON `images` ( `filename`, `version` )",
2099 "[init] can't recreate images_filename_index\n");
2100 TRY_EXEC("CREATE INDEX `images_film_id_index` ON `images` ( `film_id`, `filename` )",
2101 "[init] can't recreate images_film_id_index\n");
2102 TRY_EXEC("CREATE INDEX `images_group_id_index` ON `images` ( `group_id`, `id` )",
2103 "[init] can't recreate images_group_id_index\n");
2104 TRY_EXEC("CREATE INDEX `images_latlong_index` ON `images` ( latitude DESC, longitude DESC )",
2105 "[init] can't add images_latlong_index\n");
2106 TRY_EXEC("CREATE INDEX `images_datetime_taken` ON images (datetime_taken)",
2107 "[init] can't create images_datetime_taken\n");
2108
2109 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2110 sqlite3_exec(db->handle, "PRAGMA foreign_keys = ON", NULL, NULL, NULL);
2111 new_version = 35;
2112 }
2113 else if(version == 35)
2114 {
2115 TRY_EXEC("CREATE TABLE main.images_new (id INTEGER, filename VARCHAR, flags INTEGER)",
2116 "[init] can't create new images table\n");
2117
2118 gchar *query = g_strdup_printf("INSERT INTO `images_new` "
2119 "SELECT id, filename, flags"
2120 " FROM images"
2121 " WHERE (flags & %d == 0)",
2122 DT_IMAGE_RAW | DT_IMAGE_LDR | DT_IMAGE_HDR);
2123 TRY_EXEC(query, "[init] can't copy back from images\n");
2124
2125 TRY_PREPARE(stmt, "SELECT id, filename, flags FROM `images_new`",
2126 "[init] can't prepare selecting images flags\n");
2127
2128 while(sqlite3_step(stmt) == SQLITE_ROW)
2129 {
2130 sqlite3_stmt *stmt2;
2131 sqlite3_prepare_v2(db->handle,
2132 "UPDATE `images` SET"
2133 " (flags) = "
2134 " (?2) WHERE id = ?1",
2135 -1, &stmt2, NULL);
2136 sqlite3_bind_int(stmt2, 1, sqlite3_column_int(stmt, 0));
2137
2138 dt_image_flags_t flags = sqlite3_column_int(stmt, 2);
2139 gchar *ext = g_strrstr((const char *)sqlite3_column_text(stmt, 1), ".");
2140 flags |= dt_image_flags_from_extension(ext);
2141 sqlite3_bind_int(stmt2, 2, flags);
2142
2143 TRY_STEP(stmt2, SQLITE_DONE, "[init] can't update flags\n");
2144 sqlite3_finalize(stmt2);
2145 }
2146 sqlite3_finalize(stmt);
2147
2148 TRY_EXEC("DROP TABLE `images_new`", "[init] can't drop temp images table\n");
2149 new_version = 36;
2150 }
2151 else
2152 new_version = version; // should be the fallback so that calling code sees that we are in an infinite loop
2153
2154 // write the new version to db
2155 sqlite3_prepare_v2(db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', ?1)", -1, &stmt,
2156 NULL);
2157 sqlite3_bind_int(stmt, 1, new_version);
2158 sqlite3_step(stmt);
2159 sqlite3_finalize(stmt);
2160
2161 return new_version;
2162}
2163
2164/* do the real migration steps, returns the version the db was converted to */
2165static int _upgrade_data_schema_step(dt_database_t *db, int version)
2166{
2167 sqlite3_stmt *stmt;
2168 int new_version = version;
2169 if(version == CURRENT_DATABASE_VERSION_DATA)
2170 return version;
2171 else if(version == 0)
2172 {
2173 // this can't happen, we started with 1, but it's a good example how this function works
2174 // <do some magic to the db>
2175 new_version = 1; // the version we transformed the db to. this way it might be possible to roll back or
2176 // add fast paths
2177 }
2178 else if(version == 1)
2179 {
2180 // clang-format off
2181 // style_items:
2182 // NO TRY_EXEC has the column could be there before version 1 (master build)
2183 // TRY_EXEC("ALTER TABLE data.style_items ADD COLUMN iop_order REAL",
2184 // "[init] can't add `iop_order' column to style_items table in database\n");
2185 sqlite3_exec(db->handle, "ALTER TABLE data.style_items ADD COLUMN iop_order REAL", NULL, NULL, NULL);
2186 // clang-format on
2187 sqlite3_stmt *sel_stmt = NULL;
2188 GList *prior_v1 = dt_ioppr_get_iop_order_list_version(DT_IOP_ORDER_LEGACY);
2189 // clang-format off
2190 // create a temp table with the previous priorities
2191 TRY_EXEC("CREATE TEMPORARY TABLE iop_order_tmp (iop_order REAL, operation VARCHAR(256))",
2192 "[init] can't create temporary table for updating `data.style_items'\n");
2193 // clang-format on
2194 // fill temp table with all operations up to this release
2195 // it will be used to create the pipe and update the iop_order on history
2196 for(GList *priorities = prior_v1; priorities; priorities = g_list_next(priorities))
2197 {
2198 dt_iop_order_entry_t *prior = (dt_iop_order_entry_t *)priorities->data;
2199
2200 sqlite3_prepare_v2(
2201 db->handle,
2202 "INSERT INTO iop_order_tmp (iop_order, operation) VALUES (?1, ?2)",
2203 -1, &stmt, NULL);
2204 sqlite3_bind_double(stmt, 1, prior->o.iop_order_f);
2205 sqlite3_bind_text(stmt, 2, prior->operation, -1, SQLITE_TRANSIENT);
2206 TRY_STEP(stmt, SQLITE_DONE, "[init] can't insert default value in iop_order_tmp\n");
2207 sqlite3_finalize(stmt);
2208 }
2209 g_list_free_full(prior_v1, dt_free_gpointer);
2210 prior_v1 = NULL;
2211
2212 // do the same as for history
2213 // clang-format off
2214 TRY_EXEC("UPDATE data.style_items SET iop_order = ((("
2215 "SELECT MAX(multi_priority) FROM data.style_items style1 WHERE style1.styleid = data.style_items.styleid AND style1.operation = data.style_items.operation "
2216 ") + 1. - multi_priority) / 1000.) + "
2217 "IFNULL((SELECT iop_order FROM iop_order_tmp WHERE iop_order_tmp.operation = "
2218 "data.style_items.operation), -999999.) ",
2219 "[init] can't update iop_order in style_items table\n");
2220
2221 TRY_PREPARE(sel_stmt, "SELECT DISTINCT operation FROM data.style_items WHERE iop_order <= 0 OR iop_order IS NULL",
2222 "[init] can't prepare selecting style_items iop_order\n");
2223 // clang-format on
2224 while(sqlite3_step(sel_stmt) == SQLITE_ROW)
2225 {
2226 const char *op_name = (const char *)sqlite3_column_text(sel_stmt, 0);
2227 printf("operation %s with no iop_order while upgrading style_items in database\n", op_name);
2228 }
2229 sqlite3_finalize(sel_stmt);
2230 // clang-format off
2231 TRY_EXEC("DROP TABLE iop_order_tmp", "[init] can't drop table `iop_order_tmp' from database\n");
2232 // clang-format on
2233 new_version = 2;
2234 }
2235 else if(version == 2)
2236 {
2237 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2238
2239 // With sqlite above or equal to 3.25.0 RENAME COLUMN can be used instead of the following code
2240 // TRY_EXEC("ALTER TABLE data.tags RENAME COLUMN description TO synonyms;",
2241 // "[init] can't change tags column name from description to synonyms\n");
2242 // clang-format off
2243 TRY_EXEC("ALTER TABLE data.tags RENAME TO tmp_tags", "[init] can't rename table tags\n");
2244
2245 TRY_EXEC("CREATE TABLE data.tags (id INTEGER PRIMARY KEY, name VARCHAR, "
2246 "synonyms VARCHAR, flags INTEGER)",
2247 "[init] can't create new tags table\n");
2248
2249 TRY_EXEC("INSERT INTO data.tags (id, name, synonyms, flags) SELECT id, name, description, flags "
2250 "FROM tmp_tags",
2251 "[init] can't populate tags table from tmp_tags\n");
2252
2253 TRY_EXEC("DROP TABLE tmp_tags", "[init] can't delete table tmp_tags\n");
2254
2255 TRY_EXEC("CREATE UNIQUE INDEX data.tags_name_idx ON tags (name)",
2256 "[init] can't create tags_name_idx on tags table\n");
2257 // clang-format on
2258 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2259
2260 new_version = 3;
2261 }
2262 else if(version == 3)
2263 {
2264 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2265 // clang-format off
2266 // create a temp table to invert all multi_priority
2267 TRY_EXEC("CREATE TEMPORARY TABLE m_prio (id INTEGER, operation VARCHAR(256), prio INTEGER)",
2268 "[init] can't create temporary table for updating `history and style_items'\n");
2269
2270 TRY_EXEC("INSERT INTO m_prio SELECT styleid, operation, MAX(multi_priority)"
2271 " FROM data.style_items GROUP BY styleid, operation",
2272 "[init] can't populate m_prio\n");
2273
2274 // update multi_priority for style items and history
2275 TRY_EXEC("UPDATE data.style_items SET multi_priority = "
2276 "(SELECT prio FROM m_prio "
2277 " WHERE data.style_items.operation = operation AND data.style_items.styleid = id)"
2278 " - data.style_items.multi_priority",
2279 "[init] can't update multi_priority for style_items\n");
2280
2281 TRY_EXEC("DROP TABLE m_prio", "[init] can't drop table `m_prio' from database\n");
2282 // clang-format on
2283 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2284
2285 new_version = 4;
2286 }
2287 else if(version == 4)
2288 {
2289 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2290 // clang-format off
2291 // remove iop_order from style_item table
2292 TRY_EXEC("ALTER TABLE data.style_items RENAME TO s",
2293 "[init] can't rename style_items to s\n");
2294 TRY_EXEC("CREATE TABLE data.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
2295 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2296 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2297 "[init] can't create style_items table'\n");
2298 TRY_EXEC("INSERT INTO data.style_items SELECT styleid, num, module, operation, op_params, enabled, "
2299 " blendop_params, blendop_version, multi_priority, multi_name "
2300 "FROM s",
2301 "[init] can't populate style_items table'\n");
2302 TRY_EXEC("DROP TABLE s",
2303 "[init] can't drop table s'\n");
2304 // clang-format on
2305 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2306
2307 new_version = 5;
2308 }
2309 else if(version == 5)
2310 {
2311 sqlite3_exec(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
2312 // clang-format of
2313 // make style.id a PRIMARY KEY and add iop_list
2314 TRY_EXEC("ALTER TABLE data.styles RENAME TO s",
2315 "[init] can't rename styles to s\n");
2316 TRY_EXEC("CREATE TABLE data.styles (id INTEGER PRIMARY KEY, name VARCHAR, description VARCHAR, iop_list VARCHAR)",
2317 "[init] can't create styles table\n");
2318 TRY_EXEC("INSERT INTO data.styles SELECT id, name, description, NULL FROM s",
2319 "[init] can't populate styles table\n");
2320 TRY_EXEC("DROP TABLE s",
2321 "[init] can't drop table s\n");
2322
2323 TRY_EXEC("CREATE INDEX IF NOT EXISTS data.styles_name_index ON styles (name)",
2324 "[init] can't create styles_nmae_index\n");
2325
2326 // make style_items.styleid index
2327
2328 TRY_EXEC("CREATE INDEX IF NOT EXISTS data.style_items_styleid_index ON style_items (styleid)",
2329 "[init] can't create style_items_styleid_index\n");
2330 // clang-format on
2331 sqlite3_exec(db->handle, "COMMIT", NULL, NULL, NULL);
2332
2333 new_version = 6;
2334 }
2335 else if(version == 6)
2336 {
2337 // clang-format off
2338 TRY_EXEC("CREATE TABLE data.locations "
2339 "(tagid INTEGER PRIMARY KEY, type INTEGER, longitude REAL, latitude REAL, "
2340 "delta1 REAL, delta2 REAL, FOREIGN KEY(tagid) REFERENCES tags(id))",
2341 "[init] can't create new locations table\n");
2342 // clang-format on
2343 new_version = 7;
2344 }
2345 else if(version == 7)
2346 {
2347 // clang-format off
2348 TRY_EXEC("ALTER TABLE data.locations ADD COLUMN ratio FLOAT DEFAULT 1",
2349 "[init] can't add column `ratio' column to locations table\n");
2350 // clang-format on
2351 new_version = 8;
2352 }
2353 else if(version == 8)
2354 {
2355 // clang-format off
2356 TRY_EXEC("ALTER TABLE data.locations ADD COLUMN polygons BLOB",
2357 "[init] can't add column `polygons' column to locations table\n");
2358 // clang-format on
2359 new_version = 9;
2360 }
2361 else
2362 new_version = version; // should be the fallback so that calling code sees that we are in an infinite loop
2363
2364 // write the new version to db
2365 // clang-format off¨
2366 sqlite3_prepare_v2(db->handle, "INSERT OR REPLACE INTO data.db_info (key, value) VALUES ('version', ?1)", -1, &stmt,
2367 NULL);
2368 // clang-format on
2369 sqlite3_bind_int(stmt, 1, new_version);
2370 sqlite3_step(stmt);
2371 sqlite3_finalize(stmt);
2372
2373 return new_version;
2374}
2375
2376#undef FINALIZE
2377
2378#undef TRY_EXEC
2379#undef TRY_STEP
2380#undef TRY_PREPARE
2381
2382/* upgrade library db from 'version' to CURRENT_DATABASE_VERSION_LIBRARY. don't touch this function but
2383 * _upgrade_library_schema_step() instead. */
2384static gboolean _upgrade_library_schema(dt_database_t *db, int version)
2385{
2386 while(version < CURRENT_DATABASE_VERSION_LIBRARY)
2387 {
2388 const int new_version = _upgrade_library_schema_step(db, version);
2389 if(new_version == version)
2390 return FALSE; // we don't know how to upgrade this db. probably a bug in _upgrade_library_schema_step
2391 else
2392 version = new_version;
2393 }
2394 return TRUE;
2395}
2396
2397/* upgrade data db from 'version' to CURRENT_DATABASE_VERSION_DATA. don't touch this function but
2398 * _upgrade_data_schema_step() instead. */
2399static gboolean _upgrade_data_schema(dt_database_t *db, int version)
2400{
2401 while(version < CURRENT_DATABASE_VERSION_DATA)
2402 {
2403 const int new_version = _upgrade_data_schema_step(db, version);
2404 if(new_version == version)
2405 return FALSE; // we don't know how to upgrade this db. probably a bug in _upgrade_data_schema_step
2406 else
2407 version = new_version;
2408 }
2409 return TRUE;
2410}
2411
2412/* create the current database schema and set the version in db_info accordingly */
2413static void _create_library_schema(dt_database_t *db)
2414{
2415 sqlite3_stmt *stmt;
2417 // clang-format off
2418 sqlite3_exec(db->handle, "CREATE TABLE main.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)", NULL,
2419 NULL, NULL);
2420 sqlite3_prepare_v2(
2421 db->handle, "INSERT OR REPLACE INTO main.db_info (key, value) VALUES ('version', ?1)", -1, &stmt, NULL);
2422 // clang-format on
2423 sqlite3_bind_int(stmt, 1, CURRENT_DATABASE_VERSION_LIBRARY);
2424 sqlite3_step(stmt);
2425 sqlite3_finalize(stmt);
2427 // clang-format off
2428 sqlite3_exec(db->handle,
2429 "CREATE TABLE main.film_rolls "
2430 "(id INTEGER PRIMARY KEY, access_timestamp INTEGER, "
2431 // "folder VARCHAR(1024), external_drive VARCHAR(1024))", //
2432 // FIXME: make sure to bump CURRENT_DATABASE_VERSION_LIBRARY and add a
2433 // case to _upgrade_library_schema_step when adding this!
2434 "folder VARCHAR(1024) NOT NULL)",
2435 NULL, NULL, NULL);
2436 sqlite3_exec(db->handle, "CREATE INDEX main.film_rolls_folder_index ON film_rolls (folder)", NULL, NULL, NULL);
2438 sqlite3_exec(
2439 db->handle,
2440 "CREATE TABLE main.images (id INTEGER PRIMARY KEY AUTOINCREMENT, group_id INTEGER, film_id INTEGER, "
2441 "width INTEGER, height INTEGER, filename VARCHAR, maker VARCHAR, model VARCHAR, "
2442 "lens VARCHAR, exposure REAL, aperture REAL, iso REAL, focal_length REAL, "
2443 "focus_distance REAL, datetime_taken INTEGER, flags INTEGER, "
2444 "output_width INTEGER, output_height INTEGER, crop REAL, "
2445 "raw_parameters INTEGER, raw_denoise_threshold REAL, "
2446 "raw_auto_bright_threshold REAL, raw_black INTEGER, raw_maximum INTEGER, "
2447 "license VARCHAR, sha1sum CHAR(40), "
2448 "orientation INTEGER, histogram BLOB, lightmap BLOB, longitude REAL, "
2449 "latitude REAL, altitude REAL, color_matrix BLOB, colorspace INTEGER, version INTEGER, "
2450 "max_version INTEGER, write_timestamp INTEGER, history_end INTEGER, position INTEGER, "
2451 "aspect_ratio REAL, exposure_bias REAL, "
2452 "import_timestamp INTEGER DEFAULT -1, change_timestamp INTEGER DEFAULT -1, "
2453 "export_timestamp INTEGER DEFAULT -1, print_timestamp INTEGER DEFAULT -1, "
2454 "FOREIGN KEY(film_id) REFERENCES film_rolls(id) ON DELETE CASCADE ON UPDATE CASCADE, "
2455 "FOREIGN KEY(group_id) REFERENCES images(id) ON DELETE RESTRICT ON UPDATE CASCADE)",
2456 NULL, NULL, NULL);
2457 sqlite3_exec(db->handle, "CREATE INDEX main.images_group_id_index ON images (group_id, id)", NULL, NULL, NULL);
2458 sqlite3_exec(db->handle, "CREATE INDEX main.images_film_id_index ON images (film_id, filename)", NULL, NULL, NULL);
2459 sqlite3_exec(db->handle, "CREATE INDEX main.images_filename_index ON images (filename, version)", NULL, NULL, NULL);
2460 sqlite3_exec(db->handle, "CREATE INDEX main.image_position_index ON images (position)", NULL, NULL, NULL);
2461 sqlite3_exec(db->handle, "CREATE INDEX main.images_datetime_taken_nc ON images (datetime_taken)", NULL, NULL, NULL);
2462
2464 sqlite3_exec(db->handle, "CREATE TABLE main.selected_images (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2466 sqlite3_exec(
2467 db->handle,
2468 "CREATE TABLE main.history (imgid INTEGER, num INTEGER, module INTEGER, "
2469 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2470 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256), "
2471 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2472 NULL, NULL, NULL);
2473 sqlite3_exec(db->handle, "CREATE INDEX main.history_imgid_op_index ON history (imgid, operation)", NULL, NULL, NULL);
2474 sqlite3_exec(db->handle, "CREATE INDEX main.history_imgid_num_index ON history (imgid, num DESC)", NULL, NULL, NULL);
2476 sqlite3_exec(db->handle,
2477 "CREATE TABLE main.masks_history (imgid INTEGER, num INTEGER, formid INTEGER, form INTEGER, name VARCHAR(256), "
2478 "version INTEGER, points BLOB, points_count INTEGER, source BLOB, "
2479 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2480 NULL, NULL, NULL);
2481
2482 sqlite3_exec(db->handle,
2483 "CREATE INDEX main.masks_history_imgid_index ON masks_history (imgid, num)",
2484 NULL, NULL, NULL);
2485
2486 sqlite3_exec(db->handle, "CREATE INDEX main.images_latlong_index ON images (latitude DESC, longitude DESC)",
2487 NULL, NULL, NULL);
2488
2490 sqlite3_exec(db->handle, "CREATE TABLE main.tagged_images (imgid INTEGER, tagid INTEGER, position INTEGER, "
2491 "PRIMARY KEY (imgid, tagid),"
2492 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)", NULL, NULL, NULL);
2493 sqlite3_exec(db->handle, "CREATE INDEX main.tagged_images_tagid_index ON tagged_images (tagid)", NULL, NULL, NULL);
2494 sqlite3_exec(db->handle, "CREATE INDEX main.tagged_images_position_index ON tagged_images (position)", NULL, NULL, NULL);
2496 sqlite3_exec(db->handle, "CREATE TABLE main.color_labels (imgid INTEGER, color INTEGER)", NULL, NULL, NULL);
2497 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX main.color_labels_idx ON color_labels (imgid, color)", NULL, NULL,
2498 NULL);
2500 sqlite3_exec(db->handle, "CREATE TABLE main.meta_data (id INTEGER, key INTEGER, value VARCHAR)", NULL, NULL, NULL);
2501 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX main.metadata_index ON meta_data (id, key, value)", NULL, NULL, NULL);
2502
2503 sqlite3_exec(db->handle, "CREATE INDEX main.metadata_index_key ON meta_data (key)", NULL, NULL, NULL);
2504 sqlite3_exec(db->handle, "CREATE TABLE main.module_order (imgid INTEGER PRIMARY KEY, version INTEGER, iop_list VARCHAR)",
2505 NULL, NULL, NULL);
2506 sqlite3_exec(db->handle, "CREATE TABLE main.history_hash (imgid INTEGER PRIMARY KEY, "
2507 "basic_hash BLOB, auto_hash BLOB, current_hash BLOB, mipmap_hash BLOB, "
2508 "FOREIGN KEY(imgid) REFERENCES images(id) ON UPDATE CASCADE ON DELETE CASCADE)",
2509 NULL, NULL, NULL);
2510
2511 // v34
2512 sqlite3_exec(db->handle, "CREATE INDEX main.images_datetime_taken_nc ON images (datetime_taken COLLATE NOCASE)",
2513 NULL, NULL, NULL);
2514 sqlite3_exec(db->handle, "CREATE INDEX main.metadata_index_key ON meta_data (key)", NULL, NULL, NULL);
2515 // clang-format on
2516}
2517
2518/* create the current database schema and set the version in db_info accordingly */
2519static void _create_data_schema(dt_database_t *db)
2520{
2521 sqlite3_stmt *stmt;
2522 // clang-format off
2524 sqlite3_exec(db->handle, "CREATE TABLE data.db_info (key VARCHAR PRIMARY KEY, value VARCHAR)", NULL,
2525 NULL, NULL);
2526 sqlite3_prepare_v2(
2527 db->handle, "INSERT OR REPLACE INTO data.db_info (key, value) VALUES ('version', ?1)", -1, &stmt, NULL);
2528 sqlite3_bind_int(stmt, 1, CURRENT_DATABASE_VERSION_DATA);
2529 sqlite3_step(stmt);
2530 sqlite3_finalize(stmt);
2532 sqlite3_exec(db->handle, "CREATE TABLE data.tags (id INTEGER PRIMARY KEY, name VARCHAR, "
2533 "synonyms VARCHAR, flags INTEGER)", NULL, NULL, NULL);
2534 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX data.tags_name_idx ON tags (name)", NULL, NULL, NULL);
2536 sqlite3_exec(db->handle, "CREATE TABLE data.styles (id INTEGER PRIMARY KEY, name VARCHAR, description VARCHAR, iop_list VARCHAR)",
2537 NULL, NULL, NULL);
2538 sqlite3_exec(db->handle, "CREATE INDEX data.styles_name_index ON styles (name)", NULL, NULL, NULL);
2540 sqlite3_exec(
2541 db->handle,
2542 "CREATE TABLE data.style_items (styleid INTEGER, num INTEGER, module INTEGER, "
2543 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2544 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2545 NULL, NULL, NULL);
2546 sqlite3_exec(
2547 db->handle,
2548 "CREATE INDEX IF NOT EXISTS data.style_items_styleid_index ON style_items (styleid)",
2549 NULL, NULL, NULL);
2551 sqlite3_exec(db->handle, "CREATE TABLE data.presets (name VARCHAR, description VARCHAR, operation "
2552 "VARCHAR, op_version INTEGER, op_params BLOB, "
2553 "enabled INTEGER, blendop_params BLOB, blendop_version INTEGER, "
2554 "multi_priority INTEGER, multi_name VARCHAR(256), "
2555 "model VARCHAR, maker VARCHAR, lens VARCHAR, iso_min REAL, iso_max REAL, "
2556 "exposure_min REAL, exposure_max REAL, "
2557 "aperture_min REAL, aperture_max REAL, focal_length_min REAL, "
2558 "focal_length_max REAL, writeprotect INTEGER, "
2559 "autoapply INTEGER, filter INTEGER, def INTEGER, format INTEGER)",
2560 NULL, NULL, NULL);
2561 sqlite3_exec(db->handle, "CREATE UNIQUE INDEX data.presets_idx ON presets (name, operation, op_version)",
2562 NULL, NULL, NULL);
2564 sqlite3_exec(db->handle, "CREATE TABLE data.locations (tagid INTEGER PRIMARY KEY, "
2565 "type INTEGER, longitude REAL, latitude REAL, delta1 REAL, delta2 REAL, ratio FLOAT, polygons BLOB, "
2566 "FOREIGN KEY(tagid) REFERENCES tags(id))", NULL, NULL, NULL);
2567 // clang-format on
2568}
2569
2570// create the in-memory tables
2571// temporary stuff for some ops, need this for some reason with newer sqlite3:
2572static void _create_memory_schema(dt_database_t *db)
2573{
2574 // clang-format off
2575 sqlite3_exec(db->handle, "CREATE TABLE memory.color_labels_temp (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2576 sqlite3_exec(
2577 db->handle,
2578 "CREATE TABLE memory.collected_images (rowid INTEGER PRIMARY KEY AUTOINCREMENT, imgid INTEGER)", NULL,
2579 NULL, NULL);
2580 sqlite3_exec(
2581 db->handle,
2582 "CREATE TABLE memory.collected_backup (rowid INTEGER PRIMARY KEY AUTOINCREMENT, imgid INTEGER)", NULL,
2583 NULL, NULL); sqlite3_exec(db->handle, "CREATE TABLE memory.selected_backup (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2584 sqlite3_exec(db->handle, "CREATE TABLE memory.selected_backup (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2585 sqlite3_exec(db->handle, "CREATE TABLE memory.tmp_selection (imgid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2586 sqlite3_exec(db->handle, "CREATE TABLE memory.taglist "
2587 "(tmpid INTEGER PRIMARY KEY, id INTEGER UNIQUE ON CONFLICT IGNORE, "
2588 "count INTEGER DEFAULT 0, count2 INTEGER DEFAULT 0)",
2589 NULL, NULL, NULL);
2590 sqlite3_exec(db->handle, "CREATE TABLE memory.similar_tags (tagid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2591 sqlite3_exec(db->handle, "CREATE TABLE memory.darktable_tags (tagid INTEGER PRIMARY KEY)", NULL, NULL, NULL);
2592 sqlite3_exec(
2593 db->handle,
2594 "CREATE TABLE memory.undo_history (id INTEGER, imgid INTEGER, num INTEGER, module INTEGER, "
2595 "operation VARCHAR(256), op_params BLOB, enabled INTEGER, "
2596 "blendop_params BLOB, blendop_version INTEGER, multi_priority INTEGER, multi_name VARCHAR(256))",
2597 NULL, NULL, NULL);
2598 sqlite3_exec(
2599 db->handle,
2600 "CREATE TABLE memory.undo_masks_history (id INTEGER, imgid INTEGER, num INTEGER, formid INTEGER,"
2601 " form INTEGER, name VARCHAR(256), version INTEGER, points BLOB, points_count INTEGER, source BLOB)",
2602 NULL, NULL, NULL);
2603 sqlite3_exec(
2604 db->handle,
2605 "CREATE TABLE memory.undo_module_order (id INTEGER, imgid INTEGER, version INTEGER, iop_list VARCHAR)",
2606 NULL, NULL, NULL);
2607 sqlite3_exec(db->handle,
2608 "CREATE TABLE memory.darktable_iop_names (operation VARCHAR(256) PRIMARY KEY, name VARCHAR(256))",
2609 NULL, NULL, NULL);
2610 sqlite3_exec(db->handle,
2611 "CREATE TABLE memory.film_folder (id INTEGER PRIMARY KEY, status INTEGER)",
2612 NULL, NULL, NULL);
2613 // clang-format on
2614
2615 /* Staging area for "undo remove from library": one twin per table a removed image owns
2616 * rows in. They are created as `CREATE TABLE ... AS SELECT` over the live tables rather
2617 * than declared column by column, so their shape follows whatever main holds and a schema
2618 * migration reaches database/removed_image_repository.c without an edit there -- that file
2619 * names its columns from PRAGMA table_info() on these very tables. `WHERE 0` copies the
2620 * shape and no row; the two leading columns it prepends are the key the repository works
2621 * on. This runs after the library schema is created or upgraded, which is what makes the
2622 * SELECT resolvable. */
2623 static const char *const removed_twins[]
2624 = { "film_rolls", "images", "history", "masks_history", "module_order",
2625 "tagged_images", "color_labels", "meta_data", "history_hash" };
2626
2627 for(gsize i = 0; i < G_N_ELEMENTS(removed_twins); i++)
2628 {
2629 gchar *query = g_strdup_printf("CREATE TABLE memory.removed_%s AS"
2630 " SELECT 0 AS snap_id, 0 AS undo_imgid, * FROM main.%s WHERE 0",
2631 removed_twins[i], removed_twins[i]);
2632 sqlite3_exec(db->handle, query, NULL, NULL, NULL);
2633 g_free(query);
2634 }
2635
2636 /* Group membership is the one thing a removal changes outside the removed image's own
2637 * rows: dropping a group's leader hands the group to a survivor, rewriting a group_id that
2638 * belongs to an image nobody asked to remove. */
2639 sqlite3_exec(db->handle,
2640 "CREATE TABLE memory.removed_groups (snap_id INTEGER, undo_imgid INTEGER, id INTEGER, group_id INTEGER)",
2641 NULL, NULL, NULL);
2642}
2643
2644static void _sanitize_db(dt_database_t *db)
2645{
2646 sqlite3_stmt *stmt, *innerstmt;
2647 // clang-format off
2648 /* first let's get rid of non-utf8 tags. */
2649 sqlite3_prepare_v2(db->handle, "SELECT id, name FROM data.tags", -1, &stmt, NULL);
2650 sqlite3_prepare_v2(db->handle, "UPDATE data.tags SET name = ?1 WHERE id = ?2", -1, &innerstmt, NULL);
2651 // clang-format on
2652 while(sqlite3_step(stmt) == SQLITE_ROW)
2653 {
2654 const int id = sqlite3_column_int(stmt, 0);
2655 const char *tag = (const char *)sqlite3_column_text(stmt, 1);
2656
2657 if(!g_utf8_validate(tag, -1, NULL))
2658 {
2659 gchar *new_tag = dt_util_foo_to_utf8(tag);
2660 fprintf(stderr, "[init]: tag `%s' is not valid utf8, replacing it with `%s'\n", tag, new_tag);
2661 if(tag)
2662 {
2663 sqlite3_bind_text(innerstmt, 1, new_tag, -1, SQLITE_TRANSIENT);
2664 sqlite3_bind_int(innerstmt, 2, id);
2665 sqlite3_step(innerstmt);
2666 sqlite3_reset(innerstmt);
2667 sqlite3_clear_bindings(innerstmt);
2668 dt_free(new_tag);
2669 }
2670 }
2671 }
2672 sqlite3_finalize(stmt);
2673 sqlite3_finalize(innerstmt);
2674 // clang-format off
2675 // make sure film_roll folders don't end in "/", that will result in empty entries in the collect module
2676 sqlite3_exec(db->handle,
2677 "UPDATE main.film_rolls SET folder = substr(folder, 1, length(folder) - 1) WHERE folder LIKE '%/'",
2678 NULL, NULL, NULL);
2679 // clang-format on
2680}
2681
2682// in library we keep the names of the tags used in tagged_images. however, using that table at runtime results
2683// in some overhead not necessary so instead we just use the used_tags table to update tagged_images on startup
2684#define TRY_EXEC(_query, _message) \
2685 do \
2686 { \
2687 if(sqlite3_exec(db->handle, _query, NULL, NULL, NULL) != SQLITE_OK) \
2688 { \
2689 fprintf(stderr, _message); \
2690 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2691 FINALIZE; \
2692 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2693 return FALSE; \
2694 } \
2695 } while(0)
2696
2697#define TRY_STEP(_stmt, _expected, _message) \
2698 do \
2699 { \
2700 if(sqlite3_step(_stmt) != _expected) \
2701 { \
2702 fprintf(stderr, _message); \
2703 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2704 FINALIZE; \
2705 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2706 return FALSE; \
2707 } \
2708 } while(0)
2709
2710#define TRY_PREPARE(_stmt, _query, _message) \
2711 do \
2712 { \
2713 if(sqlite3_prepare_v2(db->handle, _query, -1, &_stmt, NULL) != SQLITE_OK) \
2714 { \
2715 fprintf(stderr, _message); \
2716 fprintf(stderr, "[init] %s\n", sqlite3_errmsg(db->handle)); \
2717 FINALIZE; \
2718 sqlite3_exec(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL); \
2719 return FALSE; \
2720 } \
2721 } while(0)
2722
2723#define FINALIZE \
2724 do \
2725 { \
2726 sqlite3_finalize(stmt); stmt = NULL; /* NULL so that finalize becomes a NOP */ \
2727 } while(0)
2728
2729#undef TRY_EXEC
2730#undef TRY_STEP
2731#undef TRY_PREPARE
2732#undef FINALIZE
2733
2734void dt_database_take_error(dt_database_error_t *error)
2735{
2736 dt_database_t *const db = _db;
2737 if(IS_NULL_PTR(db)) return;
2738 if(IS_NULL_PTR(db) || IS_NULL_PTR(error)) return;
2739
2740 // Ownership of the strings moves to the caller; the database keeps no pending error.
2741 error->lock_acquired = db->lock_acquired;
2742 error->other_pid = db->error_other_pid;
2743 error->message = db->error_message;
2744 error->dbfilename = db->error_dbfilename;
2745
2746 db->error_other_pid = 0;
2747 db->error_message = NULL;
2748 db->error_dbfilename = NULL;
2749}
2750
2751void dt_database_error_free(dt_database_error_t *error)
2752{
2753 if(IS_NULL_PTR(error)) return;
2754 dt_free(error->message);
2755 dt_free(error->dbfilename);
2756 error->message = NULL;
2757 error->dbfilename = NULL;
2758}
2759
2760int dt_database_delete_lock_files(const char *dbfilename)
2761{
2762 if(IS_NULL_PTR(dbfilename)) return -1;
2763
2764 char lck_pathname[1024];
2765 snprintf(lck_pathname, sizeof(lck_pathname), "%s.lock", dbfilename);
2766 char *lck_dirname = g_strdup(lck_pathname);
2767 char *slash_pos = g_strrstr(lck_dirname, "/");
2768 if(!IS_NULL_PTR(slash_pos)) *slash_pos = '\0';
2769
2770 int status = 0;
2771 const char *names[] = { "/data.db.lock", "/library.db.lock" };
2772 for(size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++)
2773 {
2774 char *lck_filename = g_strconcat(lck_dirname, names[i], NULL);
2775 if(g_access(lck_filename, F_OK) != -1) status += remove(lck_filename);
2776 dt_free(lck_filename);
2777 }
2778
2779 dt_free(lck_dirname);
2780 return status;
2781}
2782
2783
2784static gboolean pid_is_alive(int pid)
2785{
2786 gboolean pid_is_alive;
2787
2788#ifdef _WIN32
2789 pid_is_alive = FALSE;
2790 HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION, FALSE, pid);
2791 if(h)
2792 {
2793 wchar_t wfilename[MAX_PATH];
2794 long unsigned int n_filename = sizeof(wfilename);
2795 int ret = QueryFullProcessImageNameW(h, 0, wfilename, &n_filename);
2796 char *filename = g_utf16_to_utf8(wfilename, -1, NULL, NULL, NULL);
2797 if(ret && n_filename > 0 && filename && g_str_has_suffix(filename, "ansel.exe"))
2798 pid_is_alive = TRUE;
2799 dt_free(filename);
2800 CloseHandle(h);
2801 }
2802#else
2803 pid_is_alive = !((kill(pid, 0) == -1) && errno == ESRCH);
2804
2805#ifdef __linux__
2806 // If this is Linux, we can query /proc to see if the pid is
2807 // actually a darktable instance.
2808 if(pid_is_alive)
2809 {
2810 gchar *contents;
2811 gsize length;
2812 gchar filename[64];
2813 snprintf(filename, sizeof(filename), "/proc/%d/cmdline", pid);
2814
2815 if(g_file_get_contents("", &contents, &length, NULL))
2816 {
2817 if(strstr(contents, "ansel") == NULL)
2818 {
2819 pid_is_alive = FALSE;
2820 }
2821 dt_free(contents);
2822 }
2823 }
2824#endif
2825
2826#endif
2827
2828 return pid_is_alive;
2829}
2830
2831static gboolean _lock_single_database(dt_database_t *db, const char *dbfilename, char **lockfile)
2832{
2833 gboolean lock_acquired = FALSE;
2834 mode_t old_mode;
2835 int lock_tries = 0;
2836 gchar *pid = g_strdup_printf("%d", getpid());
2837
2838 if(!g_strcmp0(dbfilename, ":memory:"))
2839 {
2840 lock_acquired = TRUE;
2841 }
2842 else
2843 {
2844 *lockfile = g_strconcat(dbfilename, ".lock", NULL);
2845lock_again:
2846 lock_tries++;
2847 old_mode = umask(0);
2848 int fd = g_open(*lockfile, O_RDWR | O_CREAT | O_EXCL, 0666);
2849 umask(old_mode);
2850
2851 if(fd != -1) // the lockfile was successfully created - write our PID into it
2852 {
2853 if(write(fd, pid, strlen(pid) + 1) > -1) lock_acquired = TRUE;
2854 close(fd);
2855 // we created the file; if the PID could not be written it is ours to remove
2856 if(!lock_acquired) g_unlink(*lockfile);
2857 }
2858 else // the lockfile already exists - see if it's a stale one left over from a crashed instance
2859 {
2860 char buf[64];
2861 memset(buf, 0, sizeof(buf));
2862 fd = g_open(*lockfile, O_RDWR | O_CREAT, 0666);
2863
2864 if(fd == -1)
2865 {
2866 // failed to open the lock file
2867 const int err = errno;
2868 fprintf(stderr, "[init] error opening the database lock file for reading: %s\n", strerror(err));
2869 db->error_message = g_strdup_printf(_("error opening the database lock file for reading: %s"), strerror(err));
2870 }
2871 else
2872 {
2873 // successfully opened the lock file, read the PID
2874 const int bytes_read = read(fd, buf, sizeof(buf) - 1);
2875
2876 if(bytes_read > 0)
2877 {
2878 // lock file contains a PID
2879 db->error_other_pid = atoi(buf);
2880
2881 if(!pid_is_alive(db->error_other_pid))
2882 {
2883 // stale lock file - close fd before unlinking (required on Windows)
2884 close(fd);
2885 g_unlink(*lockfile);
2886
2887 if(lock_tries < 5)
2888 {
2889 goto lock_again;
2890 }
2891 else
2892 {
2893 fprintf(stderr, "[init] tried several times to acquire the database lock, giving up\n");
2894 db->error_message = g_strdup_printf(_("tried several times to acquire the database lock, giving up"));
2895 }
2896 }
2897 else
2898 {
2899 // lock file contains a live PID
2900 close(fd);
2901 fprintf(stderr, "[init] the database lock file contains a pid that seems to be alive in your system: %d\n",
2902 db->error_other_pid);
2903 db->error_message = g_strdup_printf(_("the database lock file contains a pid that seems to be alive in your system: %d"),
2904 db->error_other_pid);
2905 }
2906 }
2907 else
2908 {
2909 // lock file is empty or unreadable
2910 close(fd);
2911 fprintf(stderr, "[init] the database lock file seems to be empty\n");
2912 db->error_message = g_strdup_printf(_("the database lock file seems to be empty"));
2913 }
2914 }
2915 }
2916 }
2917
2918 dt_free(pid);
2919
2920 if(db->error_message)
2921 db->error_dbfilename = g_strdup(dbfilename);
2922
2923 /* The stored path must outlive this call ONLY when this process owns the file:
2924 * _database_free() unlinks whatever these fields point at, unconditionally, and a
2925 * lock-failed database still travels -- dt_database_open() publishes it so the caller
2926 * can read the error, and the retry loop closes it before trying again. Left set on
2927 * failure, the path points at the OTHER instance's live lock file, and that close
2928 * deletes it -- after which the retry "succeeds" and two instances write the same
2929 * database. (dt_free NULLs the field.) */
2930 if(!lock_acquired) dt_free(*lockfile);
2931
2932 return lock_acquired;
2933}
2934
2935static gboolean _lock_databases(dt_database_t *db)
2936{
2937 if(!_lock_single_database(db, db->dbfilename_data, &db->lockfile_data))
2938 return FALSE;
2939 if(!_lock_single_database(db, db->dbfilename_library, &db->lockfile_library))
2940 {
2941 // unlock data.db to not leave a stale lock file around -- and drop the path, so
2942 // _database_free() does not unlink whatever lives at that name by the time it runs
2943 if(db->lockfile_data)
2944 {
2945 g_unlink(db->lockfile_data);
2946 dt_free(db->lockfile_data);
2947 }
2948 return FALSE;
2949 }
2950 return TRUE;
2951}
2952
2953/* Forward declaration: this asks a question, and the asking machinery is defined further
2954 * down next to the handler it goes through. */
2955static dt_database_response_t _ask_upgrade(const char *dbfilename);
2956
2957void ask_for_upgrade(const gchar *dbname, const gboolean has_gui)
2958{
2959 // if there's no gui just leave
2960 if(!has_gui)
2961 {
2962 fprintf(stderr, "[init] database `%s' is out-of-date. aborting.\n", dbname);
2963 exit(1);
2964 }
2965
2966 // the database has to be upgraded, let's ask user. What that looks like -- the wording,
2967 // the warning that it may take a while on a large library, the button labels -- belongs
2968 // to whoever renders it; this file only states that the schema is out of date.
2969 if(_ask_upgrade(dbname) != DT_DATABASE_RESPONSE_PROCEED)
2970 {
2971 fprintf(stderr, "[init] we shall not update the database, aborting.\n");
2972 exit(1);
2973 }
2974}
2975
2976void dt_database_backup(const char *filename)
2977{
2978 char *version = g_strdup(darktable_package_version);
2979 int k = 0;
2980 // get plain version (no commit id)
2981 while(version[k])
2982 {
2983 if((version[k] < '0' || version[k] > '9') && (version[k] != '.'))
2984 {
2985 version[k] = '\0';
2986 break;
2987 }
2988 k++;
2989 }
2990
2991 gchar *backup = g_strdup_printf("%s-pre-%s", filename, version);
2992
2993 GError *gerror = NULL;
2994 if(!g_file_test(backup, G_FILE_TEST_EXISTS))
2995 {
2996 GFile *src = g_file_new_for_path(filename);
2997 GFile *dest = g_file_new_for_path(backup);
2998 gboolean copy_status = TRUE;
2999 if(g_file_test(filename, G_FILE_TEST_EXISTS))
3000 {
3001 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3002 if(copy_status) copy_status = g_chmod(backup, S_IRUSR) == 0;
3003 }
3004 else
3005 {
3006 // there is nothing to backup, create an empty file to prevent further backup attempts
3007 int fd = g_open(backup, O_CREAT, S_IRUSR);
3008 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3009 }
3010 if(!copy_status) fprintf(stderr, "[backup failed] %s -> %s\n", filename, backup);
3011
3012 g_object_unref(src);
3013 g_object_unref(dest);
3014 }
3015
3016 dt_free(version);
3017 dt_free(backup);
3018}
3019
3020int _get_pragma_int_val(sqlite3 *db, const char* pragma)
3021{
3022 gchar* query= g_strdup_printf("PRAGMA %s", pragma);
3023 int val = -1;
3024 sqlite3_stmt *stmt;
3025 const int rc = sqlite3_prepare_v2(db, query,-1, &stmt, NULL);
3026 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3027 {
3028 val = sqlite3_column_int(stmt, 0);
3029 }
3030 sqlite3_finalize(stmt);
3031 dt_free(query);
3032
3033 return val;
3034}
3035
3036gchar* _get_pragma_string_val(sqlite3 *db, const char* pragma)
3037{
3038 gchar* query= g_strdup_printf("PRAGMA %s", pragma);
3039 sqlite3_stmt *stmt;
3040 gchar* val = NULL;
3041 const int rc = sqlite3_prepare_v2(db, query,-1, &stmt, NULL);
3042 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3043 {
3044 val = g_strdup((const char *)sqlite3_column_text(stmt, 0));
3045 while(sqlite3_step(stmt) == SQLITE_ROW)
3046 {
3047 gchar* cur_val = g_strdup((const char *)sqlite3_column_text(stmt, 0));
3048 gchar* tmp_val = g_strdup(val);
3049 dt_free(val);
3050 val = g_strconcat(tmp_val, "\n", cur_val, NULL);
3051 dt_free(cur_val);
3052 dt_free(tmp_val);
3053 }
3054 }
3055 sqlite3_finalize(stmt);
3056 dt_free(query);
3057 return val;
3058}
3059
3060static dt_database_prompt_handler_t _prompt_handler = NULL;
3061
3066
3067/* Ask, or answer CLOSE if there is nobody to ask. Deliberately not a fallback that guesses:
3068 * deleting or restoring a database is not something to do on nobody's authority, and
3069 * neither is spending a minute vacuuming one. */
3070static dt_database_response_t _ask_user(const dt_database_prompt_context_t *context)
3071{
3072 if(IS_NULL_PTR(_prompt_handler))
3073 {
3074 fprintf(stderr, "[init] cannot ask the user about `%s': no prompt handler registered\n",
3075 context->dbfilename);
3076 return DT_DATABASE_RESPONSE_CLOSE;
3077 }
3078
3079 return _prompt_handler(context);
3080}
3081
3082/* The questions that used to be asked with gtk_dialog calls written inline in this file --
3083 * the reason `widgets/dialog.h` was included by the SQL layer at all. */
3084
3085static dt_database_response_t _ask_corrupted(const char *dbfilename, const char *quick_check,
3086 const gboolean snapshot_available)
3087{
3088 const dt_database_prompt_context_t context = { .prompt = DT_DATABASE_PROMPT_CORRUPTED,
3089 .dbfilename = dbfilename,
3090 .quick_check = quick_check,
3091 .snapshot_available = snapshot_available };
3092 return _ask_user(&context);
3093}
3094
3095static dt_database_response_t _ask_readonly(const char *dbfilename)
3096{
3097 const dt_database_prompt_context_t context = { .prompt = DT_DATABASE_PROMPT_READONLY,
3098 .dbfilename = dbfilename };
3099 return _ask_user(&context);
3100}
3101
3102static dt_database_response_t _ask_upgrade(const char *dbfilename)
3103{
3104 const dt_database_prompt_context_t context = { .prompt = DT_DATABASE_PROMPT_UPGRADE,
3105 .dbfilename = dbfilename };
3106 return _ask_user(&context);
3107}
3108
3109static dt_database_t *_database_init(const dt_database_params_t *params)
3110{
3111 const char *const alternative = params->alternative;
3112 const gboolean load_data = params->load_data;
3113 const gboolean has_gui = params->has_gui;
3114
3115 /* set the threading mode to Serialized */
3116 sqlite3_config(SQLITE_CONFIG_SERIALIZED);
3117
3118 sqlite3_initialize();
3119
3120 /* Declared before `start:` because the two corrupt-database paths jump back to it, and
3121 * the effective name must survive the retry.
3122 *
3123 * `library` is what the orchestrator configured, EXCEPT when the legacy pre-XDG file
3124 * under $HOME was just migrated -- then it is the name it was migrated to. Reading the
3125 * conf key after the migration used to do this implicitly, and getting it wrong here
3126 * means opening a path that does not exist and silently creating an empty library
3127 * beside the user's real one. */
3128 gchar *migrated_name = NULL;
3129 const char *library = params->library;
3130
3131start:
3132 if(IS_NULL_PTR(alternative))
3133 {
3134 /* migrate default database location to new default. On a corrupt-database retry,
3135 * `library` may still alias the PREVIOUS pass's migrated name -- so that allocation
3136 * must survive until a new name actually replaces it. Freeing it first and then
3137 * passing `library` into the migration reads freed memory. */
3138 gchar *previous_name = migrated_name;
3139 migrated_name = _database_migrate_to_xdg_structure(library);
3140 if(!IS_NULL_PTR(migrated_name))
3141 {
3142 library = migrated_name;
3143 dt_free(previous_name);
3144 }
3145 else
3146 {
3147 migrated_name = previous_name; // keep ownership: `library` may point into it
3148 }
3149 }
3150
3151 /* delete old mipmaps files */
3152 _database_delete_mipmaps_files();
3153
3154 /* lets construct the db filename */
3155 gchar *dbname = NULL;
3156 gchar dbfilename_library[DT_PATH_MAX] = { 0 };
3157 gchar datadir[DT_PATH_MAX] = { 0 };
3158
3159 dt_loc_get_user_config_dir(datadir, sizeof(datadir));
3160
3161 if(IS_NULL_PTR(alternative))
3162 {
3163 /* The configured library name, told to us rather than read from conf here. */
3164 dbname = g_strdup(library);
3165 if(IS_NULL_PTR(dbname))
3166 dt_concat_path_file(dbfilename_library, datadir, "library.db");
3167 else if(!strcmp(dbname, ":memory:"))
3168 g_strlcpy(dbfilename_library, dbname, sizeof(dbfilename_library));
3169 else if(dbname[0] != '/')
3170 dt_concat_path_file(dbfilename_library, datadir, dbname);
3171 else
3172 g_strlcpy(dbfilename_library, dbname, sizeof(dbfilename_library));
3173 }
3174 else
3175 {
3176 g_strlcpy(dbfilename_library, alternative, sizeof(dbfilename_library));
3177
3178 GFile *galternative = g_file_new_for_path(alternative);
3179 dbname = g_file_get_basename(galternative);
3180 g_object_unref(galternative);
3181 }
3182
3183 /* we also need a 2nd db with permanent data like presets, styles and tags */
3184 char dbfilename_data[DT_PATH_MAX] = { 0 };
3185 if(load_data)
3186 dt_concat_path_file(dbfilename_data, datadir, "data.db");
3187 else
3188 snprintf(dbfilename_data, sizeof(dbfilename_data), ":memory:");
3189
3190 /* create database */
3191 dt_database_t *db = (dt_database_t *)g_malloc0(sizeof(dt_database_t));
3192 db->dbfilename_data = g_strdup(dbfilename_data);
3193 db->dbfilename_library = g_strdup(dbfilename_library);
3194
3195 dt_atomic_set_int(&_trxid, 0);
3196 dt_atomic_set_int(&_trx_batch_level, 0);
3197 g_atomic_pointer_set(&_trx_batch_owner, NULL);
3198
3199 /* make sure the folder exists. this might not be the case for new databases */
3200 /* also check if a database backup is needed */
3201 if(g_strcmp0(dbfilename_data, ":memory:"))
3202 {
3203 char *data_path = g_path_get_dirname(dbfilename_data);
3204 g_mkdir_with_parents(data_path, 0750);
3205 dt_free(data_path);
3206 dt_database_backup(dbfilename_data);
3207 }
3208 if(g_strcmp0(dbfilename_library, ":memory:"))
3209 {
3210 char *library_path = g_path_get_dirname(dbfilename_library);
3211 g_mkdir_with_parents(library_path, 0750);
3212 dt_free(library_path);
3213 dt_database_backup(dbfilename_library);
3214 }
3215
3216 _db_print("[init sql] library: %s, data: %s\n", dbfilename_library, dbfilename_data);
3217
3218 /* having more than one instance of darktable using the same database is a bad idea */
3219 /* try to get locks for the databases */
3220 db->lock_acquired = _lock_databases(db);
3221
3222 if(!db->lock_acquired)
3223 {
3224 fprintf(stderr, "[init] database is locked, probably another process is already using it\n");
3225 dt_free(dbname);
3226 dt_free(migrated_name);
3227 return db;
3228 }
3229
3230 /* opening / creating database */
3231 if(sqlite3_open(db->dbfilename_library, &db->handle))
3232 {
3233 fprintf(stderr, "[init] could not find database ");
3234 if(dbname)
3235 fprintf(stderr, "`%s'!\n", dbname);
3236 else
3237 fprintf(stderr, "\n");
3238 fprintf(stderr, "[init] maybe your %s/anselrc is corrupt?\n", datadir);
3239 dt_loc_get_datadir(dbfilename_library, sizeof(dbfilename_library));
3240 fprintf(stderr, "[init] try `cp %s/anselrc %s/anselrc'\n", dbfilename_library, datadir);
3241 sqlite3_close(db->handle);
3242 dt_free(dbname);
3243 dt_free(migrated_name);
3244 dt_free(db->lockfile_data);
3245 dt_free(db->dbfilename_data);
3246 dt_free(db->lockfile_library);
3247 dt_free(db->dbfilename_library);
3248 dt_free(db);
3249 return NULL;
3250 }
3251
3252 _db_print("[sql] Opened database: '%s'\n", db->dbfilename_library);
3253
3254 sqlite3_stmt *stmt;
3255 int rc;
3256
3257 if(sqlite3_db_readonly(db->handle, "main") != 0)
3258 {
3259 fprintf(stderr, "%s database is read only. Abort...\n", db->dbfilename_library);
3260
3261 // Informational: there is nothing the user can decide here, only be told.
3262 _ask_readonly(db->dbfilename_library);
3263 _database_free(db);
3264 dt_free(dbname);
3265 dt_free(migrated_name);
3266 db = NULL;
3267 return NULL;
3268 }
3269
3270 _db_print("[sql] Checked that we can write in database: '%s'\n", db->dbfilename_library);
3271
3272 /* attach a memory database to db connection for use with temporary tables
3273 used during instance life time, which is discarded on exit.
3274 */
3275 sqlite3_exec(db->handle, "attach database ':memory:' as memory", NULL, NULL, NULL);
3276
3277 // attach the data database which contains presets, styles, tags and similar things not tied to single images
3278 gboolean have_data_db = load_data && g_file_test(dbfilename_data, G_FILE_TEST_EXISTS);
3279 rc = sqlite3_prepare_v2(db->handle, "ATTACH DATABASE ?1 AS data", -1, &stmt, NULL);
3280 sqlite3_bind_text(stmt, 1, dbfilename_data, -1, SQLITE_TRANSIENT);
3281 if(rc != SQLITE_OK || sqlite3_step(stmt) != SQLITE_DONE)
3282 {
3283 sqlite3_finalize(stmt);
3284 fprintf(stderr, "[init] database `%s' couldn't be opened. aborting\n", dbfilename_data);
3285 _database_free(db);
3286 db = NULL;
3287 goto error;
3288 }
3289 sqlite3_finalize(stmt);
3290
3291 _db_print("[sql] Opened database: '%s'\n", dbfilename_data);
3292
3293 // some sqlite3 config
3294 sqlite3_exec(db->handle, "PRAGMA synchronous = OFF", NULL, NULL, NULL);
3295 sqlite3_exec(db->handle, "PRAGMA journal_mode = MEMORY", NULL, NULL, NULL);
3296 sqlite3_exec(db->handle, "PRAGMA page_size = 32768", NULL, NULL, NULL);
3297
3298 // WARNING: the foreign_keys pragma must not be used, the integrity of the
3299 // database rely on it.
3300 sqlite3_exec(db->handle, "PRAGMA foreign_keys = ON", NULL, NULL, NULL);
3301
3302 _db_print("[sql] Configured database\n");
3303
3304 /* now that we got functional databases that are locked for us we can make sure that the schema is set up */
3305
3306 // first we update the data database to the latest version so that we can potentially move data from the library
3307 // over when updating that one
3308 if(!have_data_db)
3309 {
3310 _create_data_schema(db); // a brand new db it seems
3311 }
3312 else
3313 {
3314 gchar* data_status = _get_pragma_string_val(db->handle, "data.quick_check");
3315 rc = sqlite3_prepare_v2(db->handle, "select value from data.db_info where key = 'version'", -1, &stmt, NULL);
3316 if(!g_strcmp0(data_status, "ok") && rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3317 {
3318 dt_free(data_status); // status is OK and we don't need to care :)
3319
3320 // compare the version of the db with what is current for this executable
3321 const int db_version = sqlite3_column_int(stmt, 0);
3322 sqlite3_finalize(stmt);
3323 if(db_version < CURRENT_DATABASE_VERSION_DATA)
3324 {
3325 ask_for_upgrade(dbfilename_data, has_gui);
3326
3327 // older: upgrade
3328 if(!_upgrade_data_schema(db, db_version))
3329 {
3330 // we couldn't upgrade the db for some reason. bail out.
3331 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version %d to %d. aborting\n",
3332 dbfilename_data, db_version, CURRENT_DATABASE_VERSION_DATA);
3333 _database_free(db);
3334 db = NULL;
3335 goto error;
3336 }
3337 }
3338 else if(db_version > CURRENT_DATABASE_VERSION_DATA)
3339 {
3340 // newer: bail out
3341 fprintf(stderr, "[init] database version of `%s' is too new for this build of darktable. aborting\n",
3342 dbfilename_data);
3343 _database_free(db);
3344 db = NULL;
3345 goto error;
3346 }
3347 // else: the current version, do nothing
3348 }
3349 else
3350 {
3351 // oh, bad situation. the database is corrupt and can't be read!
3352 // we inform the user here and let him decide what to do: exit or delete and try again.
3353
3354 gchar* quick_check_text = NULL;
3355 if(g_strcmp0(data_status, "ok")) // data_status is not ok
3356 {
3357 quick_check_text = g_strdup_printf(_("quick_check said:\n"
3358 "%s \n"), data_status);
3359 }
3360 else
3361 {
3362 quick_check_text = g_strdup(""); // a trick;
3363 }
3364 dt_free(data_status);
3365
3366 gchar *data_snap = dt_database_get_most_recent_snap(dbfilename_data);
3367
3368 // The user's answer decides whether init aborts, restores or starts fresh, so it has to
3369 // come back as a value. Everything about how it is obtained -- and whether anybody can be
3370 // asked at all -- belongs to whoever registered the handler.
3371 const dt_database_response_t resp
3372 = _ask_corrupted(dbfilename_data, quick_check_text, data_snap != NULL);
3373
3374 dt_free(quick_check_text);
3375
3376 // sqlite3_close() leaves a connection with a live statement open, file included.
3377 sqlite3_finalize(stmt);
3378 _database_free(db);
3379 db = NULL;
3380
3381 if(resp != DT_DATABASE_RESPONSE_RESTORE && resp != DT_DATABASE_RESPONSE_DELETE)
3382 {
3383 fprintf(stderr, "[init] database `%s' is corrupt and can't be opened! either replace it from a backup or "
3384 "delete the file so that darktable can create a new one the next time. aborting\n", dbfilename_data);
3385 dt_free(data_snap);
3386 goto error;
3387 }
3388
3389 //here were sure that response is either accept (restore from snap) or reject (just delete the damaged db)
3390
3391 fprintf(stderr, "[init] deleting `%s' on user request", dbfilename_data);
3392
3393 if(g_unlink(dbfilename_data) == 0)
3394 fprintf(stderr, " ... ok\n");
3395 else
3396 fprintf(stderr, " ... failed\n");
3397
3398 if(resp == DT_DATABASE_RESPONSE_RESTORE && data_snap)
3399 {
3400 fprintf(stderr, "[init] restoring `%s' from `%s'...", dbfilename_data, data_snap);
3401 GError *gerror = NULL;
3402 if(!g_file_test(dbfilename_data, G_FILE_TEST_EXISTS))
3403 {
3404 GFile *src = g_file_new_for_path(data_snap);
3405 GFile *dest = g_file_new_for_path(dbfilename_data);
3406 gboolean copy_status = TRUE;
3407 if(g_file_test(data_snap, G_FILE_TEST_EXISTS))
3408 {
3409 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3410 if(copy_status) copy_status = g_chmod(dbfilename_data, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == 0;
3411 }
3412 else
3413 {
3414 // there is nothing to restore, create an empty file
3415 const int fd = g_open(dbfilename_data, O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
3416 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3417 }
3418 if(copy_status)
3419 fprintf(stderr, " success!\n");
3420 else
3421 fprintf(stderr, " failed!\n");
3422
3423 g_object_unref(src);
3424 g_object_unref(dest);
3425 }
3426 }
3427 dt_free(data_snap);
3428 dt_free(dbname);
3429 goto start;
3430 }
3431 }
3432
3433 _db_print("[sql] Checked database schema and migrated if needed\n");
3434
3435 gchar* libdb_status = _get_pragma_string_val(db->handle, "main.quick_check");
3436 // next we are looking at the library database
3437 // does the db contain the new 'db_info' table?
3438 rc = sqlite3_prepare_v2(db->handle, "select value from main.db_info where key = 'version'", -1, &stmt, NULL);
3439 if(!g_strcmp0(libdb_status, "ok") && rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3440 {
3441 dt_free(libdb_status);//it's ok :)
3442
3443 // compare the version of the db with what is current for this executable
3444 const int db_version = sqlite3_column_int(stmt, 0);
3445 sqlite3_finalize(stmt);
3446
3447 if(db_version < CURRENT_DATABASE_VERSION_LIBRARY)
3448 {
3449 ask_for_upgrade(dbfilename_library, has_gui);
3450
3451 // older: upgrade
3452 if(!_upgrade_library_schema(db, db_version))
3453 {
3454 // we couldn't upgrade the db for some reason. bail out.
3455 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version %d to %d. aborting\n", dbname,
3456 db_version, CURRENT_DATABASE_VERSION_LIBRARY);
3457 _database_free(db);
3458 db = NULL;
3459 goto error;
3460 }
3461 }
3462 else if(db_version > CURRENT_DATABASE_VERSION_LIBRARY)
3463 {
3464 // newer: bail out. it's better than what we did before: delete everything
3465 fprintf(stderr, "[init] database version of `%s' is too new for this build of darktable. aborting\n",
3466 dbname);
3467 _database_free(db);
3468 db = NULL;
3469 goto error;
3470 }
3471 // else: the current version, do nothing
3472 }
3473 else if(g_strcmp0(libdb_status, "ok") || rc == SQLITE_CORRUPT || rc == SQLITE_NOTADB)
3474 {
3475 // oh, bad situation. the database is corrupt and can't be read!
3476 // we inform the user here and let him decide what to do: exit or delete and try again.
3477
3478 gchar* quick_check_text = NULL;
3479 if(g_strcmp0(libdb_status, "ok")) // data_status is not ok
3480 {
3481 quick_check_text = g_strdup_printf(_("quick_check said:\n"
3482 "%s \n"), libdb_status);
3483 }
3484 else
3485 {
3486 quick_check_text = g_strdup(""); // a trick;
3487 }
3488 dt_free(libdb_status);
3489
3490 gchar *data_snap = dt_database_get_most_recent_snap(dbfilename_library);
3491
3492 // The user's answer decides whether init aborts, restores or starts fresh, so it has to
3493 // come back as a value. Everything about how it is obtained -- and whether anybody can be
3494 // asked at all -- belongs to whoever registered the handler.
3495 const dt_database_response_t resp
3496 = _ask_corrupted(dbfilename_library, quick_check_text, data_snap != NULL);
3497
3498 dt_free(quick_check_text);
3499
3500 // sqlite3_close() leaves a connection with a live statement open, file included.
3501 sqlite3_finalize(stmt);
3502 _database_free(db);
3503 db = NULL;
3504
3505 if(resp != DT_DATABASE_RESPONSE_RESTORE && resp != DT_DATABASE_RESPONSE_DELETE)
3506 {
3507 fprintf(stderr, "[init] database `%s' is corrupt and can't be opened! either replace it from a backup or "
3508 "delete the file so that darktable can create a new one the next time. aborting\n", dbfilename_library);
3509 dt_free(data_snap);
3510 goto error;
3511 }
3512
3513 //here were sure that response is either accept (restore from snap) or reject (just delete the damaged db)
3514
3515 fprintf(stderr, "[init] deleting `%s' on user request", dbfilename_library);
3516
3517 if(g_unlink(dbfilename_library) == 0)
3518 fprintf(stderr, " ... ok\n");
3519 else
3520 fprintf(stderr, " ... failed\n");
3521
3522 if(resp == DT_DATABASE_RESPONSE_RESTORE && data_snap)
3523 {
3524 fprintf(stderr, "[init] restoring `%s' from `%s'...", dbfilename_library, data_snap);
3525 GError *gerror = NULL;
3526 if(!g_file_test(dbfilename_library, G_FILE_TEST_EXISTS))
3527 {
3528 GFile *src = g_file_new_for_path(data_snap);
3529 GFile *dest = g_file_new_for_path(dbfilename_library);
3530 gboolean copy_status = TRUE;
3531 if(g_file_test(data_snap, G_FILE_TEST_EXISTS))
3532 {
3533 copy_status = g_file_copy(src, dest, G_FILE_COPY_NONE, NULL, NULL, NULL, &gerror);
3534 if(copy_status) copy_status = g_chmod(dbfilename_library, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH) == 0;
3535 }
3536 else
3537 {
3538 // there is nothing to restore, create an empty file to prevent further backup attempts
3539 const int fd = g_open(dbfilename_library, O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
3540 if(fd < 0 || !g_close(fd, &gerror)) copy_status = FALSE;
3541 }
3542 if(copy_status)
3543 fprintf(stderr, " success!\n");
3544 else
3545 fprintf(stderr, " failed!\n");
3546 g_object_unref(src);
3547 g_object_unref(dest);
3548 }
3549 }
3550 dt_free(data_snap);
3551 dt_free(dbname);
3552 goto start;
3553 }
3554 else
3555 {
3556 dt_free(libdb_status);
3557
3558 // does it contain the legacy 'settings' table?
3559 sqlite3_finalize(stmt);
3560 rc = sqlite3_prepare_v2(db->handle, "select settings from main.settings", -1, &stmt, NULL);
3561 if(rc == SQLITE_OK && sqlite3_step(stmt) == SQLITE_ROW)
3562 {
3563 // the old blob had the version as an int in the first place
3564 const void *set = sqlite3_column_blob(stmt, 0);
3565 const int db_version = *(int *)set;
3566 sqlite3_finalize(stmt);
3567 if(!_migrate_schema(db, db_version)) // bring the legacy layout to the first one known to our upgrade
3568 // path ...
3569 {
3570 // we couldn't migrate the db for some reason. bail out.
3571 fprintf(stderr, "[init] database `%s' couldn't be migrated from the legacy version %d. aborting\n",
3572 dbname, db_version);
3573 _database_free(db);
3574 db = NULL;
3575 goto error;
3576 }
3577 if(!_upgrade_library_schema(db, 1)) // ... and upgrade it
3578 {
3579 // we couldn't upgrade the db for some reason. bail out.
3580 fprintf(stderr, "[init] database `%s' couldn't be upgraded from version 1 to %d. aborting\n", dbname,
3581 CURRENT_DATABASE_VERSION_LIBRARY);
3582 _database_free(db);
3583 db = NULL;
3584 goto error;
3585 }
3586 }
3587 else
3588 {
3589 sqlite3_finalize(stmt);
3590 _create_library_schema(db); // a brand new db it seems
3591 }
3592 }
3593
3594 _db_print("[sql] Checked db_info\n");
3595
3596 // create the in-memory tables
3597 _create_memory_schema(db);
3598
3599 _db_print("[sql] Created in-memory DB\n");
3600
3601 // create a table legacy_presets with all the presets from pre-auto-apply-cleanup darktable.
3602 dt_legacy_presets_create(db->handle);
3603
3604 _db_print("[sql] Loaded legacy presets\n");
3605
3606 // drop table settings -- we don't want old versions of dt to drop our tables
3607 sqlite3_exec(db->handle, "drop table main.settings", NULL, NULL, NULL);
3608
3609 // take care of potential bad data in the db.
3610 _sanitize_db(db);
3611
3612#ifdef HAVE_ICU
3613 // check if sqlite is already icu enabled
3614 // if not enabled expected error: no such function:icu_load_collation
3615 rc = sqlite3_prepare_v2(db->handle,
3616 "SELECT icu_load_collation('en_US', 'english')",
3617 -1, &stmt, NULL);
3618 sqlite3_finalize(stmt);
3619
3620 if(rc != SQLITE_OK)
3621 {
3622 rc = sqlite3IcuInit(db->handle);
3623 if(rc != SQLITE_OK)
3624 fprintf(stderr, "[sqlite] init icu extension error %d\n", rc);
3625 }
3626#endif
3627
3628error:
3629 dt_free(dbname);
3630 dt_free(migrated_name);
3631
3632 return db;
3633}
3634
3639
3640dt_database_open_result_t dt_database_open(const dt_database_params_t *params)
3641{
3642 if(IS_NULL_PTR(params)) return DT_DATABASE_OPEN_FAILED;
3643
3644 if(!IS_NULL_PTR(_db))
3645 {
3646 fprintf(stderr, "[database] a database is already open; close it first\n");
3647 return DT_DATABASE_OPEN_FAILED;
3648 }
3649
3650 if(!_db_lock_inited)
3651 {
3652 dt_pthread_rwlock_init(&_db_lock, NULL);
3653 _db_lock_inited = TRUE;
3654 }
3655 if(!_settings_lock_inited)
3656 {
3657 dt_pthread_mutex_init(&_settings_lock, NULL);
3658 _settings_lock_inited = TRUE;
3659 }
3660
3661 /* Session constants. Read once, here -- nothing below consults the debug flags or the
3662 * configuration again. */
3663 _has_gui = params->has_gui;
3664 _verbose = params->verbose;
3665
3666 _db = _database_init(params);
3667
3668 if(IS_NULL_PTR(_db)) return DT_DATABASE_OPEN_FAILED;
3669 if(!_db->lock_acquired) return DT_DATABASE_OPEN_LOCKED;
3670
3671 return DT_DATABASE_OPEN_OK;
3672}
3673
3692
3693void dt_database_get_settings(dt_database_settings_t *settings)
3694{
3695 if(IS_NULL_PTR(settings)) return;
3696
3697 if(!_settings_lock_inited)
3698 {
3699 /* Nothing has been told to us yet: report the neutral policy rather than read a
3700 * half-initialised struct behind an uninitialised mutex. */
3701 settings->maintenance_check = NULL;
3702 settings->create_snapshot = NULL;
3703 settings->maintenance_freepage_ratio = 0;
3704 settings->keep_snapshots = 0;
3705 return;
3706 }
3707
3708 /* One snapshot under the lock. Four fields read one at a time can be a mix of old and
3709 * new -- the GUI thread replaces them from the preferences dialog while a maintenance
3710 * decision may be part-way through reading them. */
3711 dt_pthread_mutex_lock(&_settings_lock);
3712 settings->maintenance_check = g_strdup(_settings.maintenance_check);
3713 settings->create_snapshot = g_strdup(_settings.create_snapshot);
3714 settings->maintenance_freepage_ratio = _settings.maintenance_freepage_ratio;
3715 settings->keep_snapshots = _settings.keep_snapshots;
3716 dt_pthread_mutex_unlock(&_settings_lock);
3717}
3718
3719void dt_database_settings_free(dt_database_settings_t *settings)
3720{
3721 if(IS_NULL_PTR(settings)) return;
3722 dt_free(settings->maintenance_check);
3723 dt_free(settings->create_snapshot);
3724 settings->maintenance_check = NULL;
3725 settings->create_snapshot = NULL;
3726}
3727
3728/* Close a connection and release its lock files.
3729 *
3730 * Separate from dt_database_close() because _database_init() calls it on a database that
3731 * was never published -- it has no lock to take and nothing to unpublish. The error
3732 * strings are freed here, which the old dt_database_destroy() did not do; on the ordinary
3733 * shutdown path they are always NULL, but every failure path through _database_init()
3734 * sets one and then tore the database down. */
3735static void _database_free(dt_database_t *db)
3736{
3737 if(IS_NULL_PTR(db)) return;
3738
3739 sqlite3_close(db->handle);
3740 if (db->lockfile_data)
3741 {
3742 g_unlink(db->lockfile_data);
3743 dt_free(db->lockfile_data);
3744 }
3745 if (db->lockfile_library)
3746 {
3747 g_unlink(db->lockfile_library);
3748 dt_free(db->lockfile_library);
3749 }
3750 dt_free(db->dbfilename_data);
3751 dt_free(db->dbfilename_library);
3752 dt_free(db->error_message);
3753 dt_free(db->error_dbfilename);
3754 dt_free(db);
3755}
3756
3757void dt_database_close(void)
3758{
3759 /* Exclusive against the TRANSACTION writers -- begin/end and their batch variants are
3760 * the only readers-side acquirers of _db_lock today, so this waits for open transactions
3761 * and for nothing else. It does NOT wait for a repository mid-sqlite3_step on another
3762 * thread (no per-query read lock exists yet), and it cannot wait for the call sites
3763 * holding a raw connection -- nothing can, which is the whole argument for getting rid
3764 * of them. Callers must stop the workers before closing. */
3765 if(_db_lock_inited) dt_pthread_rwlock_wrlock(&_db_lock);
3766
3767 dt_database_t *const db = _db;
3768 _db = NULL;
3769
3770 if(!IS_NULL_PTR(db))
3771 {
3772 _database_free(db);
3773 sqlite3_shutdown();
3774 }
3775
3776 if(_db_lock_inited) dt_pthread_rwlock_unlock(&_db_lock);
3777}
3778
3779gboolean dt_database_is_open(void)
3780{
3781 return !IS_NULL_PTR(_db);
3782}
3783
3784const char *dt_database_get_last_error(void)
3785{
3786 if(IS_NULL_PTR(_db) || IS_NULL_PTR(_db->handle)) return "no database connection";
3787 return sqlite3_errmsg(_db->handle);
3788}
3789
3790sqlite3 *dt_database_get_sqlite3_global(void)
3791{
3792 return _db ? _db->handle : NULL;
3793}
3794
3795const gchar *dt_database_get_path(void)
3796{
3797 return _db ? _db->dbfilename_library : NULL;
3798}
3799
3800/* @p configured_db is the library name as the orchestrator has it (the "database" conf
3801 * key). When the legacy file under $HOME is moved into the XDG data directory the module
3802 * does not write that key back itself -- it reports the new name through
3803 * ::_renamed_handler and lets whoever owns the configuration persist it, and returns it
3804 * so this run opens the file that now exists. */
3805static gchar *_database_migrate_to_xdg_structure(const char *configured_db)
3806{
3807 gchar dbfilename[DT_PATH_MAX] = { 0 };
3808
3809 /* The destination must be the directory _database_init() resolves a relative library
3810 * name against -- the user config dir -- because the name this returns is exactly such a
3811 * relative name. It used to be dt_loc_get_datadir(), a fossil from when "datadir" WAS the
3812 * user's directory; today that is the read-only <prefix>/share/ansel, where the rename
3813 * fails on a system install (EACCES, or EXDEV across mounts) and would land the file
3814 * where the open path never looks on any install where it could succeed. */
3815 gchar configdir[DT_PATH_MAX] = { 0 };
3816 dt_loc_get_user_config_dir(configdir, sizeof(configdir));
3817
3818 if(configured_db && configured_db[0] != '/')
3819 {
3820 const char *homedir = getenv("HOME");
3821 snprintf(dbfilename, sizeof(dbfilename), "%s/%s", homedir, configured_db);
3822 if(g_file_test(dbfilename, G_FILE_TEST_EXISTS))
3823 {
3824 char destdbname[DT_PATH_MAX] = { 0 };
3825 dt_concat_path_file(destdbname, configdir, "library.db");
3826 if(!g_file_test(destdbname, G_FILE_TEST_EXISTS))
3827 {
3828 fprintf(stderr, "[init] moving database into new XDG directory structure\n");
3829 if(rename(dbfilename, destdbname) != 0)
3830 {
3831 /* Report the migration ONLY when it happened: the old code returned success on a
3832 * failed rename, so conf was rewritten to name a file that was never created and
3833 * the next open built an empty library while the real one sat orphaned in $HOME. */
3834 fprintf(stderr, "[init] could not move `%s' to `%s' (%s), keeping the old location\n",
3835 dbfilename, destdbname, strerror(errno));
3836 return NULL;
3837 }
3838 if(!IS_NULL_PTR(_renamed_handler)) _renamed_handler("library.db");
3839 return g_strdup("library.db");
3840 }
3841 }
3842 }
3843
3844 return NULL;
3845}
3846
3847/* delete old mipmaps files */
3848static void _database_delete_mipmaps_files()
3849{
3850 /* This migration is intended to be run only from 0.9.x to new cache in 1.0 */
3851
3852 // Directory
3853 char cachedir[DT_PATH_MAX] = { 0 }, mipmapfilename[DT_PATH_MAX] = { 0 };
3854 dt_loc_get_user_cache_dir(cachedir, sizeof(cachedir));
3855 dt_concat_path_file(mipmapfilename, cachedir, "mipmaps");
3856
3857 if(g_access(mipmapfilename, F_OK) != -1)
3858 {
3859 fprintf(stderr, "[mipmap_cache] dropping old version file: %s\n", mipmapfilename);
3860 g_unlink(mipmapfilename);
3861 dt_concat_path_file(mipmapfilename, cachedir, "mipmaps.fallback");
3862
3863 if(g_access(mipmapfilename, F_OK) != -1) g_unlink(mipmapfilename);
3864 }
3865}
3866
3867
3868
3869void dt_database_cleanup_busy_statements(void)
3870{
3871 /* was a parameter; the module owns the one connection now */
3872 const dt_database_t *const db = _db;
3873 if(IS_NULL_PTR(db)) return;
3874 sqlite3_stmt *stmt = NULL;
3875 while( (stmt = sqlite3_next_stmt(db->handle, NULL)) != NULL)
3876 {
3877 const char* sql = sqlite3_sql(stmt);
3878 if(sqlite3_stmt_busy(stmt))
3879 {
3880 _db_print("[db busy stmt] non-finalized nor stepped through statement: '%s'\n",sql);
3881 sqlite3_reset(stmt);
3882 }
3883 else {
3884 _db_print("[db busy stmt] non-finalized statement: '%s'\n",sql);
3885 }
3886 sqlite3_finalize(stmt);
3887 }
3888}
3889
3890#define ERRCHECK {if (err!=NULL) {_db_print("[db maintenance] maintenance error: '%s'\n",err); sqlite3_free(err); err=NULL;}}
3891void dt_database_perform_maintenance(void)
3892{
3893 /* was a parameter; the module owns the one connection now */
3894 const dt_database_t *const db = _db;
3895 if(IS_NULL_PTR(db)) return;
3896 char* err = NULL;
3897
3898 const int main_pre_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3899 const int main_page_size = _get_pragma_int_val(db->handle, "main.page_size");
3900 const int data_pre_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3901 const int data_page_size = _get_pragma_int_val(db->handle, "data.page_size");
3902
3903 const guint64 calc_pre_size = (main_pre_free_count*main_page_size) + (data_pre_free_count*data_page_size);
3904
3905 if(calc_pre_size == 0)
3906 {
3907 _db_print("[db maintenance] maintenance deemed unnecesary, performing only analyze.\n");
3908 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE data", NULL, NULL, &err);
3909 ERRCHECK
3910 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE main", NULL, NULL, &err);
3911 ERRCHECK
3912 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE", NULL, NULL, &err);
3913 ERRCHECK
3914 return;
3915 }
3916
3917 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM data", NULL, NULL, &err);
3918 ERRCHECK
3919 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM main", NULL, NULL, &err);
3920 ERRCHECK
3921 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE data", NULL, NULL, &err);
3922 ERRCHECK
3923 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE main", NULL, NULL, &err);
3924 ERRCHECK
3925
3926 // for some reason this is needed in some cases
3927 // in case above performed vacuum+analyze properly, this is noop.
3928 DT_DEBUG_SQLITE3_EXEC(db->handle, "VACUUM", NULL, NULL, &err);
3929 ERRCHECK
3930 DT_DEBUG_SQLITE3_EXEC(db->handle, "ANALYZE", NULL, NULL, &err);
3931 ERRCHECK
3932
3933 const int main_post_free_count = _get_pragma_int_val(db->handle, "main.freelist_count");
3934 const int data_post_free_count = _get_pragma_int_val(db->handle, "data.freelist_count");
3935
3936 const guint64 calc_post_size = (main_post_free_count*main_page_size) + (data_post_free_count*data_page_size);
3937 const gint64 bytes_freed = calc_pre_size - calc_post_size;
3938
3939 _db_print("[db maintenance] maintenance done, %" G_GINT64_FORMAT " bytes freed.\n", bytes_freed);
3940
3941 if(calc_post_size >= calc_pre_size)
3942 {
3943 _db_print("[db maintenance] maintenance problem. if no errors logged, it should work fine next time.\n");
3944 }
3945}
3946#undef ERRCHECK
3947
3948/* State the facts and take back an answer.
3949 *
3950 * The prose this used to build here -- how much would be freed, and which of three
3951 * "you'll be asked again..." sentences applies -- is the handler's to write. What the
3952 * module knows, and all it passes, is the byte count and the booleans that decide which
3953 * sentence is true. */
3954static gboolean _ask_for_maintenance(const char *maintenance_check, const gboolean closing_time,
3955 const guint64 size)
3956{
3957 if(!_has_gui)
3958 return FALSE;
3959
3960 const gboolean on_both = !g_strcmp0(maintenance_check, "on both");
3961
3962 const dt_database_prompt_context_t context
3963 = { .prompt = DT_DATABASE_PROMPT_MAINTENANCE,
3964 .dbfilename = _db->dbfilename_library,
3965 .reclaimable_bytes = size,
3966 .at_close = closing_time,
3967 .ask_on_startup = (closing_time && on_both) || !g_strcmp0(maintenance_check, "on startup"),
3968 .ask_on_close = (!closing_time && on_both) || !g_strcmp0(maintenance_check, "on close") };
3969
3970 return _ask_user(&context) == DT_DATABASE_RESPONSE_PROCEED;
3971}
3972
3973static inline gboolean _is_mem_db(void)
3974{
3975 return !g_strcmp0(_db->dbfilename_data, ":memory:") || !g_strcmp0(_db->dbfilename_library, ":memory:");
3976}
3977
3978gboolean dt_database_maybe_maintenance(const gboolean closing_time)
3979{
3980 if(IS_NULL_PTR(_db) || _is_mem_db())
3981 return FALSE;
3982
3983 dt_database_settings_t settings = { 0 };
3984 dt_database_get_settings(&settings);
3985 const char *config = settings.maintenance_check;
3986
3987 if(!g_strcmp0(config, "never"))
3988 {
3989 // early bail out on "never"
3990 _db_print("[db maintenance] please consider enabling database maintenance.\n");
3991 dt_database_settings_free(&settings);
3992 return FALSE;
3993 }
3994
3995 gboolean check_for_maintenance = FALSE;
3996 const gboolean force_maintenance = g_str_has_suffix (config, "(don't ask)");
3997
3998 if(config)
3999 {
4000 if((strstr(config, "on both")) // should cover "(don't ask) suffix
4001 || (closing_time && (strstr(config, "on close")))
4002 || (!closing_time && (strstr(config, "on startup"))))
4003 {
4004 // we have "on both/on close/on startup" setting, so - checking!
4005 _db_print("[db maintenance] checking for maintenance, due to rule: '%s'.\n", config);
4006 check_for_maintenance = TRUE;
4007 }
4008 // if the config was "never", check_for_vacuum is false.
4009 }
4010
4011 if(!check_for_maintenance)
4012 {
4013 dt_database_settings_free(&settings);
4014 return FALSE;
4015 }
4016
4017 // checking free pages
4018 const int main_free_count = _get_pragma_int_val(_db->handle, "main.freelist_count");
4019 const int main_page_count = _get_pragma_int_val(_db->handle, "main.page_count");
4020 const int main_page_size = _get_pragma_int_val(_db->handle, "main.page_size");
4021
4022 const int data_free_count = _get_pragma_int_val(_db->handle, "data.freelist_count");
4023 const int data_page_count = _get_pragma_int_val(_db->handle, "data.page_count");
4024 const int data_page_size = _get_pragma_int_val(_db->handle, "data.page_size");
4025
4026 _db_print("[db maintenance] main: [%d/%d pages], data: [%d/%d pages].\n",
4027 main_free_count, main_page_count, data_free_count, data_page_count);
4028
4029 if(main_page_count <= 0 || data_page_count <= 0)
4030 {
4031 //something's wrong with PRAGMA page_size returns. early bail.
4032 _db_print("[db maintenance] page_count <= 0 : main.page_count: %d, data.page_count: %d \n",
4033 main_page_count, data_page_count);
4034 dt_database_settings_free(&settings);
4035 return FALSE;
4036 }
4037
4038 // we don't need fine-grained percentages, so let's do ints
4039 const int main_free_percentage = (main_free_count * 100 ) / main_page_count;
4040 const int data_free_percentage = (data_free_count * 100 ) / data_page_count;
4041
4042 const int freepage_ratio = settings.maintenance_freepage_ratio;
4043 gboolean maintenance = FALSE;
4044
4045 if((main_free_percentage >= freepage_ratio)
4046 || (data_free_percentage >= freepage_ratio))
4047 {
4048 const guint64 calc_size = (main_free_count*main_page_size) + (data_free_count*data_page_size);
4049 _db_print("[db maintenance] maintenance suggested, %" G_GUINT64_FORMAT " bytes to free.\n", calc_size);
4050
4051 maintenance = force_maintenance || _ask_for_maintenance(config, closing_time, calc_size);
4052 }
4053
4054 dt_database_settings_free(&settings);
4055 return maintenance;
4056}
4057
4058void dt_database_optimize(void)
4059{
4060 /* was a parameter; the module owns the one connection now */
4061 const dt_database_t *const db = _db;
4062 if(IS_NULL_PTR(db) || _is_mem_db())
4063 return;
4064 // optimize should in most cases be no-op and have no noticeable downsides
4065 // this should be ran on every exit
4066 // see: https://www.sqlite.org/pragma.html#pragma_optimize
4067 DT_DEBUG_SQLITE3_EXEC(db->handle, "PRAGMA optimize", NULL, NULL, NULL);
4068}
4069
4070static void _print_backup_progress(int remaining, int total)
4071{
4072 // TODO if we have closing splashpage - this can be used to advance progressbar :)
4073 _db_print("[db backup] %d out of %d done\n", total - remaining, total);
4074}
4075
4076static int _backup_db(
4077 sqlite3 *src_db, /* Database handle to back up */
4078 const char *src_db_name, /* Database name to back up */
4079 const char *dest_filename, /* Name of file to back up to */
4080 void(*xProgress)(int, int) /* Progress function to invoke */
4081)
4082{
4083 sqlite3 *dest_db; /* Database connection opened on zFilename */
4084 sqlite3_backup *sb_dest; /* Backup handle used to copy data */
4085
4086 /* Open the database file identified by zFilename. */
4087 int rc = sqlite3_open(dest_filename, &dest_db);
4088
4089 if(rc == SQLITE_OK)
4090 {
4091 /* Open the sqlite3_backup object used to accomplish the transfer */
4092 sb_dest = sqlite3_backup_init(dest_db, "main", src_db, src_db_name);
4093 if(sb_dest)
4094 {
4095 _db_print("[db backup] %s to %s\n", src_db_name, dest_filename);
4096 gchar *pragma = g_strdup_printf("%s.page_count", src_db_name);
4097 const int spc = _get_pragma_int_val(src_db, pragma);
4098 dt_free(pragma);
4099 const int pc = MIN(spc, MAX(5,spc/100));
4100 do
4101 {
4102 rc = sqlite3_backup_step(sb_dest, pc);
4103 if(xProgress)
4104 xProgress(
4105 sqlite3_backup_remaining(sb_dest),
4106 sqlite3_backup_pagecount(sb_dest)
4107 );
4108 if( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_LOCKED )
4109 {
4110 sqlite3_sleep(25);
4111 }
4112 }
4113 while( rc==SQLITE_OK || rc==SQLITE_BUSY || rc==SQLITE_LOCKED );
4114
4115 /* Release resources allocated by backup_init(). */
4116 (void)sqlite3_backup_finish(sb_dest);
4117 }
4118 rc = sqlite3_errcode(dest_db);
4119 }
4120 /* Close the database connection opened on database file zFilename
4121 ** and return the result of this function. */
4122 (void)sqlite3_close(dest_db);
4123 return rc;
4124}
4125
4126gboolean dt_database_snapshot(void)
4127{
4128 /* was a parameter; the module owns the one connection now */
4129 const dt_database_t *const db = _db;
4130 if(IS_NULL_PTR(db)) return FALSE;
4131 // backing up memory db is pointelss
4132 if(_is_mem_db())
4133 return FALSE;
4134 GDateTime *date_now = g_date_time_new_now_local();
4135 gchar *date_suffix = g_date_time_format(date_now, "%Y%m%d%H%M%S");
4136 g_date_time_unref(date_now);
4137
4138 const char *file_pattern = "%s-snp-%s";
4139 const char *temp_pattern = "%s-tmp-%s";
4140
4141 gchar *lib_backup_file = g_strdup_printf(file_pattern, db->dbfilename_library, date_suffix);
4142 gchar *lib_tmpbackup_file = g_strdup_printf(temp_pattern, db->dbfilename_library, date_suffix);
4143
4144 int rc = _backup_db(db->handle, "main", lib_tmpbackup_file, _print_backup_progress);
4145 if(!(rc==SQLITE_OK))
4146 {
4147 g_unlink(lib_tmpbackup_file);
4148 dt_free(lib_tmpbackup_file);
4149 dt_free(lib_backup_file);
4150 dt_free(date_suffix);
4151 return FALSE;
4152 }
4153 g_rename(lib_tmpbackup_file, lib_backup_file);
4154 g_chmod(lib_backup_file, S_IRUSR);
4155 dt_free(lib_tmpbackup_file);
4156 dt_free(lib_backup_file);
4157
4158 gchar *dat_backup_file = g_strdup_printf(file_pattern, db->dbfilename_data, date_suffix);
4159 gchar *dat_tmpbackup_file = g_strdup_printf(temp_pattern, db->dbfilename_data, date_suffix);
4160
4161 dt_free(date_suffix);
4162
4163 rc = _backup_db(db->handle, "data", dat_tmpbackup_file, _print_backup_progress);
4164 if(!(rc==SQLITE_OK))
4165 {
4166 g_unlink(dat_tmpbackup_file);
4167 dt_free(dat_tmpbackup_file);
4168 dt_free(dat_backup_file);
4169 return FALSE;
4170 }
4171 g_rename(dat_tmpbackup_file, dat_backup_file);
4172 g_chmod(dat_backup_file, S_IRUSR);
4173 dt_free(dat_tmpbackup_file);
4174 dt_free(dat_backup_file);
4175
4176 return TRUE;
4177}
4178
4179/* Turn the "create_snapshot" policy into a snapshot interval.
4180 *
4181 * Returns TRUE when the policy answers the question outright -- "never", "on close", or
4182 * an unrecognised value -- with the answer in @p answer. Returns FALSE when the caller
4183 * has to go and look at what is on disk, with the required interval in @p interval.
4184 *
4185 * It exists so that the settings copy is taken and released in one small scope: the
4186 * function below has a dozen early returns and no business holding a couple of strings it
4187 * would have to free on each of them. */
4188static gboolean _snapshot_policy_decides(GTimeSpan *interval, gboolean *answer)
4189{
4190 dt_database_settings_t settings = { 0 };
4191 dt_database_get_settings(&settings);
4192 const char *const config = settings.create_snapshot;
4193
4194 gboolean decides = TRUE;
4195
4196 if(!g_strcmp0(config, "never"))
4197 {
4198 // early bail out on "never"
4199 _db_print("[db backup] please consider enabling database snapshots.\n");
4200 *answer = FALSE;
4201 }
4202 else if(!g_strcmp0(config, "on close"))
4203 {
4204 // early bail out on "on close"
4205 _db_print("[db backup] performing unconditional snapshot.\n");
4206 *answer = TRUE;
4207 }
4208 else if(!g_strcmp0(config, "once a day"))
4209 {
4210 *interval = G_TIME_SPAN_DAY;
4211 decides = FALSE;
4212 }
4213 else if(!g_strcmp0(config, "once a week"))
4214 {
4215 *interval = G_TIME_SPAN_DAY * 7;
4216 decides = FALSE;
4217 }
4218 else if(!g_strcmp0(config, "once a month"))
4219 {
4220 //average month ;)
4221 *interval = G_TIME_SPAN_DAY * 30;
4222 decides = FALSE;
4223 }
4224 else
4225 {
4226 // early bail out on "invalid value"
4227 _db_print("[db backup] invalid timespan requirement expecting never/on close/once a [day/week/month], got %s.\n", config);
4228 *answer = TRUE;
4229 }
4230
4231 dt_database_settings_free(&settings);
4232 return decides;
4233}
4234
4235gboolean dt_database_maybe_snapshot(void)
4236{
4237 /* was a parameter; the module owns the one connection now */
4238 const dt_database_t *const db = _db;
4239 if(IS_NULL_PTR(db)) return FALSE;
4240 if(_is_mem_db())
4241 return FALSE;
4242
4243 /* Resolve the policy into an interval and be done with the settings copy right here. */
4244 GTimeSpan span_from_last_snap_required = 0;
4245 gboolean decided_answer = FALSE;
4246 if(_snapshot_policy_decides(&span_from_last_snap_required, &decided_answer))
4247 return decided_answer;
4248
4249 //we're in trouble zone - we have to determine when was the last snapshot done (including version upgrade snapshot) :/
4250 //this could be easy if we wrote date of last successful backup to config, but that's not really an option since
4251 //backup may done as last db operation, way after config file is closed. Plus we might be mixing dates of backups for
4252 //various library.db
4253
4254 _db_print("[db backup] checking snapshots existence.\n");
4255 GFile *library = g_file_parse_name(db->dbfilename_library);
4256 GFile *parent = g_file_get_parent(library);
4257
4258 if(IS_NULL_PTR(parent))
4259 {
4260 _db_print("[db backup] couldn't get library parent!.\n");
4261 g_object_unref(library);
4262 return FALSE;
4263 }
4264
4265 GError *error = NULL;
4266 GFileEnumerator *library_dir_files = g_file_enumerate_children(parent, G_FILE_ATTRIBUTE_STANDARD_NAME "," G_FILE_ATTRIBUTE_TIME_MODIFIED, G_FILE_QUERY_INFO_NONE, NULL, &error);
4267
4268 if(IS_NULL_PTR(library_dir_files))
4269 {
4270 _db_print("[db backup] couldn't enumerate library parent: %s.\n", error->message);
4271 g_object_unref(parent);
4272 g_object_unref(library);
4273 g_error_free(error);
4274 return FALSE;
4275 }
4276
4277 gchar *lib_basename = g_file_get_basename(library);
4278 g_object_unref(library);
4279
4280 gchar *lib_snap_format = g_strdup_printf("%s-snp-", lib_basename);
4281 gchar *lib_backup_format = g_strdup_printf("%s-pre-", lib_basename);
4282 dt_free(lib_basename);
4283
4284 GFileInfo *info = NULL;
4285 guint64 last_snap = 0;
4286
4287 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4288 {
4289 const char* fname = g_file_info_get_name(info);
4290 if(g_str_has_prefix(fname, lib_snap_format) || g_str_has_prefix(fname, lib_backup_format))
4291 {
4292 _db_print("[db backup] found file: %s.\n", fname);
4293 if(last_snap == 0)
4294 {
4295 last_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4296 g_object_unref(info);
4297 continue;
4298 }
4299 const guint64 try_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4300 if(try_snap > last_snap)
4301 {
4302 last_snap = try_snap;
4303 }
4304 }
4305 g_object_unref(info);
4306 }
4307 g_object_unref(parent);
4308 dt_free(lib_snap_format);
4309 dt_free(lib_backup_format);
4310
4311 if(error)
4312 {
4313 _db_print("[db backup] problem enumerating library parent: %s.\n", error->message);
4314 g_file_enumerator_close(library_dir_files, NULL, NULL);
4315 g_object_unref(library_dir_files);
4316 g_error_free(error);
4317 return FALSE;
4318 }
4319
4320 g_file_enumerator_close(library_dir_files, NULL, NULL);
4321 g_object_unref(library_dir_files);
4322
4323 GDateTime *date_now = g_date_time_new_now_local();
4324
4325 // Even if last_snap is 0 (didn't found any snaps) it produces proper date - unix epoch
4326 GDateTime *date_last_snap = g_date_time_new_from_unix_local(last_snap);
4327
4328 gchar *now_txt = g_date_time_format(date_now, "%Y%m%d%H%M%S");
4329 gchar *ls_txt = g_date_time_format(date_last_snap, "%Y%m%d%H%M%S");
4330 _db_print("[db backup] last snap: %s; curr date: %s.\n", ls_txt, now_txt);
4331 dt_free(now_txt);
4332 dt_free(ls_txt);
4333
4334 GTimeSpan span_from_last_snap = g_date_time_difference(date_now, date_last_snap);
4335
4336 g_date_time_unref(date_now);
4337 g_date_time_unref(date_last_snap);
4338
4339 return span_from_last_snap > span_from_last_snap_required;
4340}
4341
4342/* Parse integers in the form d (week days), dd (hours etc), ddd (ordinal days) or dddd (years) */
4343static gboolean _get_iso8601_int (const gchar *text, gsize length, gint *value)
4344{
4345 gsize i;
4346 guint v = 0;
4347
4348 if (length < 1 || length > 4)
4349 return FALSE;
4350
4351 for (i = 0; i < length; i++)
4352 {
4353 const gchar c = text[i];
4354 if (c < '0' || c > '9')
4355 return FALSE;
4356 v = v * 10 + (c - '0');
4357 }
4358
4359 *value = v;
4360 return TRUE;
4361}
4362
4363static gint _db_snap_sort(gconstpointer a, gconstpointer b, gpointer user_data)
4364{
4365 const gchar* e1 = (gchar *) a;
4366 const gchar* e2 = (gchar *) b;
4367
4368 //we assume that both end with date in
4369 //"%Y%m%d%H%M%S" format
4370
4371 gchar *datepos1 = g_strrstr(e1, "-snp-");
4372 gchar *datepos2 = g_strrstr(e2, "-snp-");
4373 if(IS_NULL_PTR(datepos1) || IS_NULL_PTR(datepos2))
4374 return 0;
4375
4376 datepos1 +=5; //skip "-snp-"
4377 datepos2 +=5; //skip "-snp-"
4378
4379 int year,month,day,hour,minute,second;
4380
4381 if(!_get_iso8601_int(datepos1, 4, &year) ||
4382 !_get_iso8601_int(datepos1+4, 2, &month) ||
4383 !_get_iso8601_int(datepos1+6, 2, &day) ||
4384 !_get_iso8601_int(datepos1+8, 2, &hour) ||
4385 !_get_iso8601_int(datepos1+10, 2, &minute) ||
4386 !_get_iso8601_int(datepos1+12, 2, &second)
4387 )
4388 {
4389 return 0;
4390 }
4391
4392 GDateTime *d1=g_date_time_new_local(year, month, day, hour, minute, second);
4393
4394 if(!_get_iso8601_int(datepos2, 4, &year) ||
4395 !_get_iso8601_int(datepos2+4, 2, &month) ||
4396 !_get_iso8601_int(datepos2+6, 2, &day) ||
4397 !_get_iso8601_int(datepos2+8, 2, &hour) ||
4398 !_get_iso8601_int(datepos2+10, 2, &minute) ||
4399 !_get_iso8601_int(datepos2+12, 2, &second)
4400 )
4401 {
4402 g_date_time_unref(d1);
4403 return 0;
4404 }
4405
4406 GDateTime *d2=g_date_time_new_local(year, month, day, hour, minute, second);
4407
4408 const gint ret = g_date_time_compare(d1, d2);
4409
4410 g_date_time_unref(d1);
4411 g_date_time_unref(d2);
4412
4413 return ret;
4414}
4415
4416char **dt_database_snaps_to_remove(void)
4417{
4418 /* was a parameter; the module owns the one connection now */
4419 const dt_database_t *const db = _db;
4420 if(IS_NULL_PTR(db)) return NULL;
4421 if(_is_mem_db())
4422 return NULL;
4423
4424 dt_database_settings_t settings = { 0 };
4425 dt_database_get_settings(&settings);
4426 const int keep_snaps = settings.keep_snapshots;
4427 dt_database_settings_free(&settings);
4428
4429 if(keep_snaps < 0)
4430 return NULL;
4431
4432 _db_print("[db backup] checking snapshots existence.\n");
4433 GFile *lib_file = g_file_parse_name(db->dbfilename_library);
4434 GFile *lib_parent = g_file_get_parent(lib_file);
4435
4436 if(IS_NULL_PTR(lib_parent))
4437 {
4438 _db_print("[db backup] couldn't get library parent!.\n");
4439 g_object_unref(lib_file);
4440 return NULL;
4441 }
4442
4443 GFile *dat_file = g_file_parse_name(db->dbfilename_data);
4444 GFile *dat_parent = g_file_get_parent(dat_file);
4445
4446 if(IS_NULL_PTR(dat_parent))
4447 {
4448 _db_print("[db backup] couldn't get data parent!.\n");
4449 g_object_unref(dat_file);
4450 g_object_unref(lib_file);
4451 g_object_unref(lib_parent);
4452 }
4453
4454 gchar *lib_basename = g_file_get_basename(lib_file);
4455 g_object_unref(lib_file);
4456 gchar *lib_snap_format = g_strdup_printf("%s-snp-", lib_basename);
4457 gchar *lib_tmp_format = g_strdup_printf("%s-tmp-", lib_basename);
4458 dt_free(lib_basename);
4459
4460 gchar *dat_basename = g_file_get_basename(dat_file);
4461 g_object_unref(dat_file);
4462 gchar *dat_snap_format = g_strdup_printf("%s-snp-", dat_basename);
4463 gchar *dat_tmp_format = g_strdup_printf("%s-tmp-", dat_basename);
4464 dt_free(dat_basename);
4465
4466 GQueue *lib_snaps = g_queue_new();
4467 GQueue *dat_snaps = g_queue_new();
4468 GQueue *tmplib_snaps = g_queue_new();
4469 GQueue *tmpdat_snaps = g_queue_new();
4470
4471 if(g_file_equal(lib_parent, dat_parent))
4472 {
4473 //slight optimization if library and data are in same dir, we only have to scan one
4474 GError *error = NULL;
4475 GFileEnumerator *library_dir_files = g_file_enumerate_children(lib_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4476
4477 if(IS_NULL_PTR(library_dir_files))
4478 {
4479 _db_print("[db backup] couldn't enumerate library parent: %s.\n", error->message);
4480 g_object_unref(lib_parent);
4481 g_object_unref(dat_parent);
4482 dt_free(lib_snap_format);
4483 dt_free(dat_snap_format);
4484 dt_free(lib_tmp_format);
4485 dt_free(dat_tmp_format);
4486 g_queue_free(lib_snaps);
4487 g_queue_free(dat_snaps);
4488 g_queue_free(tmplib_snaps);
4489 g_queue_free(tmpdat_snaps);
4490 g_error_free(error);
4491 return NULL;
4492 }
4493
4494 GFileInfo *info = NULL;
4495
4496 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4497 {
4498 const char* fname = g_file_info_get_name(info);
4499 if(g_str_has_prefix(fname, lib_snap_format))
4500 {
4501 _db_print("[db backup] found file: %s.\n", fname);
4502 g_queue_insert_sorted(lib_snaps, g_strdup(fname), _db_snap_sort, NULL);
4503 }
4504 else if(g_str_has_prefix(fname, dat_snap_format))
4505 {
4506 _db_print("[db backup] found file: %s.\n", fname);
4507 g_queue_insert_sorted(dat_snaps, g_strdup(fname), _db_snap_sort, NULL);
4508 }
4509 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4510 {
4511 //we insert into single queue, since it's just dependent on parent
4512 g_queue_push_head(tmplib_snaps, g_strdup(fname));
4513 }
4514 g_object_unref(info);
4515 }
4516 dt_free(lib_snap_format);
4517 dt_free(dat_snap_format);
4518
4519 if(error)
4520 {
4521 _db_print("[db backup] problem enumerating library parent: %s.\n", error->message);
4522 g_object_unref(lib_parent);
4523 g_object_unref(dat_parent);
4524 dt_free(lib_tmp_format);
4525 dt_free(dat_tmp_format);
4526 g_queue_free_full(lib_snaps, g_free);
4527 g_queue_free_full(dat_snaps, g_free);
4528 g_queue_free_full(tmplib_snaps, g_free);
4529 g_queue_free_full(tmpdat_snaps, g_free);
4530 g_file_enumerator_close(library_dir_files, NULL, NULL);
4531 g_object_unref(library_dir_files);
4532 g_error_free(error);
4533 return NULL;
4534 }
4535 g_file_enumerator_close(library_dir_files, NULL, NULL);
4536 g_object_unref(library_dir_files);
4537 }
4538 else
4539 {
4540 //well... fun.
4541
4542 GError *error = NULL;
4543 GFileEnumerator *library_dir_files = g_file_enumerate_children(lib_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4544 if(IS_NULL_PTR(library_dir_files))
4545 {
4546 _db_print("[db backup] couldn't enumerate library parent: %s.\n", error->message);
4547 g_object_unref(lib_parent);
4548 g_object_unref(dat_parent);
4549 dt_free(lib_snap_format);
4550 dt_free(dat_snap_format);
4551 dt_free(lib_tmp_format);
4552 dt_free(dat_tmp_format);
4553 g_error_free(error);
4554 g_queue_free(lib_snaps);
4555 g_queue_free(dat_snaps);
4556 g_queue_free(tmplib_snaps);
4557 g_queue_free(tmpdat_snaps);
4558 return NULL;
4559 }
4560
4561 GFileEnumerator *data_dir_files = g_file_enumerate_children(dat_parent, G_FILE_ATTRIBUTE_STANDARD_NAME, G_FILE_QUERY_INFO_NONE, NULL, &error);
4562 if(IS_NULL_PTR(data_dir_files))
4563 {
4564 _db_print("[db backup] couldn't enumerate data parent: %s.\n", error->message);
4565 g_object_unref(lib_parent);
4566 g_object_unref(dat_parent);
4567 dt_free(lib_snap_format);
4568 dt_free(dat_snap_format);
4569 dt_free(lib_tmp_format);
4570 dt_free(dat_tmp_format);
4571 g_file_enumerator_close(library_dir_files, NULL, NULL);
4572 g_object_unref(library_dir_files);
4573 g_error_free(error);
4574 g_queue_free(lib_snaps);
4575 g_queue_free(dat_snaps);
4576 g_queue_free(tmplib_snaps);
4577 g_queue_free(tmpdat_snaps);
4578 return NULL;
4579 }
4580
4581 GFileInfo *info = NULL;
4582
4583 while ((info = g_file_enumerator_next_file(library_dir_files, NULL, &error)))
4584 {
4585 const char* fname = g_file_info_get_name(info);
4586 if(g_str_has_prefix(fname, lib_snap_format))
4587 {
4588 _db_print("[db backup] found file: %s.\n", fname);
4589 g_queue_insert_sorted(lib_snaps, g_strdup(fname), _db_snap_sort, NULL);
4590 }
4591 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4592 {
4593 // we remove all incomplete snaps matching pattern in BOTH dirs
4594 g_queue_push_head(tmplib_snaps, g_strdup(fname));
4595 }
4596 g_object_unref(info);
4597 }
4598 dt_free(lib_snap_format);
4599
4600 if(error)
4601 {
4602 _db_print("[db backup] problem enumerating library parent: %s.\n", error->message);
4603 g_object_unref(lib_parent);
4604 g_object_unref(dat_parent);
4605 dt_free(lib_tmp_format);
4606 dt_free(dat_tmp_format);
4607 g_queue_free_full(lib_snaps, g_free);
4608 g_queue_free(dat_snaps);
4609 g_queue_free_full(tmplib_snaps, g_free);
4610 g_queue_free(tmpdat_snaps);
4611 g_file_enumerator_close(library_dir_files, NULL, NULL);
4612 g_object_unref(library_dir_files);
4613 g_file_enumerator_close(data_dir_files, NULL, NULL);
4614 g_object_unref(data_dir_files);
4615 g_error_free(error);
4616 return NULL;
4617 }
4618 g_file_enumerator_close(library_dir_files, NULL, NULL);
4619 g_object_unref(library_dir_files);
4620
4621 while ((info = g_file_enumerator_next_file(data_dir_files, NULL, &error)))
4622 {
4623 const char* fname = g_file_info_get_name(info);
4624 if(g_str_has_prefix(fname, dat_snap_format))
4625 {
4626 _db_print("[db backup] found file: %s.\n", fname);
4627 g_queue_insert_sorted(dat_snaps, g_strdup(fname), _db_snap_sort, NULL);
4628 }
4629 else if(g_str_has_prefix(fname, lib_tmp_format) || g_str_has_prefix(fname, dat_tmp_format))
4630 {
4631 //we add to queue both matches - it just depends on parent
4632 g_queue_push_head(tmpdat_snaps, g_strdup(fname));
4633 }
4634 g_object_unref(info);
4635 }
4636 dt_free(dat_snap_format);
4637 dt_free(lib_tmp_format);
4638 dt_free(dat_tmp_format);
4639
4640 if(error)
4641 {
4642 _db_print("[db backup] problem enumerating data parent: %s.\n", error->message);
4643 g_object_unref(lib_parent);
4644 g_object_unref(dat_parent);
4645 g_queue_free_full(lib_snaps, g_free);
4646 g_queue_free_full(dat_snaps, g_free);
4647 g_queue_free_full(tmplib_snaps, g_free);
4648 g_queue_free_full(tmpdat_snaps, g_free);
4649 g_file_enumerator_close(data_dir_files, NULL, NULL);
4650 g_object_unref(data_dir_files);
4651 g_error_free(error);
4652 return NULL;
4653 }
4654
4655 g_file_enumerator_close(data_dir_files, NULL, NULL);
4656 g_object_unref(data_dir_files);
4657 }
4658
4659 // here we have list of snaps sorted in date order, now we have to
4660 // create from that list of snaps to be deleted and return that :D
4661
4662 GPtrArray *ret = g_ptr_array_new();
4663
4664 gchar *lib_parent_path = g_file_get_path(lib_parent);
4665 g_object_unref(lib_parent);
4666
4667 while(g_queue_get_length(lib_snaps) > keep_snaps)
4668 {
4669 gchar *head = g_queue_pop_head(lib_snaps);
4670 g_ptr_array_add(ret, g_strconcat(lib_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4671 dt_free(head);
4672 }
4673 while(!g_queue_is_empty(tmplib_snaps))
4674 {
4675 gchar *head = g_queue_pop_head(tmplib_snaps);
4676 g_ptr_array_add(ret, g_strconcat(lib_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4677 dt_free(head);
4678 }
4679 dt_free(lib_parent_path);
4680 g_queue_free_full(lib_snaps, g_free);
4681 g_queue_free_full(tmplib_snaps, g_free); // should be totally freed, but eh - this won't make doublefree
4682
4683 gchar *dat_parent_path = g_file_get_path(dat_parent);
4684 g_object_unref(dat_parent);
4685
4686 while(g_queue_get_length(dat_snaps) > keep_snaps)
4687 {
4688 gchar *head = g_queue_pop_head(dat_snaps);
4689 g_ptr_array_add(ret, g_strconcat(dat_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4690 dt_free(head);
4691 }
4692 while(!g_queue_is_empty(tmpdat_snaps))
4693 {
4694 gchar *head = g_queue_pop_head(tmpdat_snaps);
4695 g_ptr_array_add(ret, g_strconcat(dat_parent_path, G_DIR_SEPARATOR_S, head, NULL));
4696 dt_free(head);
4697 }
4698 dt_free(dat_parent_path);
4699 g_queue_free_full(dat_snaps, g_free);
4700 g_queue_free_full(tmpdat_snaps, g_free);
4701
4702 g_ptr_array_add (ret, NULL);
4703
4704 return (char**)g_ptr_array_free(ret, FALSE);
4705}
4706
4707gchar *dt_database_get_most_recent_snap(const char* db_filename)
4708{
4709 if(!g_strcmp0(db_filename, ":memory:"))
4710 return NULL;
4711
4712 _db_print("[db backup] checking snapshots existence.\n");
4713 GFile *db_file = g_file_parse_name(db_filename);
4714 GFile *parent = g_file_get_parent(db_file);
4715
4716 if(IS_NULL_PTR(parent))
4717 {
4718 _db_print("[db backup] couldn't get database parent!.\n");
4719 g_object_unref(db_file);
4720 return NULL;
4721 }
4722
4723 GError *error = NULL;
4724 GFileEnumerator *db_dir_files = g_file_enumerate_children(parent, G_FILE_ATTRIBUTE_STANDARD_NAME "," G_FILE_ATTRIBUTE_TIME_MODIFIED, G_FILE_QUERY_INFO_NONE, NULL, &error);
4725
4726 if(IS_NULL_PTR(db_dir_files))
4727 {
4728 _db_print("[db backup] couldn't enumerate database parent: %s.\n", error->message);
4729 g_object_unref(parent);
4730 g_object_unref(db_file);
4731 g_error_free(error);
4732 return NULL;
4733 }
4734
4735 gchar *db_basename = g_file_get_basename(db_file);
4736 g_object_unref(db_file);
4737
4738 gchar *db_snap_format = g_strdup_printf("%s-snp-", db_basename);
4739 gchar *db_backup_format = g_strdup_printf("%s-pre-", db_basename);
4740 dt_free(db_basename);
4741
4742 GFileInfo *info = NULL;
4743 guint64 last_snap = 0;
4744 gchar *last_snap_name = NULL;
4745
4746 while ((info = g_file_enumerator_next_file(db_dir_files, NULL, &error)))
4747 {
4748 const char* fname = g_file_info_get_name(info);
4749 if(g_str_has_prefix(fname, db_snap_format) || g_str_has_prefix(fname, db_backup_format))
4750 {
4751 _db_print("[db backup] found file: %s.\n", fname);
4752 if(last_snap == 0)
4753 {
4754 last_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4755 last_snap_name = g_strdup(fname);
4756 g_object_unref(info);
4757 continue;
4758 }
4759 guint64 try_snap = g_file_info_get_attribute_uint64(info, G_FILE_ATTRIBUTE_TIME_MODIFIED);
4760 if(try_snap > last_snap)
4761 {
4762 last_snap = try_snap;
4763 dt_free(last_snap_name);
4764 last_snap_name = g_strdup(fname);
4765 }
4766 }
4767 g_object_unref(info);
4768 }
4769 dt_free(db_snap_format);
4770 dt_free(db_backup_format);
4771
4772 if(error)
4773 {
4774 _db_print("[db backup] problem enumerating database parent: %s.\n", error->message);
4775 g_file_enumerator_close(db_dir_files, NULL, NULL);
4776 g_object_unref(db_dir_files);
4777 g_error_free(error);
4778 dt_free(last_snap_name);
4779 return NULL;
4780 }
4781
4782 g_file_enumerator_close(db_dir_files, NULL, NULL);
4783 g_object_unref(db_dir_files);
4784
4785 if(IS_NULL_PTR(last_snap_name))
4786 {
4787 g_object_unref(parent);
4788 return NULL;
4789 }
4790
4791 gchar *parent_path = g_file_get_path(parent);
4792 g_object_unref(parent);
4793
4794 gchar *ret = g_strconcat(parent_path, G_DIR_SEPARATOR_S, last_snap_name, NULL);
4795 dt_free(parent_path);
4796 dt_free(last_snap_name);
4797
4798 return ret;
4799}
4800
4801// Nested transactions support
4802//
4803// NOTE: the nested support is actually not activated (see || TRUE below). This current
4804// implementation is just a refactoring of the previous code using:
4805// - dt_database_start_transaction()
4806// - dt_database_release_transaction()
4807// - dt_database_rollback_transaction()
4808//
4809// With this refactoring we can count and check for nested transaction and unmatched
4810// transaction routines. And it has been done to help further implementation for
4811// proper threading and nested transaction support.
4812//
4813void dt_database_start_transaction_debug(void)
4814{
4815 /* was a parameter; the module owns the one connection now */
4816 const dt_database_t *const db = _db;
4817 gpointer const owner = g_thread_self();
4818 const int batch_level = dt_atomic_get_int(&_trx_batch_level);
4819 if(batch_level > 0)
4820 {
4821 // If a batch transaction is active on this thread, suppress nested BEGIN/COMMIT
4822 // to avoid per-module transaction overhead during presets initialization.
4823 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4824 {
4825 dt_atomic_add_int(&_trxid, 1);
4826 return;
4827 }
4828 }
4829
4830 if(g_atomic_pointer_get(&_trx_owner) == owner)
4831 {
4832 dt_atomic_add_int(&_trxid, 1);
4833 return;
4834 }
4835
4836 dt_pthread_rwlock_wrlock(&_db_lock);
4837 g_atomic_pointer_set(&_trx_owner, owner);
4838
4839 const int trxid = dt_atomic_add_int(&_trxid, 1);
4840
4841 // if top level a simple unamed transaction is used BEGIN / COMMIT / ROLLBACK
4842 // otherwise we use a savepoint (named transaction).
4843
4844 if(trxid == 0)
4845 {
4846 // In theads application it may be safer to use an IMMEDIATE transaction:
4847 // "BEGIN IMMEDIATE TRANSACTION"
4848 // This implies "BEGIN DEFERRED TRANSACTION", which means
4849 // no write event is dispatched to DB until the first "COMMIT"
4850 DT_DEBUG_SQLITE3_EXEC(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
4851 }
4852#ifdef USE_NESTED_TRANSACTIONS
4853 else
4854 {
4855 char SQLTRX[32] = { 0 };
4856 g_snprintf(SQLTRX, sizeof(SQLTRX), "SAVEPOINT trx%d", trxid);
4857 DT_DEBUG_SQLITE3_EXEC(db->handle, SQLTRX, NULL, NULL, NULL);
4858 }
4859#endif
4860
4861 if(trxid > MAX_NESTED_TRANSACTIONS)
4862 fprintf(stderr, "[dt_database_start_transaction] more than %d nested transaction\n", MAX_NESTED_TRANSACTIONS);
4863}
4864
4865void dt_database_release_transaction_debug(void)
4866{
4867 /* was a parameter; the module owns the one connection now */
4868 const dt_database_t *const db = _db;
4869 gpointer const owner = g_thread_self();
4870 const int batch_level = dt_atomic_get_int(&_trx_batch_level);
4871 if(batch_level > 0)
4872 {
4873 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4874 {
4875 dt_atomic_sub_int(&_trxid, 1);
4876 return;
4877 }
4878 }
4879
4880 if(g_atomic_pointer_get(&_trx_owner) != owner)
4881 {
4882 fprintf(stderr, "[dt_database_release_transaction] COMMIT from non-owner thread\n");
4883 return;
4884 }
4885
4886 const int trxid = dt_atomic_sub_int(&_trxid, 1);
4887
4888 if(trxid <= 0)
4889 fprintf(stderr, "[dt_database_release_transaction] COMMIT outside a transaction\n");
4890
4891 if(trxid == 1)
4892 {
4893 DT_DEBUG_SQLITE3_EXEC(db->handle, "COMMIT TRANSACTION", NULL, NULL, NULL);
4894 g_atomic_pointer_set(&_trx_owner, NULL);
4895 dt_pthread_rwlock_unlock(&_db_lock);
4896 }
4897#ifdef USE_NESTED_TRANSACTIONS
4898 else
4899 {
4900 char SQLTRX[64] = { 0 };
4901 g_snprintf(SQLTRX, sizeof(SQLTRX), "RELEASE SAVEPOINT trx%d", trxid - 1);
4902 DT_DEBUG_SQLITE3_EXEC(db->handle, SQLTRX, NULL, NULL, NULL);
4903 }
4904#endif
4905}
4906
4907void dt_database_rollback_transaction(void)
4908{
4909 /* was a parameter; the module owns the one connection now */
4910 const dt_database_t *const db = _db;
4911 gpointer const owner = g_thread_self();
4912 if(g_atomic_pointer_get(&_trx_owner) != owner && g_atomic_pointer_get(&_trx_batch_owner) != owner)
4913 {
4914 fprintf(stderr, "[dt_database_rollback_transaction] ROLLBACK from non-owner thread\n");
4915 return;
4916 }
4917
4918 const int trxid = dt_atomic_sub_int(&_trxid, 1);
4919
4920 if(trxid <= 0)
4921 fprintf(stderr, "[dt_database_rollback_transaction] ROLLBACK outside a transaction\n");
4922
4923 if(trxid >= 1)
4924 {
4925 DT_DEBUG_SQLITE3_EXEC(db->handle, "ROLLBACK TRANSACTION", NULL, NULL, NULL);
4926 dt_atomic_set_int(&_trxid, 0);
4927 dt_atomic_set_int(&_trx_batch_level, 0);
4928 g_atomic_pointer_set(&_trx_owner, NULL);
4929 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4930 dt_pthread_rwlock_unlock(&_db_lock);
4931 }
4932#ifdef USE_NESTED_TRANSACTIONS
4933 else
4934 {
4935 char SQLTRX[64] = { 0 };
4936 g_snprintf(SQLTRX, sizeof(SQLTRX), "ROLLBACK TRANSACTION TO SAVEPOINT trx%d", trxid - 1);
4937 DT_DEBUG_SQLITE3_EXEC(db->handle, SQLTRX, NULL, NULL, NULL);
4938 }
4939#endif
4940}
4941
4942void dt_database_begin_transaction_batch(void)
4943{
4944 /* was a parameter; the module owns the one connection now */
4945 const dt_database_t *const db = _db;
4946 gpointer const owner = g_thread_self();
4947 if(g_atomic_pointer_get(&_trx_batch_owner) == owner)
4948 {
4949 dt_atomic_add_int(&_trx_batch_level, 1);
4950 return;
4951 }
4952
4953 dt_pthread_rwlock_wrlock(&_db_lock);
4954 g_atomic_pointer_set(&_trx_owner, owner);
4955 g_atomic_pointer_set(&_trx_batch_owner, owner);
4956
4957 const int level = dt_atomic_add_int(&_trx_batch_level, 1);
4958 if(level == 0)
4959 {
4960 DT_DEBUG_SQLITE3_EXEC(db->handle, "BEGIN TRANSACTION", NULL, NULL, NULL);
4961 }
4962}
4963
4964void dt_database_end_transaction_batch(void)
4965{
4966 /* was a parameter; the module owns the one connection now */
4967 const dt_database_t *const db = _db;
4968 gpointer const owner = g_thread_self();
4969 if(g_atomic_pointer_get(&_trx_batch_owner) != owner)
4970 {
4971 _db_print("[dt_database_end_transaction_batch] COMMIT from non-owner thread\n");
4972 return;
4973 }
4974
4975 const int level = dt_atomic_sub_int(&_trx_batch_level, 1);
4976 if(level <= 0)
4977 {
4978 _db_print("[dt_database_end_transaction_batch] COMMIT outside a batch transaction\n");
4979 dt_atomic_set_int(&_trx_batch_level, 0);
4980 g_atomic_pointer_set(&_trx_owner, NULL);
4981 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4982 dt_pthread_rwlock_unlock(&_db_lock);
4983 return;
4984 }
4985
4986 if(level == 1)
4987 {
4988 DT_DEBUG_SQLITE3_EXEC(db->handle, "COMMIT TRANSACTION", NULL, NULL, NULL);
4989 g_atomic_pointer_set(&_trx_owner, NULL);
4990 g_atomic_pointer_set(&_trx_batch_owner, NULL);
4991 dt_pthread_rwlock_unlock(&_db_lock);
4992 }
4993}
4994
4995// clang-format off
4996// modelines: These editor modelines have been set for all relevant files by tools/update_modelines.py
4997// vim: shiftwidth=2 expandtab tabstop=2 cindent
4998// kate: tab-indents: off; indent-width 2; replace-tabs on; indent-mode cstyle; remove-trailing-spaces modified;
4999// clang-format on
error
static void error(char *msg)
Definition ashift_lsd.c:202
TRUE
#define TRUE
Definition ashift_lsd.c:162
FALSE
#define FALSE
Definition ashift_lsd.c:158
dt_atomic_set_int
void dt_atomic_set_int(dt_atomic_int *var, int value)
dt_atomic_get_int
int dt_atomic_get_int(dt_atomic_int *var)
dt_atomic_sub_int
int dt_atomic_sub_int(dt_atomic_int *var, int decr)
dt_atomic_add_int
int dt_atomic_add_int(dt_atomic_int *var, int incr)
dt_atomic_int
atomic_int dt_atomic_int
Definition atomic.h:68
void
typedef void((*dt_cache_allocate_t)(void *userdata, dt_cache_entry_t *entry))
kind
static const dt_adaptation_t kind
Definition chromatic_adaptation.h:125
v
const float v
Definition colorprofiles/iop_profile.h:536
i
const float i
Definition colorspaces_inline_conversions.h:441
L
const float L
Definition colorspaces_inline_conversions.h:494
n
const float n
Definition colorspaces_inline_conversions.h:679
dt_image_flags_from_extension
dt_image_flags_t dt_image_flags_from_extension(const char *extension)
Definition common/image.c:1889
darktable_package_version
const char darktable_package_version[]
_db_lock
static dt_pthread_rwlock_t _db_lock
Definition database.c:141
_lock_single_database
static gboolean _lock_single_database(dt_database_t *db, const char *dbfilename, char **lockfile)
Definition database.c:2831
_print_backup_progress
static void _print_backup_progress(int remaining, int total)
Definition database.c:4070
dt_database_set_renamed_handler
void dt_database_set_renamed_handler(dt_database_renamed_handler_t handler)
Definition database.c:3635
dt_database_settings_free
void dt_database_settings_free(dt_database_settings_t *settings)
Definition database.c:3719
dt_database_backup
void dt_database_backup(const char *filename)
Definition database.c:2976
dt_database_optimize
void dt_database_optimize(void)
Definition database.c:4058
dt_database_start_transaction_debug
void dt_database_start_transaction_debug(void)
Definition database.c:4813
TRY_PREPARE
#define TRY_PREPARE(_stmt, _query, _message)
Definition database.c:523
_settings
static dt_database_settings_t _settings
Definition database.c:147
dt_database_get_most_recent_snap
gchar * dt_database_get_most_recent_snap(const char *db_filename)
Definition database.c:4707
_trx_batch_owner
static gpointer _trx_batch_owner
Definition database.c:98
pid_is_alive
static gboolean pid_is_alive(int pid)
Definition database.c:2784
dt_database_set_settings
void dt_database_set_settings(const dt_database_settings_t *settings)
Definition database.c:3674
MAX_NESTED_TRANSACTIONS
#define MAX_NESTED_TRANSACTIONS
Definition database.c:93
dt_database_begin_transaction_batch
void dt_database_begin_transaction_batch(void)
Definition database.c:4942
_is_mem_db
static gboolean _is_mem_db(void)
Definition database.c:3973
dt_database_release_transaction_debug
void dt_database_release_transaction_debug(void)
Definition database.c:4865
ask_for_upgrade
void ask_for_upgrade(const gchar *dbname, const gboolean has_gui)
Definition database.c:2957
_get_pragma_string_val
gchar * _get_pragma_string_val(sqlite3 *db, const char *pragma)
Definition database.c:3036
_ask_readonly
static dt_database_response_t _ask_readonly(const char *dbfilename)
Definition database.c:3095
_trxid
static dt_atomic_int _trxid
Definition database.c:95
_ask_user
static dt_database_response_t _ask_user(const dt_database_prompt_context_t *context)
Definition database.c:3070
dt_database_snaps_to_remove
char ** dt_database_snaps_to_remove(void)
Definition database.c:4416
_settings_lock
static dt_pthread_mutex_t _settings_lock
Definition database.c:148
dt_database_take_error
void dt_database_take_error(dt_database_error_t *error)
Definition database.c:2734
_verbose
static gboolean _verbose
Definition database.c:153
dt_database_error_free
void dt_database_error_free(dt_database_error_t *error)
Definition database.c:2751
dt_database_get_settings
void dt_database_get_settings(dt_database_settings_t *settings)
Definition database.c:3693
_trx_owner
static gpointer _trx_owner
Definition database.c:97
_sanitize_db
static void _sanitize_db(dt_database_t *db)
Definition database.c:2644
dt_database_get_sqlite3_global
sqlite3 * dt_database_get_sqlite3_global(void)
Definition database.c:3790
dt_database_get_last_error
const char * dt_database_get_last_error(void)
Definition database.c:3784
dt_database_set_prompt_handler
void dt_database_set_prompt_handler(dt_database_prompt_handler_t handler)
Definition database.c:3062
FINALIZE
#define FINALIZE
Definition database.c:537
dt_database_close
void dt_database_close(void)
Definition database.c:3757
ERRCHECK
#define ERRCHECK
Definition database.c:3890
_migrate_schema
static gboolean _migrate_schema(dt_database_t *db, int version)
Definition database.c:186
_ask_corrupted
static dt_database_response_t _ask_corrupted(const char *dbfilename, const char *quick_check, const gboolean snapshot_available)
Definition database.c:3085
_database_delete_mipmaps_files
static void _database_delete_mipmaps_files()
Definition database.c:3848
_upgrade_library_schema_step
static int _upgrade_library_schema_step(dt_database_t *db, int version)
Definition database.c:540
TRY_EXEC
#define TRY_EXEC(_query, _message)
Definition database.c:497
dt_database_snapshot
gboolean dt_database_snapshot(void)
Definition database.c:4126
dt_database_get_path
const gchar * dt_database_get_path(void)
Definition database.c:3795
_has_gui
static gboolean _has_gui
Definition database.c:152
_ask_for_maintenance
static gboolean _ask_for_maintenance(const char *maintenance_check, const gboolean closing_time, const guint64 size)
Definition database.c:3954
_database_init
static dt_database_t * _database_init(const dt_database_params_t *params)
Definition database.c:3109
_upgrade_data_schema_step
static int _upgrade_data_schema_step(dt_database_t *db, int version)
Definition database.c:2165
dt_database_perform_maintenance
void dt_database_perform_maintenance(void)
Definition database.c:3891
dt_database_cleanup_busy_statements
void dt_database_cleanup_busy_statements(void)
Definition database.c:3869
_SQLITE3_EXEC
#define _SQLITE3_EXEC(a, b, c, d, e)
Definition database.c:177
dt_database_rollback_transaction
void dt_database_rollback_transaction(void)
Definition database.c:4907
_db_lock_inited
static gboolean _db_lock_inited
Definition database.c:142
_get_pragma_int_val
int _get_pragma_int_val(sqlite3 *db, const char *pragma)
Definition database.c:3020
_create_library_schema
static void _create_library_schema(dt_database_t *db)
Definition database.c:2413
_snapshot_policy_decides
static gboolean _snapshot_policy_decides(GTimeSpan *interval, gboolean *answer)
Definition database.c:4188
_upgrade_library_schema
static gboolean _upgrade_library_schema(dt_database_t *db, int version)
Definition database.c:2384
_backup_db
static int _backup_db(sqlite3 *src_db, const char *src_db_name, const char *dest_filename, void(*xProgress)(int, int))
Definition database.c:4076
_trx_batch_level
static dt_atomic_int _trx_batch_level
Definition database.c:96
_database_free
static void _database_free(dt_database_t *db)
Definition database.c:3735
_db_print
#define _db_print(...)
Definition database.c:161
_db
static dt_database_t * _db
Definition database.c:123
_settings_lock_inited
static gboolean _settings_lock_inited
Definition database.c:149
CURRENT_DATABASE_VERSION_LIBRARY
#define CURRENT_DATABASE_VERSION_LIBRARY
Definition database.c:89
_renamed_handler
static dt_database_renamed_handler_t _renamed_handler
Definition database.c:156
_upgrade_data_schema
static gboolean _upgrade_data_schema(dt_database_t *db, int version)
Definition database.c:2399
_create_memory_schema
static void _create_memory_schema(dt_database_t *db)
Definition database.c:2572
TRY_STEP
#define TRY_STEP(_stmt, _expected, _message)
Definition database.c:510
dt_database_maybe_maintenance
gboolean dt_database_maybe_maintenance(const gboolean closing_time)
Definition database.c:3978
dt_database_is_open
gboolean dt_database_is_open(void)
Definition database.c:3779
dt_database_open
dt_database_open_result_t dt_database_open(const dt_database_params_t *params)
Definition database.c:3640
_database_migrate_to_xdg_structure
static gchar * _database_migrate_to_xdg_structure(const char *configured_db)
Definition database.c:3805
dt_database_maybe_snapshot
gboolean dt_database_maybe_snapshot(void)
Definition database.c:4235
_ask_upgrade
static dt_database_response_t _ask_upgrade(const char *dbfilename)
Definition database.c:3102
dt_database_delete_lock_files
int dt_database_delete_lock_files(const char *dbfilename)
Definition database.c:2760
_lock_databases
static gboolean _lock_databases(dt_database_t *db)
Definition database.c:2935
dt_database_end_transaction_batch
void dt_database_end_transaction_batch(void)
Definition database.c:4964
_get_iso8601_int
static gboolean _get_iso8601_int(const gchar *text, gsize length, gint *value)
Definition database.c:4343
_create_data_schema
static void _create_data_schema(dt_database_t *db)
Definition database.c:2519
_prompt_handler
static dt_database_prompt_handler_t _prompt_handler
Definition database.c:3060
_db_snap_sort
static gint _db_snap_sort(gconstpointer a, gconstpointer b, gpointer user_data)
Definition database.c:4363
CURRENT_DATABASE_VERSION_DATA
#define CURRENT_DATABASE_VERSION_DATA
Definition database.c:90
dt_database_prompt_handler_t
dt_database_response_t(* dt_database_prompt_handler_t)(const dt_database_prompt_context_t *context)
Definition database.h:245
dt_database_renamed_handler_t
void(* dt_database_renamed_handler_t)(const char *new_library_name)
Definition database.h:158
dt_database_response_t
dt_database_response_t
Definition database.h:209
DT_DATABASE_RESPONSE_CLOSE
@ DT_DATABASE_RESPONSE_CLOSE
Definition database.h:212
DT_DATABASE_RESPONSE_PROCEED
@ DT_DATABASE_RESPONSE_PROCEED
Definition database.h:218
DT_DATABASE_RESPONSE_RESTORE
@ DT_DATABASE_RESPONSE_RESTORE
Definition database.h:214
DT_DATABASE_RESPONSE_DELETE
@ DT_DATABASE_RESPONSE_DELETE
Definition database.h:216
dt_database_open_result_t
dt_database_open_result_t
Definition database.h:74
DT_DATABASE_OPEN_FAILED
@ DT_DATABASE_OPEN_FAILED
Definition database.h:76
DT_DATABASE_OPEN_LOCKED
@ DT_DATABASE_OPEN_LOCKED
Definition database.h:82
DT_DATABASE_OPEN_OK
@ DT_DATABASE_OPEN_OK
Definition database.h:78
DT_DATABASE_PROMPT_UPGRADE
@ DT_DATABASE_PROMPT_UPGRADE
Definition database.h:202
DT_DATABASE_PROMPT_MAINTENANCE
@ DT_DATABASE_PROMPT_MAINTENANCE
Definition database.h:205
DT_DATABASE_PROMPT_CORRUPTED
@ DT_DATABASE_PROMPT_CORRUPTED
Definition database.h:199
DT_DATABASE_PROMPT_READONLY
@ DT_DATABASE_PROMPT_READONLY
Definition database.h:197
dt_datetime_exif_to_gdatetime
GDateTime * dt_datetime_exif_to_gdatetime(const char *exif, const GTimeZone *tz)
Definition datetime.c:260
dt_datetime_utc_tz
GTimeZone * dt_datetime_utc_tz(void)
Definition datetime.c:40
dt_datetime_gdatetime_to_gtimespan
GTimeSpan dt_datetime_gdatetime_to_gtimespan(GDateTime *gdt)
Definition datetime.c:459
names
GHashTable * names
GtkWidget* -> the name the application knows it by.
Definition doc_screenshot.c:112
status
GtkWidget * status
result of the last capture
Definition doc_screenshot.c:109
dt_pthread_rwlock_wrlock
static int dt_pthread_rwlock_wrlock(dt_pthread_rwlock_t *rwlock) ACQUIRE(rwlock) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:299
dt_pthread_rwlock_unlock
static int dt_pthread_rwlock_unlock(dt_pthread_rwlock_t *rwlock) RELEASE_GENERIC(rwlock) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:217
dt_pthread_mutex_unlock
static int dt_pthread_mutex_unlock(dt_pthread_mutex_t *mutex) RELEASE(mutex) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:127
dt_pthread_mutex_init
static int dt_pthread_mutex_init(dt_pthread_mutex_t *mutex, const pthread_mutexattr_t *mutexattr)
Initialise a mutex. With mutexattr NULL – which is how 54 of the 56 call sites in this tree spell it ...
Definition dtpthread.h:104
dt_pthread_rwlock_init
static int dt_pthread_rwlock_init(dt_pthread_rwlock_t *lock, const pthread_rwlockattr_t *attr)
Definition dtpthread.h:192
dt_pthread_mutex_lock
static int dt_pthread_mutex_lock(dt_pthread_mutex_t *mutex) ACQUIRE(mutex) NO_THREAD_SAFETY_ANALYSIS
Definition dtpthread.h:117
dt_loc_get_user_cache_dir
void dt_loc_get_user_cache_dir(char *cachedir, size_t bufsize)
Definition file_location.c:376
dt_loc_get_datadir
void dt_loc_get_datadir(char *datadir, size_t bufsize)
Definition file_location.c:384
dt_loc_get_user_config_dir
void dt_loc_get_user_config_dir(char *configdir, size_t bufsize)
Definition file_location.c:372
file_location.h
UNKNOWN_IMAGE
#define UNKNOWN_IMAGE
Definition image.h:78
dt_image_flags_t
dt_image_flags_t
Definition image.h:104
DT_IMAGE_RAW
@ DT_IMAGE_RAW
Definition image.h:124
DT_IMAGE_HDR
@ DT_IMAGE_HDR
Definition image.h:126
DT_IMAGE_LDR
@ DT_IMAGE_LDR
Definition image.h:122
dt_ioppr_get_iop_order_list_version
GList * dt_ioppr_get_iop_order_list_version(dt_iop_order_t version)
Return the built-in order list for a given version.
Definition iop_order.c:1066
dt_sort_iop_list_by_order_f
gint dt_sort_iop_list_by_order_f(gconstpointer a, gconstpointer b)
Compare two list nodes holding modules by iop_order.
Definition iop_order.c:939
dt_ioppr_get_iop_order_list_kind
dt_iop_order_t dt_ioppr_get_iop_order_list_kind(GList *iop_order_list)
Determine the kind of an order list by inspecting its content.
Definition iop_order.c:951
dt_ioppr_serialize_text_iop_order_list
char * dt_ioppr_serialize_text_iop_order_list(GList *iop_order_list)
Serialize an order list to a text representation.
Definition iop_order.c:2507
iop_order.h
dt_iop_order_t
dt_iop_order_t
Definition iop_order.h:149
DT_IOP_ORDER_ANSEL_RAW
@ DT_IOP_ORDER_ANSEL_RAW
Definition iop_order.h:154
DT_IOP_ORDER_LEGACY
@ DT_IOP_ORDER_LEGACY
Definition iop_order.h:151
DT_IOP_ORDER_CUSTOM
@ DT_IOP_ORDER_CUSTOM
Definition iop_order.h:150
params
static const dt_iop_drawlayer_params_t * params
Definition layers.h:80
dt_legacy_presets_create
void dt_legacy_presets_create(sqlite3 *handle)
Definition legacy_presets.c:1134
legacy_presets.h
k
float *const restrict const size_t k
Definition luminance_mask.h:81
IS_NULL_PTR
#define IS_NULL_PTR(p)
C is way too permissive with !=, == and if(var) checks, which can mean too many things depending on w...
Definition macros.h:96
dt_free_gpointer
static void dt_free_gpointer(gpointer ptr)
g_free() one pointer, with the signature GDestroyNotify wants.
Definition mem_alloc.h:184
dt_free
#define dt_free(ptr)
g_free() ptr and set it to NULL, skipping both if it is already NULL.
Definition mem_alloc.h:171
size
size_t size
Definition mipmap_cache.c:3
flags
dt_mipmap_buffer_dsc_flags flags
Definition mipmap_cache.c:4
DT_PATH_MAX
#define DT_PATH_MAX
Buffer size for a filesystem path anywhere in Ansel.
Definition paths.h:57
dt_concat_path_file
void dt_concat_path_file(char destination[4096], const char path[4096], const char *const file)
Append a constant filename to a variable, stack-based, fixed-sized, directory, and add a / in-between...
Definition darktable.c:2674
name
const char * name
Definition pdf.h:90
value
static const dt_aligned_pixel_simd_t value
Definition simd.h:144
sql_debug.h
Checked wrappers around the sqlite3_* calls: trace the statement, assert the return code,...
DT_DEBUG_SQLITE3_EXEC
#define DT_DEBUG_SQLITE3_EXEC(a, b, c, d, e)
Definition sql_debug.h:129
sqlite3IcuInit
int sqlite3IcuInit(sqlite3 *db)
Definition sqliteicu.c:523
day
static const char *const day[7]
Definition strptime.c:97
dt_database_error_t
Definition database.h:167
dt_database_params_t
Definition database.h:58
dt_database_prompt_context_t
Definition database.h:224
dt_database_prompt_context_t::prompt
dt_database_prompt_t prompt
Definition database.h:225
dt_database_prompt_context_t::dbfilename
const char * dbfilename
Definition database.h:227
dt_database_settings_t
Definition database.h:132
dt_database_settings_t::create_snapshot
char * create_snapshot
Definition database.h:138
dt_database_settings_t::maintenance_check
char * maintenance_check
Definition database.h:134
dt_database_settings_t::keep_snapshots
int keep_snapshots
Definition database.h:140
dt_database_settings_t::maintenance_freepage_ratio
int maintenance_freepage_ratio
Definition database.h:136
dt_database_t
Definition database.c:101
dt_database_t::error_other_pid
int error_other_pid
Definition database.c:114
dt_database_t::lockfile_data
gchar * lockfile_data
Definition database.c:105
dt_database_t::dbfilename_library
gchar * dbfilename_library
Definition database.c:108
dt_database_t::handle
sqlite3 * handle
Definition database.c:111
dt_database_t::lockfile_library
gchar * lockfile_library
Definition database.c:108
dt_database_t::error_message
gchar * error_message
Definition database.c:113
dt_database_t::lock_acquired
gboolean lock_acquired
Definition database.c:102
dt_database_t::dbfilename_data
gchar * dbfilename_data
Definition database.c:105
dt_database_t::error_dbfilename
gchar * error_dbfilename
Definition database.c:113
dt_iop_order_entry_t
Definition iop_order.h:160
dt_iop_order_entry_t::operation
char operation[20]
Definition iop_order.h:166
dt_iop_order_entry_t::o
union dt_iop_order_entry_t::@24 o
dt_iop_order_entry_t::iop_order_f
double iop_order_f
Definition iop_order.h:162
dt_iop_order_entry_t::instance
int32_t instance
Definition iop_order.h:167
MIN
#define MIN(a, b)
Definition thinplate.c:32
MAX
#define MAX(a, b)
Definition thinplate.c:29
dt_util_foo_to_utf8
gchar * dt_util_foo_to_utf8(const char *string)
Definition utility.c:419